def calculate_utc_range(org, input_value):
"""
Calculates datetime range in UTC, we use it to check date containment
"""
query_value = str_to_datetime(input_value, org.timezone, org.get_dayfirst(), fill_time=False)
if not query_value:
raise SearchException(_(f"Unable to parse the date '{input_value}'"))
return date_to_utc_range(query_value.date(), org)
def calculate_utc_range(org, input_value):
"""
Calculates datetime range in UTC, we use it to check date containment
"""
query_value = str_to_datetime(input_value,
org.timezone,
org.get_dayfirst(),
fill_time=False)
if not query_value:
raise SearchException(_(f"Unable to parse the date '{input_value}'"))
return date_to_utc_range(query_value.date(), org)
def _build_datetime_field_params(self, field):
lookup = self.DATETIME_LOOKUPS.get(self.comparator)
if not lookup:
raise SearchException(
_("Can't query date fields with %s") % self.comparator)
# parse as localized date
local_date = str_to_datetime(self.value,
field.org.timezone,
field.org.get_dayfirst(),
fill_time=False)
if not local_date:
raise SearchException(
_("Unable to parse the date %s") % self.value)
# get the range of UTC datetimes for this local date
utc_range = date_to_utc_range(local_date.date(), field.org)
if lookup == '<equal>':
return {
'contact_field': field,
'datetime_value__gte': utc_range[0],
'datetime_value__lt': utc_range[1]
}
elif lookup == 'lt':
return {'contact_field': field, 'datetime_value__lt': utc_range[0]}
elif lookup == 'lte':
return {'contact_field': field, 'datetime_value__lt': utc_range[1]}
elif lookup == 'gt':
return {
'contact_field': field,
'datetime_value__gte': utc_range[1]
}
elif lookup == 'gte':
return {
'contact_field': field,
'datetime_value__gte': utc_range[0]
}
def fix_value_results(FlowRun, RuleSet, Value):
cache = get_redis_connection()
# make a map of ruleset IDs to UUIDs
ruleset_id_to_uuid = {r.id: r.uuid for r in RuleSet.objects.all()}
if not ruleset_id_to_uuid:
return
# has this migration been run before but didn't complete?
highpoint = cache.get(CACHE_KEY_HIGHPOINT)
highpoint = 0 if highpoint is None else int(highpoint)
# problematic ruleset values are those which have a decimal and datetime value, as this implies that the input
# was numeric, but was still erroneously parsed as a datetime
values = Value.objects.exclude(decimal_value=None).exclude(
datetime_value=None).exclude(run=None)
# get all flow runs we need to fix
run_ids = values.values_list(
'run_id', flat=True).order_by('run_id').distinct('run_id')
if highpoint:
print("Resuming from previous highpoint at run #%d" % highpoint)
run_ids = run_ids.filter(id__gt=highpoint)
run_ids = array(str('l'), run_ids)
print("Total of %d runs need to be fixed" % len(run_ids))
num_fixed = 0
start = time.time()
for id_batch in chunk_list(run_ids, 1000):
with transaction.atomic():
batch = FlowRun.objects.filter(id__in=id_batch).order_by(
'id').select_related('org').prefetch_related('values')
for run in batch:
day_first = run.org.date_format == 'D'
# find result values with which are no longer parsed as dates
no_longer_dates = {}
for v in run.values.all():
parsed_new = str_to_datetime(v.string_value,
tz=run.org.timezone,
dayfirst=day_first)
if not parsed_new and v.datetime_value:
no_longer_dates[ruleset_id_to_uuid[
v.ruleset_id]] = v.string_value
if no_longer_dates:
results = json.loads(run.results) if run.results else {}
for key, result in six.iteritems(results):
node_uuid = result['node_uuid']
if node_uuid in no_longer_dates:
result['value'] = no_longer_dates[node_uuid]
run.results = json.dumps(results)
run.save(update_fields=('results', ))
cache.set(CACHE_KEY_HIGHPOINT, str(run.id), 60 * 60 * 24 * 7)
num_fixed += 1
fixed_per_sec = num_fixed / (time.time() - start)
# figure out estimated time remaining
num_remaining = len(run_ids) - highpoint
time_remaining = num_remaining / fixed_per_sec
finishes = timezone.now() + timedelta(seconds=time_remaining)
status = " > Updated %d runs of ~%d (%2.2f per sec) Est finish: %s" % (
num_fixed, len(run_ids), fixed_per_sec, finishes)
print(status)
print("Run results fix migration completed in %d mins" %
(int(time.time() - start) // 60))
def evaluate(self, org, contact_json, prop_map):
prop_type, field = prop_map[self.prop]
if prop_type == ContactQuery.PROP_FIELD:
field_uuid = str(field.uuid)
contact_fields = contact_json.get("fields", {})
if field.value_type == Value.TYPE_TEXT:
query_value = self.value.upper()
contact_value = contact_fields.get(field_uuid, {"text": ""}).get("text").upper()
if self.comparator == "=":
return contact_value == query_value
elif self.comparator == "!=":
return contact_value != query_value
else:
raise SearchException(_(f"Unknown text comparator: '{self.comparator}'"))
elif field.value_type == Value.TYPE_NUMBER:
query_value = self._parse_number(self.value)
number_value = contact_fields.get(field_uuid, {"number": None}).get(
"number", contact_fields.get(field_uuid, {"decimal": None}).get("decimal")
)
if number_value is None:
return False
contact_value = self._parse_number(number_value)
if self.comparator == "=":
return contact_value == query_value
elif self.comparator == ">":
return contact_value > query_value
elif self.comparator == ">=":
return contact_value >= query_value
elif self.comparator == "<":
return contact_value < query_value
elif self.comparator == "<=":
return contact_value <= query_value
else:
raise SearchException(_(f"Unknown number comparator: '{self.comparator}'"))
elif field.value_type == Value.TYPE_DATETIME:
query_value = str_to_date(self.value, field.org.get_dayfirst())
if not query_value:
raise SearchException(_(f"Unable to parse the date '{self.value}'"))
lower_bound, upper_bound = date_to_day_range_utc(query_value, org)
contact_datetime_value = contact_fields.get(field_uuid, {"datetime": None}).get("datetime")
if contact_datetime_value is None:
return False
# datetime contact values are serialized as ISO8601 timestamps in local time
contact_value = str_to_datetime(contact_datetime_value, pytz.UTC, field.org.get_dayfirst())
contact_value_utc = contact_value.astimezone(pytz.UTC)
if self.comparator == "=":
return contact_value_utc >= lower_bound and contact_value_utc < upper_bound
elif self.comparator == ">":
return contact_value_utc >= upper_bound
elif self.comparator == ">=":
return contact_value_utc >= lower_bound
elif self.comparator == "<":
return contact_value_utc < lower_bound
elif self.comparator == "<=":
return contact_value_utc < upper_bound
else:
raise SearchException(_(f"Unknown datetime comparator: '{self.comparator}'"))
elif field.value_type in (Value.TYPE_STATE, Value.TYPE_DISTRICT, Value.TYPE_WARD):
query_value = self.value.upper()
if field.value_type == Value.TYPE_WARD:
ward_value = contact_fields.get(field_uuid, {"ward": ""}).get("ward", "")
contact_value = ward_value.upper().split(" > ")[-1]
elif field.value_type == Value.TYPE_DISTRICT:
district_value = contact_fields.get(field_uuid, {"district": ""}).get("district", "")
contact_value = district_value.upper().split(" > ")[-1]
elif field.value_type == Value.TYPE_STATE:
state_value = contact_fields.get(field_uuid, {"state": ""}).get("state", "")
contact_value = state_value.upper().split(" > ")[-1]
else: # pragma: no cover
raise SearchException(_(f"Unknown location type: '{field.value_type}'"))
if self.comparator == "=":
return contact_value == query_value
elif self.comparator == "!=":
return contact_value != query_value
else:
raise SearchException(_(f"Unsupported comparator '{self.comparator}' for location field"))
else: # pragma: no cover
raise SearchException(_(f"Unrecognized contact field type '{field.value_type}'"))
elif prop_type == ContactQuery.PROP_SCHEME:
for urn in contact_json.get("urns"):
if urn.get("scheme") == field:
contact_value = urn.get("path").upper()
query_value = self.value.upper()
if self.comparator == "=":
if contact_value == query_value:
return True
elif self.comparator == "~":
if query_value in contact_value:
return True
else:
raise SearchException(_(f"Unknown urn scheme comparator: '{self.comparator}'"))
return False
elif prop_type == ContactQuery.PROP_ATTRIBUTE:
field_key = field.key
if field_key == "language":
query_value = self.value.upper()
raw_contact_value = contact_json.get("language")
if raw_contact_value is None:
contact_value = ""
else:
contact_value = raw_contact_value.upper()
if self.comparator == "=":
return contact_value == query_value
elif self.comparator == "!=":
return contact_value != query_value
else:
raise SearchException(_(f"Unknown language comparator: '{self.comparator}'"))
elif field_key == "created_on":
query_value = str_to_date(self.value, field.org.get_dayfirst())
if not query_value:
raise SearchException(_(f"Unable to parse the date '{self.value}'"))
lower_bound, upper_bound = date_to_day_range_utc(query_value, org)
# contact created_on is serialized as ISO8601 timestamp in utc time
contact_value = str_to_datetime(contact_json.get("created_on"), pytz.UTC, field.org.get_dayfirst())
contact_value_utc = contact_value.astimezone(pytz.UTC)
if self.comparator == "=":
return contact_value_utc >= lower_bound and contact_value_utc < upper_bound
elif self.comparator == ">":
return contact_value_utc >= upper_bound
elif self.comparator == ">=":
return contact_value_utc >= lower_bound
elif self.comparator == "<":
return contact_value_utc < lower_bound
elif self.comparator == "<=":
return contact_value_utc < upper_bound
else:
raise SearchException(_(f"Unknown created_on comparator: '{self.comparator}'"))
elif field_key == "name":
query_value = self.value.upper()
raw_contact_value = contact_json.get("name")
if raw_contact_value is None:
contact_value = ""
else:
contact_value = raw_contact_value.upper()
if self.comparator == "=":
return contact_value == query_value
elif self.comparator == "~":
return query_value in contact_value
elif self.comparator == "!=":
return contact_value != query_value
else: # pragma: no cover
raise SearchException(_(f"Unknown name comparator: '{self.comparator}'"))
else:
raise SearchException(_(f"No support for attribute field: '{field}'"))
else: # pragma: no cover
raise SearchException(_(f"Unrecognized contact field type '{prop_type}'"))
def evaluate(self, org, contact_json, prop_map):
prop_type, field = prop_map[self.prop]
if self.comparator.lower() in self.IS_SET_LOOKUPS:
is_set = True
elif self.comparator.lower() in self.IS_NOT_SET_LOOKUPS:
is_set = False
else: # pragma: no cover
raise SearchException(
_("Invalid operator for empty string comparison"))
if prop_type == ContactQuery.PROP_FIELD:
field_uuid = str(field.uuid)
contact_fields = contact_json.get("fields")
contact_field = contact_fields.get(field_uuid)
# contact field does not exist
if contact_field is None:
if is_set:
return False
else:
return True
else:
if field.value_type == Value.TYPE_TEXT:
contact_value = contact_field.get("text")
if is_set:
if contact_value is not None:
return True
else: # pragma: can't cover
return False
else:
if contact_value is not None:
return False
else: # pragma: can't cover
return True
elif field.value_type == Value.TYPE_NUMBER:
try:
contact_value = self._parse_number(
contact_field.get("decimal",
contact_field.get("number")))
except SearchException:
contact_value = None
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
elif field.value_type == Value.TYPE_DATETIME:
contact_value = str_to_datetime(
contact_field.get("datetime"), field.org.timezone)
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
elif field.value_type == Value.TYPE_WARD:
contact_value = contact_field.get("ward")
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
elif field.value_type == Value.TYPE_DISTRICT:
contact_value = contact_field.get("district")
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
elif field.value_type == Value.TYPE_STATE:
contact_value = contact_field.get("state")
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
else: # pragma: no cover
raise SearchException(
_(f"Unrecognized contact field type '{field.value_type}'"
))
elif prop_type == ContactQuery.PROP_SCHEME:
urn_exists = next((urn for urn in contact_json.get("urns")
if urn.get("scheme") == field), None)
if not urn_exists:
if is_set:
return False
else:
return True
else:
if is_set:
return True
else:
return False
elif prop_type == ContactQuery.PROP_ATTRIBUTE:
field_key = field.key
if field_key == "language":
contact_value = contact_json.get("language")
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
elif field_key == "name":
contact_value = contact_json.get("name")
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
else: # pragma: no cover
raise SearchException(
_(f"No support for attribute field: '{field}'"))
else: # pragma: no cover
raise SearchException(
_(f"Unrecognized contact field type '{prop_type}'"))
def evaluate(self, org, contact_json, prop_map):
prop_type, field = prop_map[self.prop]
if prop_type == ContactQuery.PROP_FIELD:
field_uuid = str(field.uuid)
contact_fields = contact_json.get("fields")
if field_uuid not in contact_fields:
return False
if field.value_type == Value.TYPE_TEXT:
query_value = self.value.upper()
contact_value = contact_fields.get(field_uuid).get(
"text").upper()
if self.comparator == "=":
return contact_value == query_value
else:
raise SearchException(
_(f"Unknown text comparator: '{self.comparator}'"))
elif field.value_type == Value.TYPE_NUMBER:
query_value = self._parse_number(self.value)
number_value = contact_fields.get(field_uuid).get(
"number",
contact_fields.get(field_uuid).get("decimal"))
if number_value is None:
return False
contact_value = self._parse_number(number_value)
if self.comparator == "=":
return contact_value == query_value
elif self.comparator == ">":
return contact_value > query_value
elif self.comparator == ">=":
return contact_value >= query_value
elif self.comparator == "<":
return contact_value < query_value
elif self.comparator == "<=":
return contact_value <= query_value
else:
raise SearchException(
_(f"Unknown number comparator: '{self.comparator}'"))
elif field.value_type == Value.TYPE_DATETIME:
query_value = str_to_datetime(self.value,
field.org.timezone,
field.org.get_dayfirst(),
fill_time=False)
if not query_value:
raise SearchException(
_(f"Unable to parse the date '{self.value}'"))
datetime_value = contact_fields.get(field_uuid).get("datetime")
if datetime_value is None:
return False
contact_value = str_to_datetime(datetime_value,
field.org.timezone)
utc_range = calculate_utc_range(org, self.value)
if self.comparator == "=":
return contact_value >= utc_range[
0] and contact_value < utc_range[1]
elif self.comparator == ">":
return contact_value >= utc_range[1]
elif self.comparator == ">=":
return contact_value >= utc_range[0]
elif self.comparator == "<":
return contact_value < utc_range[0]
elif self.comparator == "<=":
return contact_value < utc_range[1]
else:
raise SearchException(
_(f"Unknown datetime comparator: '{self.comparator}'"))
elif field.value_type in (Value.TYPE_STATE, Value.TYPE_DISTRICT,
Value.TYPE_WARD):
query_value = self.value.upper()
if field.value_type == Value.TYPE_WARD:
ward_value = contact_fields.get(field_uuid).get("ward")
if ward_value is None:
ward_value = ""
contact_value = ward_value.upper().split(" > ")[-1]
elif field.value_type == Value.TYPE_DISTRICT:
district_value = contact_fields.get(field_uuid).get(
"district")
if district_value is None:
district_value = ""
contact_value = district_value.upper().split(" > ")[-1]
elif field.value_type == Value.TYPE_STATE:
state_value = contact_fields.get(field_uuid).get("state")
if state_value is None:
state_value = ""
contact_value = state_value.upper().split(" > ")[-1]
else: # pragma: no cover
raise SearchException(
_(f"Unknown location type: '{field.value_type}'"))
if self.comparator == "=":
return contact_value == query_value
else:
raise SearchException(
_(f"Unsupported comparator '{self.comparator}' for location field"
))
else: # pragma: no cover
raise SearchException(
_(f"Unrecognized contact field type '{field.value_type}'"))
elif prop_type == ContactQuery.PROP_SCHEME:
for urn in contact_json.get("urns"):
if urn.get("scheme") == field:
contact_value = urn.get("path").upper()
query_value = self.value.upper()
if self.comparator == "=":
if contact_value == query_value:
return True
elif self.comparator == "~":
if query_value in contact_value:
return True
else:
raise SearchException(
_(f"Unknown urn scheme comparator: '{self.comparator}'"
))
return False
elif prop_type == ContactQuery.PROP_ATTRIBUTE:
field_key = field.key
if field_key == "language":
query_value = self.value.upper()
raw_contact_value = contact_json.get("language")
if raw_contact_value is None:
return False
else:
contact_value = raw_contact_value.upper()
if self.comparator == "=":
return contact_value == query_value
else:
raise SearchException(
_(f"Unknown language comparator: '{self.comparator}'"))
elif field_key == "created_on":
datetime_value = contact_json.get("created_on")
contact_value = str_to_datetime(datetime_value, org.timezone)
utc_range = calculate_utc_range(org, self.value)
if self.comparator == "=":
return contact_value >= utc_range[
0] and contact_value < utc_range[1]
elif self.comparator == ">":
return contact_value >= utc_range[1]
elif self.comparator == ">=":
return contact_value >= utc_range[0]
elif self.comparator == "<":
return contact_value < utc_range[0]
elif self.comparator == "<=":
return contact_value < utc_range[1]
else:
raise SearchException(
_(f"Unknown created_on comparator: '{self.comparator}'"
))
elif field_key == "name":
query_value = self.value.upper()
raw_contact_value = contact_json.get("name")
if raw_contact_value is None:
return False
else:
contact_value = raw_contact_value.upper()
if self.comparator == "=":
return contact_value == query_value
elif self.comparator == "~":
return query_value in contact_value
else: # pragma: no cover
raise SearchException(
_(f"Unknown name comparator: '{self.comparator}'"))
else:
raise SearchException(
_(f"No support for attribute field: '{field}'"))
else: # pragma: no cover
raise SearchException(
_(f"Unrecognized contact field type '{prop_type}'"))
def as_elasticsearch(self, org, prop_map):
prop_type, field = prop_map[self.prop]
if prop_type == ContactQuery.PROP_FIELD:
field_uuid = str(field.uuid)
es_query = es_Q("term", **{"fields.field": field_uuid})
if field.value_type == Value.TYPE_TEXT:
query_value = self.value.lower()
if self.comparator == "=":
es_query &= es_Q("term", **{"fields.text": query_value})
else:
raise SearchException(
_("Unknown text comparator: '%s'") %
(self.comparator, ))
elif field.value_type == Value.TYPE_NUMBER:
query_value = str(self._parse_number(self.value))
if self.comparator == "=":
es_query &= es_Q("match", **{"fields.number": query_value})
elif self.comparator == ">":
es_query &= es_Q("range",
**{"fields.number": {
"gt": query_value
}})
elif self.comparator == ">=":
es_query &= es_Q("range",
**{"fields.number": {
"gte": query_value
}})
elif self.comparator == "<":
es_query &= es_Q("range",
**{"fields.number": {
"lt": query_value
}})
elif self.comparator == "<=":
es_query &= es_Q("range",
**{"fields.number": {
"lte": query_value
}})
else:
raise SearchException(
_("Unknown number comparator: '%s'") %
(self.comparator, ))
elif field.value_type == Value.TYPE_DATETIME:
query_value = str_to_datetime(self.value,
field.org.timezone,
field.org.get_dayfirst(),
fill_time=False)
if not query_value:
raise SearchException(
_("Unable to parse the date '%s'") % self.value)
utc_range = date_to_utc_range(query_value.date(), field.org)
if self.comparator == "=":
es_query &= es_Q(
"range", **{
"fields.datetime": {
"gte": utc_range[0].isoformat(),
"lt": utc_range[1].isoformat()
}
})
elif self.comparator == ">":
es_query &= es_Q(
"range", **{
"fields.datetime": {
"gte": utc_range[1].isoformat()
}
})
elif self.comparator == ">=":
es_query &= es_Q(
"range", **{
"fields.datetime": {
"gte": utc_range[0].isoformat()
}
})
elif self.comparator == "<":
es_query &= es_Q(
"range", **{
"fields.datetime": {
"lt": utc_range[0].isoformat()
}
})
elif self.comparator == "<=":
es_query &= es_Q(
"range", **{
"fields.datetime": {
"lt": utc_range[1].isoformat()
}
})
else:
raise SearchException(
_("Unknown datetime comparator: '%s'") %
(self.comparator, ))
elif field.value_type in (Value.TYPE_STATE, Value.TYPE_DISTRICT,
Value.TYPE_WARD):
query_value = self.value.lower()
if field.value_type == Value.TYPE_WARD:
field_name = "fields.ward"
elif field.value_type == Value.TYPE_DISTRICT:
field_name = "fields.district"
elif field.value_type == Value.TYPE_STATE:
field_name = "fields.state"
else: # pragma: no cover
raise SearchException(
_("Unknown location type: '%s'") %
(field.value_type, ))
if self.comparator == "=":
field_name += "_keyword"
es_query &= es_Q("term", **{field_name: query_value})
else:
raise SearchException(
_("Unsupported comparator '%s' for location field") %
self.comparator)
else: # pragma: no cover
raise SearchException(
_("Unrecognized contact field type '%s'") %
field.value_type)
return es_Q("nested", path="fields", query=es_query)
elif prop_type == ContactQuery.PROP_ATTRIBUTE:
query_value = self.value.lower()
if field == "name":
if self.comparator == "=":
field_name = "name.keyword"
es_query = es_Q("term", **{field_name: query_value})
elif self.comparator == "~":
field_name = "name"
es_query = es_Q("match", **{field_name: query_value})
else:
raise SearchException(
_("Unknown attribute comparator: '%s'") %
(self.comparator, ))
elif field == "id":
es_query = es_Q("ids", **{"values": [query_value]})
else: # pragma: no cover
raise SearchException(
_("Unknown attribute field '%s'") % (field, ))
return es_query
elif prop_type == ContactQuery.PROP_SCHEME:
query_value = self.value.lower()
es_query = es_Q("term", **{"urns.scheme": field.lower()})
if org.is_anon:
return es_Q("ids", **{"values": [-1]})
else:
if self.comparator == "=":
es_query &= es_Q("term",
**{"urns.path.keyword": query_value})
elif self.comparator == "~":
es_query &= es_Q("match_phrase",
**{"urns.path": query_value})
else:
raise SearchException(
_("Unknown scheme comparator: '%s'") %
(self.comparator, ))
return es_Q("nested", path="urns", query=es_query)
else: # pragma: no cover
raise SearchException(
_("Unrecognized contact field type '%s'") % prop_type)
def evaluate(self, contact_json, prop_map):
prop_type, field = prop_map[self.prop]
if self.comparator.lower() in self.IS_SET_LOOKUPS:
is_set = True
elif self.comparator.lower() in self.IS_NOT_SET_LOOKUPS:
is_set = False
else: # pragma: no cover
raise SearchException(
_("Invalid operator for empty string comparison"))
if prop_type == ContactQuery.PROP_FIELD:
field_uuid = six.text_type(field.uuid)
contact_fields = contact_json.get('fields')
contact_field = contact_fields.get(field_uuid)
# contact field does not exist
if contact_field is None:
if is_set:
return False
else:
return True
else:
if field.value_type == Value.TYPE_TEXT:
contact_value = contact_field.get('text')
if is_set:
if contact_value is not None:
return True
else: # pragma: can't cover
return False
else:
if contact_value is not None:
return False
else: # pragma: can't cover
return True
elif field.value_type == Value.TYPE_NUMBER:
try:
contact_value = self._parse_number(
contact_field.get('decimal',
contact_field.get('number')))
except SearchException:
contact_value = None
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
elif field.value_type == Value.TYPE_DATETIME:
contact_value = str_to_datetime(
contact_field.get('datetime'), field.org.timezone)
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
elif field.value_type == Value.TYPE_WARD:
contact_value = contact_field.get('ward')
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
elif field.value_type == Value.TYPE_DISTRICT:
contact_value = contact_field.get('district')
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
elif field.value_type == Value.TYPE_STATE:
contact_value = contact_field.get('state')
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
else: # pragma: no cover
raise SearchException(
_("Unrecognized contact field type '%s'") %
field.value_type)
elif prop_type == ContactQuery.PROP_SCHEME:
urn_exists = next((urn for urn in contact_json.get('urns')
if urn.get('scheme') == field), None)
if not urn_exists:
if is_set:
return False
else:
return True
else:
if is_set:
return True
else:
return False
else:
raise SearchException(
_("Unrecognized contact field type '%s'") % prop_type)
def as_elasticsearch(self, org, prop_map):
prop_type, field = prop_map[self.prop]
if prop_type == ContactQuery.PROP_FIELD:
field_uuid = six.text_type(field.uuid)
es_query = es_Q('term', **{'fields.field': field_uuid})
if field.value_type == Value.TYPE_TEXT:
query_value = self.value.lower()
if self.comparator == '=':
es_query &= es_Q('term', **{'fields.text': query_value})
else:
raise SearchException(
_("Unknown text comparator: '%s'") %
(self.comparator, ))
elif field.value_type == Value.TYPE_NUMBER:
query_value = six.text_type(self._parse_number(self.value))
if self.comparator == '=':
es_query &= es_Q('match', **{'fields.number': query_value})
elif self.comparator == '>':
es_query &= es_Q('range',
**{'fields.number': {
'gt': query_value
}})
elif self.comparator == '>=':
es_query &= es_Q('range',
**{'fields.number': {
'gte': query_value
}})
elif self.comparator == '<':
es_query &= es_Q('range',
**{'fields.number': {
'lt': query_value
}})
elif self.comparator == '<=':
es_query &= es_Q('range',
**{'fields.number': {
'lte': query_value
}})
else:
raise SearchException(
_("Unknown number comparator: '%s'") %
(self.comparator, ))
elif field.value_type == Value.TYPE_DATETIME:
query_value = str_to_datetime(self.value,
field.org.timezone,
field.org.get_dayfirst(),
fill_time=False)
if not query_value:
raise SearchException(
_("Unable to parse the date '%s'") % self.value)
utc_range = date_to_utc_range(query_value.date(), field.org)
if self.comparator == '=':
es_query &= es_Q(
'range', **{
'fields.datetime': {
'gte': utc_range[0].isoformat(),
'lt': utc_range[1].isoformat()
}
})
elif self.comparator == '>':
es_query &= es_Q(
'range', **{
'fields.datetime': {
'gte': utc_range[1].isoformat()
}
})
elif self.comparator == '>=':
es_query &= es_Q(
'range', **{
'fields.datetime': {
'gte': utc_range[0].isoformat()
}
})
elif self.comparator == '<':
es_query &= es_Q(
'range', **{
'fields.datetime': {
'lt': utc_range[0].isoformat()
}
})
elif self.comparator == '<=':
es_query &= es_Q(
'range', **{
'fields.datetime': {
'lt': utc_range[1].isoformat()
}
})
else:
raise SearchException(
_("Unknown datetime comparator: '%s'") %
(self.comparator, ))
elif field.value_type in (Value.TYPE_STATE, Value.TYPE_DISTRICT,
Value.TYPE_WARD):
query_value = self.value.lower()
if field.value_type == Value.TYPE_WARD:
field_name = 'fields.ward'
elif field.value_type == Value.TYPE_DISTRICT:
field_name = 'fields.district'
elif field.value_type == Value.TYPE_STATE:
field_name = 'fields.state'
else: # pragma: no cover
raise SearchException(
_("Unknown location type: '%s'") %
(field.value_type, ))
if self.comparator == '=':
field_name += '.keyword'
es_query &= es_Q('term', **{field_name: query_value})
else:
raise SearchException(
_("Unsupported comparator '%s' for location field") %
self.comparator)
else: # pragma: no cover
raise SearchException(
_("Unrecognized contact field type '%s'") %
field.value_type)
return es_Q('nested', path='fields', query=es_query)
elif prop_type == ContactQuery.PROP_ATTRIBUTE:
query_value = self.value.lower()
if field == 'name':
if self.comparator == '=':
field_name = 'name.keyword'
es_query = es_Q('term', **{field_name: query_value})
elif self.comparator == '~':
field_name = 'name'
es_query = es_Q('match', **{field_name: query_value})
else:
raise SearchException(
_("Unknown attribute comparator: '%s'") %
(self.comparator, ))
elif field == 'id':
es_query = es_Q('ids', **{'values': [query_value]})
else: # pragma: no cover
raise SearchException(
_("Unknown attribute field '%s'") % (field, ))
return es_query
elif prop_type == ContactQuery.PROP_SCHEME:
query_value = self.value.lower()
es_query = es_Q('term', **{'urns.scheme': field.lower()})
if org.is_anon:
return es_Q('ids', **{'values': [-1]})
else:
if self.comparator == '=':
es_query &= es_Q('term',
**{'urns.path.keyword': query_value})
elif self.comparator == '~':
es_query &= es_Q('match_phrase',
**{'urns.path': query_value})
else:
raise SearchException(
_("Unknown scheme comparator: '%s'") %
(self.comparator, ))
return es_Q('nested', path='urns', query=es_query)
else: # pragma: no cover
raise SearchException(
_("Unrecognized contact field type '%s'") % prop_type)
def evaluate(self, contact_json, prop_map):
prop_type, field = prop_map[self.prop]
if prop_type == ContactQuery.PROP_FIELD:
field_uuid = six.text_type(field.uuid)
contact_fields = contact_json.get('fields')
if field_uuid not in contact_fields:
return False
if field.value_type == Value.TYPE_TEXT:
query_value = self.value.upper()
contact_value = contact_fields.get(field_uuid).get(
'text').upper()
if self.comparator == '=':
return contact_value == query_value
else:
raise SearchException(
_("Unknown text comparator: '%s'") %
(self.comparator, ))
elif field.value_type == Value.TYPE_NUMBER:
query_value = self._parse_number(self.value)
number_value = contact_fields.get(field_uuid).get(
'number',
contact_fields.get(field_uuid).get('decimal'))
if number_value is None:
return False
contact_value = self._parse_number(number_value)
if self.comparator == '=':
return contact_value == query_value
elif self.comparator == '>':
return contact_value > query_value
elif self.comparator == '>=':
return contact_value >= query_value
elif self.comparator == '<':
return contact_value < query_value
elif self.comparator == '<=':
return contact_value <= query_value
else:
raise SearchException(
_("Unknown number comparator: '%s'") %
(self.comparator, ))
elif field.value_type == Value.TYPE_DATETIME:
query_value = str_to_datetime(self.value,
field.org.timezone,
field.org.get_dayfirst(),
fill_time=False)
if not query_value:
raise SearchException(
_("Unable to parse the date '%s'") % self.value)
datetime_value = contact_fields.get(field_uuid).get('datetime')
if datetime_value is None:
return False
contact_value = str_to_datetime(datetime_value,
field.org.timezone)
utc_range = date_to_utc_range(query_value.date(), field.org)
if self.comparator == '=':
return contact_value >= utc_range[
0] and contact_value < utc_range[1]
elif self.comparator == '>':
return contact_value >= utc_range[1]
elif self.comparator == '>=':
return contact_value >= utc_range[0]
elif self.comparator == '<':
return contact_value < utc_range[0]
elif self.comparator == '<=':
return contact_value < utc_range[1]
else:
raise SearchException(
_("Unknown datetime comparator: '%s'") %
(self.comparator, ))
elif field.value_type in (Value.TYPE_STATE, Value.TYPE_DISTRICT,
Value.TYPE_WARD):
query_value = self.value.upper()
if field.value_type == Value.TYPE_WARD:
ward_value = contact_fields.get(field_uuid).get('ward')
if ward_value is None:
ward_value = ""
contact_value = ward_value.upper().split(' > ')[-1]
elif field.value_type == Value.TYPE_DISTRICT:
district_value = contact_fields.get(field_uuid).get(
'district')
if district_value is None:
district_value = ""
contact_value = district_value.upper().split(' > ')[-1]
elif field.value_type == Value.TYPE_STATE:
state_value = contact_fields.get(field_uuid).get('state')
if state_value is None:
state_value = ""
contact_value = state_value.upper().split(' > ')[-1]
else: # pragma: no cover
raise SearchException(
_("Unknown location type: '%s'") %
(field.value_type, ))
if self.comparator == '=':
return contact_value == query_value
else:
raise SearchException(
_("Unsupported comparator '%s' for location field") %
self.comparator)
else: # pragma: no cover
raise SearchException(
_("Unrecognized contact field type '%s'") %
field.value_type)
elif prop_type == ContactQuery.PROP_SCHEME:
for urn in contact_json.get('urns'):
if urn.get('scheme') == field:
contact_value = urn.get('path').upper()
query_value = self.value.upper()
if self.comparator == '=':
if contact_value == query_value:
return True
elif self.comparator == '~':
if query_value in contact_value:
return True
else:
raise SearchException(
_("Unknown urn scheme comparator: '%s'") %
(self.comparator, ))
return False
else:
raise SearchException(
_("Unrecognized contact field type '%s'") % prop_type)
def evaluate(self, org, contact_json, prop_map):
prop_type, field = prop_map[self.prop]
if self.comparator.lower() in self.IS_SET_LOOKUPS:
is_set = True
elif self.comparator.lower() in self.IS_NOT_SET_LOOKUPS:
is_set = False
else: # pragma: no cover
raise SearchException(_("Invalid operator for empty string comparison"))
if prop_type == ContactQuery.PROP_FIELD:
field_uuid = str(field.uuid)
contact_fields = contact_json.get("fields")
contact_field = contact_fields.get(field_uuid)
# contact field does not exist
if contact_field is None:
if is_set:
return False
else:
return True
else:
if field.value_type == Value.TYPE_TEXT:
contact_value = contact_field.get("text")
if is_set:
if contact_value is not None:
return True
else: # pragma: can't cover
return False
else:
if contact_value is not None:
return False
else: # pragma: can't cover
return True
elif field.value_type == Value.TYPE_NUMBER:
try:
contact_value = self._parse_number(contact_field.get("decimal", contact_field.get("number")))
except SearchException:
contact_value = None
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
elif field.value_type == Value.TYPE_DATETIME:
contact_value = str_to_datetime(contact_field.get("datetime"), field.org.timezone)
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
elif field.value_type == Value.TYPE_WARD:
contact_value = contact_field.get("ward")
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
elif field.value_type == Value.TYPE_DISTRICT:
contact_value = contact_field.get("district")
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
elif field.value_type == Value.TYPE_STATE:
contact_value = contact_field.get("state")
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
else: # pragma: no cover
raise SearchException(_(f"Unrecognized contact field type '{field.value_type}'"))
elif prop_type == ContactQuery.PROP_SCHEME:
urn_exists = next((urn for urn in contact_json.get("urns") if urn.get("scheme") == field), None)
if not urn_exists:
if is_set:
return False
else:
return True
else:
if is_set:
return True
else:
return False
elif prop_type == ContactQuery.PROP_ATTRIBUTE:
field_key = field.key
if field_key == "language":
contact_value = contact_json.get("language")
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
elif field_key == "name":
contact_value = contact_json.get("name")
if is_set:
if contact_value is not None:
return True
else:
return False
else:
if contact_value is not None:
return False
else:
return True
else: # pragma: no cover
raise SearchException(_(f"No support for attribute field: '{field}'"))
else: # pragma: no cover
raise SearchException(_(f"Unrecognized contact field type '{prop_type}'"))
def evaluate(self, org, contact_json, prop_map):
prop_type, field = prop_map[self.prop]
if prop_type == ContactQuery.PROP_FIELD:
field_uuid = str(field.uuid)
contact_fields = contact_json.get("fields")
if field_uuid not in contact_fields:
return False
if field.value_type == Value.TYPE_TEXT:
query_value = self.value.upper()
contact_value = contact_fields.get(field_uuid).get("text").upper()
if self.comparator == "=":
return contact_value == query_value
else:
raise SearchException(_(f"Unknown text comparator: '{self.comparator}'"))
elif field.value_type == Value.TYPE_NUMBER:
query_value = self._parse_number(self.value)
number_value = contact_fields.get(field_uuid).get(
"number", contact_fields.get(field_uuid).get("decimal")
)
if number_value is None:
return False
contact_value = self._parse_number(number_value)
if self.comparator == "=":
return contact_value == query_value
elif self.comparator == ">":
return contact_value > query_value
elif self.comparator == ">=":
return contact_value >= query_value
elif self.comparator == "<":
return contact_value < query_value
elif self.comparator == "<=":
return contact_value <= query_value
else:
raise SearchException(_(f"Unknown number comparator: '{self.comparator}'"))
elif field.value_type == Value.TYPE_DATETIME:
query_value = str_to_datetime(
self.value, field.org.timezone, field.org.get_dayfirst(), fill_time=False
)
if not query_value:
raise SearchException(_(f"Unable to parse the date '{self.value}'"))
datetime_value = contact_fields.get(field_uuid).get("datetime")
if datetime_value is None:
return False
contact_value = str_to_datetime(datetime_value, field.org.timezone)
utc_range = calculate_utc_range(org, self.value)
if self.comparator == "=":
return contact_value >= utc_range[0] and contact_value < utc_range[1]
elif self.comparator == ">":
return contact_value >= utc_range[1]
elif self.comparator == ">=":
return contact_value >= utc_range[0]
elif self.comparator == "<":
return contact_value < utc_range[0]
elif self.comparator == "<=":
return contact_value < utc_range[1]
else:
raise SearchException(_(f"Unknown datetime comparator: '{self.comparator}'"))
elif field.value_type in (Value.TYPE_STATE, Value.TYPE_DISTRICT, Value.TYPE_WARD):
query_value = self.value.upper()
if field.value_type == Value.TYPE_WARD:
ward_value = contact_fields.get(field_uuid).get("ward")
if ward_value is None:
ward_value = ""
contact_value = ward_value.upper().split(" > ")[-1]
elif field.value_type == Value.TYPE_DISTRICT:
district_value = contact_fields.get(field_uuid).get("district")
if district_value is None:
district_value = ""
contact_value = district_value.upper().split(" > ")[-1]
elif field.value_type == Value.TYPE_STATE:
state_value = contact_fields.get(field_uuid).get("state")
if state_value is None:
state_value = ""
contact_value = state_value.upper().split(" > ")[-1]
else: # pragma: no cover
raise SearchException(_(f"Unknown location type: '{field.value_type}'"))
if self.comparator == "=":
return contact_value == query_value
else:
raise SearchException(_(f"Unsupported comparator '{self.comparator}' for location field"))
else: # pragma: no cover
raise SearchException(_(f"Unrecognized contact field type '{field.value_type}'"))
elif prop_type == ContactQuery.PROP_SCHEME:
for urn in contact_json.get("urns"):
if urn.get("scheme") == field:
contact_value = urn.get("path").upper()
query_value = self.value.upper()
if self.comparator == "=":
if contact_value == query_value:
return True
elif self.comparator == "~":
if query_value in contact_value:
return True
else:
raise SearchException(_(f"Unknown urn scheme comparator: '{self.comparator}'"))
return False
elif prop_type == ContactQuery.PROP_ATTRIBUTE:
field_key = field.key
if field_key == "language":
query_value = self.value.upper()
raw_contact_value = contact_json.get("language")
if raw_contact_value is None:
return False
else:
contact_value = raw_contact_value.upper()
if self.comparator == "=":
return contact_value == query_value
else:
raise SearchException(_(f"Unknown language comparator: '{self.comparator}'"))
elif field_key == "created_on":
datetime_value = contact_json.get("created_on")
contact_value = str_to_datetime(datetime_value, org.timezone)
utc_range = calculate_utc_range(org, self.value)
if self.comparator == "=":
return contact_value >= utc_range[0] and contact_value < utc_range[1]
elif self.comparator == ">":
return contact_value >= utc_range[1]
elif self.comparator == ">=":
return contact_value >= utc_range[0]
elif self.comparator == "<":
return contact_value < utc_range[0]
elif self.comparator == "<=":
return contact_value < utc_range[1]
else:
raise SearchException(_(f"Unknown created_on comparator: '{self.comparator}'"))
elif field_key == "name":
query_value = self.value.upper()
raw_contact_value = contact_json.get("name")
if raw_contact_value is None:
return False
else:
contact_value = raw_contact_value.upper()
if self.comparator == "=":
return contact_value == query_value
elif self.comparator == "~":
return query_value in contact_value
else: # pragma: no cover
raise SearchException(_(f"Unknown name comparator: '{self.comparator}'"))
else:
raise SearchException(_(f"No support for attribute field: '{field}'"))
else: # pragma: no cover
raise SearchException(_(f"Unrecognized contact field type '{prop_type}'"))
