def calculate_utc_range(org, input_value): """ Calculates datetime range in UTC, we use it to check date containment """ query_value = str_to_datetime(input_value, org.timezone, org.get_dayfirst(), fill_time=False) if not query_value: raise SearchException(_(f"Unable to parse the date '{input_value}'")) return date_to_utc_range(query_value.date(), org)
def _build_datetime_field_params(self, field): lookup = self.DATETIME_LOOKUPS.get(self.comparator) if not lookup: raise SearchException( _("Can't query date fields with %s") % self.comparator) # parse as localized date local_date = str_to_datetime(self.value, field.org.timezone, field.org.get_dayfirst(), fill_time=False) if not local_date: raise SearchException( _("Unable to parse the date %s") % self.value) # get the range of UTC datetimes for this local date utc_range = date_to_utc_range(local_date.date(), field.org) if lookup == '<equal>': return { 'contact_field': field, 'datetime_value__gte': utc_range[0], 'datetime_value__lt': utc_range[1] } elif lookup == 'lt': return {'contact_field': field, 'datetime_value__lt': utc_range[0]} elif lookup == 'lte': return {'contact_field': field, 'datetime_value__lt': utc_range[1]} elif lookup == 'gt': return { 'contact_field': field, 'datetime_value__gte': utc_range[1] } elif lookup == 'gte': return { 'contact_field': field, 'datetime_value__gte': utc_range[0] }
def fix_value_results(FlowRun, RuleSet, Value): cache = get_redis_connection() # make a map of ruleset IDs to UUIDs ruleset_id_to_uuid = {r.id: r.uuid for r in RuleSet.objects.all()} if not ruleset_id_to_uuid: return # has this migration been run before but didn't complete? highpoint = cache.get(CACHE_KEY_HIGHPOINT) highpoint = 0 if highpoint is None else int(highpoint) # problematic ruleset values are those which have a decimal and datetime value, as this implies that the input # was numeric, but was still erroneously parsed as a datetime values = Value.objects.exclude(decimal_value=None).exclude( datetime_value=None).exclude(run=None) # get all flow runs we need to fix run_ids = values.values_list( 'run_id', flat=True).order_by('run_id').distinct('run_id') if highpoint: print("Resuming from previous highpoint at run #%d" % highpoint) run_ids = run_ids.filter(id__gt=highpoint) run_ids = array(str('l'), run_ids) print("Total of %d runs need to be fixed" % len(run_ids)) num_fixed = 0 start = time.time() for id_batch in chunk_list(run_ids, 1000): with transaction.atomic(): batch = FlowRun.objects.filter(id__in=id_batch).order_by( 'id').select_related('org').prefetch_related('values') for run in batch: day_first = run.org.date_format == 'D' # find result values with which are no longer parsed as dates no_longer_dates = {} for v in run.values.all(): parsed_new = str_to_datetime(v.string_value, tz=run.org.timezone, dayfirst=day_first) if not parsed_new and v.datetime_value: no_longer_dates[ruleset_id_to_uuid[ v.ruleset_id]] = v.string_value if no_longer_dates: results = json.loads(run.results) if run.results else {} for key, result in six.iteritems(results): node_uuid = result['node_uuid'] if node_uuid in no_longer_dates: result['value'] = no_longer_dates[node_uuid] run.results = json.dumps(results) run.save(update_fields=('results', )) cache.set(CACHE_KEY_HIGHPOINT, str(run.id), 60 * 60 * 24 * 7) num_fixed += 1 fixed_per_sec = num_fixed / (time.time() - start) # figure out estimated time remaining num_remaining = len(run_ids) - highpoint time_remaining = num_remaining / fixed_per_sec finishes = timezone.now() + timedelta(seconds=time_remaining) status = " > Updated %d runs of ~%d (%2.2f per sec) Est finish: %s" % ( num_fixed, len(run_ids), fixed_per_sec, finishes) print(status) print("Run results fix migration completed in %d mins" % (int(time.time() - start) // 60))
def evaluate(self, org, contact_json, prop_map): prop_type, field = prop_map[self.prop] if prop_type == ContactQuery.PROP_FIELD: field_uuid = str(field.uuid) contact_fields = contact_json.get("fields", {}) if field.value_type == Value.TYPE_TEXT: query_value = self.value.upper() contact_value = contact_fields.get(field_uuid, {"text": ""}).get("text").upper() if self.comparator == "=": return contact_value == query_value elif self.comparator == "!=": return contact_value != query_value else: raise SearchException(_(f"Unknown text comparator: '{self.comparator}'")) elif field.value_type == Value.TYPE_NUMBER: query_value = self._parse_number(self.value) number_value = contact_fields.get(field_uuid, {"number": None}).get( "number", contact_fields.get(field_uuid, {"decimal": None}).get("decimal") ) if number_value is None: return False contact_value = self._parse_number(number_value) if self.comparator == "=": return contact_value == query_value elif self.comparator == ">": return contact_value > query_value elif self.comparator == ">=": return contact_value >= query_value elif self.comparator == "<": return contact_value < query_value elif self.comparator == "<=": return contact_value <= query_value else: raise SearchException(_(f"Unknown number comparator: '{self.comparator}'")) elif field.value_type == Value.TYPE_DATETIME: query_value = str_to_date(self.value, field.org.get_dayfirst()) if not query_value: raise SearchException(_(f"Unable to parse the date '{self.value}'")) lower_bound, upper_bound = date_to_day_range_utc(query_value, org) contact_datetime_value = contact_fields.get(field_uuid, {"datetime": None}).get("datetime") if contact_datetime_value is None: return False # datetime contact values are serialized as ISO8601 timestamps in local time contact_value = str_to_datetime(contact_datetime_value, pytz.UTC, field.org.get_dayfirst()) contact_value_utc = contact_value.astimezone(pytz.UTC) if self.comparator == "=": return contact_value_utc >= lower_bound and contact_value_utc < upper_bound elif self.comparator == ">": return contact_value_utc >= upper_bound elif self.comparator == ">=": return contact_value_utc >= lower_bound elif self.comparator == "<": return contact_value_utc < lower_bound elif self.comparator == "<=": return contact_value_utc < upper_bound else: raise SearchException(_(f"Unknown datetime comparator: '{self.comparator}'")) elif field.value_type in (Value.TYPE_STATE, Value.TYPE_DISTRICT, Value.TYPE_WARD): query_value = self.value.upper() if field.value_type == Value.TYPE_WARD: ward_value = contact_fields.get(field_uuid, {"ward": ""}).get("ward", "") contact_value = ward_value.upper().split(" > ")[-1] elif field.value_type == Value.TYPE_DISTRICT: district_value = contact_fields.get(field_uuid, {"district": ""}).get("district", "") contact_value = district_value.upper().split(" > ")[-1] elif field.value_type == Value.TYPE_STATE: state_value = contact_fields.get(field_uuid, {"state": ""}).get("state", "") contact_value = state_value.upper().split(" > ")[-1] else: # pragma: no cover raise SearchException(_(f"Unknown location type: '{field.value_type}'")) if self.comparator == "=": return contact_value == query_value elif self.comparator == "!=": return contact_value != query_value else: raise SearchException(_(f"Unsupported comparator '{self.comparator}' for location field")) else: # pragma: no cover raise SearchException(_(f"Unrecognized contact field type '{field.value_type}'")) elif prop_type == ContactQuery.PROP_SCHEME: for urn in contact_json.get("urns"): if urn.get("scheme") == field: contact_value = urn.get("path").upper() query_value = self.value.upper() if self.comparator == "=": if contact_value == query_value: return True elif self.comparator == "~": if query_value in contact_value: return True else: raise SearchException(_(f"Unknown urn scheme comparator: '{self.comparator}'")) return False elif prop_type == ContactQuery.PROP_ATTRIBUTE: field_key = field.key if field_key == "language": query_value = self.value.upper() raw_contact_value = contact_json.get("language") if raw_contact_value is None: contact_value = "" else: contact_value = raw_contact_value.upper() if self.comparator == "=": return contact_value == query_value elif self.comparator == "!=": return contact_value != query_value else: raise SearchException(_(f"Unknown language comparator: '{self.comparator}'")) elif field_key == "created_on": query_value = str_to_date(self.value, field.org.get_dayfirst()) if not query_value: raise SearchException(_(f"Unable to parse the date '{self.value}'")) lower_bound, upper_bound = date_to_day_range_utc(query_value, org) # contact created_on is serialized as ISO8601 timestamp in utc time contact_value = str_to_datetime(contact_json.get("created_on"), pytz.UTC, field.org.get_dayfirst()) contact_value_utc = contact_value.astimezone(pytz.UTC) if self.comparator == "=": return contact_value_utc >= lower_bound and contact_value_utc < upper_bound elif self.comparator == ">": return contact_value_utc >= upper_bound elif self.comparator == ">=": return contact_value_utc >= lower_bound elif self.comparator == "<": return contact_value_utc < lower_bound elif self.comparator == "<=": return contact_value_utc < upper_bound else: raise SearchException(_(f"Unknown created_on comparator: '{self.comparator}'")) elif field_key == "name": query_value = self.value.upper() raw_contact_value = contact_json.get("name") if raw_contact_value is None: contact_value = "" else: contact_value = raw_contact_value.upper() if self.comparator == "=": return contact_value == query_value elif self.comparator == "~": return query_value in contact_value elif self.comparator == "!=": return contact_value != query_value else: # pragma: no cover raise SearchException(_(f"Unknown name comparator: '{self.comparator}'")) else: raise SearchException(_(f"No support for attribute field: '{field}'")) else: # pragma: no cover raise SearchException(_(f"Unrecognized contact field type '{prop_type}'"))
def evaluate(self, org, contact_json, prop_map): prop_type, field = prop_map[self.prop] if self.comparator.lower() in self.IS_SET_LOOKUPS: is_set = True elif self.comparator.lower() in self.IS_NOT_SET_LOOKUPS: is_set = False else: # pragma: no cover raise SearchException( _("Invalid operator for empty string comparison")) if prop_type == ContactQuery.PROP_FIELD: field_uuid = str(field.uuid) contact_fields = contact_json.get("fields") contact_field = contact_fields.get(field_uuid) # contact field does not exist if contact_field is None: if is_set: return False else: return True else: if field.value_type == Value.TYPE_TEXT: contact_value = contact_field.get("text") if is_set: if contact_value is not None: return True else: # pragma: can't cover return False else: if contact_value is not None: return False else: # pragma: can't cover return True elif field.value_type == Value.TYPE_NUMBER: try: contact_value = self._parse_number( contact_field.get("decimal", contact_field.get("number"))) except SearchException: contact_value = None if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True elif field.value_type == Value.TYPE_DATETIME: contact_value = str_to_datetime( contact_field.get("datetime"), field.org.timezone) if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True elif field.value_type == Value.TYPE_WARD: contact_value = contact_field.get("ward") if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True elif field.value_type == Value.TYPE_DISTRICT: contact_value = contact_field.get("district") if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True elif field.value_type == Value.TYPE_STATE: contact_value = contact_field.get("state") if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True else: # pragma: no cover raise SearchException( _(f"Unrecognized contact field type '{field.value_type}'" )) elif prop_type == ContactQuery.PROP_SCHEME: urn_exists = next((urn for urn in contact_json.get("urns") if urn.get("scheme") == field), None) if not urn_exists: if is_set: return False else: return True else: if is_set: return True else: return False elif prop_type == ContactQuery.PROP_ATTRIBUTE: field_key = field.key if field_key == "language": contact_value = contact_json.get("language") if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True elif field_key == "name": contact_value = contact_json.get("name") if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True else: # pragma: no cover raise SearchException( _(f"No support for attribute field: '{field}'")) else: # pragma: no cover raise SearchException( _(f"Unrecognized contact field type '{prop_type}'"))
def evaluate(self, org, contact_json, prop_map): prop_type, field = prop_map[self.prop] if prop_type == ContactQuery.PROP_FIELD: field_uuid = str(field.uuid) contact_fields = contact_json.get("fields") if field_uuid not in contact_fields: return False if field.value_type == Value.TYPE_TEXT: query_value = self.value.upper() contact_value = contact_fields.get(field_uuid).get( "text").upper() if self.comparator == "=": return contact_value == query_value else: raise SearchException( _(f"Unknown text comparator: '{self.comparator}'")) elif field.value_type == Value.TYPE_NUMBER: query_value = self._parse_number(self.value) number_value = contact_fields.get(field_uuid).get( "number", contact_fields.get(field_uuid).get("decimal")) if number_value is None: return False contact_value = self._parse_number(number_value) if self.comparator == "=": return contact_value == query_value elif self.comparator == ">": return contact_value > query_value elif self.comparator == ">=": return contact_value >= query_value elif self.comparator == "<": return contact_value < query_value elif self.comparator == "<=": return contact_value <= query_value else: raise SearchException( _(f"Unknown number comparator: '{self.comparator}'")) elif field.value_type == Value.TYPE_DATETIME: query_value = str_to_datetime(self.value, field.org.timezone, field.org.get_dayfirst(), fill_time=False) if not query_value: raise SearchException( _(f"Unable to parse the date '{self.value}'")) datetime_value = contact_fields.get(field_uuid).get("datetime") if datetime_value is None: return False contact_value = str_to_datetime(datetime_value, field.org.timezone) utc_range = calculate_utc_range(org, self.value) if self.comparator == "=": return contact_value >= utc_range[ 0] and contact_value < utc_range[1] elif self.comparator == ">": return contact_value >= utc_range[1] elif self.comparator == ">=": return contact_value >= utc_range[0] elif self.comparator == "<": return contact_value < utc_range[0] elif self.comparator == "<=": return contact_value < utc_range[1] else: raise SearchException( _(f"Unknown datetime comparator: '{self.comparator}'")) elif field.value_type in (Value.TYPE_STATE, Value.TYPE_DISTRICT, Value.TYPE_WARD): query_value = self.value.upper() if field.value_type == Value.TYPE_WARD: ward_value = contact_fields.get(field_uuid).get("ward") if ward_value is None: ward_value = "" contact_value = ward_value.upper().split(" > ")[-1] elif field.value_type == Value.TYPE_DISTRICT: district_value = contact_fields.get(field_uuid).get( "district") if district_value is None: district_value = "" contact_value = district_value.upper().split(" > ")[-1] elif field.value_type == Value.TYPE_STATE: state_value = contact_fields.get(field_uuid).get("state") if state_value is None: state_value = "" contact_value = state_value.upper().split(" > ")[-1] else: # pragma: no cover raise SearchException( _(f"Unknown location type: '{field.value_type}'")) if self.comparator == "=": return contact_value == query_value else: raise SearchException( _(f"Unsupported comparator '{self.comparator}' for location field" )) else: # pragma: no cover raise SearchException( _(f"Unrecognized contact field type '{field.value_type}'")) elif prop_type == ContactQuery.PROP_SCHEME: for urn in contact_json.get("urns"): if urn.get("scheme") == field: contact_value = urn.get("path").upper() query_value = self.value.upper() if self.comparator == "=": if contact_value == query_value: return True elif self.comparator == "~": if query_value in contact_value: return True else: raise SearchException( _(f"Unknown urn scheme comparator: '{self.comparator}'" )) return False elif prop_type == ContactQuery.PROP_ATTRIBUTE: field_key = field.key if field_key == "language": query_value = self.value.upper() raw_contact_value = contact_json.get("language") if raw_contact_value is None: return False else: contact_value = raw_contact_value.upper() if self.comparator == "=": return contact_value == query_value else: raise SearchException( _(f"Unknown language comparator: '{self.comparator}'")) elif field_key == "created_on": datetime_value = contact_json.get("created_on") contact_value = str_to_datetime(datetime_value, org.timezone) utc_range = calculate_utc_range(org, self.value) if self.comparator == "=": return contact_value >= utc_range[ 0] and contact_value < utc_range[1] elif self.comparator == ">": return contact_value >= utc_range[1] elif self.comparator == ">=": return contact_value >= utc_range[0] elif self.comparator == "<": return contact_value < utc_range[0] elif self.comparator == "<=": return contact_value < utc_range[1] else: raise SearchException( _(f"Unknown created_on comparator: '{self.comparator}'" )) elif field_key == "name": query_value = self.value.upper() raw_contact_value = contact_json.get("name") if raw_contact_value is None: return False else: contact_value = raw_contact_value.upper() if self.comparator == "=": return contact_value == query_value elif self.comparator == "~": return query_value in contact_value else: # pragma: no cover raise SearchException( _(f"Unknown name comparator: '{self.comparator}'")) else: raise SearchException( _(f"No support for attribute field: '{field}'")) else: # pragma: no cover raise SearchException( _(f"Unrecognized contact field type '{prop_type}'"))
def as_elasticsearch(self, org, prop_map): prop_type, field = prop_map[self.prop] if prop_type == ContactQuery.PROP_FIELD: field_uuid = str(field.uuid) es_query = es_Q("term", **{"fields.field": field_uuid}) if field.value_type == Value.TYPE_TEXT: query_value = self.value.lower() if self.comparator == "=": es_query &= es_Q("term", **{"fields.text": query_value}) else: raise SearchException( _("Unknown text comparator: '%s'") % (self.comparator, )) elif field.value_type == Value.TYPE_NUMBER: query_value = str(self._parse_number(self.value)) if self.comparator == "=": es_query &= es_Q("match", **{"fields.number": query_value}) elif self.comparator == ">": es_query &= es_Q("range", **{"fields.number": { "gt": query_value }}) elif self.comparator == ">=": es_query &= es_Q("range", **{"fields.number": { "gte": query_value }}) elif self.comparator == "<": es_query &= es_Q("range", **{"fields.number": { "lt": query_value }}) elif self.comparator == "<=": es_query &= es_Q("range", **{"fields.number": { "lte": query_value }}) else: raise SearchException( _("Unknown number comparator: '%s'") % (self.comparator, )) elif field.value_type == Value.TYPE_DATETIME: query_value = str_to_datetime(self.value, field.org.timezone, field.org.get_dayfirst(), fill_time=False) if not query_value: raise SearchException( _("Unable to parse the date '%s'") % self.value) utc_range = date_to_utc_range(query_value.date(), field.org) if self.comparator == "=": es_query &= es_Q( "range", **{ "fields.datetime": { "gte": utc_range[0].isoformat(), "lt": utc_range[1].isoformat() } }) elif self.comparator == ">": es_query &= es_Q( "range", **{ "fields.datetime": { "gte": utc_range[1].isoformat() } }) elif self.comparator == ">=": es_query &= es_Q( "range", **{ "fields.datetime": { "gte": utc_range[0].isoformat() } }) elif self.comparator == "<": es_query &= es_Q( "range", **{ "fields.datetime": { "lt": utc_range[0].isoformat() } }) elif self.comparator == "<=": es_query &= es_Q( "range", **{ "fields.datetime": { "lt": utc_range[1].isoformat() } }) else: raise SearchException( _("Unknown datetime comparator: '%s'") % (self.comparator, )) elif field.value_type in (Value.TYPE_STATE, Value.TYPE_DISTRICT, Value.TYPE_WARD): query_value = self.value.lower() if field.value_type == Value.TYPE_WARD: field_name = "fields.ward" elif field.value_type == Value.TYPE_DISTRICT: field_name = "fields.district" elif field.value_type == Value.TYPE_STATE: field_name = "fields.state" else: # pragma: no cover raise SearchException( _("Unknown location type: '%s'") % (field.value_type, )) if self.comparator == "=": field_name += "_keyword" es_query &= es_Q("term", **{field_name: query_value}) else: raise SearchException( _("Unsupported comparator '%s' for location field") % self.comparator) else: # pragma: no cover raise SearchException( _("Unrecognized contact field type '%s'") % field.value_type) return es_Q("nested", path="fields", query=es_query) elif prop_type == ContactQuery.PROP_ATTRIBUTE: query_value = self.value.lower() if field == "name": if self.comparator == "=": field_name = "name.keyword" es_query = es_Q("term", **{field_name: query_value}) elif self.comparator == "~": field_name = "name" es_query = es_Q("match", **{field_name: query_value}) else: raise SearchException( _("Unknown attribute comparator: '%s'") % (self.comparator, )) elif field == "id": es_query = es_Q("ids", **{"values": [query_value]}) else: # pragma: no cover raise SearchException( _("Unknown attribute field '%s'") % (field, )) return es_query elif prop_type == ContactQuery.PROP_SCHEME: query_value = self.value.lower() es_query = es_Q("term", **{"urns.scheme": field.lower()}) if org.is_anon: return es_Q("ids", **{"values": [-1]}) else: if self.comparator == "=": es_query &= es_Q("term", **{"urns.path.keyword": query_value}) elif self.comparator == "~": es_query &= es_Q("match_phrase", **{"urns.path": query_value}) else: raise SearchException( _("Unknown scheme comparator: '%s'") % (self.comparator, )) return es_Q("nested", path="urns", query=es_query) else: # pragma: no cover raise SearchException( _("Unrecognized contact field type '%s'") % prop_type)
def evaluate(self, contact_json, prop_map): prop_type, field = prop_map[self.prop] if self.comparator.lower() in self.IS_SET_LOOKUPS: is_set = True elif self.comparator.lower() in self.IS_NOT_SET_LOOKUPS: is_set = False else: # pragma: no cover raise SearchException( _("Invalid operator for empty string comparison")) if prop_type == ContactQuery.PROP_FIELD: field_uuid = six.text_type(field.uuid) contact_fields = contact_json.get('fields') contact_field = contact_fields.get(field_uuid) # contact field does not exist if contact_field is None: if is_set: return False else: return True else: if field.value_type == Value.TYPE_TEXT: contact_value = contact_field.get('text') if is_set: if contact_value is not None: return True else: # pragma: can't cover return False else: if contact_value is not None: return False else: # pragma: can't cover return True elif field.value_type == Value.TYPE_NUMBER: try: contact_value = self._parse_number( contact_field.get('decimal', contact_field.get('number'))) except SearchException: contact_value = None if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True elif field.value_type == Value.TYPE_DATETIME: contact_value = str_to_datetime( contact_field.get('datetime'), field.org.timezone) if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True elif field.value_type == Value.TYPE_WARD: contact_value = contact_field.get('ward') if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True elif field.value_type == Value.TYPE_DISTRICT: contact_value = contact_field.get('district') if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True elif field.value_type == Value.TYPE_STATE: contact_value = contact_field.get('state') if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True else: # pragma: no cover raise SearchException( _("Unrecognized contact field type '%s'") % field.value_type) elif prop_type == ContactQuery.PROP_SCHEME: urn_exists = next((urn for urn in contact_json.get('urns') if urn.get('scheme') == field), None) if not urn_exists: if is_set: return False else: return True else: if is_set: return True else: return False else: raise SearchException( _("Unrecognized contact field type '%s'") % prop_type)
def as_elasticsearch(self, org, prop_map): prop_type, field = prop_map[self.prop] if prop_type == ContactQuery.PROP_FIELD: field_uuid = six.text_type(field.uuid) es_query = es_Q('term', **{'fields.field': field_uuid}) if field.value_type == Value.TYPE_TEXT: query_value = self.value.lower() if self.comparator == '=': es_query &= es_Q('term', **{'fields.text': query_value}) else: raise SearchException( _("Unknown text comparator: '%s'") % (self.comparator, )) elif field.value_type == Value.TYPE_NUMBER: query_value = six.text_type(self._parse_number(self.value)) if self.comparator == '=': es_query &= es_Q('match', **{'fields.number': query_value}) elif self.comparator == '>': es_query &= es_Q('range', **{'fields.number': { 'gt': query_value }}) elif self.comparator == '>=': es_query &= es_Q('range', **{'fields.number': { 'gte': query_value }}) elif self.comparator == '<': es_query &= es_Q('range', **{'fields.number': { 'lt': query_value }}) elif self.comparator == '<=': es_query &= es_Q('range', **{'fields.number': { 'lte': query_value }}) else: raise SearchException( _("Unknown number comparator: '%s'") % (self.comparator, )) elif field.value_type == Value.TYPE_DATETIME: query_value = str_to_datetime(self.value, field.org.timezone, field.org.get_dayfirst(), fill_time=False) if not query_value: raise SearchException( _("Unable to parse the date '%s'") % self.value) utc_range = date_to_utc_range(query_value.date(), field.org) if self.comparator == '=': es_query &= es_Q( 'range', **{ 'fields.datetime': { 'gte': utc_range[0].isoformat(), 'lt': utc_range[1].isoformat() } }) elif self.comparator == '>': es_query &= es_Q( 'range', **{ 'fields.datetime': { 'gte': utc_range[1].isoformat() } }) elif self.comparator == '>=': es_query &= es_Q( 'range', **{ 'fields.datetime': { 'gte': utc_range[0].isoformat() } }) elif self.comparator == '<': es_query &= es_Q( 'range', **{ 'fields.datetime': { 'lt': utc_range[0].isoformat() } }) elif self.comparator == '<=': es_query &= es_Q( 'range', **{ 'fields.datetime': { 'lt': utc_range[1].isoformat() } }) else: raise SearchException( _("Unknown datetime comparator: '%s'") % (self.comparator, )) elif field.value_type in (Value.TYPE_STATE, Value.TYPE_DISTRICT, Value.TYPE_WARD): query_value = self.value.lower() if field.value_type == Value.TYPE_WARD: field_name = 'fields.ward' elif field.value_type == Value.TYPE_DISTRICT: field_name = 'fields.district' elif field.value_type == Value.TYPE_STATE: field_name = 'fields.state' else: # pragma: no cover raise SearchException( _("Unknown location type: '%s'") % (field.value_type, )) if self.comparator == '=': field_name += '.keyword' es_query &= es_Q('term', **{field_name: query_value}) else: raise SearchException( _("Unsupported comparator '%s' for location field") % self.comparator) else: # pragma: no cover raise SearchException( _("Unrecognized contact field type '%s'") % field.value_type) return es_Q('nested', path='fields', query=es_query) elif prop_type == ContactQuery.PROP_ATTRIBUTE: query_value = self.value.lower() if field == 'name': if self.comparator == '=': field_name = 'name.keyword' es_query = es_Q('term', **{field_name: query_value}) elif self.comparator == '~': field_name = 'name' es_query = es_Q('match', **{field_name: query_value}) else: raise SearchException( _("Unknown attribute comparator: '%s'") % (self.comparator, )) elif field == 'id': es_query = es_Q('ids', **{'values': [query_value]}) else: # pragma: no cover raise SearchException( _("Unknown attribute field '%s'") % (field, )) return es_query elif prop_type == ContactQuery.PROP_SCHEME: query_value = self.value.lower() es_query = es_Q('term', **{'urns.scheme': field.lower()}) if org.is_anon: return es_Q('ids', **{'values': [-1]}) else: if self.comparator == '=': es_query &= es_Q('term', **{'urns.path.keyword': query_value}) elif self.comparator == '~': es_query &= es_Q('match_phrase', **{'urns.path': query_value}) else: raise SearchException( _("Unknown scheme comparator: '%s'") % (self.comparator, )) return es_Q('nested', path='urns', query=es_query) else: # pragma: no cover raise SearchException( _("Unrecognized contact field type '%s'") % prop_type)
def evaluate(self, contact_json, prop_map): prop_type, field = prop_map[self.prop] if prop_type == ContactQuery.PROP_FIELD: field_uuid = six.text_type(field.uuid) contact_fields = contact_json.get('fields') if field_uuid not in contact_fields: return False if field.value_type == Value.TYPE_TEXT: query_value = self.value.upper() contact_value = contact_fields.get(field_uuid).get( 'text').upper() if self.comparator == '=': return contact_value == query_value else: raise SearchException( _("Unknown text comparator: '%s'") % (self.comparator, )) elif field.value_type == Value.TYPE_NUMBER: query_value = self._parse_number(self.value) number_value = contact_fields.get(field_uuid).get( 'number', contact_fields.get(field_uuid).get('decimal')) if number_value is None: return False contact_value = self._parse_number(number_value) if self.comparator == '=': return contact_value == query_value elif self.comparator == '>': return contact_value > query_value elif self.comparator == '>=': return contact_value >= query_value elif self.comparator == '<': return contact_value < query_value elif self.comparator == '<=': return contact_value <= query_value else: raise SearchException( _("Unknown number comparator: '%s'") % (self.comparator, )) elif field.value_type == Value.TYPE_DATETIME: query_value = str_to_datetime(self.value, field.org.timezone, field.org.get_dayfirst(), fill_time=False) if not query_value: raise SearchException( _("Unable to parse the date '%s'") % self.value) datetime_value = contact_fields.get(field_uuid).get('datetime') if datetime_value is None: return False contact_value = str_to_datetime(datetime_value, field.org.timezone) utc_range = date_to_utc_range(query_value.date(), field.org) if self.comparator == '=': return contact_value >= utc_range[ 0] and contact_value < utc_range[1] elif self.comparator == '>': return contact_value >= utc_range[1] elif self.comparator == '>=': return contact_value >= utc_range[0] elif self.comparator == '<': return contact_value < utc_range[0] elif self.comparator == '<=': return contact_value < utc_range[1] else: raise SearchException( _("Unknown datetime comparator: '%s'") % (self.comparator, )) elif field.value_type in (Value.TYPE_STATE, Value.TYPE_DISTRICT, Value.TYPE_WARD): query_value = self.value.upper() if field.value_type == Value.TYPE_WARD: ward_value = contact_fields.get(field_uuid).get('ward') if ward_value is None: ward_value = "" contact_value = ward_value.upper().split(' > ')[-1] elif field.value_type == Value.TYPE_DISTRICT: district_value = contact_fields.get(field_uuid).get( 'district') if district_value is None: district_value = "" contact_value = district_value.upper().split(' > ')[-1] elif field.value_type == Value.TYPE_STATE: state_value = contact_fields.get(field_uuid).get('state') if state_value is None: state_value = "" contact_value = state_value.upper().split(' > ')[-1] else: # pragma: no cover raise SearchException( _("Unknown location type: '%s'") % (field.value_type, )) if self.comparator == '=': return contact_value == query_value else: raise SearchException( _("Unsupported comparator '%s' for location field") % self.comparator) else: # pragma: no cover raise SearchException( _("Unrecognized contact field type '%s'") % field.value_type) elif prop_type == ContactQuery.PROP_SCHEME: for urn in contact_json.get('urns'): if urn.get('scheme') == field: contact_value = urn.get('path').upper() query_value = self.value.upper() if self.comparator == '=': if contact_value == query_value: return True elif self.comparator == '~': if query_value in contact_value: return True else: raise SearchException( _("Unknown urn scheme comparator: '%s'") % (self.comparator, )) return False else: raise SearchException( _("Unrecognized contact field type '%s'") % prop_type)
def evaluate(self, org, contact_json, prop_map): prop_type, field = prop_map[self.prop] if self.comparator.lower() in self.IS_SET_LOOKUPS: is_set = True elif self.comparator.lower() in self.IS_NOT_SET_LOOKUPS: is_set = False else: # pragma: no cover raise SearchException(_("Invalid operator for empty string comparison")) if prop_type == ContactQuery.PROP_FIELD: field_uuid = str(field.uuid) contact_fields = contact_json.get("fields") contact_field = contact_fields.get(field_uuid) # contact field does not exist if contact_field is None: if is_set: return False else: return True else: if field.value_type == Value.TYPE_TEXT: contact_value = contact_field.get("text") if is_set: if contact_value is not None: return True else: # pragma: can't cover return False else: if contact_value is not None: return False else: # pragma: can't cover return True elif field.value_type == Value.TYPE_NUMBER: try: contact_value = self._parse_number(contact_field.get("decimal", contact_field.get("number"))) except SearchException: contact_value = None if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True elif field.value_type == Value.TYPE_DATETIME: contact_value = str_to_datetime(contact_field.get("datetime"), field.org.timezone) if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True elif field.value_type == Value.TYPE_WARD: contact_value = contact_field.get("ward") if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True elif field.value_type == Value.TYPE_DISTRICT: contact_value = contact_field.get("district") if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True elif field.value_type == Value.TYPE_STATE: contact_value = contact_field.get("state") if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True else: # pragma: no cover raise SearchException(_(f"Unrecognized contact field type '{field.value_type}'")) elif prop_type == ContactQuery.PROP_SCHEME: urn_exists = next((urn for urn in contact_json.get("urns") if urn.get("scheme") == field), None) if not urn_exists: if is_set: return False else: return True else: if is_set: return True else: return False elif prop_type == ContactQuery.PROP_ATTRIBUTE: field_key = field.key if field_key == "language": contact_value = contact_json.get("language") if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True elif field_key == "name": contact_value = contact_json.get("name") if is_set: if contact_value is not None: return True else: return False else: if contact_value is not None: return False else: return True else: # pragma: no cover raise SearchException(_(f"No support for attribute field: '{field}'")) else: # pragma: no cover raise SearchException(_(f"Unrecognized contact field type '{prop_type}'"))
def evaluate(self, org, contact_json, prop_map): prop_type, field = prop_map[self.prop] if prop_type == ContactQuery.PROP_FIELD: field_uuid = str(field.uuid) contact_fields = contact_json.get("fields") if field_uuid not in contact_fields: return False if field.value_type == Value.TYPE_TEXT: query_value = self.value.upper() contact_value = contact_fields.get(field_uuid).get("text").upper() if self.comparator == "=": return contact_value == query_value else: raise SearchException(_(f"Unknown text comparator: '{self.comparator}'")) elif field.value_type == Value.TYPE_NUMBER: query_value = self._parse_number(self.value) number_value = contact_fields.get(field_uuid).get( "number", contact_fields.get(field_uuid).get("decimal") ) if number_value is None: return False contact_value = self._parse_number(number_value) if self.comparator == "=": return contact_value == query_value elif self.comparator == ">": return contact_value > query_value elif self.comparator == ">=": return contact_value >= query_value elif self.comparator == "<": return contact_value < query_value elif self.comparator == "<=": return contact_value <= query_value else: raise SearchException(_(f"Unknown number comparator: '{self.comparator}'")) elif field.value_type == Value.TYPE_DATETIME: query_value = str_to_datetime( self.value, field.org.timezone, field.org.get_dayfirst(), fill_time=False ) if not query_value: raise SearchException(_(f"Unable to parse the date '{self.value}'")) datetime_value = contact_fields.get(field_uuid).get("datetime") if datetime_value is None: return False contact_value = str_to_datetime(datetime_value, field.org.timezone) utc_range = calculate_utc_range(org, self.value) if self.comparator == "=": return contact_value >= utc_range[0] and contact_value < utc_range[1] elif self.comparator == ">": return contact_value >= utc_range[1] elif self.comparator == ">=": return contact_value >= utc_range[0] elif self.comparator == "<": return contact_value < utc_range[0] elif self.comparator == "<=": return contact_value < utc_range[1] else: raise SearchException(_(f"Unknown datetime comparator: '{self.comparator}'")) elif field.value_type in (Value.TYPE_STATE, Value.TYPE_DISTRICT, Value.TYPE_WARD): query_value = self.value.upper() if field.value_type == Value.TYPE_WARD: ward_value = contact_fields.get(field_uuid).get("ward") if ward_value is None: ward_value = "" contact_value = ward_value.upper().split(" > ")[-1] elif field.value_type == Value.TYPE_DISTRICT: district_value = contact_fields.get(field_uuid).get("district") if district_value is None: district_value = "" contact_value = district_value.upper().split(" > ")[-1] elif field.value_type == Value.TYPE_STATE: state_value = contact_fields.get(field_uuid).get("state") if state_value is None: state_value = "" contact_value = state_value.upper().split(" > ")[-1] else: # pragma: no cover raise SearchException(_(f"Unknown location type: '{field.value_type}'")) if self.comparator == "=": return contact_value == query_value else: raise SearchException(_(f"Unsupported comparator '{self.comparator}' for location field")) else: # pragma: no cover raise SearchException(_(f"Unrecognized contact field type '{field.value_type}'")) elif prop_type == ContactQuery.PROP_SCHEME: for urn in contact_json.get("urns"): if urn.get("scheme") == field: contact_value = urn.get("path").upper() query_value = self.value.upper() if self.comparator == "=": if contact_value == query_value: return True elif self.comparator == "~": if query_value in contact_value: return True else: raise SearchException(_(f"Unknown urn scheme comparator: '{self.comparator}'")) return False elif prop_type == ContactQuery.PROP_ATTRIBUTE: field_key = field.key if field_key == "language": query_value = self.value.upper() raw_contact_value = contact_json.get("language") if raw_contact_value is None: return False else: contact_value = raw_contact_value.upper() if self.comparator == "=": return contact_value == query_value else: raise SearchException(_(f"Unknown language comparator: '{self.comparator}'")) elif field_key == "created_on": datetime_value = contact_json.get("created_on") contact_value = str_to_datetime(datetime_value, org.timezone) utc_range = calculate_utc_range(org, self.value) if self.comparator == "=": return contact_value >= utc_range[0] and contact_value < utc_range[1] elif self.comparator == ">": return contact_value >= utc_range[1] elif self.comparator == ">=": return contact_value >= utc_range[0] elif self.comparator == "<": return contact_value < utc_range[0] elif self.comparator == "<=": return contact_value < utc_range[1] else: raise SearchException(_(f"Unknown created_on comparator: '{self.comparator}'")) elif field_key == "name": query_value = self.value.upper() raw_contact_value = contact_json.get("name") if raw_contact_value is None: return False else: contact_value = raw_contact_value.upper() if self.comparator == "=": return contact_value == query_value elif self.comparator == "~": return query_value in contact_value else: # pragma: no cover raise SearchException(_(f"Unknown name comparator: '{self.comparator}'")) else: raise SearchException(_(f"No support for attribute field: '{field}'")) else: # pragma: no cover raise SearchException(_(f"Unrecognized contact field type '{prop_type}'"))