def check_elasticsearch_lag(): if settings.ELASTICSEARCH_URL: from temba.utils.es import ES, ModelESSearch # get the modified_on of the last synced contact res = (ModelESSearch( model=Contact, index="contacts").params(size=1).sort("-modified_on_mu").source( include=["modified_on", "id"]).using(ES).execute()) es_hits = res["hits"]["hits"] if es_hits: # if we have elastic results, make sure they aren't more than five minutes behind db_contact = Contact.objects.filter( is_test=False).order_by("-modified_on").first() es_modified_on = iso8601.parse_date( es_hits[0]["_source"]["modified_on"], pytz.utc) es_id = es_hits[0]["_source"]["id"] # no db contact is an error, ES should be empty as well if not db_contact: logger.error( "db empty but ElasticSearch has contacts. Newest ES(id: %d, modified_on: %s)", es_id, es_modified_on, ) return False # check the lag between the two, shouldn't be more than 5 minutes if db_contact.modified_on - es_modified_on > timedelta(minutes=5): logger.error( "drift between ElasticSearch and DB. Newest DB(id: %d, modified_on: %s) Newest ES(id: %d, modified_on: %s)", db_contact.id, db_contact.modified_on, es_id, es_modified_on, ) return False else: # we don't have any ES hits, get our oldest db contact, check it is less than five minutes old db_contact = Contact.objects.filter( is_test=False).order_by("modified_on").first() if db_contact and timezone.now( ) - db_contact.modified_on > timedelta(minutes=5): logger.error( "ElasticSearch empty with DB contacts older than five minutes. Oldest DB(id: %d, modified_on: %s)", db_contact.id, db_contact.modified_on, ) return False return True
def contact_es_search(org, text, base_group=None, sort_struct=None): """ Returns ES query """ if not base_group: base_group = org.cached_all_contacts_group if not sort_struct: sort_field = "-id" else: if sort_struct["field_type"] == "field": sort_field = { sort_struct["field_path"]: { "order": sort_struct["sort_direction"], "nested": { "path": "fields", "filter": { "term": { "fields.field": sort_struct["field_uuid"] } } }, } } else: sort_field = { sort_struct["field_name"]: { "order": sort_struct["sort_direction"] } } es_filter = es_Q( "bool", filter=[ # es_Q('term', is_blocked=False), # es_Q('term', is_stopped=False), es_Q("term", org_id=org.id), es_Q("term", groups=str(base_group.uuid)), ], ) if text: parsed = parse_query(text, as_anon=org.is_anon) es_match = parsed.as_elasticsearch(org) else: parsed = None es_match = es_Q() return ( (ModelESSearch(model=Contact, index="contacts").params( routing=org.id).query(es_match & es_filter).sort(sort_field)), parsed, )
def contact_es_search(org, text, base_group=None): """ Returns ES query """ if not base_group: base_group = org.cached_all_contacts_group es_filter = es_Q( 'bool', filter=[ # es_Q('term', is_blocked=False), # es_Q('term', is_stopped=False), es_Q('term', org_id=org.id), es_Q('term', groups=six.text_type(base_group.uuid)) ]) parsed = parse_query(text, as_anon=org.is_anon) es_match = parsed.as_elasticsearch(org) return (ModelESSearch(model=Contact, index='contacts').params( routing=org.id).query(es_match & es_filter).sort('-modified_on_mu')), parsed
def omnibox_mixed_search(org, search, types): """ Performs a mixed group, contact and URN search, returning the first N matches of each type. """ search_terms = search.split(" ") if search else None search_types = types or (SEARCH_ALL_GROUPS, SEARCH_CONTACTS, SEARCH_URNS) per_type_limit = 25 results = [] if SEARCH_ALL_GROUPS in search_types or SEARCH_STATIC_GROUPS in search_types: groups = ContactGroup.get_user_groups(org, ready_only=True) # exclude dynamic groups if not searching all groups if SEARCH_ALL_GROUPS not in search_types: groups = groups.filter(query=None) if search: groups = term_search(groups, ("name__icontains", ), search_terms) results += list(groups.order_by(Upper("name"))[:per_type_limit]) if SEARCH_CONTACTS in search_types: sort_struct = { "field_type": "attribute", "sort_direction": "asc", "field_name": "name.keyword" } try: search_id = int(search) except (ValueError, TypeError): search_id = None if org.is_anon and search_id is not None: search_text = f"id = {search_id}" elif search: search_text = " AND ".join(f"name ~ {search_term}" for search_term in search_terms) else: search_text = None from temba.utils.es import ES try: search_object, _ = contact_es_search(org, search_text, sort_struct=sort_struct) es_search = search_object.source( fields=("id", )).using(ES)[:per_type_limit].execute() contact_ids = list(mapEStoDB(Contact, es_search, only_ids=True)) es_results = Contact.objects.filter(id__in=contact_ids).order_by( Upper("name")) results += list(es_results[:per_type_limit]) Contact.bulk_cache_initialize(org, contacts=results) except SearchException: # ignore SearchException pass if SEARCH_URNS in search_types: # only include URNs that are send-able from temba.channels.models import Channel allowed_schemes = org.get_schemes(Channel.ROLE_SEND) from temba.utils.es import ES, ModelESSearch if search: # we use trigrams on Elasticsearch, minimum required length for a term is 3 filtered_search_terms = ( search_term for search_term in search_terms if search_term != "" # and len(search_term) >= 3 ) must_condition = [{ "match_phrase": { "urns.path": search_term } } for search_term in filtered_search_terms] else: must_condition = [] es_query = { "query": { "bool": { "filter": [ { "term": { "org_id": org.id } }, { "term": { "groups": str(org.cached_all_contacts_group.uuid) } }, ], "must": [{ "nested": { "path": "urns", "query": { "bool": { "must": must_condition, "should": [{ "term": { "urns.scheme": scheme } } for scheme in allowed_schemes], } }, } }], } }, "sort": [{ "name.keyword": { "order": "asc" } }], } if not org.is_anon: search_object = ModelESSearch( model=Contact, index="contacts").from_dict(es_query).params(routing=org.id) es_search = search_object.source( fields=("id", )).using(ES)[:per_type_limit].execute() es_results = mapEStoDB(Contact, es_search, only_ids=True) # get ContactURNs for filtered Contacts urns = ContactURN.objects.filter(contact_id__in=list(es_results)) # we got max `per_type_limit` contacts, but each contact can have multiple URNs and we need to limit the # results to the per type limit results += list( urns.prefetch_related("contact").order_by( Upper("path"))[:per_type_limit]) return results # sorted(results, key=lambda o: o.name if hasattr(o, 'name') else o.path)
def omnibox_mixed_search(org, search, types): """ Performs a mixed group, contact and URN search, returning the first N matches of each type. """ search_terms = search.split(" ") if search else None search_types = types or (SEARCH_ALL_GROUPS, SEARCH_CONTACTS, SEARCH_URNS) per_type_limit = 25 results = [] if SEARCH_ALL_GROUPS in search_types or SEARCH_STATIC_GROUPS in search_types: groups = ContactGroup.get_user_groups(org) # exclude dynamic groups if not searching all groups if SEARCH_ALL_GROUPS not in search_types: groups = groups.filter(query=None) if search: groups = term_search(groups, ("name__icontains",), search_terms) results += list(groups.order_by(Upper("name"))[:per_type_limit]) if SEARCH_CONTACTS in search_types: sort_struct = {"field_type": "attribute", "sort_direction": "asc", "field_name": "name.keyword"} try: search_id = int(search) except (ValueError, TypeError): search_id = None if org.is_anon and search_id is not None: search_text = f"id = {search_id}" elif search: # we use trigrams on Elasticsearch, minimum required length for a term is 3 filtered_search_terms = ( search_term for search_term in search_terms if search_term != "" and len(search_term) >= 3 ) search_text = " AND ".join(f"name ~ {search_term}" for search_term in filtered_search_terms) else: search_text = None from temba.utils.es import ES try: search_object, _ = contact_es_search(org, search_text, sort_struct=sort_struct) es_search = search_object.source(fields=("id",)).using(ES)[:per_type_limit].execute() contact_ids = list(mapEStoDB(Contact, es_search, only_ids=True)) es_results = Contact.objects.filter(id__in=contact_ids).order_by(Upper("name")) results += list(es_results[:per_type_limit]) Contact.bulk_cache_initialize(org, contacts=results) except SearchException: # ignore SearchException pass if SEARCH_URNS in search_types: # only include URNs that are send-able from temba.channels.models import Channel allowed_schemes = org.get_schemes(Channel.ROLE_SEND) from temba.utils.es import ES, ModelESSearch if search: # we use trigrams on Elasticsearch, minimum required length for a term is 3 filtered_search_terms = ( search_term for search_term in search_terms if search_term != "" and len(search_term) >= 3 ) must_condition = [{"match_phrase": {"urns.path": search_term}} for search_term in filtered_search_terms] else: must_condition = [] es_query = { "query": { "bool": { "filter": [ {"term": {"org_id": org.id}}, {"term": {"groups": str(org.cached_all_contacts_group.uuid)}}, ], "must": [ { "nested": { "path": "urns", "query": { "bool": { "must": must_condition, "should": [{"term": {"urns.scheme": scheme}} for scheme in allowed_schemes], } }, } } ], } }, "sort": [{"name.keyword": {"order": "asc"}}], } if not org.is_anon: search_object = ModelESSearch(model=Contact, index="contacts").from_dict(es_query).params(routing=org.id) es_search = search_object.source(fields=("id",)).using(ES)[:per_type_limit].execute() es_results = mapEStoDB(Contact, es_search, only_ids=True) # get ContactURNs for filtered Contacts urns = ContactURN.objects.filter(contact_id__in=list(es_results)) # we got max `per_type_limit` contacts, but each contact can have multiple URNs and we need to limit the # results to the per type limit results += list(urns.prefetch_related("contact").order_by(Upper("path"))[:per_type_limit]) return results # sorted(results, key=lambda o: o.name if hasattr(o, 'name') else o.path)