def check_elasticsearch_lag():
if settings.ELASTICSEARCH_URL:
from temba.utils.es import ES, ModelESSearch
# get the modified_on of the last synced contact
res = (ModelESSearch(
model=Contact,
index="contacts").params(size=1).sort("-modified_on_mu").source(
include=["modified_on", "id"]).using(ES).execute())
es_hits = res["hits"]["hits"]
if es_hits:
# if we have elastic results, make sure they aren't more than five minutes behind
db_contact = Contact.objects.filter(
is_test=False).order_by("-modified_on").first()
es_modified_on = iso8601.parse_date(
es_hits[0]["_source"]["modified_on"], pytz.utc)
es_id = es_hits[0]["_source"]["id"]
# no db contact is an error, ES should be empty as well
if not db_contact:
logger.error(
"db empty but ElasticSearch has contacts. Newest ES(id: %d, modified_on: %s)",
es_id,
es_modified_on,
)
return False
# check the lag between the two, shouldn't be more than 5 minutes
if db_contact.modified_on - es_modified_on > timedelta(minutes=5):
logger.error(
"drift between ElasticSearch and DB. Newest DB(id: %d, modified_on: %s) Newest ES(id: %d, modified_on: %s)",
db_contact.id,
db_contact.modified_on,
es_id,
es_modified_on,
)
return False
else:
# we don't have any ES hits, get our oldest db contact, check it is less than five minutes old
db_contact = Contact.objects.filter(
is_test=False).order_by("modified_on").first()
if db_contact and timezone.now(
) - db_contact.modified_on > timedelta(minutes=5):
logger.error(
"ElasticSearch empty with DB contacts older than five minutes. Oldest DB(id: %d, modified_on: %s)",
db_contact.id,
db_contact.modified_on,
)
return False
return True
def contact_es_search(org, text, base_group=None, sort_struct=None):
"""
Returns ES query
"""
if not base_group:
base_group = org.cached_all_contacts_group
if not sort_struct:
sort_field = "-id"
else:
if sort_struct["field_type"] == "field":
sort_field = {
sort_struct["field_path"]: {
"order": sort_struct["sort_direction"],
"nested": {
"path": "fields",
"filter": {
"term": {
"fields.field": sort_struct["field_uuid"]
}
}
},
}
}
else:
sort_field = {
sort_struct["field_name"]: {
"order": sort_struct["sort_direction"]
}
}
es_filter = es_Q(
"bool",
filter=[
# es_Q('term', is_blocked=False),
# es_Q('term', is_stopped=False),
es_Q("term", org_id=org.id),
es_Q("term", groups=str(base_group.uuid)),
],
)
if text:
parsed = parse_query(text, as_anon=org.is_anon)
es_match = parsed.as_elasticsearch(org)
else:
parsed = None
es_match = es_Q()
return (
(ModelESSearch(model=Contact, index="contacts").params(
routing=org.id).query(es_match & es_filter).sort(sort_field)),
parsed,
)
def contact_es_search(org, text, base_group=None):
"""
Returns ES query
"""
if not base_group:
base_group = org.cached_all_contacts_group
es_filter = es_Q(
'bool',
filter=[
# es_Q('term', is_blocked=False),
# es_Q('term', is_stopped=False),
es_Q('term', org_id=org.id),
es_Q('term', groups=six.text_type(base_group.uuid))
])
parsed = parse_query(text, as_anon=org.is_anon)
es_match = parsed.as_elasticsearch(org)
return (ModelESSearch(model=Contact, index='contacts').params(
routing=org.id).query(es_match
& es_filter).sort('-modified_on_mu')), parsed
def omnibox_mixed_search(org, search, types):
"""
Performs a mixed group, contact and URN search, returning the first N matches of each type.
"""
search_terms = search.split(" ") if search else None
search_types = types or (SEARCH_ALL_GROUPS, SEARCH_CONTACTS, SEARCH_URNS)
per_type_limit = 25
results = []
if SEARCH_ALL_GROUPS in search_types or SEARCH_STATIC_GROUPS in search_types:
groups = ContactGroup.get_user_groups(org, ready_only=True)
# exclude dynamic groups if not searching all groups
if SEARCH_ALL_GROUPS not in search_types:
groups = groups.filter(query=None)
if search:
groups = term_search(groups, ("name__icontains", ), search_terms)
results += list(groups.order_by(Upper("name"))[:per_type_limit])
if SEARCH_CONTACTS in search_types:
sort_struct = {
"field_type": "attribute",
"sort_direction": "asc",
"field_name": "name.keyword"
}
try:
search_id = int(search)
except (ValueError, TypeError):
search_id = None
if org.is_anon and search_id is not None:
search_text = f"id = {search_id}"
elif search:
search_text = " AND ".join(f"name ~ {search_term}"
for search_term in search_terms)
else:
search_text = None
from temba.utils.es import ES
try:
search_object, _ = contact_es_search(org,
search_text,
sort_struct=sort_struct)
es_search = search_object.source(
fields=("id", )).using(ES)[:per_type_limit].execute()
contact_ids = list(mapEStoDB(Contact, es_search, only_ids=True))
es_results = Contact.objects.filter(id__in=contact_ids).order_by(
Upper("name"))
results += list(es_results[:per_type_limit])
Contact.bulk_cache_initialize(org, contacts=results)
except SearchException:
# ignore SearchException
pass
if SEARCH_URNS in search_types:
# only include URNs that are send-able
from temba.channels.models import Channel
allowed_schemes = org.get_schemes(Channel.ROLE_SEND)
from temba.utils.es import ES, ModelESSearch
if search:
# we use trigrams on Elasticsearch, minimum required length for a term is 3
filtered_search_terms = (
search_term for search_term in search_terms
if search_term != "" # and len(search_term) >= 3
)
must_condition = [{
"match_phrase": {
"urns.path": search_term
}
} for search_term in filtered_search_terms]
else:
must_condition = []
es_query = {
"query": {
"bool": {
"filter": [
{
"term": {
"org_id": org.id
}
},
{
"term": {
"groups":
str(org.cached_all_contacts_group.uuid)
}
},
],
"must": [{
"nested": {
"path": "urns",
"query": {
"bool": {
"must":
must_condition,
"should": [{
"term": {
"urns.scheme": scheme
}
} for scheme in allowed_schemes],
}
},
}
}],
}
},
"sort": [{
"name.keyword": {
"order": "asc"
}
}],
}
if not org.is_anon:
search_object = ModelESSearch(
model=Contact,
index="contacts").from_dict(es_query).params(routing=org.id)
es_search = search_object.source(
fields=("id", )).using(ES)[:per_type_limit].execute()
es_results = mapEStoDB(Contact, es_search, only_ids=True)
# get ContactURNs for filtered Contacts
urns = ContactURN.objects.filter(contact_id__in=list(es_results))
# we got max `per_type_limit` contacts, but each contact can have multiple URNs and we need to limit the
# results to the per type limit
results += list(
urns.prefetch_related("contact").order_by(
Upper("path"))[:per_type_limit])
return results # sorted(results, key=lambda o: o.name if hasattr(o, 'name') else o.path)
def omnibox_mixed_search(org, search, types):
"""
Performs a mixed group, contact and URN search, returning the first N matches of each type.
"""
search_terms = search.split(" ") if search else None
search_types = types or (SEARCH_ALL_GROUPS, SEARCH_CONTACTS, SEARCH_URNS)
per_type_limit = 25
results = []
if SEARCH_ALL_GROUPS in search_types or SEARCH_STATIC_GROUPS in search_types:
groups = ContactGroup.get_user_groups(org)
# exclude dynamic groups if not searching all groups
if SEARCH_ALL_GROUPS not in search_types:
groups = groups.filter(query=None)
if search:
groups = term_search(groups, ("name__icontains",), search_terms)
results += list(groups.order_by(Upper("name"))[:per_type_limit])
if SEARCH_CONTACTS in search_types:
sort_struct = {"field_type": "attribute", "sort_direction": "asc", "field_name": "name.keyword"}
try:
search_id = int(search)
except (ValueError, TypeError):
search_id = None
if org.is_anon and search_id is not None:
search_text = f"id = {search_id}"
elif search:
# we use trigrams on Elasticsearch, minimum required length for a term is 3
filtered_search_terms = (
search_term for search_term in search_terms if search_term != "" and len(search_term) >= 3
)
search_text = " AND ".join(f"name ~ {search_term}" for search_term in filtered_search_terms)
else:
search_text = None
from temba.utils.es import ES
try:
search_object, _ = contact_es_search(org, search_text, sort_struct=sort_struct)
es_search = search_object.source(fields=("id",)).using(ES)[:per_type_limit].execute()
contact_ids = list(mapEStoDB(Contact, es_search, only_ids=True))
es_results = Contact.objects.filter(id__in=contact_ids).order_by(Upper("name"))
results += list(es_results[:per_type_limit])
Contact.bulk_cache_initialize(org, contacts=results)
except SearchException:
# ignore SearchException
pass
if SEARCH_URNS in search_types:
# only include URNs that are send-able
from temba.channels.models import Channel
allowed_schemes = org.get_schemes(Channel.ROLE_SEND)
from temba.utils.es import ES, ModelESSearch
if search:
# we use trigrams on Elasticsearch, minimum required length for a term is 3
filtered_search_terms = (
search_term for search_term in search_terms if search_term != "" and len(search_term) >= 3
)
must_condition = [{"match_phrase": {"urns.path": search_term}} for search_term in filtered_search_terms]
else:
must_condition = []
es_query = {
"query": {
"bool": {
"filter": [
{"term": {"org_id": org.id}},
{"term": {"groups": str(org.cached_all_contacts_group.uuid)}},
],
"must": [
{
"nested": {
"path": "urns",
"query": {
"bool": {
"must": must_condition,
"should": [{"term": {"urns.scheme": scheme}} for scheme in allowed_schemes],
}
},
}
}
],
}
},
"sort": [{"name.keyword": {"order": "asc"}}],
}
if not org.is_anon:
search_object = ModelESSearch(model=Contact, index="contacts").from_dict(es_query).params(routing=org.id)
es_search = search_object.source(fields=("id",)).using(ES)[:per_type_limit].execute()
es_results = mapEStoDB(Contact, es_search, only_ids=True)
# get ContactURNs for filtered Contacts
urns = ContactURN.objects.filter(contact_id__in=list(es_results))
# we got max `per_type_limit` contacts, but each contact can have multiple URNs and we need to limit the
# results to the per type limit
results += list(urns.prefetch_related("contact").order_by(Upper("path"))[:per_type_limit])
return results # sorted(results, key=lambda o: o.name if hasattr(o, 'name') else o.path)