def to_es_filter(self):
if self.value is None:
return None
return filters.date_range(self.filter.field,
lt=self.value.startdate,
gt=self.value.enddate)
def _get_case_counts_by_user(domain,
datespan,
case_types=None,
is_opened=True,
user_ids=None,
export=False):
date_field = 'opened_on' if is_opened else 'closed_on'
user_field = 'opened_by' if is_opened else 'closed_by'
es_instance = ES_EXPORT_INSTANCE if export else ES_DEFAULT_INSTANCE
case_query = (CaseES(es_instance_alias=es_instance).domain(domain).filter(
filters.date_range(
date_field,
gte=datespan.startdate.date(),
lte=datespan.enddate.date(),
)).terms_aggregation(user_field, 'by_user').size(0))
if case_types:
case_query = case_query.case_type(case_types)
else:
case_query = case_query.filter(
filters.NOT(case_type_filter('commcare-user')))
if user_ids:
case_query = case_query.filter(filters.term(user_field, user_ids))
return case_query.run().aggregations.by_user.counts_by_bucket()
def test_nested_filter(self):
json_output = {
"query": {
"filtered": {
"filter": {
"and": [
{"nested": {
"path": "actions",
"filter": {
"range": {
"actions.date": {
"gte": "2015-01-01",
"lt": "2015-02-01"
}
}
}
}},
{"match_all": {}}
]
},
"query": {"match_all": {}}
}
},
"size": SIZE_LIMIT
}
start, end = date(2015, 1, 1), date(2015, 2, 1)
query = (HQESQuery('cases')
.nested("actions",
filters.date_range("actions.date", gte=start, lt=end)))
self.checkQuery(query, json_output)
def test_nested_filter(self):
json_output = {
"query": {
"filtered": {
"filter": {
"and": [{
"nested": {
"path": "actions",
"filter": {
"range": {
"actions.date": {
"gte": "2015-01-01",
"lt": "2015-02-01"
}
}
}
}
}, {
"match_all": {}
}]
},
"query": {
"match_all": {}
}
}
},
"size": SIZE_LIMIT
}
start, end = date(2015, 1, 1), date(2015, 2, 1)
query = (HQESQuery('cases').nested(
"actions", filters.date_range("actions.date", gte=start, lt=end)))
self.checkQuery(query, json_output)
def _get_case_counts_by_user(domain, datespan, case_types=None, is_opened=True, user_ids=None, export=False):
date_field = 'opened_on' if is_opened else 'closed_on'
user_field = 'opened_by' if is_opened else 'closed_by'
es_instance = ES_EXPORT_INSTANCE if export else ES_DEFAULT_INSTANCE
case_query = (CaseES(es_instance_alias=es_instance)
.domain(domain)
.filter(
filters.date_range(
date_field,
gte=datespan.startdate.date(),
lte=datespan.enddate.date(),
)
)
.terms_aggregation(user_field, 'by_user')
.size(0))
if case_types:
case_query = case_query.case_type(case_types)
else:
case_query = case_query.filter(filters.NOT(case_type_filter('commcare-user')))
if user_ids:
case_query = case_query.filter(filters.term(user_field, user_ids))
return case_query.run().aggregations.by_user.counts_by_bucket()
def _get_case_counts_by_user(domain, datespan, case_types=None, is_opened=True, owner_ids=None):
date_field = 'opened_on' if is_opened else 'closed_on'
user_field = 'opened_by' if is_opened else 'closed_by'
case_query = (CaseES()
.domain(domain)
.filter(
filters.date_range(
date_field,
gte=datespan.startdate.date(),
lte=datespan.enddate.date(),
)
)
.terms_aggregation(user_field, 'by_user')
.size(0))
if case_types:
case_query = case_query.case_type(case_types)
else:
case_query = case_query.filter(filters.NOT(case_type_filter('commcare-user')))
if owner_ids:
case_query = case_query.filter(filters.term(user_field, owner_ids))
return case_query.run().aggregations.by_user.counts_by_bucket()
def _get_case_counts_by_user(domain, datespan, case_types=None, is_opened=True, owner_ids=None):
date_field = 'opened_on' if is_opened else 'closed_on'
user_field = 'opened_by' if is_opened else 'closed_by'
case_query = (CaseES()
.domain(domain)
.filter(
filters.date_range(
date_field,
gte=datespan.startdate.date(),
lte=datespan.enddate.date(),
)
)
.terms_aggregation(user_field, 'by_user')
.size(0))
if case_types:
case_query = case_query.case_type(case_types)
else:
case_query = case_query.filter(filters.NOT(case_type_filter('commcare-user')))
if owner_ids:
case_query = case_query.filter(filters.term(user_field, owner_ids))
return case_query.run().aggregations.by_user.counts_by_bucket()
def get_domains_to_update_es_filter():
"""
Returns ES filter to filter domains that are never updated or
domains that haven't been updated since a week or domains that
have been updated within last week but have new form submissions
in the last day.
"""
last_week = datetime.utcnow() - timedelta(days=7)
more_than_a_week_ago = filters.date_range('cp_last_updated', lt=last_week)
less_than_a_week_ago = filters.date_range('cp_last_updated', gte=last_week)
not_updated = filters.missing('cp_last_updated')
domains_submitted_today = (FormES().submitted(
gte=datetime.utcnow() - timedelta(days=1)).terms_aggregation(
'domain', 'domain').size(0).run().aggregations.domain.keys)
return filters.OR(
not_updated, more_than_a_week_ago,
filters.AND(less_than_a_week_ago, domains_submitted_today))
def get_domains_to_update_es_filter():
"""
Returns ES filter to filter domains that are never updated or
domains that haven't been updated since a week or domains that
have been updated within last week but have new form submissions
in the last day.
"""
last_week = datetime.utcnow() - timedelta(days=7)
more_than_a_week_ago = filters.date_range('cp_last_updated', lt=last_week)
less_than_a_week_ago = filters.date_range('cp_last_updated', gte=last_week)
not_updated = filters.missing('cp_last_updated')
domains_submitted_today = (FormES().submitted(gte=datetime.utcnow() - timedelta(days=1))
.terms_aggregation('domain', 'domain').size(0).run().aggregations.domain.keys)
return filters.OR(
not_updated,
more_than_a_week_ago,
filters.AND(less_than_a_week_ago, filters.term('name', domains_submitted_today))
)
def to_es_filter(self):
# TODO: support the array and null operators defined at top of class
if self._is_dyn_date():
start_date, end_date = get_daterange_start_end_dates(self.value['operator'], *self.value['operand'])
return filters.date_range(self.filter.field, gt=start_date, lt=end_date)
elif self._is_null():
return filters.missing(self.filter.field)
else:
terms = [v.value.lower() for v in self.value]
return filters.term(self.filter.field, terms)
def consume_params(self, raw_params):
start = raw_params.pop(self.start_param, None)
end = raw_params.pop(self.end_param, None)
if start:
start = validate_date(start)
if end:
end = validate_date(end)
if start or end:
# Note that dates are already in a string format when they arrive as query params
return filters.date_range(self.term, gte=start, lte=end)
def consume_params(self, raw_params):
start = raw_params.pop(self.start_param, None)
end = raw_params.pop(self.end_param, None)
if start:
validate_date(start)
if end:
validate_date(end)
if start or end:
# Note that dates are already in a string format when they arrive as query params
return filters.date_range(self.field, gte=start, lte=end)
def to_es_filter(self):
# TODO: support the array and null operators defined at top of class
if self._is_dyn_date():
start_date, end_date = get_daterange_start_end_dates(self.value['operator'], *self.value['operand'])
return filters.date_range(self.filter.field, gt=start_date, lt=end_date)
elif self._is_null():
return filters.missing(self.filter.field)
elif self._is_list():
terms = [v.value for v in self.value['operand']]
return filters.term(self.filter.field, terms)
else:
return self._scalar_filter.es(self.filter.field, self.value['operand'])
def deactivate_mobile_workers_by_inactivity(self, domain):
date_of_inactivity = datetime.datetime.utcnow() - datetime.timedelta(
days=self.inactivity_period)
user_query = (UserES().domain(domain).mobile_users().is_active(
).created(lte=date_of_inactivity).filter(
filters.OR(
filters.date_range(
"reporting_metadata.last_submission_for_user.submission_date",
lte=date_of_inactivity),
filters.missing(
"reporting_metadata.last_submission_for_user.submission_date"
))).source(['_id']))
user_ids = [u['_id'] for u in user_query.run().hits]
for chunked_ids in chunked(user_ids, 100):
bulk_auto_deactivate_commcare_users(chunked_ids, domain)
def _get_case_counts_by_user(domain, datespan, case_types=None, is_opened=True):
date_field = 'opened_on' if is_opened else 'closed_on'
user_field = 'opened_by' if is_opened else 'closed_by'
case_query = (CaseES()
.domain(domain)
.filter(
filters.date_range(
date_field,
gte=datespan.startdate.date(),
lte=datespan.enddate.date(),
)
)
.terms_aggregation(user_field, 'by_user')
.size(1))
if case_types:
case_query = case_query.filter({"terms": {"type.exact": case_types}})
return case_query.run().aggregations.by_user.counts_by_bucket()
def to_es_filter(self):
if self.value is None:
return None
return filters.date_range(self.filter.field, lt=self.value.startdate, gt=self.value.enddate)
