def test_query_completed_date(self): domain = 'test-completed-{}'.format(self.test_id) early = datetime.datetime(2015, 12, 5) later = datetime.datetime(2015, 12, 8) self._ship_forms_to_es( 2 * [TestFormMetadata(domain=domain, time_end=early)] + 1 * [TestFormMetadata(domain=domain, time_end=later)]) base_qs = FormES().domain(domain) self.assertEqual(3, base_qs.run().total) # test gt/gte self.assertEqual( 3, base_qs.completed(gt=early - datetime.timedelta(days=1)).run().total) self.assertEqual(3, base_qs.completed(gte=early).run().total) self.assertEqual(1, base_qs.completed(gt=early).run().total) self.assertEqual(1, base_qs.completed(gte=later).run().total) self.assertEqual(0, base_qs.completed(gt=later).run().total) # test lt/lte self.assertEqual( 3, base_qs.completed(lt=later + datetime.timedelta(days=1)).run().total) self.assertEqual(3, base_qs.completed(lte=later).run().total) self.assertEqual(2, base_qs.completed(lt=later).run().total) self.assertEqual(2, base_qs.completed(lte=early).run().total) self.assertEqual(0, base_qs.completed(lt=early).run().total) # test both self.assertEqual(0, base_qs.completed(gt=early, lt=later).run().total)
def get_all_user_ids_submitted(domain, app_ids=None): query = FormES().domain(domain).aggregation(TermsAggregation("user_id", "form.meta.userID")).size(0) if app_ids: query = query.app(app_ids) return query.run().aggregations.user_id.buckets_dict.keys()
def _get_form_counts_by_user(domain, datespan, is_submission_time, user_ids=None): form_query = FormES().domain(domain).filter(filters.NOT(xmlns_filter(SYSTEM_FORM_XMLNS))) if is_submission_time: form_query = form_query.submitted(gte=datespan.startdate.date(), lte=datespan.enddate.date()) else: form_query = form_query.completed(gte=datespan.startdate.date(), lte=datespan.enddate.date()) if user_ids: form_query = form_query.user_id(user_ids) form_query = form_query.user_aggregation().size(0) return form_query.run().aggregations.user.counts_by_bucket()
def get_last_submission_time_for_user(domain, user_id, datespan): form_query = FormES() \ .domain(domain) \ .user_id([user_id]) \ .completed(gte=datespan.startdate.date(), lte=datespan.enddate.date()) \ .sort("form.meta.timeEnd", desc=True) \ .size(1) results = form_query.run().hits def convert_to_date(date): return string_to_datetime(date).date() if date else None return convert_to_date(results[0]['form']['meta']['timeEnd'] if results else None)
def _get_form_counts_by_user(domain, datespan, is_submission_time): form_query = FormES().domain(domain) if is_submission_time: form_query = (form_query .submitted(gte=datespan.startdate.date(), lte=datespan.enddate.date())) else: form_query = (form_query .completed(gte=datespan.startdate.date(), lte=datespan.enddate.date())) form_query = (form_query .user_aggregation() .size(1)) return form_query.run().aggregations.user.counts_by_bucket()
def _get_form_counts_by_user(domain, datespan, is_submission_time, user_ids=None): form_query = FormES().domain(domain).filter(filters.NOT(xmlns_filter(SYSTEM_FORM_XMLNS))) if is_submission_time: form_query = (form_query .submitted(gte=datespan.startdate.date(), lte=datespan.enddate.date())) else: form_query = (form_query .completed(gte=datespan.startdate.date(), lte=datespan.enddate.date())) if user_ids: form_query = form_query.user_id(user_ids) form_query = (form_query .user_aggregation() .size(0)) return form_query.run().aggregations.user.counts_by_bucket()
def _get_form_counts_by_user(domain, datespan, is_submission_time, user_ids=None, export=False): es_instance = ES_EXPORT_INSTANCE if export else ES_DEFAULT_INSTANCE form_query = FormES(es_instance_alias=es_instance).domain(domain) for xmlns in SYSTEM_FORM_XMLNS_MAP.keys(): form_query = form_query.filter(filters.NOT(xmlns_filter(xmlns))) if is_submission_time: form_query = (form_query.submitted(gte=datespan.startdate.date(), lte=datespan.enddate.date())) else: form_query = (form_query.completed(gte=datespan.startdate.date(), lte=datespan.enddate.date())) if user_ids: form_query = form_query.user_id(user_ids) form_query = (form_query.user_aggregation().size(0)) return form_query.run().aggregations.user.counts_by_bucket()
def _get_form_counts_by_user(domain, datespan, is_submission_time, user_ids=None, export=False): es_instance = ES_EXPORT_INSTANCE if export else ES_DEFAULT_INSTANCE form_query = FormES(es_instance_alias=es_instance).domain(domain) for xmlns in SYSTEM_FORM_XMLNS_MAP.keys(): form_query = form_query.filter(filters.NOT(xmlns_filter(xmlns))) if is_submission_time: form_query = (form_query .submitted(gte=datespan.startdate.date(), lte=datespan.enddate.date())) else: form_query = (form_query .completed(gte=datespan.startdate.date(), lte=datespan.enddate.date())) if user_ids: form_query = form_query.user_id(user_ids) form_query = (form_query .user_aggregation() .size(0)) return form_query.run().aggregations.user.counts_by_bucket()
def _get_form_counts_by_date(domain, user_ids, datespan, timezone, is_submission_time): form_query = FormES().domain(domain).user_id(user_ids).filter(filters.NOT(xmlns_filter(SYSTEM_FORM_XMLNS))) if is_submission_time: form_query = form_query.submitted( gte=datespan.startdate.date(), lte=datespan.enddate.date() ).submitted_histogram(timezone.zone) else: form_query = form_query.completed( gte=datespan.startdate.date(), lte=datespan.enddate.date() ).completed_histogram(timezone.zone) form_query = form_query.size(0) results = form_query.run().aggregations.date_histogram.buckets_list # Convert timestamp into timezone aware dateime. Must divide timestamp by 1000 since python's # fromtimestamp takes a timestamp in seconds, whereas elasticsearch's timestamp is in milliseconds results = map( lambda result: (datetime.fromtimestamp(result.key / 1000).date().isoformat(), result.doc_count), results ) return dict(results)
def test_query_completed_date(self): domain = 'test-completed-{}'.format(self.test_id) early = datetime.datetime(2015, 12, 5) later = datetime.datetime(2015, 12, 8) self._ship_forms_to_es( 2 * [TestFormMetadata(domain=domain, time_end=early)] + 1 * [TestFormMetadata(domain=domain, time_end=later)] ) base_qs = FormES().domain(domain) self.assertEqual(3, base_qs.run().total) # test gt/gte self.assertEqual(3, base_qs.completed(gt=early - datetime.timedelta(days=1)).run().total) self.assertEqual(3, base_qs.completed(gte=early).run().total) self.assertEqual(1, base_qs.completed(gt=early).run().total) self.assertEqual(1, base_qs.completed(gte=later).run().total) self.assertEqual(0, base_qs.completed(gt=later).run().total) # test lt/lte self.assertEqual(3, base_qs.completed(lt=later + datetime.timedelta(days=1)).run().total) self.assertEqual(3, base_qs.completed(lte=later).run().total) self.assertEqual(2, base_qs.completed(lt=later).run().total) self.assertEqual(2, base_qs.completed(lte=early).run().total) self.assertEqual(0, base_qs.completed(lt=early).run().total) # test both self.assertEqual(0, base_qs.completed(gt=early, lt=later).run().total)
def get_last_form_submission_for_xmlns(domain, xmlns): query = FormES().domain(domain).xmlns(xmlns).sort("received_on", desc=True).size(1) if query.run().hits: return query.run().hits[0] return None