def _send_case_to_es(self, domain=None, owner_id=None, user_id=None, case_type=None, opened_on=None, closed_on=None): case = CommCareCaseSQL( case_id=uuid.uuid4().hex, domain=domain or self.domain, owner_id=owner_id or self.owner_id, modified_by=user_id or self.user_id, type=case_type or self.case_type, opened_on=opened_on or datetime.now(), opened_by=user_id or self.user_id, closed_on=closed_on, closed_by=user_id or self.user_id, server_modified_on=datetime.utcnow(), closed=bool(closed_on) ) case.track_create(CaseTransaction( type=CaseTransaction.TYPE_FORM, form_id=uuid.uuid4().hex, case=case, server_date=opened_on, )) es_case = transform_case_for_elasticsearch(case.to_json()) send_to_elasticsearch('cases', es_case) self.es.indices.refresh(CASE_INDEX) return case
def update_user_in_es(sender, couch_user, **kwargs): """ Automatically sync the user to elastic directly on save or delete """ from corehq.pillows.user import transform_user_for_elasticsearch send_to_elasticsearch("users", transform_user_for_elasticsearch(couch_user.to_json()), delete=couch_user.to_be_deleted())
def _send_form_to_es( self, domain=None, completion_time=None, received_on=None, attachment_dict=None, **metadata_kwargs): attachment_dict = attachment_dict or {} metadata = TestFormMetadata( domain=domain or self.domain, time_end=completion_time or datetime.utcnow(), received_on=received_on or datetime.utcnow(), ) for attr, value in metadata_kwargs.items(): setattr(metadata, attr, value) form_pair = make_es_ready_form(metadata) if attachment_dict: form_pair.wrapped_form.external_blobs = {name: BlobMetaRef(**meta) for name, meta in six.iteritems(attachment_dict)} form_pair.json_form['external_blobs'] = attachment_dict es_form = transform_xform_for_elasticsearch(form_pair.json_form) send_to_elasticsearch('forms', es_form) self.es.indices.refresh(XFORM_INDEX_INFO.index) return form_pair
def _send_case_to_es(self, domain=None, owner_id=None, user_id=None, case_type=None, opened_on=None, closed_on=None): actions = [CommCareCaseAction( action_type=CASE_ACTION_CREATE, date=opened_on, )] case = CommCareCase( _id=uuid.uuid4().hex, domain=domain or self.domain, owner_id=owner_id or self.owner_id, user_id=user_id or self.user_id, type=case_type or self.case_type, opened_on=opened_on or datetime.now(), opened_by=user_id or self.user_id, closed_on=closed_on, closed_by=user_id or self.user_id, actions=actions, ) send_to_elasticsearch('cases', case.to_json()) self.es.indices.refresh(CASE_INDEX_INFO.index) return case
def update_calculated_properties(): results = DomainES().is_snapshot(False).fields(["name", "_id"]).run().hits all_stats = _all_domain_stats() for r in results: dom = r["name"] calced_props = { "_id": r["_id"], "cp_n_web_users": int(all_stats["web_users"][dom]), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"][dom]), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_users_submitted_form": total_distinct_users([dom]), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_cases": int(all_stats["cases"][dom]), "cp_n_forms": int(all_stats["forms"][dom]), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), "cp_last_updated": json_format_datetime(datetime.utcnow()), "cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")), "cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")), "cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)), "cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)), "cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)), "cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)), } if calced_props['cp_first_form'] == 'No forms': del calced_props['cp_first_form'] del calced_props['cp_last_form'] send_to_elasticsearch("domains", calced_props)
def _ship_forms_to_es(cls, metadatas): for form_metadata in metadatas: form_metadata = form_metadata or TestFormMetadata() form_pair = make_es_ready_form(form_metadata) cls.forms.append(form_pair) send_to_elasticsearch('forms', form_pair.json_form) # have to refresh the index to make sure changes show up cls.es.indices.refresh(XFORM_INDEX_INFO.index)
def setUpClass(cls): super(TestFilterDslLookups, cls).setUpClass() with trap_extra_setup(ConnectionError): cls.es = get_es_new() initialize_index_and_mapping(cls.es, CASE_SEARCH_INDEX_INFO) cls.child_case_id = 'margaery' cls.parent_case_id = 'mace' cls.grandparent_case_id = 'olenna' cls.domain = "Tyrell" factory = CaseFactory(domain=cls.domain) grandparent_case = CaseStructure( case_id=cls.grandparent_case_id, attrs={ 'create': True, 'case_type': 'grandparent', 'update': { "name": "Olenna", "alias": "Queen of thorns", "house": "Tyrell", }, }) parent_case = CaseStructure( case_id=cls.parent_case_id, attrs={ 'create': True, 'case_type': 'parent', 'update': { "name": "Mace", "house": "Tyrell", }, }, indices=[CaseIndex( grandparent_case, identifier='mother', relationship='child', )]) child_case = CaseStructure( case_id=cls.child_case_id, attrs={ 'create': True, 'case_type': 'child', 'update': { "name": "Margaery", "house": "Tyrell", }, }, indices=[CaseIndex( parent_case, identifier='father', relationship='extension', )], ) for case in factory.create_or_update_cases([child_case]): send_to_elasticsearch('case_search', transform_case_for_elasticsearch(case.to_json())) cls.es.indices.refresh(CASE_SEARCH_INDEX_INFO.index)
def check_elasticsearch(): doc = {"_id": "elasticsearch-service-check", "date": datetime.datetime.now().isoformat()} send_to_elasticsearch("groups", doc) time.sleep(1) hits = GroupES().remove_default_filters().doc_id(doc["_id"]).run().hits send_to_elasticsearch("groups", doc, delete=True) # clean up if doc in hits: return ServiceStatus(True, "Successfully sent a doc to ES and read it back") return ServiceStatus(False, "Something went wrong sending a doc to ES")
def _send_form_to_es(self, domain, **metadata_kwargs): metadata = TestFormMetadata(domain=domain, time_end=datetime.utcnow(), received_on=datetime.utcnow()) for attr, value in metadata_kwargs.items(): setattr(metadata, attr, value) form_pair = make_es_ready_form(metadata) send_to_elasticsearch("forms", form_pair.json_form) self.es.indices.refresh(XFORM_INDEX_INFO.index) return form_pair
def update_calculated_properties(): results = DomainES().is_snapshot(False).fields(["name", "_id"]).run().hits all_stats = _all_domain_stats() for r in results: dom = r["name"] try: calced_props = { "_id": r["_id"], "cp_n_web_users": int(all_stats["web_users"].get(dom, 0)), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"].get(dom, 0)), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_users_submitted_form": total_distinct_users([dom]), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_30_day_cases": int(CALC_FNS["cases_in_last"](dom, 30)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_90_day_cases": int(CALC_FNS["cases_in_last"](dom, 90)), "cp_n_cases": int(all_stats["cases"].get(dom, 0)), "cp_n_forms": int(all_stats["forms"].get(dom, 0)), "cp_n_forms_30_d": int(CALC_FNS["forms_in_last"](dom, 30)), "cp_n_forms_60_d": int(CALC_FNS["forms_in_last"](dom, 60)), "cp_n_forms_90_d": int(CALC_FNS["forms_in_last"](dom, 90)), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), "cp_last_updated": json_format_datetime(datetime.utcnow()), "cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")), "cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")), "cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)), "cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)), "cp_n_sms_60_d": int(CALC_FNS["sms_in_last"](dom, 60)), "cp_n_sms_90_d": int(CALC_FNS["sms_in_last"](dom, 90)), "cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)), "cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)), "cp_n_sms_in_30_d": int(CALC_FNS["sms_in_in_last"](dom, 30)), "cp_n_sms_in_60_d": int(CALC_FNS["sms_in_in_last"](dom, 60)), "cp_n_sms_in_90_d": int(CALC_FNS["sms_in_in_last"](dom, 90)), "cp_n_sms_out_30_d": int(CALC_FNS["sms_out_in_last"](dom, 30)), "cp_n_sms_out_60_d": int(CALC_FNS["sms_out_in_last"](dom, 60)), "cp_n_sms_out_90_d": int(CALC_FNS["sms_out_in_last"](dom, 90)), "cp_n_j2me_30_d": int(CALC_FNS["j2me_forms_in_last"](dom, 30)), "cp_n_j2me_60_d": int(CALC_FNS["j2me_forms_in_last"](dom, 60)), "cp_n_j2me_90_d": int(CALC_FNS["j2me_forms_in_last"](dom, 90)), "cp_j2me_90_d_bool": int(CALC_FNS["j2me_forms_in_last_bool"](dom, 90)), "cp_300th_form": CALC_FNS["300th_form_submission"](dom) } if calced_props['cp_first_form'] is None: del calced_props['cp_first_form'] if calced_props['cp_last_form'] is None: del calced_props['cp_last_form'] if calced_props['cp_300th_form'] is None: del calced_props['cp_300th_form'] send_to_elasticsearch("domains", calced_props) except Exception, e: notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
def _create_es_user(es_client, user_id, domain): user = CommCareUser( _id=user_id, domain=domain, username='******', first_name='Harry', last_name='Casual', ) send_to_elasticsearch('users', user.to_json()) es_client.indices.refresh(USER_INDEX) return user
def _send_group_to_es(self, _id=None): group = Group( domain=self.domain, name=self.group_name, case_sharing=self.case_sharing, reporting=self.reporting, _id=_id or uuid.uuid4().hex, ) send_to_elasticsearch('groups', group.to_json()) self.es.indices.refresh(GROUP_INDEX_INFO.index) return group
def _send_group_to_es(self, _id=None, users=None): group = Group( domain=self.domain, name='narcos', users=users or [], case_sharing=False, reporting=True, _id=_id or uuid.uuid4().hex, ) send_to_elasticsearch('groups', group.to_json()) self.es.indices.refresh(GROUP_INDEX_INFO.index) return group
def _create_es_user(es_client, user_id, domain): user = CommCareUser( _id=user_id, domain=domain, username='******', first_name='Harry', last_name='Casual', is_active=True, ) send_to_elasticsearch('users', user.to_json()) es_client.indices.refresh(USER_INDEX) return user
def _send_user_to_es(self, _id=None, is_active=True): user = CommCareUser( domain=self.domain, username=self.username, _id=_id or uuid.uuid4().hex, is_active=is_active, first_name=self.first_name, last_name=self.last_name, ) send_to_elasticsearch('users', user.to_json()) self.es.indices.refresh(USER_INDEX) return user
def _send_ledger_to_es(self, section_id, case_id, entry_id, balance): ledger = { "_id": uuid.uuid4().hex, "domain": self.domain, "section_id": section_id, "case_id": case_id, "entry_id": entry_id, "balance": balance, } send_to_elasticsearch('ledgers', ledger) self.es.indices.refresh(self.es_index_info.index) return ledger
def make_checkin_case(self, properties=None): properties = properties if properties is not None else {"username": self.mobile_worker.raw_username} checkin_case = CaseFactory(self.domain).create_case( case_type="checkin", owner_id=self.mobile_worker.get_id, update=properties, ) send_to_elasticsearch( "case_search", transform_case_for_elasticsearch(checkin_case.to_json()) ) self.es.indices.refresh(CASE_SEARCH_INDEX_INFO.index) return checkin_case
def test_get_all_commcare_users_by_domain(self): from corehq.util.elastic import ensure_index_deleted from corehq.elastic import get_es_new, send_to_elasticsearch from corehq.pillows.mappings.user_mapping import USER_INDEX_INFO, USER_INDEX from pillowtop.es_utils import initialize_index_and_mapping es = get_es_new() ensure_index_deleted(USER_INDEX) initialize_index_and_mapping(es, USER_INDEX_INFO) send_to_elasticsearch('users', self.ccuser_1.to_json()) send_to_elasticsearch('users', self.ccuser_2.to_json()) es.indices.refresh(USER_INDEX) usernames = lambda users: [u.username for u in users] # if no filters are passed, should return all cc-users in the domain self.assertItemsEqual( usernames(get_commcare_users_by_filters(self.ccdomain.name, {})), usernames([self.ccuser_2, self.ccuser_1])) self.assertEqual( get_commcare_users_by_filters(self.ccdomain.name, {}, count_only=True), 2) # can search by username self.assertItemsEqual( usernames( get_commcare_users_by_filters(self.ccdomain.name, {'search_string': 'user_1'})), [self.ccuser_1.username]) self.assertEqual( get_commcare_users_by_filters(self.ccdomain.name, {'search_string': 'user_1'}, count_only=True), 1) # can search by role_id self.assertItemsEqual( usernames( get_commcare_users_by_filters( self.ccdomain.name, {'role_id': self.custom_role._id})), [self.ccuser_2.username]) self.assertEqual( get_commcare_users_by_filters(self.ccdomain.name, {'role_id': self.custom_role._id}, count_only=True), 1) # can search by location self.assertItemsEqual( usernames( get_commcare_users_by_filters(self.ccdomain.name, {'location_id': self.loc1._id})), [self.ccuser_1.username]) self.assertEqual( get_commcare_users_by_filters(self.ccdomain.name, {'location_id': self.loc1._id}, count_only=True), 1) ensure_index_deleted(USER_INDEX)
def _setup_es_for_data(cls): cls.es = get_es_new() cls.es_indices = [XFORM_INDEX_INFO] for index_info in cls.es_indices: initialize_index_and_mapping(cls.es, index_info) for form in cls.forms + cls.forms_b: es_form = transform_xform_for_elasticsearch(form) send_to_elasticsearch('forms', es_form) for index_info in cls.es_indices: cls.es.indices.refresh(index_info.index)
def _send_cases_to_es(self, cases, refetch_doc=True): for case in cases: if refetch_doc: es_case = transform_case_for_elasticsearch( CaseDocumentStore(case.domain, case.type).get_document(case.case_id)) else: es_case = transform_case_for_elasticsearch(case.to_json()) send_to_elasticsearch('cases', es_case) self.elasticsearch.indices.refresh(CASE_INDEX_INFO.index) self.cases_to_delete_from_es.update(case.case_id for case in cases)
def _send_group_to_es(self, _id=None, users=None): group = Group( domain=self.domain, name='narcos', users=users or [], case_sharing=False, reporting=False, _id=_id or uuid.uuid4().hex, ) send_to_elasticsearch('groups', group.to_json()) self.es.indices.refresh(GROUP_INDEX_INFO.index) return group
def save_to_es_analytics_db(domain, received_on, app_id, device_id, user_id, username=None): metadata = TestFormMetadata( domain=domain, time_end=received_on, received_on=received_on, app_id=app_id, user_id=user_id, device_id=device_id, username=username ) form_pair = make_es_ready_form(metadata) send_to_elasticsearch('forms', form_pair.json_form)
def test_associated_usercase_closed(self): usercase_ids = self.case_accessor.get_case_ids_in_domain( type=USERCASE_TYPE) for usercase_id in usercase_ids: CaseFactory(self.domain).close_case(usercase_id) usercase = self.case_accessor.get_case(usercase_id) send_to_elasticsearch( "case_search", transform_case_for_elasticsearch(usercase.to_json())) self.es.indices.refresh(CASE_SEARCH_INDEX_INFO.index) self.assertTrue(associated_user_cases_closed(self.checkin_case, None))
def _send_form_to_es(self, domain, **metadata_kwargs): metadata = TestFormMetadata( domain=domain, time_end=datetime.utcnow(), received_on=datetime.utcnow(), ) for attr, value in metadata_kwargs.items(): setattr(metadata, attr, value) form_pair = make_es_ready_form(metadata) send_to_elasticsearch('forms', form_pair.json_form) self.es.indices.refresh(XFORM_INDEX_INFO.index) return form_pair
def check_elasticsearch(): doc = { '_id': 'elasticsearch-service-check', 'date': datetime.datetime.now().isoformat() } send_to_elasticsearch('groups', doc) time.sleep(1) hits = GroupES().remove_default_filters().doc_id(doc['_id']).run().hits send_to_elasticsearch('groups', doc, delete=True) # clean up if doc in hits: return ServiceStatus(True, "Successfully sent a doc to ES and read it back") return ServiceStatus(False, "Something went wrong sending a doc to ES")
def check_elasticsearch(): cluster_health = check_es_cluster_health() if cluster_health != 'green': return ServiceStatus(False, "Cluster health at %s" % cluster_health) doc = {'_id': 'elasticsearch-service-check', 'date': datetime.datetime.now().isoformat()} send_to_elasticsearch('groups', doc) refresh_elasticsearch_index('groups') hits = GroupES().remove_default_filters().doc_id(doc['_id']).run().hits send_to_elasticsearch('groups', doc, delete=True) # clean up if doc in hits: return ServiceStatus(True, "Successfully sent a doc to ES and read it back") return ServiceStatus(False, "Something went wrong sending a doc to ES")
def _send_user_to_es(self, _id=None, username=None, user_data=None): user = CommCareUser( domain=self.domain, username=username or self.username, _id=_id or uuid.uuid4().hex, first_name=self.first_name, last_name=self.last_name, user_data=user_data or {}, is_active=True, ) with patch('corehq.pillows.user.Group.by_user', return_value=[]): send_to_elasticsearch('users', transform_user_for_elasticsearch(user.to_json())) self.es.indices.refresh(USER_INDEX) return user
def _send_user_to_es(self, _id=None, username=None, user_data=None): user = CommCareUser( domain=self.domain, username=username or self.username, _id=_id or uuid.uuid4().hex, first_name=self.first_name, last_name=self.last_name, user_data=user_data or {}, is_active=True, ) with patch('corehq.pillows.user.get_group_id_name_map_by_user', return_value=[]): send_to_elasticsearch('users', transform_user_for_elasticsearch(user.to_json())) self.es.indices.refresh(USER_INDEX) return user
def save_to_es_analytics_db(domain, received_on, app_id, device_id, user_id, username=None): metadata = TestFormMetadata(domain=domain, time_end=received_on, received_on=received_on, app_id=app_id, user_id=user_id, device_id=device_id, username=username) form_pair = make_es_ready_form(metadata) send_to_elasticsearch('forms', form_pair.json_form)
def setUpClass(cls): super(CallCenterLocationOwnerOptionsViewTest, cls).setUpClass() with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"): es = get_es_new() ensure_index_deleted(USER_INDEX_INFO.index) ensure_index_deleted(GROUP_INDEX_INFO.index) initialize_index_and_mapping(es, USER_INDEX_INFO) initialize_index_and_mapping(es, GROUP_INDEX_INFO) # Create domain cls.domain = create_domain(TEST_DOMAIN) cls.domain.save() CALL_CENTER_LOCATION_OWNERS.set(cls.domain.name, True, NAMESPACE_DOMAIN) cls.username = "******" cls.password = "******" cls.web_user = WebUser.create(cls.domain.name, cls.username, cls.password) cls.web_user.save() # Create case sharing groups cls.groups = [] for i in range(2): group = Group(domain=TEST_DOMAIN, name="group{}".format(i), case_sharing=True) group.save() send_to_elasticsearch('groups', group.to_json()) cls.groups.append(group) es.indices.refresh(GROUP_INDEX_INFO.index) cls.group_ids = {g._id for g in cls.groups} # Create locations LocationType.objects.get_or_create( domain=cls.domain.name, name=LOCATION_TYPE, shares_cases=True, ) cls.locations = [ make_loc('loc{}'.format(i), type=LOCATION_TYPE, domain=TEST_DOMAIN) for i in range(4) ] cls.location_ids = {l._id for l in cls.locations} # Create users cls.users = [CommCareUser.create(TEST_DOMAIN, 'user{}'.format(i), '***') for i in range(3)] for user in cls.users: send_to_elasticsearch('users', user.to_json()) es.indices.refresh(USER_INDEX_INFO.index) cls.user_ids = {u._id for u in cls.users}
def setUpClass(cls): super(ExportsFormsAnalyticsTest, cls).setUpClass() from casexml.apps.case.tests.util import delete_all_xforms from corehq.apps.app_manager.models import Application, Module, Form delete_all_xforms() with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"): cls.es = get_es_new() initialize_index_and_mapping(cls.es, XFORM_INDEX_INFO) cls.domain = 'exports_forms_analytics_domain' cls.app_id_1 = 'a' + uuid.uuid4().hex cls.app_id_2 = 'b' + uuid.uuid4().hex cls.xmlns_1 = 'my://crazy.xmlns/' cls.xmlns_2 = 'my://crazy.xmlns/app' cls.apps = [ Application(_id=cls.app_id_2, domain=cls.domain, modules=[Module(forms=[Form(xmlns=cls.xmlns_2)])]) ] for app in cls.apps: app.save() cls.forms = [ create_form_for_test(domain=cls.domain, app_id=cls.app_id_1, xmlns=cls.xmlns_1, save=False), create_form_for_test(domain=cls.domain, app_id=cls.app_id_1, xmlns=cls.xmlns_1, save=False), create_form_for_test(domain=cls.domain, app_id=cls.app_id_2, xmlns=cls.xmlns_2, save=False), ] cls.error_forms = [ create_form_for_test(domain=cls.domain, state=XFormInstance.ERROR, save=False) ] cls.all_forms = cls.forms + cls.error_forms for form in cls.all_forms: elastic_form = transform_xform_for_elasticsearch(form.to_json()) send_to_elasticsearch('forms', elastic_form) cls.es.indices.refresh(XFORM_INDEX_INFO.alias)
def setUpClass(cls): super(CallCenterLocationOwnerOptionsViewTest, cls).setUpClass() with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"): es = get_es_new() ensure_index_deleted(USER_INDEX_INFO.index) ensure_index_deleted(GROUP_INDEX_INFO.index) initialize_index_and_mapping(es, USER_INDEX_INFO) initialize_index_and_mapping(es, GROUP_INDEX_INFO) # Create domain cls.domain = create_domain(TEST_DOMAIN) cls.domain.save() CALL_CENTER_LOCATION_OWNERS.set(cls.domain.name, True, NAMESPACE_DOMAIN) cls.username = "******" cls.password = "******" cls.web_user = WebUser.create(cls.domain.name, cls.username, cls.password, None, None) cls.web_user.save() # Create case sharing groups cls.groups = [] for i in range(2): group = Group(domain=TEST_DOMAIN, name="group{}".format(i), case_sharing=True) group.save() send_to_elasticsearch('groups', group.to_json()) cls.groups.append(group) es.indices.refresh(GROUP_INDEX_INFO.index) cls.group_ids = {g._id for g in cls.groups} # Create locations LocationType.objects.get_or_create( domain=cls.domain.name, name=LOCATION_TYPE, shares_cases=True, ) cls.locations = [ make_loc('loc{}'.format(i), type=LOCATION_TYPE, domain=TEST_DOMAIN) for i in range(4) ] cls.location_ids = {l._id for l in cls.locations} # Create users cls.users = [CommCareUser.create(TEST_DOMAIN, 'user{}'.format(i), '***', None, None) for i in range(3)] for user in cls.users: send_to_elasticsearch('users', user.to_json()) es.indices.refresh(USER_INDEX_INFO.index) cls.user_ids = {u._id for u in cls.users}
def check_elasticsearch(): cluster_health = check_es_cluster_health() if cluster_health != 'green': return ServiceStatus(False, "Cluster health at %s" % cluster_health) doc = {'_id': 'elasticsearch-service-check-{}'.format(random_hex()[:7]), 'date': datetime.datetime.now().isoformat()} try: send_to_elasticsearch('groups', doc) refresh_elasticsearch_index('groups') hits = GroupES().remove_default_filters().doc_id(doc['_id']).run().hits if doc in hits: return ServiceStatus(True, "Successfully sent a doc to ES and read it back") else: return ServiceStatus(False, "Something went wrong sending a doc to ES") finally: send_to_elasticsearch('groups', doc, delete=True) # clean up
def setUpClass(cls): cls.domain = 'test-languages' with trap_extra_setup(ConnectionError): cls.es = get_es_new() initialize_index_and_mapping(cls.es, APP_INDEX_INFO) cls.app1 = Application.new_app(cls.domain, 'My Application 1') cls.app1.langs = ['en', 'es'] cls.app1.save() send_to_elasticsearch('apps', cls.app1.to_json()) cls.app2 = Application.new_app(cls.domain, 'My Application 2') cls.app2.langs = ['fr'] cls.app2.save() send_to_elasticsearch('apps', cls.app2.to_json()) cls.es.indices.refresh(APP_INDEX_INFO.index)
def setUpClass(cls): cls.domain = 'test-languages' with trap_extra_setup(ConnectionError): cls.es = get_es_new() initialize_index_and_mapping(cls.es, APP_INDEX_INFO) cls.app1 = Application.new_app(cls.domain, 'My Application 1', APP_V2) cls.app1.langs = ['en', 'es'] cls.app1.save() send_to_elasticsearch('apps', cls.app1.to_json()) cls.app2 = Application.new_app(cls.domain, 'My Application 2', APP_V2) cls.app2.langs = ['fr'] cls.app2.save() send_to_elasticsearch('apps', cls.app2.to_json()) cls.es.indices.refresh(APP_INDEX_INFO.index)
def _create_domain(name, cc_enabled, cc_use_fixtures, cc_case_type, cc_case_owner_id, use_location_as_owner): with drop_connected_signals(commcare_domain_post_save): domain = Domain( _id=uuid.uuid4().hex, name=name, is_active=True, date_created=datetime.utcnow(), ) domain.call_center_config.enabled = cc_enabled domain.call_center_config.use_fixtures = cc_use_fixtures domain.call_center_config.case_type = cc_case_type domain.call_center_config.case_owner_id = cc_case_owner_id domain.call_center_config.use_user_location_as_owner = use_location_as_owner send_to_elasticsearch( 'domains', doc=domain.to_json(), )
def test_case_by_identifier(self): self._send_case_to_es(case_type='ccuser') case = self._send_case_to_es() case.external_id = '123' case.save() case = CaseAccessors(self.domain).get_case(case.case_id) case_json = case.to_json() case_json['contact_phone_number'] = '234' es_case = transform_case_for_elasticsearch(case_json) send_to_elasticsearch('cases', es_case) self.es.indices.refresh(CASE_INDEX) self.assertEqual( get_case_by_identifier(self.domain, case.case_id).case_id, case.case_id) self.assertEqual( get_case_by_identifier(self.domain, '234').case_id, case.case_id) self.assertEqual( get_case_by_identifier(self.domain, '123').case_id, case.case_id)
def test_users_assigned(self): self.es = get_es_new() ensure_index_deleted(USER_INDEX) initialize_index_and_mapping(self.es, USER_INDEX_INFO) user = CommCareUser.create(self.domain, 'pentagon', '*****', None, None, metadata={ PROFILE_SLUG: self.profile5.id, }) self.addCleanup(user.delete, self.domain, deleted_by=None) send_to_elasticsearch('users', transform_user_for_elasticsearch(user.to_json())) self.es.indices.refresh(USER_INDEX) self.assertFalse(self.profile3.has_users_assigned) self.assertEqual([], self.profile3.user_ids_assigned()) self.assertTrue(self.profile5.has_users_assigned) self.assertEqual([user._id], self.profile5.user_ids_assigned()) ensure_index_deleted(USER_INDEX)
def test_get_all_commcare_users_by_domain(self): from corehq.util.elastic import ensure_index_deleted from corehq.elastic import get_es_new, send_to_elasticsearch from corehq.pillows.mappings.user_mapping import USER_INDEX_INFO, USER_INDEX from pillowtop.es_utils import initialize_index_and_mapping es = get_es_new() ensure_index_deleted(USER_INDEX) initialize_index_and_mapping(es, USER_INDEX_INFO) send_to_elasticsearch('users', self.ccuser_1.to_json()) send_to_elasticsearch('users', self.ccuser_2.to_json()) es.indices.refresh(USER_INDEX) usernames = lambda users: [u.username for u in users] # if no filters are passed, should return all cc-users in the domain self.assertItemsEqual( usernames(get_commcare_users_by_filters(self.ccdomain.name, {})), usernames([self.ccuser_2, self.ccuser_1]) ) self.assertEqual( get_commcare_users_by_filters(self.ccdomain.name, {}, count_only=True), 2 ) # can search by username self.assertItemsEqual( usernames(get_commcare_users_by_filters(self.ccdomain.name, {'search_string': 'user_1'})), [self.ccuser_1.username] ) self.assertEqual( get_commcare_users_by_filters(self.ccdomain.name, {'search_string': 'user_1'}, count_only=True), 1 ) # can search by role_id self.assertItemsEqual( usernames(get_commcare_users_by_filters(self.ccdomain.name, {'role_id': self.custom_role._id})), [self.ccuser_2.username] ) self.assertEqual( get_commcare_users_by_filters(self.ccdomain.name, {'role_id': self.custom_role._id}, count_only=True), 1 ) ensure_index_deleted(USER_INDEX)
def update_calculated_properties(): results = DomainES().fields(["name", "_id", "cp_last_updated"]).scroll() all_stats = all_domain_stats() for r in results: dom = r["name"] try: last_form_submission = CALC_FNS["last_form_submission"](dom, False) if _skip_updating_domain_stats(r.get("cp_last_updated"), last_form_submission): continue props = calced_props(dom, r["_id"], all_stats) if props['cp_first_form'] is None: del props['cp_first_form'] if props['cp_last_form'] is None: del props['cp_last_form'] if props['cp_300th_form'] is None: del props['cp_300th_form'] send_to_elasticsearch("domains", props) except Exception, e: notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
def update_calculated_properties(): results = DomainES().fields(["name", "_id", "cp_last_updated"]).scroll() all_stats = all_domain_stats() for r in results: dom = r["name"] try: last_form_submission = CALC_FNS["last_form_submission"](dom, False) if _skip_updating_domain_stats(r.get("cp_last_updated"), last_form_submission): continue props = calced_props(dom, r["_id"], all_stats) if props['cp_first_form'] is None: del props['cp_first_form'] if props['cp_last_form'] is None: del props['cp_last_form'] if props['cp_300th_form'] is None: del props['cp_300th_form'] send_to_elasticsearch("domains", props, es_merge_update=True) except Exception as e: notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
def setUpClass(cls): form_pillow = XFormPillow(online=False) case_pillow = CasePillow(online=False) cls.pillows = [form_pillow, case_pillow] es = get_es_new() with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"): for pillow in cls.pillows: completely_initialize_pillow_index(pillow) initialize_index_and_mapping(es, GROUP_INDEX_INFO) case = new_case(closed=True) case_pillow.send_robust(case.to_json()) case = new_case(closed=False) case_pillow.send_robust(case.to_json()) case = new_case(closed=True, owner_id="foo") case_pillow.send_robust(case.to_json()) case = new_case(closed=False, owner_id="bar") case_pillow.send_robust(case.to_json()) group = Group(_id=uuid.uuid4().hex, users=["foo", "bar"]) cls.group_id = group._id send_to_elasticsearch('groups', group.to_json()) form = new_form(form={"meta": {"userID": None}}) form_pillow.send_robust(form.to_json()) form = new_form(form={"meta": {"userID": ""}}) form_pillow.send_robust(form.to_json()) form = new_form(form={"meta": {"deviceID": "abc"}}) form_pillow.send_robust(form.to_json()) form = new_form(form={"meta": {"userID": uuid.uuid4().hex}}) form_pillow.send_robust(form.to_json()) for pillow in cls.pillows: pillow.get_es_new().indices.refresh(pillow.es_index) es.indices.refresh(GROUP_INDEX_INFO.index)
def setUpClass(cls): with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"): es = get_es_new() cls.tearDownClass() initialize_index_and_mapping(es, CASE_INDEX_INFO) initialize_index_and_mapping(es, GROUP_INDEX_INFO) initialize_index_and_mapping(es, XFORM_INDEX_INFO) case = new_case(closed=True) send_to_elasticsearch('cases', case.to_json()) case = new_case(closed=False) send_to_elasticsearch('cases', case.to_json()) case = new_case(closed=True, owner_id="foo") send_to_elasticsearch('cases', case.to_json()) case = new_case(closed=False, owner_id="bar") send_to_elasticsearch('cases', case.to_json()) group = Group(_id=uuid.uuid4().hex, users=["foo", "bar"]) cls.group_id = group._id send_to_elasticsearch('groups', group.to_json()) form = new_form(form={"meta": {"userID": None}}) send_to_elasticsearch('forms', form.to_json()) form = new_form(form={"meta": {"userID": ""}}) send_to_elasticsearch('forms', form.to_json()) form = new_form(form={"meta": {"deviceID": "abc"}}) send_to_elasticsearch('forms', form.to_json()) form = new_form(form={"meta": {"userID": uuid.uuid4().hex}}) send_to_elasticsearch('forms', form.to_json()) es.indices.refresh(CASE_INDEX_INFO.index) es.indices.refresh(XFORM_INDEX_INFO.index) es.indices.refresh(GROUP_INDEX_INFO.index)
def setUpClass(cls): from casexml.apps.case.tests.util import delete_all_xforms from corehq.apps.app_manager.models import Application, Module, Form delete_all_xforms() with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"): cls.es = get_es_new() initialize_index_and_mapping(cls.es, XFORM_INDEX_INFO) cls.domain = 'exports_forms_analytics_domain' cls.app_id_1 = 'a' + uuid.uuid4().hex cls.app_id_2 = 'b' + uuid.uuid4().hex cls.xmlns_1 = 'my://crazy.xmlns/' cls.xmlns_2 = 'my://crazy.xmlns/app' cls.apps = [ Application(_id=cls.app_id_2, domain=cls.domain, modules=[Module(forms=[Form(xmlns=cls.xmlns_2)])]) ] for app in cls.apps: app.save() cls.forms = [ XFormInstance(domain=cls.domain, app_id=cls.app_id_1, xmlns=cls.xmlns_1), XFormInstance(domain=cls.domain, app_id=cls.app_id_1, xmlns=cls.xmlns_1), XFormInstance(domain=cls.domain, app_id=cls.app_id_2, xmlns=cls.xmlns_2), ] cls.error_forms = [XFormError(domain=cls.domain)] cls.all_forms = cls.forms + cls.error_forms for form in cls.all_forms: form.save() send_to_elasticsearch('forms', form.to_json()) cls.es.indices.refresh(XFORM_INDEX_INFO.index) update_analytics_indexes()
def setUpClass(cls): super().setUpClass() # Set up domains cls.domains = [ create_domain('state'), create_domain('county'), create_domain('staging'), ] # Set up users cls.web_user = WebUser.create('state', 'webu', 'badpassword', None, None, email='*****@*****.**', is_admin=True) cls.web_user.add_domain_membership('county') cls.web_user.add_domain_membership('staging') cls.web_user.save() cls.mobile_users = [ CommCareUser.create('state', "state_u", "123", None, None), CommCareUser.create("county", "county_u", "123", None, None), CommCareUser.create("staging", "staging_u", "123", None, None), ] # Set up permissions create_enterprise_permissions(cls.web_user.username, 'state', ['county'], ['staging']) cls.es = get_es_new() ensure_index_deleted(USER_INDEX) initialize_index_and_mapping(cls.es, USER_INDEX_INFO) for user_obj in cls.mobile_users: send_to_elasticsearch( 'users', transform_user_for_elasticsearch(user_obj.to_json())) cls.es.indices.refresh(USER_INDEX)
def setUpClass(cls): ensure_index_deleted(LEDGER_INDEX_INFO.index) es = get_es_new() initialize_index_and_mapping(es, LEDGER_INDEX_INFO) cls.expected_combos = { ('stock', 'product_a'), ('stock', 'product_b'), ('consumption', 'product_a'), ('consumption', 'product_c'), } for section, entry in cls.expected_combos: ledger = LedgerValue( domain=cls.domain, case_id=uuid.uuid4().hex, section_id=section, entry_id=entry, ) ledger_json = ledger.to_json() ledger_json['_id'] = ledger.ledger_reference.as_id() send_to_elasticsearch('ledgers', doc=ledger_json) es.indices.refresh(LEDGER_INDEX_INFO.index)
def _setup_case(self, cases=None): modify_date = datetime.utcnow() cases = cases or [(None, None)] for owner_id, case_id in cases: kwargs = {} if owner_id: kwargs['owner_id'] = owner_id if case_id: kwargs['_id'] = case_id backend_case = CommCareCase(server_modified_on=modify_date, domain=self.domain.name, **kwargs) backend_case.save() self.addCleanup(backend_case.delete) translated_doc = transform_case_for_elasticsearch( backend_case.to_json()) send_to_elasticsearch('cases', translated_doc) self.es.indices.refresh(CASE_INDEX_INFO.index) return backend_case
def update_calculated_properties(): results = DomainES().filter( get_domains_to_update_es_filter() ).fields(["name", "_id"]).run().hits all_stats = all_domain_stats() for r in results: dom = r["name"] domain_obj = Domain.get_by_name(dom) if not domain_obj: send_to_elasticsearch("domains", r, delete=True) continue try: props = calced_props(domain_obj, r["_id"], all_stats) if props['cp_first_form'] is None: del props['cp_first_form'] if props['cp_last_form'] is None: del props['cp_last_form'] if props['cp_300th_form'] is None: del props['cp_300th_form'] send_to_elasticsearch("domains", props, es_merge_update=True) except Exception as e: notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
def setUpClass(cls): ensure_index_deleted(LEDGER_INDEX_INFO.index) es = get_es_new() initialize_index_and_mapping(es, LEDGER_INDEX_INFO) cls.expected_combos = { ('stock', 'product_a'), ('stock', 'product_b'), ('consumption', 'product_a'), ('consumption', 'product_c'), } for section, entry in cls.expected_combos: ledger = LedgerValue( domain=cls.domain, case_id=uuid.uuid4().hex, section_id=section, entry_id=entry, ) ledger_json = ledger.to_json(include_location_id=False) ledger_json['_id'] = ledger.ledger_reference.as_id() send_to_elasticsearch('ledgers', doc=ledger_json) es.indices.refresh(LEDGER_INDEX_INFO.index)
def test_fetching_xform_cases(self): # Create an xform that touches a case case_id = uuid.uuid4().hex form = submit_case_blocks( CaseBlock.deprecated_init( case_id=case_id, create=True, ).as_text(), self.domain.name)[0] send_to_elasticsearch( 'forms', transform_xform_for_elasticsearch(form.to_json())) self.es.indices.refresh(XFORM_INDEX_INFO.index) # Fetch the xform through the API response = self._assert_auth_get_resource( self.single_endpoint(form.form_id) + "?cases__full=true") self.assertEqual(response.status_code, 200) cases = json.loads(response.content)['cases'] # Confirm that the case appears in the resource self.assertEqual(len(cases), 1) self.assertEqual(cases[0]['id'], case_id)
def setUpClass(cls): super(ExportsFormsAnalyticsTest, cls).setUpClass() from casexml.apps.case.tests.util import delete_all_xforms from corehq.apps.app_manager.models import Application, Module, Form delete_all_xforms() with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"): cls.es = get_es_new() initialize_index_and_mapping(cls.es, XFORM_INDEX_INFO) cls.domain = 'exports_forms_analytics_domain' cls.app_id_1 = 'a' + uuid.uuid4().hex cls.app_id_2 = 'b' + uuid.uuid4().hex cls.xmlns_1 = 'my://crazy.xmlns/' cls.xmlns_2 = 'my://crazy.xmlns/app' cls.apps = [ Application(_id=cls.app_id_2, domain=cls.domain, modules=[Module(forms=[Form(xmlns=cls.xmlns_2)])]) ] for app in cls.apps: app.save() cls.forms = [ XFormInstance(domain=cls.domain, app_id=cls.app_id_1, xmlns=cls.xmlns_1), XFormInstance(domain=cls.domain, app_id=cls.app_id_1, xmlns=cls.xmlns_1), XFormInstance(domain=cls.domain, app_id=cls.app_id_2, xmlns=cls.xmlns_2), ] cls.error_forms = [XFormError(domain=cls.domain)] cls.all_forms = cls.forms + cls.error_forms for form in cls.all_forms: form.save() send_to_elasticsearch('forms', form.to_json()) cls.es.indices.refresh(XFORM_INDEX_INFO.index) update_analytics_indexes()
def test_archived_forms(self): xmlns1 = 'https://xmlns1' xmlns2 = 'https://xmlns2' forms = self._send_forms([(xmlns1, None), (xmlns2, None)]) # archive update = forms[0].to_json() update['doc_type'] = 'xformarchived' send_to_elasticsearch('forms', transform_xform_for_elasticsearch(update)) self.es.indices.refresh(XFORM_INDEX_INFO.index) # archived form should not be included by default response = self._assert_auth_get_resource(self.list_endpoint) self.assertEqual(response.status_code, 200) api_forms = json.loads(response.content)['objects'] self.assertEqual(len(api_forms), 1) api_form = api_forms[0] self.assertEqual(api_form['form']['@xmlns'], xmlns2) # archived form should be included response = self._assert_auth_get_resource( '%s?%s' % (self.list_endpoint, urlencode({'include_archived': 'true'}))) self.assertEqual(response.status_code, 200) api_forms = json.loads(response.content)['objects'] self.assertEqual(len(api_forms), 2)
def setUpClass(cls): super(ExportTest, cls).setUpClass() with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"): cls.es = get_es_new() initialize_index_and_mapping(cls.es, CASE_INDEX_INFO) case = new_case(_id='robin', name='batman', foo="apple", bar="banana", date='2016-4-24') send_to_elasticsearch('cases', case.to_json()) case = new_case(owner_id="some_other_owner", foo="apple", bar="banana", date='2016-4-04') send_to_elasticsearch('cases', case.to_json()) case = new_case(type="some_other_type", foo="apple", bar="banana") send_to_elasticsearch('cases', case.to_json()) case = new_case(closed=True, foo="apple", bar="banana") send_to_elasticsearch('cases', case.to_json()) cls.es.indices.refresh(CASE_INDEX_INFO.index) cache.clear()
def test_create(self): group = Group({"name": "test"}) group.save() self.addCleanup(group.delete) self.assertEqual(0, len(CommCareUser.by_domain(self.domain.name))) user_json = { "username": "******", "password": "******", "first_name": "John", "last_name": "Doe", "email": "*****@*****.**", "language": "en", "phone_numbers": [ "+50253311399", "50253314588" ], "groups": [ group._id ], "user_data": { "chw_id": "13/43/DFA" } } response = self._assert_auth_post_resource(self.list_endpoint, json.dumps(user_json), content_type='application/json') self.assertEqual(response.status_code, 201) [user_back] = CommCareUser.by_domain(self.domain.name) self.addCleanup(user_back.delete, deleted_by=None) self.addCleanup(lambda: send_to_elasticsearch('users', user_back.to_json(), delete=True)) self.assertEqual(user_back.username, "jdoe") self.assertEqual(user_back.first_name, "John") self.assertEqual(user_back.last_name, "Doe") self.assertEqual(user_back.email, "*****@*****.**") self.assertEqual(user_back.language, "en") self.assertEqual(user_back.get_group_ids()[0], group._id) self.assertEqual(user_back.user_data["chw_id"], "13/43/DFA") self.assertEqual(user_back.default_phone_number, "50253311399")
def delete_sms_in_es(sms_doc): send_to_elasticsearch("sms", sms_doc, delete=True) refresh_elasticsearch_index('sms')
def update_user_in_es(sender, couch_user, **kwargs): """ Automatically sync the user to elastic directly on save or delete """ send_to_elasticsearch("users", couch_user.to_json(), delete=couch_user.to_be_deleted())
def add_demo_user_to_user_index(): send_to_elasticsearch("users", {"_id": "demo_user", "username": "******", "doc_type": "DemoUser"})