def _send_form_to_es(self, domain=None, completion_time=None, received_on=None, attachment_dict=None, **metadata_kwargs): attachment_dict = attachment_dict or {} metadata = TestFormMetadata( domain=domain or self.domain, time_end=completion_time or datetime.utcnow(), received_on=received_on or datetime.utcnow(), ) for attr, value in metadata_kwargs.items(): setattr(metadata, attr, value) form_pair = make_es_ready_form(metadata) if attachment_dict: form_pair.wrapped_form.external_blobs = { name: BlobMetaRef(**meta) for name, meta in attachment_dict.items() } form_pair.json_form['external_blobs'] = attachment_dict es_form = transform_xform_for_elasticsearch(form_pair.json_form) send_to_elasticsearch('forms', es_form) self.es.indices.refresh(XFORM_INDEX_INFO.index) return form_pair
def _send_form_to_es( self, domain=None, completion_time=None, received_on=None, attachment_dict=None, **metadata_kwargs): attachment_dict = attachment_dict or {} metadata = TestFormMetadata( domain=domain or self.domain, time_end=completion_time or datetime.utcnow(), received_on=received_on or datetime.utcnow(), ) for attr, value in metadata_kwargs.items(): setattr(metadata, attr, value) form_pair = make_es_ready_form(metadata) if attachment_dict: form_pair.wrapped_form.external_blobs = {name: BlobMetaRef(**meta) for name, meta in six.iteritems(attachment_dict)} form_pair.json_form['external_blobs'] = attachment_dict es_form = transform_xform_for_elasticsearch(form_pair.json_form) send_to_elasticsearch('forms', es_form) self.es.indices.refresh(XFORM_INDEX_INFO.index) return form_pair
def _ship_forms_to_es(cls, metadatas): for form_metadata in metadatas: form_metadata = form_metadata or TestFormMetadata() form_pair = make_es_ready_form(form_metadata) cls.forms.append(form_pair) cls.pillow.change_transport(form_pair.json_form) # have to refresh the index to make sure changes show up cls.pillow.get_es_new().indices.refresh(cls.pillow.es_index)
def _ship_forms_to_es(cls, metadatas): for form_metadata in metadatas: form_metadata = form_metadata or TestFormMetadata() form_pair = make_es_ready_form(form_metadata) cls.forms.append(form_pair) send_to_elasticsearch('forms', form_pair.json_form) # have to refresh the index to make sure changes show up cls.es.indices.refresh(XFORM_INDEX_INFO.index)
def _send_form_to_es(self, domain, **metadata_kwargs): metadata = TestFormMetadata(domain=domain, time_end=datetime.utcnow(), received_on=datetime.utcnow()) for attr, value in metadata_kwargs.items(): setattr(metadata, attr, value) form_pair = make_es_ready_form(metadata) send_to_elasticsearch("forms", form_pair.json_form) self.es.indices.refresh(XFORM_INDEX_INFO.index) return form_pair
def _send_form_to_es(self, domain=None, completion_time=None, received_on=None): metadata = TestFormMetadata( domain=domain or self.domain, time_end=completion_time or datetime.utcnow(), received_on=received_on or datetime.utcnow(), ) form_pair = make_es_ready_form(metadata) self.pillow.change_transport(form_pair.json_form) self.pillow.get_es_new().indices.refresh(self.pillow.es_index) return form_pair
def save_to_es_analytics_db(domain, received_on, app_id, device_id, user_id, username=None): metadata = TestFormMetadata( domain=domain, time_end=received_on, received_on=received_on, app_id=app_id, user_id=user_id, device_id=device_id, username=username ) form_pair = make_es_ready_form(metadata) send_to_elasticsearch('forms', form_pair.json_form)
def _send_form_to_es(self, domain, **metadata_kwargs): metadata = TestFormMetadata( domain=domain, time_end=datetime.utcnow(), received_on=datetime.utcnow(), ) for attr, value in metadata_kwargs.items(): setattr(metadata, attr, value) form_pair = make_es_ready_form(metadata) send_to_elasticsearch('forms', form_pair.json_form) self.es.indices.refresh(XFORM_INDEX_INFO.index) return form_pair
def _send_form_to_es(self, domain, **metadata_kwargs): metadata = TestFormMetadata( domain=domain, time_end=datetime.utcnow(), received_on=datetime.utcnow(), ) for attr, value in metadata_kwargs.items(): setattr(metadata, attr, value) form_pair = make_es_ready_form(metadata) self._pillow_process_form(form_pair) es = get_es_new() es.indices.refresh(self.es_index) return form_pair
def _send_form_to_es(self, domain=None, completion_time=None, received_on=None, **metadata_kwargs): metadata = TestFormMetadata( domain=domain or self.domain, time_end=completion_time or datetime.utcnow(), received_on=received_on or datetime.utcnow(), ) for attr, value in metadata_kwargs.items(): setattr(metadata, attr, value) form_pair = make_es_ready_form(metadata) self._pillow_process_form(form_pair) es = get_es_new() es.indices.refresh(XFORM_INDEX) return form_pair
def save_to_es_analytics_db(domain, received_on, app_id, device_id, user_id, username=None): metadata = TestFormMetadata( domain=domain, time_end=received_on, received_on=received_on, app_id=app_id, user_id=user_id, device_id=device_id, username=username ) form_pair = make_es_ready_form(metadata) pillow = XFormPillow() pillow.change_transport(form_pair.json_form)
def save_to_es_analytics_db(domain, received_on, app_id, device_id, user_id, username=None): metadata = TestFormMetadata(domain=domain, time_end=received_on, received_on=received_on, app_id=app_id, user_id=user_id, device_id=device_id, username=username) form_pair = make_es_ready_form(metadata) send_to_elasticsearch('forms', form_pair.json_form)
def setUpClass(cls): super().setUpClass() cls.es = get_es_new() initialize_index_and_mapping(cls.es, USER_INDEX_INFO) initialize_index_and_mapping(cls.es, XFORM_INDEX_INFO) today = datetime.datetime.utcnow() one_year_ago = add_months_to_date(today.date(), -12) enterprise_plan = get_enterprise_software_plan() cls.billing_account = get_enterprise_account() cls.domains = [ create_domain('test-emw-settings-001'), create_domain('test-emw-settings-002'), ] add_domains_to_enterprise_account(cls.billing_account, cls.domains, enterprise_plan, one_year_ago) cls.emw_settings = EnterpriseMobileWorkerSettings.objects.create( account=cls.billing_account, enable_auto_deactivation=True, ) cls.active_user1 = CommCareUser.create( domain=cls.domains[0].name, username='******', password='******', created_by=None, created_via=None, is_active=True, ) cls.active_user2 = CommCareUser.create( domain=cls.domains[0].name, username='******', password='******', created_by=None, created_via=None, is_active=True, ) cls.active_user3 = CommCareUser.create( domain=cls.domains[1].name, username='******', password='******', created_by=None, created_via=None, is_active=True, ) cls.active_user4 = CommCareUser.create( domain=cls.domains[1].name, username='******', password='******', created_by=None, created_via=None, is_active=True, ) cls.active_user5 = CommCareUser.create( domain=cls.domains[1].name, username='******', password='******', created_by=None, created_via=None, is_active=True, ) cls.active_user5.created_on = today - datetime.timedelta( days=cls.emw_settings.inactivity_period) cls.active_user5.save() cls.active_user6 = CommCareUser.create( domain=cls.domains[1].name, username='******', password='******', created_by=None, created_via=None, is_active=True, ) cls.users = [ cls.active_user1, cls.active_user2, cls.active_user3, cls.active_user4, cls.active_user5, cls.active_user6, CommCareUser.create(domain=cls.domains[0].name, username='******', password='******', created_by=None, created_via=None, is_active=False), CommCareUser.create(domain=cls.domains[1].name, username='******', password='******', created_by=None, created_via=None, is_active=False), ] form_submissions = [ (TestFormMetadata( domain=cls.domains[0].name, received_on=today - datetime.timedelta( days=cls.emw_settings.inactivity_period - 1), user_id=cls.active_user1.user_id, username=cls.active_user1.username, ), cls.active_user1), (TestFormMetadata( domain=cls.domains[0].name, received_on=today - datetime.timedelta(days=cls.emw_settings.inactivity_period), user_id=cls.active_user2.user_id, username=cls.active_user2.username, ), cls.active_user2), (TestFormMetadata( domain=cls.domains[1].name, received_on=today - datetime.timedelta( days=cls.emw_settings.inactivity_period - 10), user_id=cls.active_user3.user_id, username=cls.active_user3.username, ), cls.active_user3), (TestFormMetadata( domain=cls.domains[1].name, received_on=today - datetime.timedelta( days=cls.emw_settings.inactivity_period + 1), user_id=cls.active_user6.user_id, username=cls.active_user6.username, ), cls.active_user6), ] for form_metadata, user in form_submissions: # ensure users are as old as the received_on dates of their submissions user.created_on = form_metadata.received_on user.save() form_pair = make_es_ready_form(form_metadata) send_to_elasticsearch('forms', form_pair.json_form) mark_latest_submission(form_metadata.domain, user, form_metadata.app_id, "build-id", "2", {'deviceID': 'device-id'}, form_metadata.received_on) for user in cls.users: fresh_user = CommCareUser.get_by_user_id(user.user_id) elastic_user = transform_user_for_elasticsearch( fresh_user.to_json()) send_to_elasticsearch('users', elastic_user) cls.es.indices.refresh(USER_INDEX_INFO.alias) cls.es.indices.refresh(XFORM_INDEX_INFO.alias)