def _send_form_to_es(self,
domain=None,
completion_time=None,
received_on=None,
attachment_dict=None,
**metadata_kwargs):
attachment_dict = attachment_dict or {}
metadata = TestFormMetadata(
domain=domain or self.domain,
time_end=completion_time or datetime.utcnow(),
received_on=received_on or datetime.utcnow(),
)
for attr, value in metadata_kwargs.items():
setattr(metadata, attr, value)
form_pair = make_es_ready_form(metadata)
if attachment_dict:
form_pair.wrapped_form.external_blobs = {
name: BlobMetaRef(**meta)
for name, meta in attachment_dict.items()
}
form_pair.json_form['external_blobs'] = attachment_dict
es_form = transform_xform_for_elasticsearch(form_pair.json_form)
send_to_elasticsearch('forms', es_form)
self.es.indices.refresh(XFORM_INDEX_INFO.index)
return form_pair
def _send_form_to_es(
self,
domain=None,
completion_time=None,
received_on=None,
attachment_dict=None,
**metadata_kwargs):
attachment_dict = attachment_dict or {}
metadata = TestFormMetadata(
domain=domain or self.domain,
time_end=completion_time or datetime.utcnow(),
received_on=received_on or datetime.utcnow(),
)
for attr, value in metadata_kwargs.items():
setattr(metadata, attr, value)
form_pair = make_es_ready_form(metadata)
if attachment_dict:
form_pair.wrapped_form.external_blobs = {name: BlobMetaRef(**meta)
for name, meta in six.iteritems(attachment_dict)}
form_pair.json_form['external_blobs'] = attachment_dict
es_form = transform_xform_for_elasticsearch(form_pair.json_form)
send_to_elasticsearch('forms', es_form)
self.es.indices.refresh(XFORM_INDEX_INFO.index)
return form_pair
def _ship_forms_to_es(cls, metadatas):
for form_metadata in metadatas:
form_metadata = form_metadata or TestFormMetadata()
form_pair = make_es_ready_form(form_metadata)
cls.forms.append(form_pair)
cls.pillow.change_transport(form_pair.json_form)
# have to refresh the index to make sure changes show up
cls.pillow.get_es_new().indices.refresh(cls.pillow.es_index)
def _ship_forms_to_es(cls, metadatas):
for form_metadata in metadatas:
form_metadata = form_metadata or TestFormMetadata()
form_pair = make_es_ready_form(form_metadata)
cls.forms.append(form_pair)
send_to_elasticsearch('forms', form_pair.json_form)
# have to refresh the index to make sure changes show up
cls.es.indices.refresh(XFORM_INDEX_INFO.index)
def _ship_forms_to_es(cls, metadatas):
for form_metadata in metadatas:
form_metadata = form_metadata or TestFormMetadata()
form_pair = make_es_ready_form(form_metadata)
cls.forms.append(form_pair)
send_to_elasticsearch('forms', form_pair.json_form)
# have to refresh the index to make sure changes show up
cls.es.indices.refresh(XFORM_INDEX_INFO.index)
def _send_form_to_es(self, domain, **metadata_kwargs):
metadata = TestFormMetadata(domain=domain, time_end=datetime.utcnow(), received_on=datetime.utcnow())
for attr, value in metadata_kwargs.items():
setattr(metadata, attr, value)
form_pair = make_es_ready_form(metadata)
send_to_elasticsearch("forms", form_pair.json_form)
self.es.indices.refresh(XFORM_INDEX_INFO.index)
return form_pair
def _send_form_to_es(self, domain=None, completion_time=None, received_on=None):
metadata = TestFormMetadata(
domain=domain or self.domain,
time_end=completion_time or datetime.utcnow(),
received_on=received_on or datetime.utcnow(),
)
form_pair = make_es_ready_form(metadata)
self.pillow.change_transport(form_pair.json_form)
self.pillow.get_es_new().indices.refresh(self.pillow.es_index)
return form_pair
def save_to_es_analytics_db(domain, received_on, app_id, device_id, user_id, username=None):
metadata = TestFormMetadata(
domain=domain,
time_end=received_on,
received_on=received_on,
app_id=app_id,
user_id=user_id,
device_id=device_id,
username=username
)
form_pair = make_es_ready_form(metadata)
send_to_elasticsearch('forms', form_pair.json_form)
def _send_form_to_es(self, domain, **metadata_kwargs):
metadata = TestFormMetadata(
domain=domain,
time_end=datetime.utcnow(),
received_on=datetime.utcnow(),
)
for attr, value in metadata_kwargs.items():
setattr(metadata, attr, value)
form_pair = make_es_ready_form(metadata)
send_to_elasticsearch('forms', form_pair.json_form)
self.es.indices.refresh(XFORM_INDEX_INFO.index)
return form_pair
def _send_form_to_es(self, domain, **metadata_kwargs):
metadata = TestFormMetadata(
domain=domain,
time_end=datetime.utcnow(),
received_on=datetime.utcnow(),
)
for attr, value in metadata_kwargs.items():
setattr(metadata, attr, value)
form_pair = make_es_ready_form(metadata)
self._pillow_process_form(form_pair)
es = get_es_new()
es.indices.refresh(self.es_index)
return form_pair
def _send_form_to_es(self, domain=None, completion_time=None, received_on=None, **metadata_kwargs):
metadata = TestFormMetadata(
domain=domain or self.domain,
time_end=completion_time or datetime.utcnow(),
received_on=received_on or datetime.utcnow(),
)
for attr, value in metadata_kwargs.items():
setattr(metadata, attr, value)
form_pair = make_es_ready_form(metadata)
self._pillow_process_form(form_pair)
es = get_es_new()
es.indices.refresh(XFORM_INDEX)
return form_pair
def save_to_es_analytics_db(domain, received_on, app_id, device_id, user_id, username=None):
metadata = TestFormMetadata(
domain=domain,
time_end=received_on,
received_on=received_on,
app_id=app_id,
user_id=user_id,
device_id=device_id,
username=username
)
form_pair = make_es_ready_form(metadata)
pillow = XFormPillow()
pillow.change_transport(form_pair.json_form)
def save_to_es_analytics_db(domain,
received_on,
app_id,
device_id,
user_id,
username=None):
metadata = TestFormMetadata(domain=domain,
time_end=received_on,
received_on=received_on,
app_id=app_id,
user_id=user_id,
device_id=device_id,
username=username)
form_pair = make_es_ready_form(metadata)
send_to_elasticsearch('forms', form_pair.json_form)
def setUpClass(cls):
super().setUpClass()
cls.es = get_es_new()
initialize_index_and_mapping(cls.es, USER_INDEX_INFO)
initialize_index_and_mapping(cls.es, XFORM_INDEX_INFO)
today = datetime.datetime.utcnow()
one_year_ago = add_months_to_date(today.date(), -12)
enterprise_plan = get_enterprise_software_plan()
cls.billing_account = get_enterprise_account()
cls.domains = [
create_domain('test-emw-settings-001'),
create_domain('test-emw-settings-002'),
]
add_domains_to_enterprise_account(cls.billing_account, cls.domains,
enterprise_plan, one_year_ago)
cls.emw_settings = EnterpriseMobileWorkerSettings.objects.create(
account=cls.billing_account,
enable_auto_deactivation=True,
)
cls.active_user1 = CommCareUser.create(
domain=cls.domains[0].name,
username='******',
password='******',
created_by=None,
created_via=None,
is_active=True,
)
cls.active_user2 = CommCareUser.create(
domain=cls.domains[0].name,
username='******',
password='******',
created_by=None,
created_via=None,
is_active=True,
)
cls.active_user3 = CommCareUser.create(
domain=cls.domains[1].name,
username='******',
password='******',
created_by=None,
created_via=None,
is_active=True,
)
cls.active_user4 = CommCareUser.create(
domain=cls.domains[1].name,
username='******',
password='******',
created_by=None,
created_via=None,
is_active=True,
)
cls.active_user5 = CommCareUser.create(
domain=cls.domains[1].name,
username='******',
password='******',
created_by=None,
created_via=None,
is_active=True,
)
cls.active_user5.created_on = today - datetime.timedelta(
days=cls.emw_settings.inactivity_period)
cls.active_user5.save()
cls.active_user6 = CommCareUser.create(
domain=cls.domains[1].name,
username='******',
password='******',
created_by=None,
created_via=None,
is_active=True,
)
cls.users = [
cls.active_user1,
cls.active_user2,
cls.active_user3,
cls.active_user4,
cls.active_user5,
cls.active_user6,
CommCareUser.create(domain=cls.domains[0].name,
username='******',
password='******',
created_by=None,
created_via=None,
is_active=False),
CommCareUser.create(domain=cls.domains[1].name,
username='******',
password='******',
created_by=None,
created_via=None,
is_active=False),
]
form_submissions = [
(TestFormMetadata(
domain=cls.domains[0].name,
received_on=today - datetime.timedelta(
days=cls.emw_settings.inactivity_period - 1),
user_id=cls.active_user1.user_id,
username=cls.active_user1.username,
), cls.active_user1),
(TestFormMetadata(
domain=cls.domains[0].name,
received_on=today -
datetime.timedelta(days=cls.emw_settings.inactivity_period),
user_id=cls.active_user2.user_id,
username=cls.active_user2.username,
), cls.active_user2),
(TestFormMetadata(
domain=cls.domains[1].name,
received_on=today - datetime.timedelta(
days=cls.emw_settings.inactivity_period - 10),
user_id=cls.active_user3.user_id,
username=cls.active_user3.username,
), cls.active_user3),
(TestFormMetadata(
domain=cls.domains[1].name,
received_on=today - datetime.timedelta(
days=cls.emw_settings.inactivity_period + 1),
user_id=cls.active_user6.user_id,
username=cls.active_user6.username,
), cls.active_user6),
]
for form_metadata, user in form_submissions:
# ensure users are as old as the received_on dates of their submissions
user.created_on = form_metadata.received_on
user.save()
form_pair = make_es_ready_form(form_metadata)
send_to_elasticsearch('forms', form_pair.json_form)
mark_latest_submission(form_metadata.domain, user,
form_metadata.app_id, "build-id", "2",
{'deviceID': 'device-id'},
form_metadata.received_on)
for user in cls.users:
fresh_user = CommCareUser.get_by_user_id(user.user_id)
elastic_user = transform_user_for_elasticsearch(
fresh_user.to_json())
send_to_elasticsearch('users', elastic_user)
cls.es.indices.refresh(USER_INDEX_INFO.alias)
cls.es.indices.refresh(XFORM_INDEX_INFO.alias)
