def finish_handle(self):
filepath = os.path.join(settings.FILEPATH, 'corehq', 'pillows',
'mappings', 'xform_mapping.py')
xform_pillow = XFormPillow(create_index=False)
#check current index
aliased_indices = xform_pillow.check_alias()
current_index = xform_pillow.es_index
sys.stderr.write("current index:\n")
sys.stderr.write('XFORM_INDEX="%s"\n' % current_index)
#regenerate the mapping dict
mapping = xform_mapping.XFORM_MAPPING
xform_pillow.default_mapping = mapping
delattr(xform_pillow, '_calc_meta_cache')
calc_index = "%s_%s" % (xform_pillow.es_index_prefix,
xform_pillow.calc_meta())
if calc_index not in aliased_indices and calc_index != current_index:
sys.stderr.write(
"\n\tWarning, current index %s is not aliased at the moment\n"
% current_index)
sys.stderr.write("\tCurrent live aliased index: %s\n\n" %
(','.join(aliased_indices)))
if calc_index != current_index:
sys.stderr.write(
"XFORM_INDEX hash has changed, please update \n\t%s\n\tXFORM_INDEX property with the line below:\n"
% filepath)
sys.stdout.write('XFORM_INDEX="%s"\n' % calc_index)
else:
sys.stderr.write("XFORM_INDEX unchanged\n")
def finish_handle(self):
filepath = os.path.join(settings.FILEPATH, "corehq", "pillows", "mappings", "xform_mapping.py")
xform_pillow = XFormPillow(create_index=False)
# current index
# check current index
aliased_indices = xform_pillow.check_alias()
# current_index = '%s_%s' % (xform_pillow.es_index_prefix, xform_pillow.calc_meta())
current_index = xform_pillow.es_index
sys.stderr.write("current index:\n")
sys.stderr.write('XFORM_INDEX="%s"\n' % current_index)
# regenerate the mapping dict
mapping = xform_mapping.XFORM_MAPPING
xform_pillow.default_mapping = mapping
delattr(xform_pillow, "_calc_meta_cache")
# xform_pillow.calc_meta.reset_cache()
calc_index = "%s_%s" % (xform_pillow.es_index_prefix, xform_pillow.calc_meta())
if calc_index not in aliased_indices and calc_index != current_index:
sys.stderr.write("\n\tWarning, current index %s is not aliased at the moment\n" % current_index)
sys.stderr.write("\tCurrent live aliased index: %s\n\n" % (",".join(aliased_indices)))
if calc_index != current_index:
sys.stderr.write(
"XFORM_INDEX hash has changed, please update \n\t%s\n\tXFORM_INDEX property with the line below:\n"
% filepath
)
sys.stdout.write('XFORM_INDEX="%s"\n' % calc_index)
else:
sys.stderr.write("XFORM_INDEX unchanged\n")
class XFormPillowTest(TestCase):
domain = 'xform-pillowtest-domain'
def setUp(self):
FormProcessorTestUtils.delete_all_xforms()
with trap_extra_setup(ConnectionError):
self.pillow = XFormPillow()
self.elasticsearch = self.pillow.get_es_new()
delete_es_index(self.pillow.es_index)
def tearDown(self):
ensure_index_deleted(self.pillow.es_index)
def test_xform_pillow_couch(self):
metadata = TestFormMetadata(domain=self.domain)
form = get_form_ready_to_save(metadata)
FormProcessorInterface(domain=self.domain).save_processed_models([form])
self.pillow.process_changes(since=0, forever=False)
self.elasticsearch.indices.refresh(self.pillow.es_index)
results = FormES().run()
self.assertEqual(1, results.total)
form_doc = results.hits[0]
self.assertEqual(self.domain, form_doc['domain'])
self.assertEqual(metadata.xmlns, form_doc['xmlns'])
self.assertEqual('XFormInstance', form_doc['doc_type'])
form.delete()
@override_settings(TESTS_SHOULD_USE_SQL_BACKEND=True)
def test_xform_pillow_sql(self):
consumer = get_test_kafka_consumer(topics.FORM_SQL)
# have to get the seq id before the change is processed
kafka_seq = consumer.offsets()['fetch'][(topics.FORM_SQL, 0)]
metadata = TestFormMetadata(domain=self.domain)
form = get_form_ready_to_save(metadata, is_db_test=True)
form_processor = FormProcessorInterface(domain=self.domain)
form_processor.save_processed_models([form])
# confirm change made it to kafka
message = consumer.next()
change_meta = change_meta_from_kafka_message(message.value)
self.assertEqual(form.form_id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
# send to elasticsearch
sql_pillow = get_sql_xform_to_elasticsearch_pillow()
sql_pillow.process_changes(since=kafka_seq, forever=False)
self.elasticsearch.indices.refresh(self.pillow.es_index)
# confirm change made it to elasticserach
results = FormES().run()
self.assertEqual(1, results.total)
form_doc = results.hits[0]
self.assertEqual(self.domain, form_doc['domain'])
self.assertEqual(metadata.xmlns, form_doc['xmlns'])
self.assertEqual('XFormInstance', form_doc['doc_type'])
def testXFormPillowSingleCaseProcess(self):
"""
Test that xform pillow can process and cleanup a single xform with a case submission
"""
xform = XFORM_SINGLE_CASE
pillow = XFormPillow(create_index=False, online=False)
changed = pillow.change_transform(xform)
self.assertIsNone(changed['form']['case'].get('@date_modified'))
self.assertIsNotNone(xform['form']['case']['@date_modified'])
def testXFormMapping(self):
"""
Verify that a simple case doc will yield the basic mapping
"""
pillow = XFormPillow(create_index=False, online=False)
t1 = pillow.get_mapping_from_type(XFORM_SINGLE_CASE)
t2 = pillow.get_mapping_from_type(XFORM_MULTI_CASES)
self.assertEqual(t1, t2)
def testXFormPillowSingleCaseProcess(self):
"""
Test that xform pillow can process and cleanup a single xform with a case submission
"""
xform = XFORM_SINGLE_CASE
pillow = XFormPillow(create_index=False, online=False)
changed = pillow.change_transform(xform)
self.assertIsNone(changed['form']['case'].get('@date_modified'))
self.assertIsNotNone(xform['form']['case']['@date_modified'])
def testXFormMapping(self):
"""
Verify that a simple case doc will yield the basic mapping
"""
pillow = XFormPillow(create_index=False, online=False)
t1 = pillow.get_mapping_from_type(XFORM_SINGLE_CASE)
t2 = pillow.get_mapping_from_type(XFORM_MULTI_CASES)
self.assertEqual(t1, t2)
def testXFormPillowListCaseProcess(self):
"""
Test that xform pillow can process and cleanup a single xform with a list of cases in it
"""
xform = XFORM_MULTI_CASES
pillow = XFormPillow(create_index=False, online=False)
changed = pillow.change_transform(xform)
changed_cases = extract_case_blocks(changed)
orig_cases = extract_case_blocks(xform)
[self.assertIsNotNone(x['@date_modified']) for x in orig_cases]
[self.assertIsNone(x.get('@date_modified')) for x in changed_cases]
def testXFormPillowListCaseProcess(self):
"""
Test that xform pillow can process and cleanup a single xform with a list of cases in it
"""
xform = XFORM_MULTI_CASES
pillow = XFormPillow(online=False)
changed = pillow.change_transform(xform)
changed_cases = extract_case_blocks(changed)
orig_cases = extract_case_blocks(xform)
[self.assertIsNotNone(x['@date_modified']) for x in orig_cases]
[self.assertIsNone(x.get('@date_modified')) for x in changed_cases]
def save_to_es_analytics_db(domain, received_on, app_id, device_id, user_id, username=None):
metadata = TestFormMetadata(
domain=domain,
time_end=received_on,
received_on=received_on,
app_id=app_id,
user_id=user_id,
device_id=device_id,
username=username
)
form_pair = make_es_ready_form(metadata)
pillow = XFormPillow()
pillow.change_transport(form_pair.json_form)
def test_xformPillowTransform(self):
"""
Test to make sure report xform and reportxform pillows strip the appVersion dict to match the
mappings
"""
pillows = [ReportXFormPillow(online=False),XFormPillow(online=False)]
bad_appVersion = {
"_id": "foo",
"domain": settings.ES_XFORM_FULL_INDEX_DOMAINS[0],
"form": {
"meta": {
"@xmlns": "http://openrosa.org/jr/xforms",
"username": "******",
"instanceID": "foo",
"userID": "some_user_id",
"timeEnd": "2013-09-20T01:33:12Z",
"appVersion": {
"@xmlns": "http://commcarehq.org/xforms",
"#text": "CCODK:\"2.5.1\"(11126). v236 CC2.5b[11126] on April-15-2013"
},
"timeStart": "2013-09-19T01:13:20Z",
"deviceID": "somedevice"
}
}
}
for pillow in pillows:
cleaned = pillow.change_transform(bad_appVersion)
self.assertFalse(isinstance(cleaned['form']['meta']['appVersion'], dict))
self.assertTrue(isinstance(cleaned['form']['meta']['appVersion'], str))
self.assertTrue(cleaned['form']['meta']['appVersion'], "CCODK:\"2.5.1\"(11126). v236 CC2.5b[11126] on April-15-2013")
def test_get_list(self):
"""
Any form in the appropriate domain should be in the list from the API.
"""
# The actual infrastructure involves saving to CouchDB, having PillowTop
# read the changes and write it to ElasticSearch.
# In order to test just the API code, we set up a fake XFormES (this should
# really be a parameter to the XFormInstanceResource constructor)
# and write the translated form directly; we are not trying to test
# the ptop infrastructure.
#the pillow is set to offline mode - elasticsearch not needed to validate
pillow = XFormPillow(online=False)
fake_xform_es = FakeXFormES()
v0_4.MOCK_XFORM_ES = fake_xform_es
backend_form = XFormInstance(xmlns='fake-xmlns',
domain=self.domain.name,
received_on=datetime.utcnow(),
form={
'#type': 'fake-type',
'@xmlns': 'fake-xmlns'
})
backend_form.save()
translated_doc = pillow.change_transform(backend_form.to_json())
fake_xform_es.add_doc(translated_doc['_id'], translated_doc)
self.client.login(username=self.username, password=self.password)
response = self.client.get(self.list_endpoint)
self.assertEqual(response.status_code, 200)
api_forms = simplejson.loads(response.content)['objects']
self.assertEqual(len(api_forms), 1)
api_form = api_forms[0]
self.assertEqual(api_form['form']['@xmlns'], backend_form.xmlns)
self.assertEqual(api_form['received_on'],
backend_form.received_on.isoformat())
backend_form.delete()
def test_get_list(self):
"""
Any form in the appropriate domain should be in the list from the API.
"""
# The actual infrastructure involves saving to CouchDB, having PillowTop
# read the changes and write it to ElasticSearch.
# In order to test just the API code, we set up a fake XFormES (this should
# really be a parameter to the XFormInstanceResource constructor)
# and write the translated form directly; we are not trying to test
# the ptop infrastructure.
#the pillow is set to offline mode - elasticsearch not needed to validate
pillow = XFormPillow(online=False)
fake_xform_es = FakeXFormES()
v0_4.MOCK_XFORM_ES = fake_xform_es
backend_form = XFormInstance(xmlns = 'fake-xmlns',
domain = self.domain.name,
received_on = datetime.utcnow(),
form = {
'#type': 'fake-type',
'@xmlns': 'fake-xmlns'
})
backend_form.save()
translated_doc = pillow.change_transform(backend_form.to_json())
fake_xform_es.add_doc(translated_doc['_id'], translated_doc)
self.client.login(username=self.username, password=self.password)
response = self.client.get(self.list_endpoint)
self.assertEqual(response.status_code, 200)
api_forms = simplejson.loads(response.content)['objects']
self.assertEqual(len(api_forms), 1)
api_form = api_forms[0]
self.assertEqual(api_form['form']['@xmlns'], backend_form.xmlns)
self.assertEqual(api_form['received_on'], backend_form.received_on.isoformat())
backend_form.delete()
def setUpClass(cls):
form_pillow = XFormPillow(online=False)
case_pillow = CasePillow(online=False)
cls.pillows = [form_pillow, case_pillow]
es = get_es_new()
with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"):
for pillow in cls.pillows:
completely_initialize_pillow_index(pillow)
initialize_index_and_mapping(es, GROUP_INDEX_INFO)
case = new_case(closed=True)
case_pillow.send_robust(case.to_json())
case = new_case(closed=False)
case_pillow.send_robust(case.to_json())
case = new_case(closed=True, owner_id="foo")
case_pillow.send_robust(case.to_json())
case = new_case(closed=False, owner_id="bar")
case_pillow.send_robust(case.to_json())
group = Group(_id=uuid.uuid4().hex, users=["foo", "bar"])
cls.group_id = group._id
send_to_elasticsearch('groups', group.to_json())
form = new_form(form={"meta": {"userID": None}})
form_pillow.send_robust(form.to_json())
form = new_form(form={"meta": {"userID": ""}})
form_pillow.send_robust(form.to_json())
form = new_form(form={"meta": {"deviceID": "abc"}})
form_pillow.send_robust(form.to_json())
form = new_form(form={"meta": {"userID": uuid.uuid4().hex}})
form_pillow.send_robust(form.to_json())
for pillow in cls.pillows:
pillow.get_es_new().indices.refresh(pillow.es_index)
es.indices.refresh(GROUP_INDEX_INFO.index)
def setUp(self):
FormProcessorTestUtils.delete_all_xforms()
with trap_extra_setup(ConnectionError):
self.pillow = XFormPillow()
self.elasticsearch = self.pillow.get_es_new()
delete_es_index(self.pillow.es_index)
def setUp(self):
FormProcessorTestUtils.delete_all_xforms()
self.pillow = XFormPillow()
self.elasticsearch = self.pillow.get_es_new()
delete_es_index(self.pillow.es_index)
