def _send_to_es_and_check(self,
doc,
update=False,
es_merge_update=False,
delete=False,
esgetter=None):
if update and es_merge_update:
old_doc = self.es_interface.get_doc(self.es_alias,
TEST_INDEX_INFO.type,
doc['_id'])
send_to_elasticsearch(TEST_INDEX_INFO,
doc_type=TEST_INDEX_INFO.type,
doc_id=doc['_id'],
es_getter=esgetter or get_es_new,
name='test',
data=doc,
es_merge_update=es_merge_update,
delete=delete)
if not delete:
self.assertEqual(1, get_doc_count(self.es, self.index))
es_doc = self.es_interface.get_doc(self.es_alias,
TEST_INDEX_INFO.type,
doc['_id'])
if es_merge_update:
old_doc.update(es_doc)
for prop in doc:
self.assertEqual(doc[prop], old_doc[prop])
else:
for prop in doc:
self.assertEqual(doc[prop], es_doc[prop])
self.assertTrue(all(prop in doc for prop in es_doc))
else:
self.assertEqual(0, get_doc_count(self.es, self.index))
def _send_to_es_and_check(self, doc, update=False, es_merge_update=False,
delete=False, esgetter=None):
if update and es_merge_update:
old_doc = self.es.get_source(self.index, TEST_INDEX_INFO.type, doc['_id'])
send_to_elasticsearch(
index=self.index,
doc_type=TEST_INDEX_INFO.type,
doc_id=doc['_id'],
es_getter=esgetter or get_es_new,
name='test',
data=doc,
update=update,
es_merge_update=es_merge_update,
delete=delete,
except_on_failure=True,
retries=1
)
if not delete:
self.assertEqual(1, get_doc_count(self.es, self.index))
es_doc = self.es.get_source(self.index, TEST_INDEX_INFO.type, doc['_id'])
if es_merge_update:
old_doc.update(es_doc)
for prop in doc:
self.assertEqual(doc[prop], old_doc[prop])
else:
for prop in doc:
self.assertEqual(doc[prop], es_doc[prop])
self.assertTrue(all(prop in doc for prop in es_doc))
else:
self.assertEqual(0, get_doc_count(self.es, self.index))
def _send_to_es_and_check(self,
doc,
update=False,
delete=False,
esgetter=None):
send_to_elasticsearch(index=self.index,
doc_type=TEST_INDEX_INFO.type,
doc_id=doc['_id'],
es_getter=esgetter or get_es_new,
name='test',
data=doc,
update=update,
delete=delete,
except_on_failure=True,
retries=1)
if not delete:
self.assertEqual(1, get_doc_count(self.es, self.index))
es_doc = self.es.get_source(self.index, TEST_INDEX_INFO.type,
doc['_id'])
for prop in doc:
self.assertEqual(doc[prop], es_doc[prop])
self.assertTrue(all(prop in doc for prop in es_doc))
else:
self.assertEqual(0, get_doc_count(self.es, self.index))
def test_assume_alias(self):
initialize_index_and_mapping(self.es, TEST_INDEX_INFO)
doc_id = uuid.uuid4().hex
doc = {'_id': doc_id, 'doc_type': 'CommCareCase', 'type': 'mother'}
send_to_elasticsearch(self.index, TEST_INDEX_INFO.type, doc_id, get_es_new, 'test', doc)
self.assertEqual(1, get_doc_count(self.es, self.index))
assume_alias(self.es, self.index, TEST_INDEX_INFO.alias)
es_doc = self.es.get_source(TEST_INDEX_INFO.alias, TEST_INDEX_INFO.type, doc_id)
for prop in doc:
self.assertEqual(doc[prop], es_doc[prop])
def test_assume_alias(self):
initialize_index_and_mapping(self.es, TEST_INDEX_INFO)
doc_id = uuid.uuid4().hex
doc = {'_id': doc_id, 'doc_type': 'CommCareCase', 'type': 'mother'}
send_to_elasticsearch(self.index, TEST_INDEX_INFO.type, doc_id,
get_es_new, 'test', doc)
self.assertEqual(1, get_doc_count(self.es, self.index))
assume_alias(self.es, self.index, TEST_INDEX_INFO.alias)
es_doc = self.es_interface.get_doc(TEST_INDEX_INFO.alias,
TEST_INDEX_INFO.type, doc_id)
for prop in doc:
self.assertEqual(doc[prop], es_doc[prop])
def test_missing_merge(self):
doc = {'_id': uuid.uuid4().hex, 'doc_type': 'MyCoolDoc', 'property': 'bar'}
send_to_elasticsearch(
TEST_INDEX_INFO,
doc_type=TEST_INDEX_INFO.type,
doc_id=doc['_id'],
es_getter=get_es_new,
name='test',
data=doc,
es_merge_update=True,
)
self.assertEqual(0, get_doc_count(self.es, self.index))
def _send_to_es(self, doc, es_merge_update=False, delete=False,
esgetter=None):
send_to_elasticsearch(
TEST_INDEX_INFO,
doc_type=TEST_INDEX_INFO.type,
doc_id=doc['_id'],
es_getter=esgetter or get_es_new,
name='test',
data=doc,
es_merge_update=es_merge_update,
delete=delete
)
def _set_up_sms_es(self):
sms_doc = {
'_id': 'some_sms_id',
'domain': self.domain.name,
'direction': INCOMING,
'date': json_format_datetime(datetime.datetime.utcnow()),
'doc_type': SMS_INDEX_INFO.type,
}
send_to_elasticsearch(
alias=SMS_INDEX_INFO.alias,
doc_type=SMS_INDEX_INFO.type,
doc_id=sms_doc['_id'],
es_getter=get_es_new,
name='ElasticProcessor',
data=sms_doc,
update=False,
)
refresh_elasticsearch_index('sms')
def _send_to_es_mock_errors(self, exception, retries):
doc = {'_id': uuid.uuid4().hex, 'doc_type': 'MyCoolDoc', 'property': 'bar'}
with mock.patch("pillowtop.processors.elastic._propagate_failure", return_value=False), \
mock.patch("pillowtop.processors.elastic._retries", return_value=retries), \
mock.patch("pillowtop.processors.elastic._sleep_between_retries"), \
mock.patch("pillowtop.processors.elastic._get_es_interface") as _get_es_interface, \
capture_log_output("pillowtop") as log:
es_interface = mock.Mock()
es_interface.index_doc.side_effect = exception
_get_es_interface.return_value = es_interface
send_to_elasticsearch(
TEST_INDEX_INFO,
doc_type=TEST_INDEX_INFO.type,
doc_id=doc['_id'],
es_getter=None,
name='test',
data=doc,
es_merge_update=False,
delete=False
)
return log.get_output()
