def change_meta_from_ledger_v1(stock_state, deleted=False):
return ChangeMeta(
document_id=stock_state.pk,
data_source_type=data_sources.LEDGER_V1,
data_source_name='ledger-v1', # todo: this isn't really needed.
domain=stock_state.domain,
is_deletion=deleted,
)
def create_error(change, message='message', attempts=0, pillow=None, ex_class=None):
change.metadata = ChangeMeta(
data_source_type='couch', data_source_name='test_commcarehq', document_id=change.id
)
error = PillowError.get_or_create(change, pillow or FakePillow())
for n in range(0, attempts):
error.add_attempt(*get_ex_tb(message, ex_class=ex_class))
return error
def change_meta_from_hard_delete(document, data_source_type, data_source_name):
return ChangeMeta(
document_id=document['_id'],
document_rev=document.get('_rev', None),
data_source_type=data_source_type,
data_source_name=data_source_name,
is_deletion=True,
)
def change_meta_from_ledger_v2(ledger_ref, domain, deleted=False):
return ChangeMeta(
document_id=ledger_ref.as_id(),
data_source_type=data_sources.LEDGER_V2,
data_source_name='ledger-v2', # todo: this isn't really needed.
domain=domain,
is_deletion=deleted,
)
def _change_meta_from_ledger_reference(domain, ledger_reference):
return ChangeMeta(
document_id=ledger_reference.as_id(),
data_source_type=data_sources.LEDGER_V2,
data_source_name='ledger-v2', # todo: this isn't really needed.
domain=domain,
is_deletion=False,
)
def publish_form_deleted(domain, form_id):
producer.send_change(topics.FORM_SQL, ChangeMeta(
document_id=form_id,
data_source_type=data_sources.FORM_SQL,
data_source_name='form-sql',
document_type='XFormInstance-Deleted',
domain=domain,
is_deletion=True,
))
def _test_success(self, auto_flush):
kafka_producer = ChangeProducer(auto_flush=auto_flush)
with capture_log_output(KAFKA_AUDIT_LOGGER) as logs:
meta = ChangeMeta(document_id=uuid.uuid4().hex, data_source_type='dummy-type',
data_source_name='dummy-name')
kafka_producer.send_change(topics.CASE, meta)
if not auto_flush:
kafka_producer.flush()
self._check_logs(logs, meta.document_id, [CHANGE_PRE_SEND, CHANGE_SENT])
def _change_meta_from_ledger_reference(domain, ledger_reference, source_name):
return ChangeMeta(
document_id=ledger_reference.as_id(),
data_source_type=data_sources.SOURCE_SQL,
data_source_name=source_name,
document_type=topics.LEDGER,
domain=domain,
is_deletion=False,
)
def change_meta_from_ledger_v1(stock_state, deleted=False):
return ChangeMeta(
document_id=stock_state.pk,
data_source_type=data_sources.SOURCE_SQL,
data_source_name=data_sources.LEDGER_V1,
document_type=topics.LEDGER,
domain=stock_state.domain,
is_deletion=deleted,
)
def change_meta_from_ledger_v2(ledger_ref, domain, deleted=False):
return ChangeMeta(
document_id=ledger_ref.as_id(),
data_source_type=data_sources.SOURCE_SQL,
data_source_name=data_sources.LEDGER_V2,
document_type=topics.LEDGER,
domain=domain,
is_deletion=deleted,
)
def publish_case_deleted(domain, case_id):
producer.send_change(topics.CASE_SQL, ChangeMeta(
document_id=case_id,
data_source_type=data_sources.CASE_SQL,
data_source_name='case-sql', # todo: this isn't really needed.
document_type='CommCareCase-Deleted',
domain=domain,
is_deletion=True,
))
def publish_location_saved(domain, location_id, is_deletion=False):
change_meta = ChangeMeta(
document_id=location_id,
data_source_type='location',
data_source_name='location',
document_type=LOCATION_DOC_TYPE,
domain=domain,
is_deletion=is_deletion,
)
producer.send_change(topics.LOCATION, change_meta)
def change_meta_from_sql_case(case):
return ChangeMeta(
document_id=case.case_id,
data_source_type=data_sources.SOURCE_SQL,
data_source_name=data_sources.CASE_SQL,
document_type='CommCareCase',
document_subtype=case.type,
domain=case.domain,
is_deletion=case.is_deleted,
)
def change_meta_from_sql_form(form):
return ChangeMeta(
document_id=form.form_id,
data_source_type=data_sources.SOURCE_SQL,
data_source_name=data_sources.FORM_SQL,
document_type=form.doc_type,
document_subtype=form.xmlns,
domain=form.domain,
is_deletion=form.is_deleted,
)
def _change_meta(data_row, source):
return ChangeMeta(
document_id=data_row.doc_id,
data_source_type=data_sources.SOURCE_SQL,
data_source_name=source,
document_type=data_row.doc_type,
document_subtype=data_row.doc_subtype,
domain=data_row.domain,
is_deletion=False,
)
def _change_meta_for_sql_form_record(domain, form_record):
return ChangeMeta(
document_id=form_record.doc_id,
data_source_type=data_sources.SOURCE_SQL,
data_source_name=data_sources.FORM_SQL,
document_type=FORM_DOC_TYPE,
document_subtype=form_record.doc_subtype,
domain=domain,
is_deletion=False,
)
def change_meta_from_sql_case(case):
return ChangeMeta(
document_id=case.case_id,
data_source_type=data_sources.CASE_SQL,
data_source_name='case-sql', # todo: this isn't really needed.
document_type='CommCareCase',
document_subtype=case.type,
domain=case.domain,
is_deletion=case.is_deleted,
)
def _change_meta_for_sql_case(domain, case_id, case_type):
return ChangeMeta(
document_id=case_id,
data_source_type=data_sources.SOURCE_SQL,
data_source_name=data_sources.CASE_SQL,
document_type=CASE_DOC_TYPE,
document_subtype=case_type,
domain=domain,
is_deletion=False,
)
def change_meta_from_sql_form(form):
return ChangeMeta(
document_id=form.form_id,
data_source_type=data_sources.FORM_SQL,
data_source_name='form-sql', # todo: this isn't really needed.
document_type=form.doc_type,
document_subtype=form.xmlns,
domain=form.domain,
is_deletion=form.is_deleted,
)
def change_meta_from_synclog(synclog):
from corehq.apps.change_feed import data_sources
return ChangeMeta(
document_id=synclog.synclog_id.hex,
data_source_type=data_sources.SYNCLOG_SQL,
data_source_name=data_sources.SYNCLOG_SQL,
document_type='SYNCLOG_SQL',
document_subtype=None,
domain=synclog.domain,
is_deletion=False,
)
def _changes_from_ids(self, case_ids):
return [
Change(id=case_id,
sequence_id=None,
document_store=CaseDocumentStore('domain'),
metadata=ChangeMeta(document_id=case_id,
domain='domain',
data_source_type='sql',
data_source_name='case-sql'))
for case_id in case_ids
]
def publish_case_deleted(domain, case_id):
producer.send_change(
topics.CASE_SQL,
ChangeMeta(
document_id=case_id,
data_source_type=data_sources.SOURCE_SQL,
data_source_name=data_sources.CASE_SQL,
document_type='CommCareCase-Deleted',
domain=domain,
is_deletion=True,
))
def create_change_meta(doc_type, domain, doc_id, doc_rev):
return ChangeMeta(
document_id=doc_id,
document_rev=doc_rev,
data_source_type=DATA_SOURCE_TYPE,
data_source_name=DATA_SOURCE_NAME,
document_type=doc_type,
document_subtype=None, # should be case.type or form.xmlns, but not used anywhere
domain=domain,
is_deletion=False,
)
def change_meta_from_sms(sms):
from corehq.apps.change_feed import data_sources
return ChangeMeta(
document_id=sms.couch_id,
data_source_type=data_sources.SMS,
data_source_name=data_sources.SMS,
document_type='SMS',
document_subtype=None,
domain=sms.domain,
is_deletion=False,
)
def _changes_from_ids(self, case_ids):
return [
Change(id=case_id,
sequence_id=None,
document_store=CaseDocumentStore(self.domain),
metadata=ChangeMeta(document_id=case_id,
domain=self.domain,
data_source_type=SOURCE_COUCH,
data_source_name='commcarehq'))
for case_id in case_ids
]
def publish_location_saved(domain, location_id, is_deletion=False):
from corehq.apps.change_feed import data_sources
change_meta = ChangeMeta(
document_id=location_id,
data_source_type=data_sources.SOURCE_SQL,
data_source_name=data_sources.LOCATION,
document_type=LOCATION_DOC_TYPE,
domain=domain,
is_deletion=is_deletion,
)
producer.send_change(topics.LOCATION, change_meta)
def doc_to_change(doc):
return Change(id=doc['_id'],
sequence_id='0',
document=doc,
metadata=ChangeMeta(
document_id=doc['_id'],
data_source_type=data_sources.COUCH,
data_source_name=CommCareCase.get_db().dbname,
document_type=doc['doc_type'],
document_subtype=doc.get('type'),
domain=doc['domain'],
is_deletion=False,
))
def doc_to_change(doc):
return Change(id=doc['_id'],
sequence_id='0',
document=doc,
metadata=ChangeMeta(
document_id=doc['_id'],
data_source_type=data_sources.SOURCE_COUCH,
data_source_name=data_sources.CASE_SQL,
document_type=doc['doc_type'],
document_subtype=doc.get('type'),
domain=doc['domain'],
is_deletion=False,
))
def test_error_asynchronous(self):
kafka_producer = ChangeProducer(auto_flush=False)
future = Future()
kafka_producer.producer.send = Mock(return_value=future)
meta = ChangeMeta(
document_id=uuid.uuid4().hex, data_source_type='dummy-type', data_source_name='dummy-name'
)
with capture_log_output(KAFKA_AUDIT_LOGGER) as logs:
kafka_producer.send_change(topics.CASE, meta)
future.failure(Exception())
self._check_logs(logs, meta.document_id, [CHANGE_PRE_SEND, CHANGE_ERROR])
def test_error_synchronous(self):
kafka_producer = ChangeProducer()
future = Future()
future.get = Mock(side_effect=Exception())
kafka_producer.producer.send = Mock(return_value=future)
meta = ChangeMeta(
document_id=uuid.uuid4().hex, data_source_type='dummy-type', data_source_name='dummy-name'
)
with capture_log_output(KAFKA_AUDIT_LOGGER) as logs:
with self.assertRaises(Exception):
kafka_producer.send_change(topics.CASE, meta)
self._check_logs(logs, meta.document_id, [CHANGE_PRE_SEND, CHANGE_ERROR])
