def test_test_local_domain_sql_backend_override_overrides(self):
domain_name = uuid.uuid4().hex
create_domain(domain_name)
self.assertFalse(should_use_sql_backend(domain_name))
set_local_domain_sql_backend_override(domain_name)
self.assertTrue(should_use_sql_backend(domain_name))
clear_local_domain_sql_backend_override(domain_name)
self.assertFalse(should_use_sql_backend(domain_name))
def location_map_case_id(user):
if should_use_sql_backend(user.domain):
user_id = user.user_id
if isinstance(user_id, unicode):
user_id = user_id.encode('utf8')
return uuid.uuid5(const.MOBILE_WORKER_UUID_NS, user_id).hex
return 'user-owner-mapping-' + user.user_id
def test_fire_signals(self):
from corehq.apps.receiverwrapper.tests.test_submit_errors import failing_signal_handler
case_id = uuid.uuid4().hex
form_id = uuid.uuid4().hex
with failing_signal_handler('signal death'):
submit_case_blocks(
CaseBlock(case_id=case_id, create=True, case_type='box').as_string().decode('utf-8'),
self.domain,
form_id=form_id
)
form = self.formdb.get_form(form_id)
with catch_signal(successful_form_received) as form_handler, catch_signal(case_post_save) as case_handler:
submit_form_locally(
instance=form.get_xml(),
domain=self.domain,
)
case = self.casedb.get_case(case_id)
if should_use_sql_backend(self.domain):
self.assertEqual(form, form_handler.call_args[1]['xform'])
self.assertEqual(case, case_handler.call_args[1]['case'])
else:
signal_form = form_handler.call_args[1]['xform']
self.assertEqual(form.form_id, signal_form.form_id)
self.assertEqual(form.get_rev, signal_form.get_rev)
signal_case = case_handler.call_args[1]['case']
self.assertEqual(case.case_id, signal_case.case_id)
self.assertEqual(case.get_rev, signal_case.get_rev)
def test_archiving_only_form(self):
"""
Checks that archiving the only form associated with the case archives
the case and unarchiving unarchives it.
"""
case_id = _post_util(create=True, p1='p1-1', p2='p2-1')
case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN)
case = case_accessors.get_case(case_id)
self.assertFalse(case.is_deleted)
if should_use_sql_backend(REBUILD_TEST_DOMAIN):
self.assertEqual(1, len(case.actions))
else:
self.assertEqual(2, len(case.actions))
[form_id] = case.xform_ids
form = FormAccessors(REBUILD_TEST_DOMAIN).get_form(form_id)
form.archive()
case = case_accessors.get_case(case_id)
self.assertTrue(case.is_deleted)
# should just have the 'rebuild' action
self.assertEqual(1, len(case.actions))
self.assertTrue(case.actions[0].is_case_rebuild)
form.unarchive()
case = case_accessors.get_case(case_id)
self.assertFalse(case.is_deleted)
self.assertEqual(3, len(case.actions))
self.assertTrue(case.actions[-1].is_case_rebuild)
def test_simple_delete(self):
factory = CaseFactory()
case = factory.create_case()
[case] = factory.create_or_update_case(
CaseStructure(case_id=case.case_id, attrs={'update': {'foo': 'bar'}})
)
self.assertIsNotNone(self.casedb.get_case(case.case_id))
self.assertEqual(2, len(case.xform_ids))
for form_id in case.xform_ids:
self.assertIsNotNone(self.formdb.get_form(form_id))
with capture_kafka_changes_context(topics.FORM_SQL, topics.CASE_SQL) as change_context:
safe_hard_delete(case)
if should_use_sql_backend(case.domain):
self.assertEqual(3, len(change_context.changes))
expected_ids = {case.case_id} | set(case.xform_ids)
self.assertEqual(expected_ids, {change.id for change in change_context.changes})
for change in change_context.changes:
self.assertTrue(change.deleted)
with self.assertRaises(CaseNotFound):
self.casedb.get_case(case.case_id)
for form_id in case.xform_ids:
with self.assertRaises(XFormNotFound):
self.formdb.get_form(form_id)
def test_ledger_pillow(self):
factory = CaseFactory(domain=self.domain)
case = factory.create_case()
consumer = get_test_kafka_consumer(topics.LEDGER)
# have to get the seq id before the change is processed
kafka_seq = get_topic_offset(topics.LEDGER)
from corehq.apps.commtrack.tests.util import get_single_balance_block
from corehq.apps.hqcase.utils import submit_case_blocks
submit_case_blocks([
get_single_balance_block(case.case_id, self.product_id, 100)],
self.domain
)
ref = UniqueLedgerReference(case.case_id, 'stock', self.product_id)
# confirm change made it to kafka
message = consumer.next()
change_meta = change_meta_from_kafka_message(message.value)
if should_use_sql_backend(self.domain):
self.assertEqual(ref.as_id(), change_meta.document_id)
else:
from corehq.apps.commtrack.models import StockState
state = StockState.objects.all()
self.assertEqual(1, len(state))
self.assertEqual(state[0].pk, change_meta.document_id) #
self.assertEqual(self.domain, change_meta.domain)
# send to elasticsearch
self.pillow.process_changes(since=kafka_seq, forever=False)
self.elasticsearch.indices.refresh(LEDGER_INDEX_INFO.index)
# confirm change made it to elasticserach
self._assert_ledger_in_es(ref)
def _populate_results(self, case_id_list):
if should_use_sql_backend(self.domain):
base_results = [CaseAPIResult(domain=self.domain, couch_doc=case, id_only=self.ids_only)
for case in self.case_accessors.iter_cases(case_id_list)]
else:
base_results = [CaseAPIResult(domain=self.domain, couch_doc=case, id_only=self.ids_only)
for case in iter_cases(case_id_list, self.strip_history, self.wrap)]
return base_results
def _create_models_for_stock_report_helper(self, form, stock_report_helper):
processing_result = StockProcessingResult(form, stock_report_helpers=[stock_report_helper])
processing_result.populate_models()
if should_use_sql_backend(self.domain_name):
from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL
LedgerAccessorSQL.save_ledger_values(processing_result.models_to_save)
else:
processing_result.commit()
def get_domain_case_change_provider(domains):
change_providers = []
for domain in domains:
if should_use_sql_backend(domain):
change_providers.append(SqlDomainCaseChangeProvider(domain))
else:
change_providers.append(get_couch_domain_case_change_provider(domain))
return CompositeChangeProvider(change_providers)
def _assert_case_revision(self, rev_number, last_modified, expect_modified=False):
if should_use_sql_backend(self.domain):
self.assertEqual(
expect_modified,
CaseAccessorSQL.case_modified_since(self.case_id, last_modified)
)
else:
doc = self._get_case()
self.assertTrue(doc['_rev'].startswith('%s-' % rev_number))
def __init__(self, domain):
from corehq.apps.commtrack.models import StockState
assert not should_use_sql_backend(domain), "Only non-SQL backend supported"
self.domain = domain
def _doc_gen_fn(obj):
return obj.to_json()
super(LedgerV1DocumentStore, self).__init__(StockState, _doc_gen_fn)
def location_map_case_id(user):
if should_use_sql_backend(user.domain):
user_id = user.user_id
if isinstance(user_id, six.text_type) and six.PY2:
user_id = user_id.encode('utf8')
case_id = uuid.uuid5(const.MOBILE_WORKER_UUID_NS, user_id).hex
if six.PY2:
case_id = case_id.decode('utf-8')
return case_id
return 'user-owner-mapping-' + user.user_id
def create_case(domain, case_type, **kwargs):
case = CaseFactory(domain).create_case(case_type=case_type, **kwargs)
try:
yield case
finally:
if should_use_sql_backend(domain):
CaseAccessorSQL.hard_delete_cases(domain, [case.case_id])
else:
case.delete()
def test_archive_last_form(self):
initial_amounts = [(p._id, float(100)) for p in self.products]
self.submit_xml_form(
balance_submission(initial_amounts),
timestamp=datetime.utcnow() + timedelta(-30)
)
final_amounts = [(p._id, float(50)) for i, p in enumerate(self.products)]
second_form_id = self.submit_xml_form(balance_submission(final_amounts))
ledger_accessors = LedgerAccessors(self.domain.name)
def _assert_initial_state():
if should_use_sql_backend(self.domain):
self.assertEqual(3, len(self._get_all_ledger_transactions(Q(form_id=second_form_id))))
else:
self.assertEqual(1, StockReport.objects.filter(form_id=second_form_id).count())
# 6 = 3 stockonhand and 3 inferred consumption txns
self.assertEqual(6, StockTransaction.objects.filter(report__form_id=second_form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for lv in ledger_values:
self.assertEqual(50, lv.stock_on_hand)
self.assertEqual(
round(float(lv.daily_consumption), 2),
1.67
)
# check initial setup
_assert_initial_state()
# archive and confirm commtrack data is deleted
form = FormAccessors(self.domain.name).get_form(second_form_id)
with process_pillow_changes('LedgerToElasticsearchPillow'):
form.archive()
if should_use_sql_backend(self.domain):
self.assertEqual(0, len(self._get_all_ledger_transactions(Q(form_id=second_form_id))))
else:
self.assertEqual(0, StockReport.objects.filter(form_id=second_form_id).count())
self.assertEqual(0, StockTransaction.objects.filter(report__form_id=second_form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for state in ledger_values:
# balance should be reverted to 100 in the StockState
self.assertEqual(100, int(state.stock_on_hand))
# consumption should be none since there will only be 1 data point
self.assertIsNone(state.daily_consumption)
# unarchive and confirm commtrack data is restored
with process_pillow_changes('LedgerToElasticsearchPillow'):
form.unarchive()
_assert_initial_state()
def _assert_initial_state():
if should_use_sql_backend(self.domain.name):
self.assertEqual(3, LedgerTransaction.objects.filter(form_id=form_id).count())
else:
self.assertEqual(1, StockReport.objects.filter(form_id=form_id).count())
self.assertEqual(3, StockTransaction.objects.filter(report__form_id=form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for state in ledger_values:
self.assertEqual(100, int(state.stock_on_hand))
def _update_case(domain, case_id, server_modified_on, last_visit_date=None):
accessors = CaseAccessors(domain)
case = accessors.get_case(case_id)
case.server_modified_on = server_modified_on
if last_visit_date:
set_case_property_directly(case, 'last_visit_date', last_visit_date.strftime('%Y-%m-%d'))
if should_use_sql_backend(domain):
CaseAccessorSQL.save_case(case)
else:
# can't call case.save() since it overrides the server_modified_on property
CommCareCase.get_db().save_doc(case.to_json())
def get_domain_form_change_provider(domains):
sql_domains = {domain for domain in domains if should_use_sql_backend(domain)}
couch_domains = set(domains) - sql_domains
return CompositeChangeProvider([
SqlDomainXFormChangeProvider(sql_domains),
CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=couch_domains,
doc_types=all_known_formlike_doc_types()
),
])
def _assert_transactions(self, values, ignore_ordering=False):
if should_use_sql_backend(DOMAIN):
txs = LedgerAccessorSQL.get_ledger_transactions_for_case(self.case.case_id)
self.assertEqual(len(values), len(txs))
if ignore_ordering:
values = sorted(values, key=lambda v: (v.type, v.product_id))
txs = sorted(txs, key=lambda t: (t.type, t.entry_id))
for expected, tx in zip(values, txs):
self.assertEqual(expected.type, tx.type)
self.assertEqual(expected.product_id, tx.entry_id)
self.assertEqual('stock', tx.section_id)
self.assertEqual(expected.delta, tx.delta)
self.assertEqual(expected.updated_balance, tx.updated_balance)
def test_ledger_pillow_sql(self):
factory = CaseFactory(domain=self.domain)
case = factory.create_case()
consumer = get_test_kafka_consumer(topics.LEDGER)
# have to get the seq id before the change is processed
kafka_seq = consumer.offsets()['fetch'][(topics.LEDGER, 0)]
from corehq.apps.commtrack.tests import get_single_balance_block
from corehq.apps.hqcase.utils import submit_case_blocks
submit_case_blocks([
get_single_balance_block(case.case_id, self.product_id, 100)],
self.domain
)
ref = UniqueLedgerReference(case.case_id, 'stock', self.product_id)
# confirm change made it to kafka
message = consumer.next()
change_meta = change_meta_from_kafka_message(message.value)
if should_use_sql_backend(self.domain):
self.assertEqual(ref.as_id(), change_meta.document_id)
else:
from corehq.apps.commtrack.models import StockState
state = StockState.objects.all()
self.assertEqual(1, len(state))
self.assertEqual(state[0].pk, change_meta.document_id) #
self.assertEqual(self.domain, change_meta.domain)
# send to elasticsearch
self.pillow.process_changes(since=kafka_seq, forever=False)
self.elasticsearch.indices.refresh(LEDGER_INDEX_INFO.index)
# confirm change made it to elasticserach
results = self.elasticsearch.search(
LEDGER_INDEX_INFO.index,
LEDGER_INDEX_INFO.type, body={
"query": {
"bool": {
"must": [{
"match_all": {}
}]
}
}
}
)
self.assertEqual(1, results['hits']['total'])
ledger_doc = results['hits']['hits'][0]['_source']
self.assertEqual(self.domain, ledger_doc['domain'])
self.assertEqual(ref.case_id, ledger_doc['case_id'])
self.assertEqual(ref.section_id, ledger_doc['section_id'])
self.assertEqual(ref.entry_id, ledger_doc['entry_id'])
def delete_all_ledgers(domain):
if should_use_sql_backend(domain):
for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain):
transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id)
form_ids = {tx.form_id for tx in transactions}
for form_id in form_ids:
LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id)
LedgerAccessorSQL.delete_ledger_values(case_id)
else:
from casexml.apps.stock.models import StockReport
from casexml.apps.stock.models import StockTransaction
stock_report_ids = StockReport.objects.filter(domain=domain).values_list('id', flat=True)
StockReport.objects.filter(domain=domain).delete()
StockTransaction.objects.filter(report_id__in=stock_report_ids).delete()
def _check_initial_state(case):
self.assertTrue(case.closed)
self.assertEqual(closed_by, case.closed_by)
self.assertEqual(closed_on, case.closed_on)
self.assertEqual(case.get_case_property('p1'), 'p1-1') # original
self.assertEqual(case.get_case_property('p2'), 'p2-2') # updated in second post
self.assertEqual(case.get_case_property('p3'), 'p3-2') # new in second post
self.assertEqual(case.get_case_property('p4'), 'p4-3') # updated in third post
self.assertEqual(case.get_case_property('p5'), 'p5-3') # new in third post
if should_use_sql_backend(REBUILD_TEST_DOMAIN):
# SQL stores one transaction per form
self.assertEqual(3, len(primary_actions(case))) # create + update + close
else:
self.assertEqual(5, len(primary_actions(case))) # create + 3 updates + close
def get_case_properties_for_case_type(domain, case_type):
if should_use_sql_backend(domain):
from corehq.apps.export.models import CaseExportDataSchema
from corehq.apps.export.const import MAIN_TABLE
schema = CaseExportDataSchema.generate_schema_from_builds(
domain,
case_type,
)
group_schemas = [gs for gs in schema.group_schemas if gs.path == MAIN_TABLE]
if group_schemas:
return sorted(set([item.path[0] for item in group_schemas[0].items]))
else:
from corehq.apps.hqcase.dbaccessors import get_case_properties
return get_case_properties(domain, case_type)
def _with_case(domain, case_type, last_modified):
with drop_connected_signals(case_post_save):
case = CaseFactory(domain).create_case(case_type=case_type)
_update_case(domain, case.case_id, last_modified)
accessors = CaseAccessors(domain)
case = accessors.get_case(case.case_id)
try:
yield case
finally:
if should_use_sql_backend(domain):
CaseAccessorSQL.hard_delete_cases(domain, [case.case_id])
else:
case.delete()
def run_case_update_rules_for_domain(domain, now=None):
now = now or datetime.utcnow()
run_record = DomainCaseRuleRun.objects.create(
domain=domain,
started_on=datetime.utcnow(),
status=DomainCaseRuleRun.STATUS_RUNNING,
)
if should_use_sql_backend(domain):
for db in get_db_aliases_for_partitioned_query():
run_case_update_rules_for_domain_and_db.delay(domain, now, run_record.pk, db=db)
else:
# explicitly pass db=None so that the serial task decorator has access to db in the key generation
run_case_update_rules_for_domain_and_db.delay(domain, now, run_record.pk, db=None)
def _get_daily_consumption_for_ledger(ledger):
from corehq.apps.commtrack.consumption import get_consumption_for_ledger_json
daily_consumption = get_consumption_for_ledger_json(ledger)
if should_use_sql_backend(ledger['domain']):
from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL
ledger_value = LedgerAccessorSQL.get_ledger_value(
ledger['case_id'], ledger['section_id'], ledger['entry_id']
)
ledger_value.daily_consumption = daily_consumption
LedgerAccessorSQL.save_ledger_values([ledger_value])
else:
from corehq.apps.commtrack.models import StockState
StockState.objects.filter(pk=ledger['_id']).update(daily_consumption=daily_consumption)
return daily_consumption
def _get_case_properties(doc_dict):
domain = doc_dict.get('domain')
assert domain
base_case_properties = [
{'key': base_case_property.key, 'value': base_case_property.value_getter(doc_dict)}
for base_case_property in list(SPECIAL_CASE_PROPERTIES_MAP.values())
]
if should_use_sql_backend(domain):
dynamic_case_properties = OrderedDict(doc_dict['case_json'])
else:
dynamic_case_properties = CommCareCase.wrap(doc_dict).dynamic_case_properties()
dynamic_mapping = [{'key': key, VALUE: value} for key, value in six.iteritems(dynamic_case_properties)]
return base_case_properties + dynamic_mapping
def _assert_initial_state():
if should_use_sql_backend(self.domain.name):
self.assertEqual(3, LedgerTransaction.objects.filter(form_id=second_form_id).count())
else:
self.assertEqual(1, StockReport.objects.filter(form_id=second_form_id).count())
# 6 = 3 stockonhand and 3 inferred consumption txns
self.assertEqual(6, StockTransaction.objects.filter(report__form_id=second_form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for lv in ledger_values:
self.assertEqual(50, lv.stock_on_hand)
self.assertEqual(
round(float(lv.daily_consumption), 2),
1.67
)
def _get_case_properties(doc_dict):
domain = doc_dict.get('domain')
assert domain
base_case_properties = [
{'key': 'name', 'value': doc_dict.get('name')},
{'key': 'external_id', 'value': doc_dict.get('external_id')}
]
if should_use_sql_backend(domain):
dynamic_case_properties = OrderedDict(doc_dict['case_json'])
else:
dynamic_case_properties = CommCareCase.wrap(doc_dict).dynamic_case_properties()
return base_case_properties + [
{'key': key, 'value': value}
for key, value in dynamic_case_properties.iteritems()
]
def handle(self, domain, doc_type, **options):
if not should_use_sql_backend(domain):
raise CommandError('Only SQL backends currently supported')
handlers = {
'xforminstance': _compare_xforms,
'commcarecase': _compare_cases,
'commcareuser': _compare_mobile_users,
}
try:
diff = handlers[doc_type.lower()](domain, doc_type)
except KeyError:
raise CommandError('Unsupported doc type. Use on of: {}'.format(', '.join(handlers)))
self.stdout.write('{} "{}" docs are missing from ES'.format(len(diff), doc_type))
for doc_id in diff:
self.stdout.write(doc_id)
def check_product_stock(self, case, product_id, expected_soh, expected_qty, section_id='stock'):
if not isinstance(expected_qty, Decimal):
expected_qty = Decimal(str(expected_qty))
if not isinstance(expected_soh, Decimal):
expected_soh = Decimal(str(expected_soh))
latest_trans = LedgerAccessors(self.domain.name).get_latest_transaction(
case.case_id, section_id, product_id
)
self.assertIsNotNone(latest_trans)
self.assertEqual(section_id, latest_trans.section_id)
self.assertEqual(expected_soh, latest_trans.stock_on_hand)
if should_use_sql_backend(self.domain.name):
if latest_trans.type == LedgerTransaction.TYPE_TRANSFER:
self.assertEqual(int(expected_qty), latest_trans.delta)
else:
self.assertEqual(expected_qty, latest_trans.quantity)
def test_balance_consumption(self):
initial = float(100)
initial_amounts = [(p._id, initial) for p in self.products]
self.submit_xml_form(balance_submission(initial_amounts))
final_amounts = [(p._id, float(50 - 10*i)) for i, p in enumerate(self.products)]
self.submit_xml_form(balance_submission(final_amounts))
for product, amt in final_amounts:
self.check_product_stock(self.sp, product, amt, 0)
inferred = amt - initial
if should_use_sql_backend(self.domain):
sql_txn = LedgerAccessors(self.domain.name).get_latest_transaction(
self.sp.case_id, 'stock', product
)
self.assertEqual(inferred, sql_txn.delta)
else:
inferred_txn = StockTransaction.objects.get(
case_id=self.sp.case_id, product_id=product, subtype=stockconst.TRANSACTION_SUBTYPE_INFERRED
)
self.assertEqual(Decimal(str(inferred)), inferred_txn.quantity)
self.assertEqual(Decimal(str(amt)), inferred_txn.stock_on_hand)
self.assertEqual(stockconst.TRANSACTION_TYPE_CONSUMPTION, inferred_txn.type)
def build(self):
limit_to_db = self.options.pop('limit_to_db', None)
domain = self.options.pop('domain')
if not case_search_enabled_for_domain(domain):
raise CaseSearchNotEnabledException(
"{} does not have case search enabled".format(domain))
assert should_use_sql_backend(
domain), '{} can only be used with SQL domains'.format(self.slug)
iteration_key = "CaseSearchResumableToElasticsearchPillow_{}_reindexer_{}_{}".format(
CASE_SEARCH_INDEX_INFO.index, limit_to_db or 'all', domain
or 'all')
limit_db_aliases = [limit_to_db] if limit_to_db else None
accessor = CaseReindexAccessor(domain=domain,
limit_db_aliases=limit_db_aliases)
doc_provider = SqlDocumentProvider(iteration_key, accessor)
return ResumableBulkElasticPillowReindexer(
doc_provider,
elasticsearch=get_es_new(),
index_info=CASE_SEARCH_INDEX_INFO,
doc_transform=transform_case_for_elasticsearch,
**self.options)
def _assert_initial_state():
if should_use_sql_backend(self.domain):
self.assertEqual(
3,
len(
self._get_all_ledger_transactions(
Q(form_id=second_form_id))))
else:
self.assertEqual(
1,
StockReport.objects.filter(form_id=second_form_id).count())
# 6 = 3 stockonhand and 3 inferred consumption txns
self.assertEqual(
6,
StockTransaction.objects.filter(
report__form_id=second_form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(
self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for lv in ledger_values:
self.assertEqual(50, lv.stock_on_hand)
self.assertEqual(round(float(lv.daily_consumption), 2), 1.67)
def test_archive_only_form(self):
# check no data in stock states
ledger_accessors = LedgerAccessors(self.domain.name)
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(0, len(ledger_values))
initial_amounts = [(p._id, float(100)) for p in self.products]
form_id = self.submit_xml_form(balance_submission(initial_amounts))
# check that we made stuff
def _assert_initial_state():
if should_use_sql_backend(self.domain):
self.assertEqual(3, LedgerTransaction.objects.filter(form_id=form_id).count())
else:
self.assertEqual(1, StockReport.objects.filter(form_id=form_id).count())
self.assertEqual(3, StockTransaction.objects.filter(report__form_id=form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for state in ledger_values:
self.assertEqual(100, int(state.stock_on_hand))
_assert_initial_state()
# archive and confirm commtrack data is cleared
form = FormAccessors(self.domain.name).get_form(form_id)
form.archive()
self.assertEqual(0, len(ledger_accessors.get_ledger_values_for_case(self.sp.case_id)))
if should_use_sql_backend(self.domain):
self.assertEqual(0, LedgerTransaction.objects.filter(form_id=form_id).count())
else:
self.assertEqual(0, StockReport.objects.filter(form_id=form_id).count())
self.assertEqual(0, StockTransaction.objects.filter(report__form_id=form_id).count())
# unarchive and confirm commtrack data is restored
form.unarchive()
_assert_initial_state()
def test_archive_last_form(self):
initial_amounts = [(p._id, float(100)) for p in self.products]
self.submit_xml_form(balance_submission(initial_amounts),
timestamp=datetime.utcnow() + timedelta(-30))
final_amounts = [(p._id, float(50))
for i, p in enumerate(self.products)]
second_form_id = self.submit_xml_form(
balance_submission(final_amounts))
ledger_accessors = LedgerAccessors(self.domain.name)
def _assert_initial_state():
if should_use_sql_backend(self.domain):
self.assertEqual(
3,
LedgerTransaction.objects.filter(
form_id=second_form_id).count())
else:
self.assertEqual(
1,
StockReport.objects.filter(form_id=second_form_id).count())
# 6 = 3 stockonhand and 3 inferred consumption txns
self.assertEqual(
6,
StockTransaction.objects.filter(
report__form_id=second_form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(
self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for lv in ledger_values:
self.assertEqual(50, lv.stock_on_hand)
self.assertEqual(round(float(lv.daily_consumption), 2), 1.67)
# check initial setup
_assert_initial_state()
# archive and confirm commtrack data is deleted
form = FormAccessors(self.domain.name).get_form(second_form_id)
with process_kafka_changes('LedgerToElasticsearchPillow'):
form.archive()
if should_use_sql_backend(self.domain):
self.assertEqual(
0,
LedgerTransaction.objects.filter(
form_id=second_form_id).count())
else:
self.assertEqual(
0,
StockReport.objects.filter(form_id=second_form_id).count())
self.assertEqual(
0,
StockTransaction.objects.filter(
report__form_id=second_form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(
self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for state in ledger_values:
# balance should be reverted to 100 in the StockState
self.assertEqual(100, int(state.stock_on_hand))
# consumption should be none since there will only be 1 data point
self.assertIsNone(state.daily_consumption)
# unarchive and confirm commtrack data is restored
with process_kafka_changes('LedgerToElasticsearchPillow'):
form.unarchive()
_assert_initial_state()
def _assert_no_stock_transactions():
if should_use_sql_backend(self.domain):
self.assertEqual(0, LedgerTransaction.objects.count())
else:
self.assertEqual(0, StockTransaction.objects.count())
expected_soh,
expected_qty,
section_id='stock'):
if not isinstance(expected_qty, Decimal):
expected_qty = Decimal(str(expected_qty))
if not isinstance(expected_soh, Decimal):
expected_soh = Decimal(str(expected_soh))
latest_trans = LedgerAccessors(
self.domain.name).get_latest_transaction(case.case_id, section_id,
product_id)
self.assertIsNotNone(latest_trans)
self.assertEqual(section_id, latest_trans.section_id)
self.assertEqual(expected_soh, latest_trans.stock_on_hand)
if should_use_sql_backend(self.domain):
if latest_trans.type == LedgerTransaction.TYPE_TRANSFER:
self.assertEqual(int(expected_qty), latest_trans.delta)
else:
self.assertEqual(expected_qty, latest_trans.quantity)
class CommTrackBalanceTransferTest(CommTrackSubmissionTest):
@run_with_all_backends
def test_balance_submit(self):
amounts = [(p._id, float(i * 10)) for i, p in enumerate(self.products)]
self.submit_xml_form(balance_submission(amounts))
for product, amt in amounts:
self.check_product_stock(self.sp, product, amt, 0)
@run_with_all_backends
def test_form_archiving(self):
now = datetime.utcnow()
# make sure we timestamp everything so they have the right order
case_id = _post_util(create=True,
p1='p1-1',
p2='p2-1',
form_extras={'received_on': now})
_post_util(case_id=case_id,
p2='p2-2',
p3='p3-2',
p4='p4-2',
form_extras={'received_on': now + timedelta(seconds=1)})
_post_util(case_id=case_id,
p4='p4-3',
p5='p5-3',
close=True,
form_extras={'received_on': now + timedelta(seconds=2)})
case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN)
case = case_accessors.get_case(case_id)
closed_by = case.closed_by
closed_on = case.closed_on
self.assertNotEqual('', closed_by)
self.assertNotEqual(None, closed_on)
def _check_initial_state(case):
self.assertTrue(case.closed)
self.assertEqual(closed_by, case.closed_by)
self.assertEqual(closed_on, case.closed_on)
self.assertEqual(case.get_case_property('p1'), 'p1-1') # original
self.assertEqual(case.get_case_property('p2'),
'p2-2') # updated in second post
self.assertEqual(case.get_case_property('p3'),
'p3-2') # new in second post
self.assertEqual(case.get_case_property('p4'),
'p4-3') # updated in third post
self.assertEqual(case.get_case_property('p5'),
'p5-3') # new in third post
if should_use_sql_backend(REBUILD_TEST_DOMAIN):
# SQL stores one transaction per form
self.assertEqual(3, len(
primary_actions(case))) # create + update + close
else:
self.assertEqual(5, len(
primary_actions(case))) # create + 3 updates + close
_check_initial_state(case)
# verify xform/action states
[f1, f2, f3] = case.xform_ids
if should_use_sql_backend(REBUILD_TEST_DOMAIN):
[create, update, close] = case.actions
self.assertEqual(f1, create.form_id)
self.assertEqual(f2, update.form_id)
self.assertEqual(f3, close.form_id)
else:
[create, u1, u2, u3, close] = case.actions
self.assertEqual(f1, create.form_id)
self.assertEqual(f1, u1.form_id)
self.assertEqual(f2, u2.form_id)
self.assertEqual(f3, u3.form_id)
# todo: should this be the behavior for archiving the create form?
form_acessors = FormAccessors(REBUILD_TEST_DOMAIN)
f1_doc = form_acessors.get_form(f1)
with capture_kafka_changes_context(topics.CASE_SQL) as change_context:
f1_doc.archive()
if should_use_sql_backend(case.domain):
self.assertEqual([case.case_id],
[change.id for change in change_context.changes])
class CaseRebuildTest(TestCase, CaseRebuildTestMixin):
@classmethod
def setUpClass(cls):
super(CaseRebuildTest, cls).setUpClass()
delete_all_cases()
def test_rebuild_empty(self):
self.assertEqual(
None,
rebuild_case_from_forms('anydomain', 'notarealid', RebuildWithReason(reason='test'))
)
def test_archiving_only_form(self):
"""
Checks that archiving the only form associated with the case archives
the case and unarchiving unarchives it.
"""
case_id = _post_util(create=True, p1='p1-1', p2='p2-1')
case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN)
case = case_accessors.get_case(case_id)
self.assertFalse(case.is_deleted)
if should_use_sql_backend(REBUILD_TEST_DOMAIN):
self.assertEqual(1, len(case.actions))
else:
self.assertEqual(2, len(case.actions))
[form_id] = case.xform_ids
form = FormAccessors(REBUILD_TEST_DOMAIN).get_form(form_id)
form.archive()
case = case_accessors.get_case(case_id)
self.assertTrue(case.is_deleted)
# should just have the 'rebuild' action
self.assertEqual(1, len(case.actions))
self.assertTrue(case.actions[0].is_case_rebuild)
form.unarchive()
case = case_accessors.get_case(case_id)
self.assertFalse(case.is_deleted)
self.assertEqual(3, len(case.actions))
self.assertTrue(case.actions[-1].is_case_rebuild)
def test_form_archiving(self):
now = datetime.utcnow()
# make sure we timestamp everything so they have the right order
case_id = _post_util(create=True, p1='p1-1', p2='p2-1',
form_extras={'received_on': now})
_post_util(case_id=case_id, p2='p2-2', p3='p3-2', p4='p4-2',
form_extras={'received_on': now + timedelta(seconds=1)})
_post_util(case_id=case_id, p4='p4-3', p5='p5-3', close=True,
form_extras={'received_on': now + timedelta(seconds=2)})
case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN)
case = case_accessors.get_case(case_id)
closed_by = case.closed_by
closed_on = case.closed_on
self.assertNotEqual('', closed_by)
self.assertNotEqual(None, closed_on)
def _check_initial_state(case):
self.assertTrue(case.closed)
self.assertEqual(closed_by, case.closed_by)
self.assertEqual(closed_on, case.closed_on)
self.assertEqual(case.get_case_property('p1'), 'p1-1') # original
self.assertEqual(case.get_case_property('p2'), 'p2-2') # updated in second post
self.assertEqual(case.get_case_property('p3'), 'p3-2') # new in second post
self.assertEqual(case.get_case_property('p4'), 'p4-3') # updated in third post
self.assertEqual(case.get_case_property('p5'), 'p5-3') # new in third post
if should_use_sql_backend(REBUILD_TEST_DOMAIN):
# SQL stores one transaction per form
self.assertEqual(3, len(primary_actions(case))) # create + update + close
else:
self.assertEqual(5, len(primary_actions(case))) # create + 3 updates + close
_check_initial_state(case)
# verify xform/action states
[f1, f2, f3] = case.xform_ids
if should_use_sql_backend(REBUILD_TEST_DOMAIN):
[create, update, close] = case.actions
self.assertEqual(f1, create.form_id)
self.assertEqual(f2, update.form_id)
self.assertEqual(f3, close.form_id)
else:
[create, u1, u2, u3, close] = case.actions
self.assertEqual(f1, create.form_id)
self.assertEqual(f1, u1.form_id)
self.assertEqual(f2, u2.form_id)
self.assertEqual(f3, u3.form_id)
# todo: should this be the behavior for archiving the create form?
form_acessors = FormAccessors(REBUILD_TEST_DOMAIN)
f1_doc = form_acessors.get_form(f1)
with capture_kafka_changes_context(topics.CASE_SQL) as change_context:
f1_doc.archive()
if should_use_sql_backend(case.domain):
self.assertEqual([case.case_id], [change.id for change in change_context.changes])
case = case_accessors.get_case(case_id)
if should_use_sql_backend(REBUILD_TEST_DOMAIN):
self.assertEqual(2, len(primary_actions(case)))
else:
self.assertEqual(3, len(primary_actions(case)))
[u2, u3] = case.xform_ids
self.assertEqual(f2, u2)
self.assertEqual(f3, u3)
self.assertTrue(case.closed) # no change
self.assertFalse('p1' in case.dynamic_case_properties()) # should disappear entirely
self.assertEqual(case.get_case_property('p2'), 'p2-2') # no change
self.assertEqual(case.get_case_property('p3'), 'p3-2') # no change
self.assertEqual(case.get_case_property('p4'), 'p4-3') # no change
self.assertEqual(case.get_case_property('p5'), 'p5-3') # no change
def _reset(form_id):
form_doc = form_acessors.get_form(form_id)
form_doc.unarchive()
case = case_accessors.get_case(case_id)
_check_initial_state(case)
_reset(f1)
f2_doc = form_acessors.get_form(f2)
f2_doc.archive()
case = case_accessors.get_case(case_id)
if should_use_sql_backend(REBUILD_TEST_DOMAIN):
self.assertEqual(2, len(primary_actions(case)))
else:
self.assertEqual(4, len(primary_actions(case)))
[u1, u3] = case.xform_ids
self.assertEqual(f1, u1)
self.assertEqual(f3, u3)
self.assertTrue(case.closed) # no change
self.assertEqual(case.get_case_property('p1'), 'p1-1') # original
self.assertEqual(case.get_case_property('p2'), 'p2-1') # loses second form update
self.assertFalse('p3' in case.dynamic_case_properties()) # should disappear entirely
self.assertEqual(case.get_case_property('p4'), 'p4-3') # no change
self.assertEqual(case.get_case_property('p5'), 'p5-3') # no change
_reset(f2)
f3_doc = form_acessors.get_form(f3)
f3_doc.archive()
case = case_accessors.get_case(case_id)
if should_use_sql_backend(REBUILD_TEST_DOMAIN):
self.assertEqual(2, len(primary_actions(case)))
else:
self.assertEqual(3, len(primary_actions(case)))
[u1, u2] = case.xform_ids
self.assertEqual(f1, u1)
self.assertEqual(f2, u2)
self.assertFalse(case.closed) # reopened!
self.assertEqual('', case.closed_by)
self.assertEqual(None, case.closed_on)
self.assertEqual(case.get_case_property('p1'), 'p1-1') # original
self.assertEqual(case.get_case_property('p2'), 'p2-2') # original
self.assertEqual(case.get_case_property('p3'), 'p3-2') # new in second post
self.assertEqual(case.get_case_property('p4'), 'p4-2') # loses third form update
self.assertFalse('p5' in case.dynamic_case_properties()) # should disappear entirely
_reset(f3)
def __init__(self, domain):
if not should_use_sql_backend(domain):
raise UnexpectedBackend(
"Only SQL backend supported: {}".format(domain))
self.domain = domain
self.ledger_accessors = LedgerAccessorSQL
def bug_report(req):
report = dict([(key, req.POST.get(key, '')) for key in (
'subject',
'username',
'domain',
'url',
'message',
'app_id',
'cc',
'email',
'500traceback',
'sentry_id',
)])
report['user_agent'] = req.META['HTTP_USER_AGENT']
report['datetime'] = datetime.utcnow()
try:
couch_user = req.couch_user
full_name = couch_user.full_name
if couch_user.is_commcare_user():
email = report['email']
else:
email = couch_user.get_email()
except Exception:
full_name = None
email = report['email']
report['full_name'] = full_name
report['email'] = email or report['username']
if report['domain']:
domain = report['domain']
elif len(couch_user.domains) == 1:
# This isn't a domain page, but the user has only one domain, so let's use that
domain = couch_user.domains[0]
else:
domain = "<no domain>"
message = (u"username: {username}\n"
u"full name: {full_name}\n"
u"domain: {domain}\n"
u"url: {url}\n"
u"datetime: {datetime}\n"
u"User Agent: {user_agent}\n").format(**report)
domain_object = Domain.get_by_name(domain) if report['domain'] else None
if domain_object:
current_project_description = domain_object.project_description if domain_object else None
new_project_description = req.POST.get('project_description')
if (domain_object and req.couch_user.is_domain_admin(domain=domain)
and new_project_description
and current_project_description != new_project_description):
domain_object.project_description = new_project_description
domain_object.save()
matching_subscriptions = Subscription.objects.filter(
is_active=True,
subscriber__domain=domain,
)
if len(matching_subscriptions) >= 1:
software_plan = matching_subscriptions[0].plan_version
else:
software_plan = u'domain has no active subscription'
message += ((
u"software plan: {software_plan}\n"
u"Is self start: {self_started}\n"
u"Feature Flags: {feature_flags}\n"
u"Feature Previews: {feature_previews}\n"
u"Is scale backend: {scale_backend}\n"
u"Has Support Hand-off Info: {has_handoff_info}\n"
u"Internal Project Information: {internal_info_link}\n"
u"Project description: {project_description}\n"
u"Sentry Error: {sentry_error}\n").format(
software_plan=software_plan,
self_started=domain_object.internal.self_started,
feature_flags=toggles.toggles_dict(username=report['username'],
domain=domain).keys(),
feature_previews=feature_previews.previews_dict(domain).keys(),
scale_backend=should_use_sql_backend(domain),
has_handoff_info=bool(domain_object.internal.partner_contact),
internal_info_link=reverse('domain_internal_settings',
args=[domain],
absolute=True),
project_description=domain_object.project_description,
sentry_error='{}{}'.format(
getattr(settings, 'SENTRY_QUERY_URL'),
report['sentry_id'])))
subject = u'{subject} ({domain})'.format(subject=report['subject'],
domain=domain)
cc = report['cc'].strip().split(",")
cc = filter(None, cc)
if full_name and not any([c in full_name for c in '<>"']):
reply_to = u'"{full_name}" <{email}>'.format(**report)
else:
reply_to = report['email']
# if the person looks like a commcare user, fogbugz can't reply
# to their email, so just use the default
if settings.HQ_ACCOUNT_ROOT in reply_to:
reply_to = settings.SERVER_EMAIL
message += u"Message:\n\n{message}\n".format(message=report['message'])
if req.POST.get('five-hundred-report'):
extra_message = ("This messge was reported from a 500 error page! "
"Please fix this ASAP (as if you wouldn't anyway)...")
traceback_info = cache.cache.get(report['500traceback'])
cache.cache.delete(report['500traceback'])
traceback_info = "Traceback of this 500: \n%s" % traceback_info
message = "%s \n\n %s \n\n %s" % (message, extra_message,
traceback_info)
email = EmailMessage(subject=subject,
body=message,
to=settings.BUG_REPORT_RECIPIENTS,
headers={'Reply-To': reply_to},
cc=cc)
uploaded_file = req.FILES.get('report_issue')
if uploaded_file:
filename = uploaded_file.name
content = uploaded_file.read()
email.attach(filename=filename, content=content)
# only fake the from email if it's an @dimagi.com account
if re.search('@dimagi\.com$', report['username']):
email.from_email = report['username']
else:
email.from_email = settings.CCHQ_BUG_REPORT_EMAIL
email.send(fail_silently=False)
if req.POST.get('five-hundred-report'):
messages.success(
req,
"Your CommCare HQ Issue Report has been sent. We are working quickly to resolve this problem."
)
return HttpResponseRedirect(reverse('homepage'))
return HttpResponse()
def _ledgers_per_case(self):
results = (LedgerES(es_instance_alias=ES_EXPORT_INSTANCE).domain(
self.domain).aggregation(
TermsAggregation('by_case', 'case_id',
size=100)).size(0).run())
ledgers_per_case = results.aggregations.by_case
case_ids = set()
ledger_counts = []
for case_id, ledger_count in ledgers_per_case.counts_by_bucket().items(
):
case_ids.add(case_id)
ledger_counts.append(ledger_count)
if not case_ids:
self.stdout.write("Domain has no ledgers")
return
avg_ledgers_per_case = sum(ledger_counts) // len(case_ids)
case_types_result = CaseES(es_instance_alias=ES_EXPORT_INSTANCE)\
.domain(self.domain).case_ids(case_ids)\
.aggregation(TermsAggregation('types', 'type'))\
.size(0).run()
case_types = case_types_result.aggregations.types.keys
self.stdout.write('\nCase Types with Ledgers')
for type_ in case_types:
self._print_value(
'case_type', type_,
CaseES().domain(self.domain).case_type(type_).count())
if should_use_sql_backend(self.domain):
db_name = get_db_aliases_for_partitioned_query()[
0] # just query one shard DB
results = (CommCareCaseSQL.objects.using(db_name).filter(
domain=self.domain, closed=True, type=type_).annotate(
lifespan=F('closed_on') - F('opened_on')).annotate(
avg_lifespan=Avg('lifespan')).values(
'avg_lifespan', flat=True))
self._print_value('Average lifespan for "%s" cases' % type_,
results[0]['avg_lifespan'])
self._cases_created_per_user_per_month(type_)
self._print_value('Average ledgers per case', avg_ledgers_per_case)
if should_use_sql_backend(self.domain):
stats = defaultdict(list)
for db_name, case_ids_p in split_list_by_db_partition(case_ids):
transactions_per_case_per_month = (
LedgerTransaction.objects.using(db_name).filter(
case_id__in=case_ids).annotate(
m=Month('server_date'),
y=Year('server_date')).values(
'case_id', 'y',
'm').annotate(count=Count('id')))
for row in transactions_per_case_per_month:
month = date(row['y'], row['m'], 1)
stats[month].append(row['count'])
else:
transactions_per_case_per_month = (StockTransaction.objects.filter(
case_id__in=case_ids).annotate(
m=Month('report__date'), y=Year('report__date')).values(
'case_id', 'y', 'm').annotate(count=Count('id')))
stats = defaultdict(list)
for row in transactions_per_case_per_month:
month = date(row['y'], row['m'], 1)
stats[month].append(row['count'])
final_stats = []
for month, transaction_count_list in sorted(list(stats.items()),
key=lambda r: r[0]):
final_stats.append(
(month.isoformat(),
sum(transaction_count_list) // len(transaction_count_list)))
self._print_table(['Month', 'Ledgers updated per case'], final_stats)
self.assertEqual(f2, u2.form_id)
self.assertEqual(f3, u3.form_id)
# todo: should this be the behavior for archiving the create form?
form_acessors = FormAccessors(REBUILD_TEST_DOMAIN)
f1_doc = form_acessors.get_form(f1)
with capture_kafka_changes_context(topics.CASE_SQL) as change_context:
f1_doc.archive()
if should_use_sql_backend(case.domain):
self.assertEqual([case.case_id],
[change.id for change in change_context.changes])
case = case_accessors.get_case(case_id)
if should_use_sql_backend(REBUILD_TEST_DOMAIN):
self.assertEqual(2, len(primary_actions(case)))
else:
self.assertEqual(3, len(primary_actions(case)))
[u2, u3] = case.xform_ids
self.assertEqual(f2, u2)
self.assertEqual(f3, u3)
self.assertTrue(case.closed) # no change
self.assertFalse(
'p1'
in case.dynamic_case_properties()) # should disappear entirely
self.assertEqual(case.get_case_property('p2'), 'p2-2') # no change
self.assertEqual(case.get_case_property('p3'), 'p3-2') # no change
self.assertEqual(case.get_case_property('p4'), 'p4-3') # no change
def reprocess_form(form, save=True, lock_form=True):
interface = FormProcessorInterface(form.domain)
lock = interface.acquire_lock_for_xform(
form.form_id) if lock_form else None
with LockManager(form, lock):
logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain)
# reset form state prior to processing
if should_use_sql_backend(form.domain):
form.state = XFormInstanceSQL.NORMAL
else:
form.doc_type = 'XFormInstance'
cache = interface.casedb_cache(domain=form.domain,
lock=True,
deleted_ok=True,
xforms=[form])
with cache as casedb:
try:
case_stock_result = SubmissionPost.process_xforms_for_cases(
[form], casedb)
except (IllegalCaseId, UsesReferrals, MissingProductId,
PhoneDateValueError, InvalidCaseIndex,
CaseValueError) as e:
error_message = '{}: {}'.format(
type(e).__name__, six.text_type(e))
form = interface.xformerror_from_xform_instance(
form, error_message)
return ReprocessingResult(form, [], [], error_message)
form.initial_processing_complete = True
form.problem = None
stock_result = case_stock_result.stock_result
assert stock_result.populated
cases = case_stock_result.case_models
_log_changes(cases, stock_result.models_to_save,
stock_result.models_to_delete)
ledgers = []
if should_use_sql_backend(form.domain):
cases_needing_rebuild = _get_case_ids_needing_rebuild(
form, cases)
ledgers = stock_result.models_to_save
ledgers_updated = {
ledger.ledger_reference
for ledger in ledgers if ledger.is_saved()
}
if save:
for case in cases:
CaseAccessorSQL.save_case(case)
LedgerAccessorSQL.save_ledger_values(ledgers)
FormAccessorSQL.update_form_problem_and_state(form)
FormProcessorSQL._publish_changes(
ProcessedForms(form, None), cases, stock_result)
# rebuild cases and ledgers that were affected
for case in cases:
if case.case_id in cases_needing_rebuild:
logger.info('Rebuilding case: %s', case.case_id)
if save:
# only rebuild cases that were updated
detail = FormReprocessRebuild(form_id=form.form_id)
interface.hard_rebuild_case(case.case_id,
detail,
lock=False)
for ledger in ledgers:
if ledger.ledger_reference in ledgers_updated:
logger.info('Rebuilding ledger: %s',
ledger.ledger_reference)
if save:
# only rebuild updated ledgers
interface.ledger_processor.rebuild_ledger_state(
**ledger.ledger_reference._asdict())
else:
def get_primary_db_case_ids(domain, doc_type, startdate, enddate):
if should_use_sql_backend(domain):
return get_sql_case_ids(domain, doc_type, startdate, enddate)
else:
# date filtering not supported for couch
return set(get_doc_ids_in_domain_by_type(domain, doc_type, CommCareCase.get_db()))
def test_reprocess_unfinished_submission_ledger_rebuild(self):
from corehq.apps.commtrack.tests.util import get_single_balance_block
case_id = uuid.uuid4().hex
form_ids = []
form_ids.append(
submit_case_blocks([
CaseBlock(case_id=case_id, create=True,
case_type='shop').as_string(),
get_single_balance_block(case_id, 'product1', 100),
], self.domain)[0].form_id)
with _patch_save_to_raise_error(self):
submit_case_blocks(
get_single_balance_block(case_id, 'product1', 50), self.domain)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
saved=False).all()
self.assertEqual(1, len(stubs))
form_ids.append(stubs[0].xform_id)
# submit another form afterwards
form_ids.append(
submit_case_blocks(
get_single_balance_block(case_id, 'product1', 25),
self.domain)[0].form_id)
ledgers = self.ledgerdb.get_ledger_values_for_case(case_id)
self.assertEqual(1, len(ledgers))
self.assertEqual(25, ledgers[0].balance)
ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(
case_id)
if should_use_sql_backend(self.domain):
self.assertEqual(2, len(ledger_transactions))
else:
# includes extra consumption transaction
self.assertEqual(3, len(ledger_transactions))
# should rebuild ledger transactions
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(1, len(result.cases))
self.assertEqual(1, len(result.ledgers))
ledgers = self.ledgerdb.get_ledger_values_for_case(case_id)
self.assertEqual(1, len(ledgers)) # still only 1
self.assertEqual(25, ledgers[0].balance)
ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(
case_id)
if should_use_sql_backend(self.domain):
self.assertEqual(3, len(ledger_transactions))
# make sure transactions are in correct order
self.assertEqual(form_ids,
[trans.form_id for trans in ledger_transactions])
self.assertEqual(100, ledger_transactions[0].updated_balance)
self.assertEqual(100, ledger_transactions[0].delta)
self.assertEqual(50, ledger_transactions[1].updated_balance)
self.assertEqual(-50, ledger_transactions[1].delta)
self.assertEqual(25, ledger_transactions[2].updated_balance)
self.assertEqual(-25, ledger_transactions[2].delta)
else:
self.assertEqual(3, len(ledger_transactions))
self.assertEqual(
form_ids,
[trans.report.form_id for trans in ledger_transactions])
self.assertEqual(100, ledger_transactions[0].stock_on_hand)
self.assertEqual(50, ledger_transactions[1].stock_on_hand)
self.assertEqual(25, ledger_transactions[2].stock_on_hand)
def get_primary_db_case_counts(domain):
if should_use_sql_backend(domain):
return _get_sql_cases_by_doc_type(domain)
else:
return _get_couch_cases_by_doc_type(domain)
def bug_report(req):
report = dict([(key, req.POST.get(key, '')) for key in (
'subject',
'username',
'domain',
'url',
'message',
'app_id',
'cc',
'email',
'500traceback',
)])
domain_object = Domain.get_by_name(report['domain'])
current_project_description = domain_object.project_description
new_project_description = req.POST.get('project_description')
if (req.couch_user.is_domain_admin(domain=report['domain'])
and new_project_description
and current_project_description != new_project_description):
domain_object.project_description = new_project_description
domain_object.save()
report['user_agent'] = req.META['HTTP_USER_AGENT']
report['datetime'] = datetime.utcnow()
report['feature_flags'] = toggles.toggles_dict(
username=report['username'], domain=report['domain']).keys()
report['feature_previews'] = feature_previews.previews_dict(
report['domain']).keys()
report['scale_backend'] = should_use_sql_backend(
report['domain']) if report['domain'] else False
report['project_description'] = domain_object.project_description
try:
couch_user = req.couch_user
full_name = couch_user.full_name
if couch_user.is_commcare_user():
email = report['email']
else:
email = couch_user.get_email()
except Exception:
full_name = None
email = report['email']
report['full_name'] = full_name
report['email'] = email or report['username']
matching_subscriptions = Subscription.objects.filter(
is_active=True,
subscriber__domain=report['domain'],
)
if len(matching_subscriptions) >= 1:
report['software_plan'] = matching_subscriptions[0].plan_version
else:
report['software_plan'] = u'domain has no active subscription'
subject = u'{subject} ({domain})'.format(**report)
message = (u"username: {username}\n"
u"full name: {full_name}\n"
u"domain: {domain}\n"
u"software plan: {software_plan}\n"
u"url: {url}\n"
u"datetime: {datetime}\n"
u"User Agent: {user_agent}\n"
u"Feature Flags: {feature_flags}\n"
u"Feature Previews: {feature_previews}\n"
u"Is scale backend: {scale_backend}\n"
u"Project description: {project_description}\n"
u"Message:\n\n"
u"{message}\n").format(**report)
cc = report['cc'].strip().split(",")
cc = filter(None, cc)
if full_name and not any([c in full_name for c in '<>"']):
reply_to = u'"{full_name}" <{email}>'.format(**report)
else:
reply_to = report['email']
# if the person looks like a commcare user, fogbugz can't reply
# to their email, so just use the default
if settings.HQ_ACCOUNT_ROOT in reply_to:
reply_to = settings.SERVER_EMAIL
if req.POST.get('five-hundred-report'):
extra_message = ("This messge was reported from a 500 error page! "
"Please fix this ASAP (as if you wouldn't anyway)...")
traceback_info = cache.cache.get(report['500traceback'])
cache.cache.delete(report['500traceback'])
traceback_info = "Traceback of this 500: \n%s" % traceback_info
message = "%s \n\n %s \n\n %s" % (message, extra_message,
traceback_info)
email = EmailMessage(subject=subject,
body=message,
to=settings.BUG_REPORT_RECIPIENTS,
headers={'Reply-To': reply_to},
cc=cc)
uploaded_file = req.FILES.get('report_issue')
if uploaded_file:
filename = uploaded_file.name
content = uploaded_file.read()
email.attach(filename=filename, content=content)
# only fake the from email if it's an @dimagi.com account
if re.search('@dimagi\.com$', report['username']):
email.from_email = report['username']
else:
email.from_email = settings.CCHQ_BUG_REPORT_EMAIL
email.send(fail_silently=False)
if req.POST.get('five-hundred-report'):
messages.success(
req,
"Your CommCare HQ Issue Report has been sent. We are working quickly to resolve this problem."
)
return HttpResponseRedirect(reverse('homepage'))
return HttpResponse()
def location_map_case_id(user):
if should_use_sql_backend(user.domain):
user_id = user.user_id
case_id = uuid.uuid5(const.MOBILE_WORKER_UUID_NS, user_id).hex
return case_id
return 'user-owner-mapping-' + user.user_id
def test_test_local_domain_sql_backend_override_overrides(self):
domain_name = uuid.uuid4().hex
create_domain(domain_name)
self.assertFalse(should_use_sql_backend(domain_name))
set_local_domain_sql_backend_override(domain_name)
self.assertTrue(should_use_sql_backend(domain_name))
def __init__(self, domain):
assert should_use_sql_backend(
domain), "Only SQL backend supported: {}".format(domain)
self.domain = domain
self.ledger_accessors = LedgerAccessorSQL
def iter_document_ids(self):
if should_use_sql_backend(self.domain):
accessor = LedgerReindexAccessor(self.domain)
return iter_all_ids(accessor)
else:
return iter(self._couch_iterator())
def _save_case(domain, case):
if should_use_sql_backend(domain):
CaseAccessorSQL.save_case(case)
else:
# can't call case.save() since it overrides the server_modified_on property
CommCareCase.get_db().save_doc(case.to_json())
from casexml.apps.case.mock import CaseFactory
from contextlib import contextmanager
from corehq.apps.data_interfaces.models import AutomaticUpdateRule
from corehq.form_processor.backends.sql.dbaccessors import CaseAccessorSQL
from corehq.form_processor.utils.general import should_use_sql_backend
@contextmanager
def create_case(domain, case_type, **kwargs):
case = CaseFactory(domain).create_case(case_type=case_type, **kwargs)
try:
yield case
finally:
if should_use_sql_backend(domain):
CaseAccessorSQL.hard_delete_cases(domain, [case.case_id])
else:
case.delete()
def create_empty_rule(domain, workflow):
return AutomaticUpdateRule.objects.create(
domain=domain,
name='test',
case_type='person',
active=True,
deleted=False,
filter_on_server_modified=False,
server_modified_boundary=None,
migrated=True,
workflow=workflow,
def __init__(self, domain):
assert should_use_sql_backend(domain), "Only SQL backend supported"
self.domain = domain
self.ledger_accessors = LedgerAccessorSQL
self.case_accessors = CaseAccessorSQL
def post(self, req, *args, **kwargs):
report = dict([(key, req.POST.get(key, '')) for key in (
'subject',
'username',
'domain',
'url',
'message',
'app_id',
'cc',
'email',
'500traceback',
'sentry_id',
)])
try:
couch_user = req.couch_user
full_name = couch_user.full_name
if couch_user.is_commcare_user():
email = report['email']
else:
email = couch_user.get_email()
except Exception:
full_name = None
email = report['email']
report['full_name'] = full_name
report['email'] = email or report['username']
if report['domain']:
domain = report['domain']
elif len(couch_user.domains) == 1:
# This isn't a domain page, but the user has only one domain, so let's use that
domain = couch_user.domains[0]
else:
domain = "<no domain>"
message = ("username: {username}\n"
"full name: {full_name}\n"
"domain: {domain}\n"
"url: {url}\n").format(**report)
domain_object = Domain.get_by_name(
domain) if report['domain'] else None
debug_context = {
'datetime':
datetime.utcnow(),
'self_started':
'<unknown>',
'scale_backend':
'<unknown>',
'has_handoff_info':
'<unknown>',
'project_description':
'<unknown>',
'sentry_error':
'{}{}'.format(getattr(settings, 'SENTRY_QUERY_URL', ''),
report['sentry_id'])
}
if domain_object:
current_project_description = domain_object.project_description if domain_object else None
new_project_description = req.POST.get('project_description')
if (domain_object and req.couch_user.is_domain_admin(domain=domain)
and new_project_description and
current_project_description != new_project_description):
domain_object.project_description = new_project_description
domain_object.save()
message += (("software plan: {software_plan}\n").format(
software_plan=Subscription.get_subscribed_plan_by_domain(
domain), ))
debug_context.update({
'self_started':
domain_object.internal.self_started,
'scale_backend':
should_use_sql_backend(domain),
'has_handoff_info':
bool(domain_object.internal.partner_contact),
'project_description':
domain_object.project_description,
})
subject = '{subject} ({domain})'.format(subject=report['subject'],
domain=domain)
cc = [el for el in report['cc'].strip().split(",") if el]
if full_name and not any([c in full_name for c in '<>"']):
reply_to = '"{full_name}" <{email}>'.format(**report)
else:
reply_to = report['email']
# if the person looks like a commcare user, fogbugz can't reply
# to their email, so just use the default
if settings.HQ_ACCOUNT_ROOT in reply_to:
reply_to = settings.SERVER_EMAIL
message += "Message:\n\n{message}\n".format(message=report['message'])
if req.POST.get('five-hundred-report'):
extra_message = (
"This message was reported from a 500 error page! "
"Please fix this ASAP (as if you wouldn't anyway)...")
extra_debug_info = (
"datetime: {datetime}\n"
"Is self start: {self_started}\n"
"Is scale backend: {scale_backend}\n"
"Has Support Hand-off Info: {has_handoff_info}\n"
"Project description: {project_description}\n"
"Sentry Error: {sentry_error}\n").format(**debug_context)
traceback_info = cache.cache.get(
report['500traceback']) or 'No traceback info available'
cache.cache.delete(report['500traceback'])
message = "\n\n".join(
[message, extra_debug_info, extra_message, traceback_info])
email = EmailMessage(subject=subject,
body=message,
to=self.recipients,
headers={'Reply-To': reply_to},
cc=cc)
uploaded_file = req.FILES.get('report_issue')
if uploaded_file:
filename = uploaded_file.name
content = uploaded_file.read()
email.attach(filename=filename, content=content)
# only fake the from email if it's an @dimagi.com account
is_icds_env = settings.SERVER_ENVIRONMENT in settings.ICDS_ENVS
if re.search(r'@dimagi\.com$', report['username']) and not is_icds_env:
email.from_email = report['username']
else:
email.from_email = settings.CCHQ_BUG_REPORT_EMAIL
email.send(fail_silently=False)
if req.POST.get('five-hundred-report'):
messages.success(
req,
"Your CommCare HQ Issue Report has been sent. We are working quickly to resolve this problem."
)
return HttpResponseRedirect(reverse('homepage'))
return HttpResponse()
def get_primary_db_form_counts(domain, startdate=None, enddate=None):
if should_use_sql_backend(domain):
return _get_sql_forms_by_doc_type(domain, startdate, enddate)
else:
return _get_couch_forms_by_doc_type(domain)
