def test_test_local_domain_sql_backend_override_overrides(self): domain_name = uuid.uuid4().hex create_domain(domain_name) self.assertFalse(should_use_sql_backend(domain_name)) set_local_domain_sql_backend_override(domain_name) self.assertTrue(should_use_sql_backend(domain_name)) clear_local_domain_sql_backend_override(domain_name) self.assertFalse(should_use_sql_backend(domain_name))
def location_map_case_id(user): if should_use_sql_backend(user.domain): user_id = user.user_id if isinstance(user_id, unicode): user_id = user_id.encode('utf8') return uuid.uuid5(const.MOBILE_WORKER_UUID_NS, user_id).hex return 'user-owner-mapping-' + user.user_id
def test_fire_signals(self): from corehq.apps.receiverwrapper.tests.test_submit_errors import failing_signal_handler case_id = uuid.uuid4().hex form_id = uuid.uuid4().hex with failing_signal_handler('signal death'): submit_case_blocks( CaseBlock(case_id=case_id, create=True, case_type='box').as_string().decode('utf-8'), self.domain, form_id=form_id ) form = self.formdb.get_form(form_id) with catch_signal(successful_form_received) as form_handler, catch_signal(case_post_save) as case_handler: submit_form_locally( instance=form.get_xml(), domain=self.domain, ) case = self.casedb.get_case(case_id) if should_use_sql_backend(self.domain): self.assertEqual(form, form_handler.call_args[1]['xform']) self.assertEqual(case, case_handler.call_args[1]['case']) else: signal_form = form_handler.call_args[1]['xform'] self.assertEqual(form.form_id, signal_form.form_id) self.assertEqual(form.get_rev, signal_form.get_rev) signal_case = case_handler.call_args[1]['case'] self.assertEqual(case.case_id, signal_case.case_id) self.assertEqual(case.get_rev, signal_case.get_rev)
def test_archiving_only_form(self): """ Checks that archiving the only form associated with the case archives the case and unarchiving unarchives it. """ case_id = _post_util(create=True, p1='p1-1', p2='p2-1') case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN) case = case_accessors.get_case(case_id) self.assertFalse(case.is_deleted) if should_use_sql_backend(REBUILD_TEST_DOMAIN): self.assertEqual(1, len(case.actions)) else: self.assertEqual(2, len(case.actions)) [form_id] = case.xform_ids form = FormAccessors(REBUILD_TEST_DOMAIN).get_form(form_id) form.archive() case = case_accessors.get_case(case_id) self.assertTrue(case.is_deleted) # should just have the 'rebuild' action self.assertEqual(1, len(case.actions)) self.assertTrue(case.actions[0].is_case_rebuild) form.unarchive() case = case_accessors.get_case(case_id) self.assertFalse(case.is_deleted) self.assertEqual(3, len(case.actions)) self.assertTrue(case.actions[-1].is_case_rebuild)
def test_simple_delete(self): factory = CaseFactory() case = factory.create_case() [case] = factory.create_or_update_case( CaseStructure(case_id=case.case_id, attrs={'update': {'foo': 'bar'}}) ) self.assertIsNotNone(self.casedb.get_case(case.case_id)) self.assertEqual(2, len(case.xform_ids)) for form_id in case.xform_ids: self.assertIsNotNone(self.formdb.get_form(form_id)) with capture_kafka_changes_context(topics.FORM_SQL, topics.CASE_SQL) as change_context: safe_hard_delete(case) if should_use_sql_backend(case.domain): self.assertEqual(3, len(change_context.changes)) expected_ids = {case.case_id} | set(case.xform_ids) self.assertEqual(expected_ids, {change.id for change in change_context.changes}) for change in change_context.changes: self.assertTrue(change.deleted) with self.assertRaises(CaseNotFound): self.casedb.get_case(case.case_id) for form_id in case.xform_ids: with self.assertRaises(XFormNotFound): self.formdb.get_form(form_id)
def test_ledger_pillow(self): factory = CaseFactory(domain=self.domain) case = factory.create_case() consumer = get_test_kafka_consumer(topics.LEDGER) # have to get the seq id before the change is processed kafka_seq = get_topic_offset(topics.LEDGER) from corehq.apps.commtrack.tests.util import get_single_balance_block from corehq.apps.hqcase.utils import submit_case_blocks submit_case_blocks([ get_single_balance_block(case.case_id, self.product_id, 100)], self.domain ) ref = UniqueLedgerReference(case.case_id, 'stock', self.product_id) # confirm change made it to kafka message = consumer.next() change_meta = change_meta_from_kafka_message(message.value) if should_use_sql_backend(self.domain): self.assertEqual(ref.as_id(), change_meta.document_id) else: from corehq.apps.commtrack.models import StockState state = StockState.objects.all() self.assertEqual(1, len(state)) self.assertEqual(state[0].pk, change_meta.document_id) # self.assertEqual(self.domain, change_meta.domain) # send to elasticsearch self.pillow.process_changes(since=kafka_seq, forever=False) self.elasticsearch.indices.refresh(LEDGER_INDEX_INFO.index) # confirm change made it to elasticserach self._assert_ledger_in_es(ref)
def _populate_results(self, case_id_list): if should_use_sql_backend(self.domain): base_results = [CaseAPIResult(domain=self.domain, couch_doc=case, id_only=self.ids_only) for case in self.case_accessors.iter_cases(case_id_list)] else: base_results = [CaseAPIResult(domain=self.domain, couch_doc=case, id_only=self.ids_only) for case in iter_cases(case_id_list, self.strip_history, self.wrap)] return base_results
def _create_models_for_stock_report_helper(self, form, stock_report_helper): processing_result = StockProcessingResult(form, stock_report_helpers=[stock_report_helper]) processing_result.populate_models() if should_use_sql_backend(self.domain_name): from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL LedgerAccessorSQL.save_ledger_values(processing_result.models_to_save) else: processing_result.commit()
def get_domain_case_change_provider(domains): change_providers = [] for domain in domains: if should_use_sql_backend(domain): change_providers.append(SqlDomainCaseChangeProvider(domain)) else: change_providers.append(get_couch_domain_case_change_provider(domain)) return CompositeChangeProvider(change_providers)
def _assert_case_revision(self, rev_number, last_modified, expect_modified=False): if should_use_sql_backend(self.domain): self.assertEqual( expect_modified, CaseAccessorSQL.case_modified_since(self.case_id, last_modified) ) else: doc = self._get_case() self.assertTrue(doc['_rev'].startswith('%s-' % rev_number))
def __init__(self, domain): from corehq.apps.commtrack.models import StockState assert not should_use_sql_backend(domain), "Only non-SQL backend supported" self.domain = domain def _doc_gen_fn(obj): return obj.to_json() super(LedgerV1DocumentStore, self).__init__(StockState, _doc_gen_fn)
def location_map_case_id(user): if should_use_sql_backend(user.domain): user_id = user.user_id if isinstance(user_id, six.text_type) and six.PY2: user_id = user_id.encode('utf8') case_id = uuid.uuid5(const.MOBILE_WORKER_UUID_NS, user_id).hex if six.PY2: case_id = case_id.decode('utf-8') return case_id return 'user-owner-mapping-' + user.user_id
def create_case(domain, case_type, **kwargs): case = CaseFactory(domain).create_case(case_type=case_type, **kwargs) try: yield case finally: if should_use_sql_backend(domain): CaseAccessorSQL.hard_delete_cases(domain, [case.case_id]) else: case.delete()
def test_archive_last_form(self): initial_amounts = [(p._id, float(100)) for p in self.products] self.submit_xml_form( balance_submission(initial_amounts), timestamp=datetime.utcnow() + timedelta(-30) ) final_amounts = [(p._id, float(50)) for i, p in enumerate(self.products)] second_form_id = self.submit_xml_form(balance_submission(final_amounts)) ledger_accessors = LedgerAccessors(self.domain.name) def _assert_initial_state(): if should_use_sql_backend(self.domain): self.assertEqual(3, len(self._get_all_ledger_transactions(Q(form_id=second_form_id)))) else: self.assertEqual(1, StockReport.objects.filter(form_id=second_form_id).count()) # 6 = 3 stockonhand and 3 inferred consumption txns self.assertEqual(6, StockTransaction.objects.filter(report__form_id=second_form_id).count()) ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id) self.assertEqual(3, len(ledger_values)) for lv in ledger_values: self.assertEqual(50, lv.stock_on_hand) self.assertEqual( round(float(lv.daily_consumption), 2), 1.67 ) # check initial setup _assert_initial_state() # archive and confirm commtrack data is deleted form = FormAccessors(self.domain.name).get_form(second_form_id) with process_pillow_changes('LedgerToElasticsearchPillow'): form.archive() if should_use_sql_backend(self.domain): self.assertEqual(0, len(self._get_all_ledger_transactions(Q(form_id=second_form_id)))) else: self.assertEqual(0, StockReport.objects.filter(form_id=second_form_id).count()) self.assertEqual(0, StockTransaction.objects.filter(report__form_id=second_form_id).count()) ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id) self.assertEqual(3, len(ledger_values)) for state in ledger_values: # balance should be reverted to 100 in the StockState self.assertEqual(100, int(state.stock_on_hand)) # consumption should be none since there will only be 1 data point self.assertIsNone(state.daily_consumption) # unarchive and confirm commtrack data is restored with process_pillow_changes('LedgerToElasticsearchPillow'): form.unarchive() _assert_initial_state()
def _assert_initial_state(): if should_use_sql_backend(self.domain.name): self.assertEqual(3, LedgerTransaction.objects.filter(form_id=form_id).count()) else: self.assertEqual(1, StockReport.objects.filter(form_id=form_id).count()) self.assertEqual(3, StockTransaction.objects.filter(report__form_id=form_id).count()) ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id) self.assertEqual(3, len(ledger_values)) for state in ledger_values: self.assertEqual(100, int(state.stock_on_hand))
def _update_case(domain, case_id, server_modified_on, last_visit_date=None): accessors = CaseAccessors(domain) case = accessors.get_case(case_id) case.server_modified_on = server_modified_on if last_visit_date: set_case_property_directly(case, 'last_visit_date', last_visit_date.strftime('%Y-%m-%d')) if should_use_sql_backend(domain): CaseAccessorSQL.save_case(case) else: # can't call case.save() since it overrides the server_modified_on property CommCareCase.get_db().save_doc(case.to_json())
def get_domain_form_change_provider(domains): sql_domains = {domain for domain in domains if should_use_sql_backend(domain)} couch_domains = set(domains) - sql_domains return CompositeChangeProvider([ SqlDomainXFormChangeProvider(sql_domains), CouchDomainDocTypeChangeProvider( couch_db=XFormInstance.get_db(), domains=couch_domains, doc_types=all_known_formlike_doc_types() ), ])
def _assert_transactions(self, values, ignore_ordering=False): if should_use_sql_backend(DOMAIN): txs = LedgerAccessorSQL.get_ledger_transactions_for_case(self.case.case_id) self.assertEqual(len(values), len(txs)) if ignore_ordering: values = sorted(values, key=lambda v: (v.type, v.product_id)) txs = sorted(txs, key=lambda t: (t.type, t.entry_id)) for expected, tx in zip(values, txs): self.assertEqual(expected.type, tx.type) self.assertEqual(expected.product_id, tx.entry_id) self.assertEqual('stock', tx.section_id) self.assertEqual(expected.delta, tx.delta) self.assertEqual(expected.updated_balance, tx.updated_balance)
def test_ledger_pillow_sql(self): factory = CaseFactory(domain=self.domain) case = factory.create_case() consumer = get_test_kafka_consumer(topics.LEDGER) # have to get the seq id before the change is processed kafka_seq = consumer.offsets()['fetch'][(topics.LEDGER, 0)] from corehq.apps.commtrack.tests import get_single_balance_block from corehq.apps.hqcase.utils import submit_case_blocks submit_case_blocks([ get_single_balance_block(case.case_id, self.product_id, 100)], self.domain ) ref = UniqueLedgerReference(case.case_id, 'stock', self.product_id) # confirm change made it to kafka message = consumer.next() change_meta = change_meta_from_kafka_message(message.value) if should_use_sql_backend(self.domain): self.assertEqual(ref.as_id(), change_meta.document_id) else: from corehq.apps.commtrack.models import StockState state = StockState.objects.all() self.assertEqual(1, len(state)) self.assertEqual(state[0].pk, change_meta.document_id) # self.assertEqual(self.domain, change_meta.domain) # send to elasticsearch self.pillow.process_changes(since=kafka_seq, forever=False) self.elasticsearch.indices.refresh(LEDGER_INDEX_INFO.index) # confirm change made it to elasticserach results = self.elasticsearch.search( LEDGER_INDEX_INFO.index, LEDGER_INDEX_INFO.type, body={ "query": { "bool": { "must": [{ "match_all": {} }] } } } ) self.assertEqual(1, results['hits']['total']) ledger_doc = results['hits']['hits'][0]['_source'] self.assertEqual(self.domain, ledger_doc['domain']) self.assertEqual(ref.case_id, ledger_doc['case_id']) self.assertEqual(ref.section_id, ledger_doc['section_id']) self.assertEqual(ref.entry_id, ledger_doc['entry_id'])
def delete_all_ledgers(domain): if should_use_sql_backend(domain): for case_id in CaseAccessorSQL.get_case_ids_in_domain(domain): transactions = LedgerAccessorSQL.get_ledger_transactions_for_case(case_id) form_ids = {tx.form_id for tx in transactions} for form_id in form_ids: LedgerAccessorSQL.delete_ledger_transactions_for_form([case_id], form_id) LedgerAccessorSQL.delete_ledger_values(case_id) else: from casexml.apps.stock.models import StockReport from casexml.apps.stock.models import StockTransaction stock_report_ids = StockReport.objects.filter(domain=domain).values_list('id', flat=True) StockReport.objects.filter(domain=domain).delete() StockTransaction.objects.filter(report_id__in=stock_report_ids).delete()
def _check_initial_state(case): self.assertTrue(case.closed) self.assertEqual(closed_by, case.closed_by) self.assertEqual(closed_on, case.closed_on) self.assertEqual(case.get_case_property('p1'), 'p1-1') # original self.assertEqual(case.get_case_property('p2'), 'p2-2') # updated in second post self.assertEqual(case.get_case_property('p3'), 'p3-2') # new in second post self.assertEqual(case.get_case_property('p4'), 'p4-3') # updated in third post self.assertEqual(case.get_case_property('p5'), 'p5-3') # new in third post if should_use_sql_backend(REBUILD_TEST_DOMAIN): # SQL stores one transaction per form self.assertEqual(3, len(primary_actions(case))) # create + update + close else: self.assertEqual(5, len(primary_actions(case))) # create + 3 updates + close
def get_case_properties_for_case_type(domain, case_type): if should_use_sql_backend(domain): from corehq.apps.export.models import CaseExportDataSchema from corehq.apps.export.const import MAIN_TABLE schema = CaseExportDataSchema.generate_schema_from_builds( domain, case_type, ) group_schemas = [gs for gs in schema.group_schemas if gs.path == MAIN_TABLE] if group_schemas: return sorted(set([item.path[0] for item in group_schemas[0].items])) else: from corehq.apps.hqcase.dbaccessors import get_case_properties return get_case_properties(domain, case_type)
def _with_case(domain, case_type, last_modified): with drop_connected_signals(case_post_save): case = CaseFactory(domain).create_case(case_type=case_type) _update_case(domain, case.case_id, last_modified) accessors = CaseAccessors(domain) case = accessors.get_case(case.case_id) try: yield case finally: if should_use_sql_backend(domain): CaseAccessorSQL.hard_delete_cases(domain, [case.case_id]) else: case.delete()
def run_case_update_rules_for_domain(domain, now=None): now = now or datetime.utcnow() run_record = DomainCaseRuleRun.objects.create( domain=domain, started_on=datetime.utcnow(), status=DomainCaseRuleRun.STATUS_RUNNING, ) if should_use_sql_backend(domain): for db in get_db_aliases_for_partitioned_query(): run_case_update_rules_for_domain_and_db.delay(domain, now, run_record.pk, db=db) else: # explicitly pass db=None so that the serial task decorator has access to db in the key generation run_case_update_rules_for_domain_and_db.delay(domain, now, run_record.pk, db=None)
def _get_daily_consumption_for_ledger(ledger): from corehq.apps.commtrack.consumption import get_consumption_for_ledger_json daily_consumption = get_consumption_for_ledger_json(ledger) if should_use_sql_backend(ledger['domain']): from corehq.form_processor.backends.sql.dbaccessors import LedgerAccessorSQL ledger_value = LedgerAccessorSQL.get_ledger_value( ledger['case_id'], ledger['section_id'], ledger['entry_id'] ) ledger_value.daily_consumption = daily_consumption LedgerAccessorSQL.save_ledger_values([ledger_value]) else: from corehq.apps.commtrack.models import StockState StockState.objects.filter(pk=ledger['_id']).update(daily_consumption=daily_consumption) return daily_consumption
def _get_case_properties(doc_dict): domain = doc_dict.get('domain') assert domain base_case_properties = [ {'key': base_case_property.key, 'value': base_case_property.value_getter(doc_dict)} for base_case_property in list(SPECIAL_CASE_PROPERTIES_MAP.values()) ] if should_use_sql_backend(domain): dynamic_case_properties = OrderedDict(doc_dict['case_json']) else: dynamic_case_properties = CommCareCase.wrap(doc_dict).dynamic_case_properties() dynamic_mapping = [{'key': key, VALUE: value} for key, value in six.iteritems(dynamic_case_properties)] return base_case_properties + dynamic_mapping
def _assert_initial_state(): if should_use_sql_backend(self.domain.name): self.assertEqual(3, LedgerTransaction.objects.filter(form_id=second_form_id).count()) else: self.assertEqual(1, StockReport.objects.filter(form_id=second_form_id).count()) # 6 = 3 stockonhand and 3 inferred consumption txns self.assertEqual(6, StockTransaction.objects.filter(report__form_id=second_form_id).count()) ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id) self.assertEqual(3, len(ledger_values)) for lv in ledger_values: self.assertEqual(50, lv.stock_on_hand) self.assertEqual( round(float(lv.daily_consumption), 2), 1.67 )
def _get_case_properties(doc_dict): domain = doc_dict.get('domain') assert domain base_case_properties = [ {'key': 'name', 'value': doc_dict.get('name')}, {'key': 'external_id', 'value': doc_dict.get('external_id')} ] if should_use_sql_backend(domain): dynamic_case_properties = OrderedDict(doc_dict['case_json']) else: dynamic_case_properties = CommCareCase.wrap(doc_dict).dynamic_case_properties() return base_case_properties + [ {'key': key, 'value': value} for key, value in dynamic_case_properties.iteritems() ]
def handle(self, domain, doc_type, **options): if not should_use_sql_backend(domain): raise CommandError('Only SQL backends currently supported') handlers = { 'xforminstance': _compare_xforms, 'commcarecase': _compare_cases, 'commcareuser': _compare_mobile_users, } try: diff = handlers[doc_type.lower()](domain, doc_type) except KeyError: raise CommandError('Unsupported doc type. Use on of: {}'.format(', '.join(handlers))) self.stdout.write('{} "{}" docs are missing from ES'.format(len(diff), doc_type)) for doc_id in diff: self.stdout.write(doc_id)
def check_product_stock(self, case, product_id, expected_soh, expected_qty, section_id='stock'): if not isinstance(expected_qty, Decimal): expected_qty = Decimal(str(expected_qty)) if not isinstance(expected_soh, Decimal): expected_soh = Decimal(str(expected_soh)) latest_trans = LedgerAccessors(self.domain.name).get_latest_transaction( case.case_id, section_id, product_id ) self.assertIsNotNone(latest_trans) self.assertEqual(section_id, latest_trans.section_id) self.assertEqual(expected_soh, latest_trans.stock_on_hand) if should_use_sql_backend(self.domain.name): if latest_trans.type == LedgerTransaction.TYPE_TRANSFER: self.assertEqual(int(expected_qty), latest_trans.delta) else: self.assertEqual(expected_qty, latest_trans.quantity)
def test_balance_consumption(self): initial = float(100) initial_amounts = [(p._id, initial) for p in self.products] self.submit_xml_form(balance_submission(initial_amounts)) final_amounts = [(p._id, float(50 - 10*i)) for i, p in enumerate(self.products)] self.submit_xml_form(balance_submission(final_amounts)) for product, amt in final_amounts: self.check_product_stock(self.sp, product, amt, 0) inferred = amt - initial if should_use_sql_backend(self.domain): sql_txn = LedgerAccessors(self.domain.name).get_latest_transaction( self.sp.case_id, 'stock', product ) self.assertEqual(inferred, sql_txn.delta) else: inferred_txn = StockTransaction.objects.get( case_id=self.sp.case_id, product_id=product, subtype=stockconst.TRANSACTION_SUBTYPE_INFERRED ) self.assertEqual(Decimal(str(inferred)), inferred_txn.quantity) self.assertEqual(Decimal(str(amt)), inferred_txn.stock_on_hand) self.assertEqual(stockconst.TRANSACTION_TYPE_CONSUMPTION, inferred_txn.type)
def build(self): limit_to_db = self.options.pop('limit_to_db', None) domain = self.options.pop('domain') if not case_search_enabled_for_domain(domain): raise CaseSearchNotEnabledException( "{} does not have case search enabled".format(domain)) assert should_use_sql_backend( domain), '{} can only be used with SQL domains'.format(self.slug) iteration_key = "CaseSearchResumableToElasticsearchPillow_{}_reindexer_{}_{}".format( CASE_SEARCH_INDEX_INFO.index, limit_to_db or 'all', domain or 'all') limit_db_aliases = [limit_to_db] if limit_to_db else None accessor = CaseReindexAccessor(domain=domain, limit_db_aliases=limit_db_aliases) doc_provider = SqlDocumentProvider(iteration_key, accessor) return ResumableBulkElasticPillowReindexer( doc_provider, elasticsearch=get_es_new(), index_info=CASE_SEARCH_INDEX_INFO, doc_transform=transform_case_for_elasticsearch, **self.options)
def _assert_initial_state(): if should_use_sql_backend(self.domain): self.assertEqual( 3, len( self._get_all_ledger_transactions( Q(form_id=second_form_id)))) else: self.assertEqual( 1, StockReport.objects.filter(form_id=second_form_id).count()) # 6 = 3 stockonhand and 3 inferred consumption txns self.assertEqual( 6, StockTransaction.objects.filter( report__form_id=second_form_id).count()) ledger_values = ledger_accessors.get_ledger_values_for_case( self.sp.case_id) self.assertEqual(3, len(ledger_values)) for lv in ledger_values: self.assertEqual(50, lv.stock_on_hand) self.assertEqual(round(float(lv.daily_consumption), 2), 1.67)
def test_archive_only_form(self): # check no data in stock states ledger_accessors = LedgerAccessors(self.domain.name) ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id) self.assertEqual(0, len(ledger_values)) initial_amounts = [(p._id, float(100)) for p in self.products] form_id = self.submit_xml_form(balance_submission(initial_amounts)) # check that we made stuff def _assert_initial_state(): if should_use_sql_backend(self.domain): self.assertEqual(3, LedgerTransaction.objects.filter(form_id=form_id).count()) else: self.assertEqual(1, StockReport.objects.filter(form_id=form_id).count()) self.assertEqual(3, StockTransaction.objects.filter(report__form_id=form_id).count()) ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id) self.assertEqual(3, len(ledger_values)) for state in ledger_values: self.assertEqual(100, int(state.stock_on_hand)) _assert_initial_state() # archive and confirm commtrack data is cleared form = FormAccessors(self.domain.name).get_form(form_id) form.archive() self.assertEqual(0, len(ledger_accessors.get_ledger_values_for_case(self.sp.case_id))) if should_use_sql_backend(self.domain): self.assertEqual(0, LedgerTransaction.objects.filter(form_id=form_id).count()) else: self.assertEqual(0, StockReport.objects.filter(form_id=form_id).count()) self.assertEqual(0, StockTransaction.objects.filter(report__form_id=form_id).count()) # unarchive and confirm commtrack data is restored form.unarchive() _assert_initial_state()
def test_archive_last_form(self): initial_amounts = [(p._id, float(100)) for p in self.products] self.submit_xml_form(balance_submission(initial_amounts), timestamp=datetime.utcnow() + timedelta(-30)) final_amounts = [(p._id, float(50)) for i, p in enumerate(self.products)] second_form_id = self.submit_xml_form( balance_submission(final_amounts)) ledger_accessors = LedgerAccessors(self.domain.name) def _assert_initial_state(): if should_use_sql_backend(self.domain): self.assertEqual( 3, LedgerTransaction.objects.filter( form_id=second_form_id).count()) else: self.assertEqual( 1, StockReport.objects.filter(form_id=second_form_id).count()) # 6 = 3 stockonhand and 3 inferred consumption txns self.assertEqual( 6, StockTransaction.objects.filter( report__form_id=second_form_id).count()) ledger_values = ledger_accessors.get_ledger_values_for_case( self.sp.case_id) self.assertEqual(3, len(ledger_values)) for lv in ledger_values: self.assertEqual(50, lv.stock_on_hand) self.assertEqual(round(float(lv.daily_consumption), 2), 1.67) # check initial setup _assert_initial_state() # archive and confirm commtrack data is deleted form = FormAccessors(self.domain.name).get_form(second_form_id) with process_kafka_changes('LedgerToElasticsearchPillow'): form.archive() if should_use_sql_backend(self.domain): self.assertEqual( 0, LedgerTransaction.objects.filter( form_id=second_form_id).count()) else: self.assertEqual( 0, StockReport.objects.filter(form_id=second_form_id).count()) self.assertEqual( 0, StockTransaction.objects.filter( report__form_id=second_form_id).count()) ledger_values = ledger_accessors.get_ledger_values_for_case( self.sp.case_id) self.assertEqual(3, len(ledger_values)) for state in ledger_values: # balance should be reverted to 100 in the StockState self.assertEqual(100, int(state.stock_on_hand)) # consumption should be none since there will only be 1 data point self.assertIsNone(state.daily_consumption) # unarchive and confirm commtrack data is restored with process_kafka_changes('LedgerToElasticsearchPillow'): form.unarchive() _assert_initial_state()
def _assert_no_stock_transactions(): if should_use_sql_backend(self.domain): self.assertEqual(0, LedgerTransaction.objects.count()) else: self.assertEqual(0, StockTransaction.objects.count())
expected_soh, expected_qty, section_id='stock'): if not isinstance(expected_qty, Decimal): expected_qty = Decimal(str(expected_qty)) if not isinstance(expected_soh, Decimal): expected_soh = Decimal(str(expected_soh)) latest_trans = LedgerAccessors( self.domain.name).get_latest_transaction(case.case_id, section_id, product_id) self.assertIsNotNone(latest_trans) self.assertEqual(section_id, latest_trans.section_id) self.assertEqual(expected_soh, latest_trans.stock_on_hand) if should_use_sql_backend(self.domain): if latest_trans.type == LedgerTransaction.TYPE_TRANSFER: self.assertEqual(int(expected_qty), latest_trans.delta) else: self.assertEqual(expected_qty, latest_trans.quantity) class CommTrackBalanceTransferTest(CommTrackSubmissionTest): @run_with_all_backends def test_balance_submit(self): amounts = [(p._id, float(i * 10)) for i, p in enumerate(self.products)] self.submit_xml_form(balance_submission(amounts)) for product, amt in amounts: self.check_product_stock(self.sp, product, amt, 0) @run_with_all_backends
def test_form_archiving(self): now = datetime.utcnow() # make sure we timestamp everything so they have the right order case_id = _post_util(create=True, p1='p1-1', p2='p2-1', form_extras={'received_on': now}) _post_util(case_id=case_id, p2='p2-2', p3='p3-2', p4='p4-2', form_extras={'received_on': now + timedelta(seconds=1)}) _post_util(case_id=case_id, p4='p4-3', p5='p5-3', close=True, form_extras={'received_on': now + timedelta(seconds=2)}) case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN) case = case_accessors.get_case(case_id) closed_by = case.closed_by closed_on = case.closed_on self.assertNotEqual('', closed_by) self.assertNotEqual(None, closed_on) def _check_initial_state(case): self.assertTrue(case.closed) self.assertEqual(closed_by, case.closed_by) self.assertEqual(closed_on, case.closed_on) self.assertEqual(case.get_case_property('p1'), 'p1-1') # original self.assertEqual(case.get_case_property('p2'), 'p2-2') # updated in second post self.assertEqual(case.get_case_property('p3'), 'p3-2') # new in second post self.assertEqual(case.get_case_property('p4'), 'p4-3') # updated in third post self.assertEqual(case.get_case_property('p5'), 'p5-3') # new in third post if should_use_sql_backend(REBUILD_TEST_DOMAIN): # SQL stores one transaction per form self.assertEqual(3, len( primary_actions(case))) # create + update + close else: self.assertEqual(5, len( primary_actions(case))) # create + 3 updates + close _check_initial_state(case) # verify xform/action states [f1, f2, f3] = case.xform_ids if should_use_sql_backend(REBUILD_TEST_DOMAIN): [create, update, close] = case.actions self.assertEqual(f1, create.form_id) self.assertEqual(f2, update.form_id) self.assertEqual(f3, close.form_id) else: [create, u1, u2, u3, close] = case.actions self.assertEqual(f1, create.form_id) self.assertEqual(f1, u1.form_id) self.assertEqual(f2, u2.form_id) self.assertEqual(f3, u3.form_id) # todo: should this be the behavior for archiving the create form? form_acessors = FormAccessors(REBUILD_TEST_DOMAIN) f1_doc = form_acessors.get_form(f1) with capture_kafka_changes_context(topics.CASE_SQL) as change_context: f1_doc.archive() if should_use_sql_backend(case.domain): self.assertEqual([case.case_id], [change.id for change in change_context.changes])
class CaseRebuildTest(TestCase, CaseRebuildTestMixin): @classmethod def setUpClass(cls): super(CaseRebuildTest, cls).setUpClass() delete_all_cases() def test_rebuild_empty(self): self.assertEqual( None, rebuild_case_from_forms('anydomain', 'notarealid', RebuildWithReason(reason='test')) ) def test_archiving_only_form(self): """ Checks that archiving the only form associated with the case archives the case and unarchiving unarchives it. """ case_id = _post_util(create=True, p1='p1-1', p2='p2-1') case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN) case = case_accessors.get_case(case_id) self.assertFalse(case.is_deleted) if should_use_sql_backend(REBUILD_TEST_DOMAIN): self.assertEqual(1, len(case.actions)) else: self.assertEqual(2, len(case.actions)) [form_id] = case.xform_ids form = FormAccessors(REBUILD_TEST_DOMAIN).get_form(form_id) form.archive() case = case_accessors.get_case(case_id) self.assertTrue(case.is_deleted) # should just have the 'rebuild' action self.assertEqual(1, len(case.actions)) self.assertTrue(case.actions[0].is_case_rebuild) form.unarchive() case = case_accessors.get_case(case_id) self.assertFalse(case.is_deleted) self.assertEqual(3, len(case.actions)) self.assertTrue(case.actions[-1].is_case_rebuild) def test_form_archiving(self): now = datetime.utcnow() # make sure we timestamp everything so they have the right order case_id = _post_util(create=True, p1='p1-1', p2='p2-1', form_extras={'received_on': now}) _post_util(case_id=case_id, p2='p2-2', p3='p3-2', p4='p4-2', form_extras={'received_on': now + timedelta(seconds=1)}) _post_util(case_id=case_id, p4='p4-3', p5='p5-3', close=True, form_extras={'received_on': now + timedelta(seconds=2)}) case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN) case = case_accessors.get_case(case_id) closed_by = case.closed_by closed_on = case.closed_on self.assertNotEqual('', closed_by) self.assertNotEqual(None, closed_on) def _check_initial_state(case): self.assertTrue(case.closed) self.assertEqual(closed_by, case.closed_by) self.assertEqual(closed_on, case.closed_on) self.assertEqual(case.get_case_property('p1'), 'p1-1') # original self.assertEqual(case.get_case_property('p2'), 'p2-2') # updated in second post self.assertEqual(case.get_case_property('p3'), 'p3-2') # new in second post self.assertEqual(case.get_case_property('p4'), 'p4-3') # updated in third post self.assertEqual(case.get_case_property('p5'), 'p5-3') # new in third post if should_use_sql_backend(REBUILD_TEST_DOMAIN): # SQL stores one transaction per form self.assertEqual(3, len(primary_actions(case))) # create + update + close else: self.assertEqual(5, len(primary_actions(case))) # create + 3 updates + close _check_initial_state(case) # verify xform/action states [f1, f2, f3] = case.xform_ids if should_use_sql_backend(REBUILD_TEST_DOMAIN): [create, update, close] = case.actions self.assertEqual(f1, create.form_id) self.assertEqual(f2, update.form_id) self.assertEqual(f3, close.form_id) else: [create, u1, u2, u3, close] = case.actions self.assertEqual(f1, create.form_id) self.assertEqual(f1, u1.form_id) self.assertEqual(f2, u2.form_id) self.assertEqual(f3, u3.form_id) # todo: should this be the behavior for archiving the create form? form_acessors = FormAccessors(REBUILD_TEST_DOMAIN) f1_doc = form_acessors.get_form(f1) with capture_kafka_changes_context(topics.CASE_SQL) as change_context: f1_doc.archive() if should_use_sql_backend(case.domain): self.assertEqual([case.case_id], [change.id for change in change_context.changes]) case = case_accessors.get_case(case_id) if should_use_sql_backend(REBUILD_TEST_DOMAIN): self.assertEqual(2, len(primary_actions(case))) else: self.assertEqual(3, len(primary_actions(case))) [u2, u3] = case.xform_ids self.assertEqual(f2, u2) self.assertEqual(f3, u3) self.assertTrue(case.closed) # no change self.assertFalse('p1' in case.dynamic_case_properties()) # should disappear entirely self.assertEqual(case.get_case_property('p2'), 'p2-2') # no change self.assertEqual(case.get_case_property('p3'), 'p3-2') # no change self.assertEqual(case.get_case_property('p4'), 'p4-3') # no change self.assertEqual(case.get_case_property('p5'), 'p5-3') # no change def _reset(form_id): form_doc = form_acessors.get_form(form_id) form_doc.unarchive() case = case_accessors.get_case(case_id) _check_initial_state(case) _reset(f1) f2_doc = form_acessors.get_form(f2) f2_doc.archive() case = case_accessors.get_case(case_id) if should_use_sql_backend(REBUILD_TEST_DOMAIN): self.assertEqual(2, len(primary_actions(case))) else: self.assertEqual(4, len(primary_actions(case))) [u1, u3] = case.xform_ids self.assertEqual(f1, u1) self.assertEqual(f3, u3) self.assertTrue(case.closed) # no change self.assertEqual(case.get_case_property('p1'), 'p1-1') # original self.assertEqual(case.get_case_property('p2'), 'p2-1') # loses second form update self.assertFalse('p3' in case.dynamic_case_properties()) # should disappear entirely self.assertEqual(case.get_case_property('p4'), 'p4-3') # no change self.assertEqual(case.get_case_property('p5'), 'p5-3') # no change _reset(f2) f3_doc = form_acessors.get_form(f3) f3_doc.archive() case = case_accessors.get_case(case_id) if should_use_sql_backend(REBUILD_TEST_DOMAIN): self.assertEqual(2, len(primary_actions(case))) else: self.assertEqual(3, len(primary_actions(case))) [u1, u2] = case.xform_ids self.assertEqual(f1, u1) self.assertEqual(f2, u2) self.assertFalse(case.closed) # reopened! self.assertEqual('', case.closed_by) self.assertEqual(None, case.closed_on) self.assertEqual(case.get_case_property('p1'), 'p1-1') # original self.assertEqual(case.get_case_property('p2'), 'p2-2') # original self.assertEqual(case.get_case_property('p3'), 'p3-2') # new in second post self.assertEqual(case.get_case_property('p4'), 'p4-2') # loses third form update self.assertFalse('p5' in case.dynamic_case_properties()) # should disappear entirely _reset(f3)
def __init__(self, domain): if not should_use_sql_backend(domain): raise UnexpectedBackend( "Only SQL backend supported: {}".format(domain)) self.domain = domain self.ledger_accessors = LedgerAccessorSQL
def bug_report(req): report = dict([(key, req.POST.get(key, '')) for key in ( 'subject', 'username', 'domain', 'url', 'message', 'app_id', 'cc', 'email', '500traceback', 'sentry_id', )]) report['user_agent'] = req.META['HTTP_USER_AGENT'] report['datetime'] = datetime.utcnow() try: couch_user = req.couch_user full_name = couch_user.full_name if couch_user.is_commcare_user(): email = report['email'] else: email = couch_user.get_email() except Exception: full_name = None email = report['email'] report['full_name'] = full_name report['email'] = email or report['username'] if report['domain']: domain = report['domain'] elif len(couch_user.domains) == 1: # This isn't a domain page, but the user has only one domain, so let's use that domain = couch_user.domains[0] else: domain = "<no domain>" message = (u"username: {username}\n" u"full name: {full_name}\n" u"domain: {domain}\n" u"url: {url}\n" u"datetime: {datetime}\n" u"User Agent: {user_agent}\n").format(**report) domain_object = Domain.get_by_name(domain) if report['domain'] else None if domain_object: current_project_description = domain_object.project_description if domain_object else None new_project_description = req.POST.get('project_description') if (domain_object and req.couch_user.is_domain_admin(domain=domain) and new_project_description and current_project_description != new_project_description): domain_object.project_description = new_project_description domain_object.save() matching_subscriptions = Subscription.objects.filter( is_active=True, subscriber__domain=domain, ) if len(matching_subscriptions) >= 1: software_plan = matching_subscriptions[0].plan_version else: software_plan = u'domain has no active subscription' message += (( u"software plan: {software_plan}\n" u"Is self start: {self_started}\n" u"Feature Flags: {feature_flags}\n" u"Feature Previews: {feature_previews}\n" u"Is scale backend: {scale_backend}\n" u"Has Support Hand-off Info: {has_handoff_info}\n" u"Internal Project Information: {internal_info_link}\n" u"Project description: {project_description}\n" u"Sentry Error: {sentry_error}\n").format( software_plan=software_plan, self_started=domain_object.internal.self_started, feature_flags=toggles.toggles_dict(username=report['username'], domain=domain).keys(), feature_previews=feature_previews.previews_dict(domain).keys(), scale_backend=should_use_sql_backend(domain), has_handoff_info=bool(domain_object.internal.partner_contact), internal_info_link=reverse('domain_internal_settings', args=[domain], absolute=True), project_description=domain_object.project_description, sentry_error='{}{}'.format( getattr(settings, 'SENTRY_QUERY_URL'), report['sentry_id']))) subject = u'{subject} ({domain})'.format(subject=report['subject'], domain=domain) cc = report['cc'].strip().split(",") cc = filter(None, cc) if full_name and not any([c in full_name for c in '<>"']): reply_to = u'"{full_name}" <{email}>'.format(**report) else: reply_to = report['email'] # if the person looks like a commcare user, fogbugz can't reply # to their email, so just use the default if settings.HQ_ACCOUNT_ROOT in reply_to: reply_to = settings.SERVER_EMAIL message += u"Message:\n\n{message}\n".format(message=report['message']) if req.POST.get('five-hundred-report'): extra_message = ("This messge was reported from a 500 error page! " "Please fix this ASAP (as if you wouldn't anyway)...") traceback_info = cache.cache.get(report['500traceback']) cache.cache.delete(report['500traceback']) traceback_info = "Traceback of this 500: \n%s" % traceback_info message = "%s \n\n %s \n\n %s" % (message, extra_message, traceback_info) email = EmailMessage(subject=subject, body=message, to=settings.BUG_REPORT_RECIPIENTS, headers={'Reply-To': reply_to}, cc=cc) uploaded_file = req.FILES.get('report_issue') if uploaded_file: filename = uploaded_file.name content = uploaded_file.read() email.attach(filename=filename, content=content) # only fake the from email if it's an @dimagi.com account if re.search('@dimagi\.com$', report['username']): email.from_email = report['username'] else: email.from_email = settings.CCHQ_BUG_REPORT_EMAIL email.send(fail_silently=False) if req.POST.get('five-hundred-report'): messages.success( req, "Your CommCare HQ Issue Report has been sent. We are working quickly to resolve this problem." ) return HttpResponseRedirect(reverse('homepage')) return HttpResponse()
def _ledgers_per_case(self): results = (LedgerES(es_instance_alias=ES_EXPORT_INSTANCE).domain( self.domain).aggregation( TermsAggregation('by_case', 'case_id', size=100)).size(0).run()) ledgers_per_case = results.aggregations.by_case case_ids = set() ledger_counts = [] for case_id, ledger_count in ledgers_per_case.counts_by_bucket().items( ): case_ids.add(case_id) ledger_counts.append(ledger_count) if not case_ids: self.stdout.write("Domain has no ledgers") return avg_ledgers_per_case = sum(ledger_counts) // len(case_ids) case_types_result = CaseES(es_instance_alias=ES_EXPORT_INSTANCE)\ .domain(self.domain).case_ids(case_ids)\ .aggregation(TermsAggregation('types', 'type'))\ .size(0).run() case_types = case_types_result.aggregations.types.keys self.stdout.write('\nCase Types with Ledgers') for type_ in case_types: self._print_value( 'case_type', type_, CaseES().domain(self.domain).case_type(type_).count()) if should_use_sql_backend(self.domain): db_name = get_db_aliases_for_partitioned_query()[ 0] # just query one shard DB results = (CommCareCaseSQL.objects.using(db_name).filter( domain=self.domain, closed=True, type=type_).annotate( lifespan=F('closed_on') - F('opened_on')).annotate( avg_lifespan=Avg('lifespan')).values( 'avg_lifespan', flat=True)) self._print_value('Average lifespan for "%s" cases' % type_, results[0]['avg_lifespan']) self._cases_created_per_user_per_month(type_) self._print_value('Average ledgers per case', avg_ledgers_per_case) if should_use_sql_backend(self.domain): stats = defaultdict(list) for db_name, case_ids_p in split_list_by_db_partition(case_ids): transactions_per_case_per_month = ( LedgerTransaction.objects.using(db_name).filter( case_id__in=case_ids).annotate( m=Month('server_date'), y=Year('server_date')).values( 'case_id', 'y', 'm').annotate(count=Count('id'))) for row in transactions_per_case_per_month: month = date(row['y'], row['m'], 1) stats[month].append(row['count']) else: transactions_per_case_per_month = (StockTransaction.objects.filter( case_id__in=case_ids).annotate( m=Month('report__date'), y=Year('report__date')).values( 'case_id', 'y', 'm').annotate(count=Count('id'))) stats = defaultdict(list) for row in transactions_per_case_per_month: month = date(row['y'], row['m'], 1) stats[month].append(row['count']) final_stats = [] for month, transaction_count_list in sorted(list(stats.items()), key=lambda r: r[0]): final_stats.append( (month.isoformat(), sum(transaction_count_list) // len(transaction_count_list))) self._print_table(['Month', 'Ledgers updated per case'], final_stats)
self.assertEqual(f2, u2.form_id) self.assertEqual(f3, u3.form_id) # todo: should this be the behavior for archiving the create form? form_acessors = FormAccessors(REBUILD_TEST_DOMAIN) f1_doc = form_acessors.get_form(f1) with capture_kafka_changes_context(topics.CASE_SQL) as change_context: f1_doc.archive() if should_use_sql_backend(case.domain): self.assertEqual([case.case_id], [change.id for change in change_context.changes]) case = case_accessors.get_case(case_id) if should_use_sql_backend(REBUILD_TEST_DOMAIN): self.assertEqual(2, len(primary_actions(case))) else: self.assertEqual(3, len(primary_actions(case))) [u2, u3] = case.xform_ids self.assertEqual(f2, u2) self.assertEqual(f3, u3) self.assertTrue(case.closed) # no change self.assertFalse( 'p1' in case.dynamic_case_properties()) # should disappear entirely self.assertEqual(case.get_case_property('p2'), 'p2-2') # no change self.assertEqual(case.get_case_property('p3'), 'p3-2') # no change self.assertEqual(case.get_case_property('p4'), 'p4-3') # no change
def reprocess_form(form, save=True, lock_form=True): interface = FormProcessorInterface(form.domain) lock = interface.acquire_lock_for_xform( form.form_id) if lock_form else None with LockManager(form, lock): logger.info('Reprocessing form: %s (%s)', form.form_id, form.domain) # reset form state prior to processing if should_use_sql_backend(form.domain): form.state = XFormInstanceSQL.NORMAL else: form.doc_type = 'XFormInstance' cache = interface.casedb_cache(domain=form.domain, lock=True, deleted_ok=True, xforms=[form]) with cache as casedb: try: case_stock_result = SubmissionPost.process_xforms_for_cases( [form], casedb) except (IllegalCaseId, UsesReferrals, MissingProductId, PhoneDateValueError, InvalidCaseIndex, CaseValueError) as e: error_message = '{}: {}'.format( type(e).__name__, six.text_type(e)) form = interface.xformerror_from_xform_instance( form, error_message) return ReprocessingResult(form, [], [], error_message) form.initial_processing_complete = True form.problem = None stock_result = case_stock_result.stock_result assert stock_result.populated cases = case_stock_result.case_models _log_changes(cases, stock_result.models_to_save, stock_result.models_to_delete) ledgers = [] if should_use_sql_backend(form.domain): cases_needing_rebuild = _get_case_ids_needing_rebuild( form, cases) ledgers = stock_result.models_to_save ledgers_updated = { ledger.ledger_reference for ledger in ledgers if ledger.is_saved() } if save: for case in cases: CaseAccessorSQL.save_case(case) LedgerAccessorSQL.save_ledger_values(ledgers) FormAccessorSQL.update_form_problem_and_state(form) FormProcessorSQL._publish_changes( ProcessedForms(form, None), cases, stock_result) # rebuild cases and ledgers that were affected for case in cases: if case.case_id in cases_needing_rebuild: logger.info('Rebuilding case: %s', case.case_id) if save: # only rebuild cases that were updated detail = FormReprocessRebuild(form_id=form.form_id) interface.hard_rebuild_case(case.case_id, detail, lock=False) for ledger in ledgers: if ledger.ledger_reference in ledgers_updated: logger.info('Rebuilding ledger: %s', ledger.ledger_reference) if save: # only rebuild updated ledgers interface.ledger_processor.rebuild_ledger_state( **ledger.ledger_reference._asdict()) else:
def get_primary_db_case_ids(domain, doc_type, startdate, enddate): if should_use_sql_backend(domain): return get_sql_case_ids(domain, doc_type, startdate, enddate) else: # date filtering not supported for couch return set(get_doc_ids_in_domain_by_type(domain, doc_type, CommCareCase.get_db()))
def test_reprocess_unfinished_submission_ledger_rebuild(self): from corehq.apps.commtrack.tests.util import get_single_balance_block case_id = uuid.uuid4().hex form_ids = [] form_ids.append( submit_case_blocks([ CaseBlock(case_id=case_id, create=True, case_type='shop').as_string(), get_single_balance_block(case_id, 'product1', 100), ], self.domain)[0].form_id) with _patch_save_to_raise_error(self): submit_case_blocks( get_single_balance_block(case_id, 'product1', 50), self.domain) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) form_ids.append(stubs[0].xform_id) # submit another form afterwards form_ids.append( submit_case_blocks( get_single_balance_block(case_id, 'product1', 25), self.domain)[0].form_id) ledgers = self.ledgerdb.get_ledger_values_for_case(case_id) self.assertEqual(1, len(ledgers)) self.assertEqual(25, ledgers[0].balance) ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case( case_id) if should_use_sql_backend(self.domain): self.assertEqual(2, len(ledger_transactions)) else: # includes extra consumption transaction self.assertEqual(3, len(ledger_transactions)) # should rebuild ledger transactions result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(1, len(result.cases)) self.assertEqual(1, len(result.ledgers)) ledgers = self.ledgerdb.get_ledger_values_for_case(case_id) self.assertEqual(1, len(ledgers)) # still only 1 self.assertEqual(25, ledgers[0].balance) ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case( case_id) if should_use_sql_backend(self.domain): self.assertEqual(3, len(ledger_transactions)) # make sure transactions are in correct order self.assertEqual(form_ids, [trans.form_id for trans in ledger_transactions]) self.assertEqual(100, ledger_transactions[0].updated_balance) self.assertEqual(100, ledger_transactions[0].delta) self.assertEqual(50, ledger_transactions[1].updated_balance) self.assertEqual(-50, ledger_transactions[1].delta) self.assertEqual(25, ledger_transactions[2].updated_balance) self.assertEqual(-25, ledger_transactions[2].delta) else: self.assertEqual(3, len(ledger_transactions)) self.assertEqual( form_ids, [trans.report.form_id for trans in ledger_transactions]) self.assertEqual(100, ledger_transactions[0].stock_on_hand) self.assertEqual(50, ledger_transactions[1].stock_on_hand) self.assertEqual(25, ledger_transactions[2].stock_on_hand)
def get_primary_db_case_counts(domain): if should_use_sql_backend(domain): return _get_sql_cases_by_doc_type(domain) else: return _get_couch_cases_by_doc_type(domain)
def bug_report(req): report = dict([(key, req.POST.get(key, '')) for key in ( 'subject', 'username', 'domain', 'url', 'message', 'app_id', 'cc', 'email', '500traceback', )]) domain_object = Domain.get_by_name(report['domain']) current_project_description = domain_object.project_description new_project_description = req.POST.get('project_description') if (req.couch_user.is_domain_admin(domain=report['domain']) and new_project_description and current_project_description != new_project_description): domain_object.project_description = new_project_description domain_object.save() report['user_agent'] = req.META['HTTP_USER_AGENT'] report['datetime'] = datetime.utcnow() report['feature_flags'] = toggles.toggles_dict( username=report['username'], domain=report['domain']).keys() report['feature_previews'] = feature_previews.previews_dict( report['domain']).keys() report['scale_backend'] = should_use_sql_backend( report['domain']) if report['domain'] else False report['project_description'] = domain_object.project_description try: couch_user = req.couch_user full_name = couch_user.full_name if couch_user.is_commcare_user(): email = report['email'] else: email = couch_user.get_email() except Exception: full_name = None email = report['email'] report['full_name'] = full_name report['email'] = email or report['username'] matching_subscriptions = Subscription.objects.filter( is_active=True, subscriber__domain=report['domain'], ) if len(matching_subscriptions) >= 1: report['software_plan'] = matching_subscriptions[0].plan_version else: report['software_plan'] = u'domain has no active subscription' subject = u'{subject} ({domain})'.format(**report) message = (u"username: {username}\n" u"full name: {full_name}\n" u"domain: {domain}\n" u"software plan: {software_plan}\n" u"url: {url}\n" u"datetime: {datetime}\n" u"User Agent: {user_agent}\n" u"Feature Flags: {feature_flags}\n" u"Feature Previews: {feature_previews}\n" u"Is scale backend: {scale_backend}\n" u"Project description: {project_description}\n" u"Message:\n\n" u"{message}\n").format(**report) cc = report['cc'].strip().split(",") cc = filter(None, cc) if full_name and not any([c in full_name for c in '<>"']): reply_to = u'"{full_name}" <{email}>'.format(**report) else: reply_to = report['email'] # if the person looks like a commcare user, fogbugz can't reply # to their email, so just use the default if settings.HQ_ACCOUNT_ROOT in reply_to: reply_to = settings.SERVER_EMAIL if req.POST.get('five-hundred-report'): extra_message = ("This messge was reported from a 500 error page! " "Please fix this ASAP (as if you wouldn't anyway)...") traceback_info = cache.cache.get(report['500traceback']) cache.cache.delete(report['500traceback']) traceback_info = "Traceback of this 500: \n%s" % traceback_info message = "%s \n\n %s \n\n %s" % (message, extra_message, traceback_info) email = EmailMessage(subject=subject, body=message, to=settings.BUG_REPORT_RECIPIENTS, headers={'Reply-To': reply_to}, cc=cc) uploaded_file = req.FILES.get('report_issue') if uploaded_file: filename = uploaded_file.name content = uploaded_file.read() email.attach(filename=filename, content=content) # only fake the from email if it's an @dimagi.com account if re.search('@dimagi\.com$', report['username']): email.from_email = report['username'] else: email.from_email = settings.CCHQ_BUG_REPORT_EMAIL email.send(fail_silently=False) if req.POST.get('five-hundred-report'): messages.success( req, "Your CommCare HQ Issue Report has been sent. We are working quickly to resolve this problem." ) return HttpResponseRedirect(reverse('homepage')) return HttpResponse()
def location_map_case_id(user): if should_use_sql_backend(user.domain): user_id = user.user_id case_id = uuid.uuid5(const.MOBILE_WORKER_UUID_NS, user_id).hex return case_id return 'user-owner-mapping-' + user.user_id
def test_test_local_domain_sql_backend_override_overrides(self): domain_name = uuid.uuid4().hex create_domain(domain_name) self.assertFalse(should_use_sql_backend(domain_name)) set_local_domain_sql_backend_override(domain_name) self.assertTrue(should_use_sql_backend(domain_name))
def __init__(self, domain): assert should_use_sql_backend( domain), "Only SQL backend supported: {}".format(domain) self.domain = domain self.ledger_accessors = LedgerAccessorSQL
def iter_document_ids(self): if should_use_sql_backend(self.domain): accessor = LedgerReindexAccessor(self.domain) return iter_all_ids(accessor) else: return iter(self._couch_iterator())
def _save_case(domain, case): if should_use_sql_backend(domain): CaseAccessorSQL.save_case(case) else: # can't call case.save() since it overrides the server_modified_on property CommCareCase.get_db().save_doc(case.to_json())
from casexml.apps.case.mock import CaseFactory from contextlib import contextmanager from corehq.apps.data_interfaces.models import AutomaticUpdateRule from corehq.form_processor.backends.sql.dbaccessors import CaseAccessorSQL from corehq.form_processor.utils.general import should_use_sql_backend @contextmanager def create_case(domain, case_type, **kwargs): case = CaseFactory(domain).create_case(case_type=case_type, **kwargs) try: yield case finally: if should_use_sql_backend(domain): CaseAccessorSQL.hard_delete_cases(domain, [case.case_id]) else: case.delete() def create_empty_rule(domain, workflow): return AutomaticUpdateRule.objects.create( domain=domain, name='test', case_type='person', active=True, deleted=False, filter_on_server_modified=False, server_modified_boundary=None, migrated=True, workflow=workflow,
def __init__(self, domain): assert should_use_sql_backend(domain), "Only SQL backend supported" self.domain = domain self.ledger_accessors = LedgerAccessorSQL self.case_accessors = CaseAccessorSQL
def post(self, req, *args, **kwargs): report = dict([(key, req.POST.get(key, '')) for key in ( 'subject', 'username', 'domain', 'url', 'message', 'app_id', 'cc', 'email', '500traceback', 'sentry_id', )]) try: couch_user = req.couch_user full_name = couch_user.full_name if couch_user.is_commcare_user(): email = report['email'] else: email = couch_user.get_email() except Exception: full_name = None email = report['email'] report['full_name'] = full_name report['email'] = email or report['username'] if report['domain']: domain = report['domain'] elif len(couch_user.domains) == 1: # This isn't a domain page, but the user has only one domain, so let's use that domain = couch_user.domains[0] else: domain = "<no domain>" message = ("username: {username}\n" "full name: {full_name}\n" "domain: {domain}\n" "url: {url}\n").format(**report) domain_object = Domain.get_by_name( domain) if report['domain'] else None debug_context = { 'datetime': datetime.utcnow(), 'self_started': '<unknown>', 'scale_backend': '<unknown>', 'has_handoff_info': '<unknown>', 'project_description': '<unknown>', 'sentry_error': '{}{}'.format(getattr(settings, 'SENTRY_QUERY_URL', ''), report['sentry_id']) } if domain_object: current_project_description = domain_object.project_description if domain_object else None new_project_description = req.POST.get('project_description') if (domain_object and req.couch_user.is_domain_admin(domain=domain) and new_project_description and current_project_description != new_project_description): domain_object.project_description = new_project_description domain_object.save() message += (("software plan: {software_plan}\n").format( software_plan=Subscription.get_subscribed_plan_by_domain( domain), )) debug_context.update({ 'self_started': domain_object.internal.self_started, 'scale_backend': should_use_sql_backend(domain), 'has_handoff_info': bool(domain_object.internal.partner_contact), 'project_description': domain_object.project_description, }) subject = '{subject} ({domain})'.format(subject=report['subject'], domain=domain) cc = [el for el in report['cc'].strip().split(",") if el] if full_name and not any([c in full_name for c in '<>"']): reply_to = '"{full_name}" <{email}>'.format(**report) else: reply_to = report['email'] # if the person looks like a commcare user, fogbugz can't reply # to their email, so just use the default if settings.HQ_ACCOUNT_ROOT in reply_to: reply_to = settings.SERVER_EMAIL message += "Message:\n\n{message}\n".format(message=report['message']) if req.POST.get('five-hundred-report'): extra_message = ( "This message was reported from a 500 error page! " "Please fix this ASAP (as if you wouldn't anyway)...") extra_debug_info = ( "datetime: {datetime}\n" "Is self start: {self_started}\n" "Is scale backend: {scale_backend}\n" "Has Support Hand-off Info: {has_handoff_info}\n" "Project description: {project_description}\n" "Sentry Error: {sentry_error}\n").format(**debug_context) traceback_info = cache.cache.get( report['500traceback']) or 'No traceback info available' cache.cache.delete(report['500traceback']) message = "\n\n".join( [message, extra_debug_info, extra_message, traceback_info]) email = EmailMessage(subject=subject, body=message, to=self.recipients, headers={'Reply-To': reply_to}, cc=cc) uploaded_file = req.FILES.get('report_issue') if uploaded_file: filename = uploaded_file.name content = uploaded_file.read() email.attach(filename=filename, content=content) # only fake the from email if it's an @dimagi.com account is_icds_env = settings.SERVER_ENVIRONMENT in settings.ICDS_ENVS if re.search(r'@dimagi\.com$', report['username']) and not is_icds_env: email.from_email = report['username'] else: email.from_email = settings.CCHQ_BUG_REPORT_EMAIL email.send(fail_silently=False) if req.POST.get('five-hundred-report'): messages.success( req, "Your CommCare HQ Issue Report has been sent. We are working quickly to resolve this problem." ) return HttpResponseRedirect(reverse('homepage')) return HttpResponse()
def get_primary_db_form_counts(domain, startdate=None, enddate=None): if should_use_sql_backend(domain): return _get_sql_forms_by_doc_type(domain, startdate, enddate) else: return _get_couch_forms_by_doc_type(domain)