def get_current_ledger_value(self, unique_ledger_reference): latest_txn = StockTransaction.latest( case_id=unique_ledger_reference.case_id, section_id=unique_ledger_reference.section_id, product_id=unique_ledger_reference.entry_id, ) return latest_txn.stock_on_hand if latest_txn else 0
def get_current_ledger_state(case_ids, ensure_form_id=False): """ Given a list of cases returns a dict of all current ledger data of the following format: { "case_id": { "section_id": { "product_id": StockState, "product_id": StockState, ... }, ... }, ... } :param ensure_form_id: Set to True to make sure return StockState have the ``last_modified_form_id`` field populated """ from corehq.apps.commtrack.models import StockState if not case_ids: return {} states = StockState.objects.filter(case_id__in=case_ids) ret = {case_id: {} for case_id in case_ids} for state in states: sections = ret[state.case_id].setdefault(state.section_id, {}) sections[state.product_id] = state if ensure_form_id and not state.last_modified_form_id: transaction = StockTransaction.latest(state.case_id, state.section_id, state.product_id) if transaction is not None: state.last_modified_form_id = transaction.report.form_id state.save() return ret
def get_valid_reports(self, data): filtered_transactions = [] excluded_products = [] for product_id, transactions in get_transactions_by_product( data['transactions']).iteritems(): begin_soh = None end_soh = None receipt = 0 for transaction in transactions: if begin_soh is None: sql_location = SQLLocation.objects.get( location_id=transaction.location_id) latest = StockTransaction.latest( sql_location.supply_point_id, SECTION_TYPE_STOCK, transaction.product_id) begin_soh = 0 if latest: begin_soh = float(latest.stock_on_hand) if transaction.action == 'receipts': receipt += float(transaction.quantity) elif not end_soh: end_soh = float(transaction.quantity) if end_soh > begin_soh + receipt: excluded_products.append(transaction.product_id) else: filtered_transactions.append(transaction) if excluded_products: message = ERROR_MESSAGE.format(products_list=', '.join([ SQLProduct.objects.get(product_id=product_id).code for product_id in set(excluded_products) ])) self.respond(message) return filtered_transactions
def get_current_ledger_state(case_ids, ensure_form_id=False): """ Given a list of cases returns a dict of all current ledger data of the following format: { "case_id": { "section_id": { "product_id": StockState, "product_id": StockState, ... }, ... }, ... } :param ensure_form_id: Set to True to make sure return StockState have the ``last_modified_form_id`` field populated """ from corehq.apps.commtrack.models import StockState if not case_ids: return {} states = StockState.objects.filter( case_id__in=case_ids ) ret = {case_id: {} for case_id in case_ids} for state in states: sections = ret[state.case_id].setdefault(state.section_id, {}) sections[state.product_id] = state if ensure_form_id and not state.last_modified_form_id: transaction = StockTransaction.latest(state.case_id, state.section_id, state.product_id) if transaction is not None: state.last_modified_form_id = transaction.report.form_id state.save() return ret
def get_data(self): sp_ids = get_relevant_supply_point_ids( self.domain, self.active_location ) products = Product.by_domain(self.domain) if self.program_id: products = filter( lambda product: product.program_id == self.program_id, products ) for sp_id in sp_ids: for product in products: loc = SupplyPointCase.get(sp_id).location last_transaction = StockTransaction.latest( sp_id, STOCK_SECTION_TYPE, product._id ) yield { 'loc_id': loc._id, 'loc_path': loc.path, 'name': loc.name, 'type': loc.location_type, 'reporting_status': reporting_status( last_transaction, self.start_date, self.end_date ), 'geo': loc._geopoint, }
def get_valid_reports(self, data): filtered_transactions = [] excluded_products = [] for product_id, transactions in get_transactions_by_product(data["transactions"]).iteritems(): begin_soh = None end_soh = None receipt = 0 for transaction in transactions: if begin_soh is None: sql_location = SQLLocation.objects.get(location_id=transaction.location_id) latest = StockTransaction.latest( sql_location.supply_point_id, SECTION_TYPE_STOCK, transaction.product_id ) begin_soh = 0 if latest: begin_soh = float(latest.stock_on_hand) if transaction.action == "receipts": receipt += float(transaction.quantity) elif not end_soh: end_soh = float(transaction.quantity) if end_soh > begin_soh + receipt: excluded_products.append(transaction.product_id) else: filtered_transactions.append(transaction) if excluded_products: message = ERROR_MESSAGE.format( products_list=", ".join( [SQLProduct.objects.get(product_id=product_id).code for product_id in set(excluded_products)] ) ) self.respond(message) return filtered_transactions
def get_stock_state_for_transaction(transaction): from corehq.apps.commtrack.models import StockState from corehq.apps.locations.models import SQLLocation from corehq.apps.products.models import SQLProduct # todo: in the worst case, this function makes # - three calls to couch (for the case, domain, and commtrack config) # - four postgres queries (transacitons, product, location, and state) # - one postgres write (to save the state) # and that doesn't even include the consumption calc, which can do a whole # bunch more work and hit the database. sql_product = SQLProduct.objects.get(product_id=transaction.product_id) try: domain_name = transaction.__domain except AttributeError: domain_name = sql_product.domain try: sql_location = SQLLocation.objects.get(supply_point_id=transaction.case_id) except SQLLocation.DoesNotExist: sql_location = None try: state = StockState.include_archived.get( section_id=transaction.section_id, case_id=transaction.case_id, product_id=transaction.product_id, ) except StockState.DoesNotExist: state = StockState( section_id=transaction.section_id, case_id=transaction.case_id, product_id=transaction.product_id, sql_product=sql_product, sql_location=sql_location, ) # we may not be saving the latest transaction so make sure we use that # todo: this should change to server date latest_transaction = StockTransaction.latest( case_id=transaction.case_id, section_id=transaction.section_id, product_id=transaction.product_id ) if latest_transaction != transaction: logging.warning( 'Just fired signal for a stale stock transaction. Domain: {}, instance: {},latest was {}'.format( domain_name, transaction, latest_transaction ) ) transaction = latest_transaction state.last_modified_date = transaction.report.server_date state.last_modified_form_id = transaction.report.form_id state.stock_on_hand = transaction.stock_on_hand # so you don't have to look it up again in the signal receivers if domain_name: state.__domain = domain_name return state
def _get_ledger(self, unique_ledger_reference): try: return StockTransaction.latest( case_id=unique_ledger_reference.case_id, section_id=unique_ledger_reference.section_id, product_id=unique_ledger_reference.entry_id, ) except StockTransaction.DoesNotExist: return None
def _get_ledger(self, unique_ledger_reference): try: return StockTransaction.latest( case_id=unique_ledger_reference.case_id, section_id=unique_ledger_reference.section_id, product_id=unique_ledger_reference.entry_id, ) except StockTransaction.DoesNotExist: return None
def check_stock_models(self, case, product_id, expected_soh, expected_qty, section_id): if not isinstance(expected_qty, Decimal): expected_qty = Decimal(str(expected_qty)) if not isinstance(expected_soh, Decimal): expected_soh = Decimal(str(expected_soh)) latest_trans = StockTransaction.latest(case._id, section_id, product_id) self.assertIsNotNone(latest_trans) self.assertEqual(section_id, latest_trans.section_id) self.assertEqual(expected_soh, latest_trans.stock_on_hand) self.assertEqual(expected_qty, latest_trans.quantity)
def sync_stock_transactions_for_facility(domain, endpoint, facility, checkpoint, date, limit=1000, offset=0): """ Syncs stock data from StockTransaction objects in ILSGateway to StockTransaction objects in HQ """ has_next = True next_url = "" section_id = 'stock' supply_point = facility case = get_supply_point_case_in_domain_by_id(domain, supply_point) if not case: return location_id = case.location_id save_stock_data_checkpoint(checkpoint, 'stock_transaction', limit, offset, date, location_id, True) products_saved = set() while has_next: meta, stocktransactions = endpoint.get_stocktransactions( next_url_params=next_url, limit=limit, offset=offset, filters={ 'supply_point': supply_point, 'date__gte': date, 'date__lte': checkpoint.start_date } ) # set the checkpoint right before the data we are about to process meta_limit = meta.get('limit') or limit meta_offset = meta.get('offset') or offset save_stock_data_checkpoint( checkpoint, 'stock_transaction', meta_limit, meta_offset, date, location_id, True ) transactions_to_add = [] with transaction.atomic(): for stocktransaction in stocktransactions: transactions = sync_stock_transaction(stocktransaction, domain, case, bulk=True) transactions_to_add.extend(transactions) products_saved.update(map(lambda x: x.product_id, transactions)) if transactions_to_add: # Doesn't send signal StockTransaction.objects.bulk_create(transactions_to_add) if not meta.get('next', False): has_next = False else: next_url = meta['next'].split('?')[1] for product in products_saved: # if we saved anything rebuild the stock state object by firing the signal # on the last transaction for each product last_st = StockTransaction.latest(case.get_id, section_id, product) update_stock_state_for_transaction(last_st)
def get_current_ledger_transactions(case_id): """ Given a case returns a dict of all current ledger data. { "section_id": { "product_id": StockTransaction, "product_id": StockTransaction, ... }, ... } """ from corehq.apps.commtrack.models import StockState results = StockState.objects.filter(case_id=case_id).values_list('case_id', 'section_id', 'product_id') ret = {} for case_id, section_id, product_id in results: sections = ret.setdefault(section_id, {}) sections[product_id] = StockTransaction.latest(case_id, section_id, product_id) return ret
def get_current_ledger_transactions(case_id): """ Given a case returns a dict of all current ledger data. { "section_id": { "product_id": StockTransaction, "product_id": StockTransaction, ... }, ... } """ from corehq.apps.commtrack.models import StockState results = StockState.objects.filter(case_id=case_id).values_list( 'case_id', 'section_id', 'product_id') ret = {} for case_id, section_id, product_id in results: sections = ret.setdefault(section_id, {}) sections[product_id] = StockTransaction.latest(case_id, section_id, product_id) return ret
def get_latest_transaction(case_id, section_id, entry_id): return StockTransaction.latest(case_id, section_id, entry_id)
def _get_stats(self): stock_state = StockState.objects.get(**self._stock_state_key) latest_txn = StockTransaction.latest(**self._stock_state_key) all_txns = StockTransaction.get_ordered_transactions_for_stock( **self._stock_state_key) return stock_state, latest_txn, all_txns
def get_stock_payload(self, syncop): cases = [e.case for e in syncop.actual_cases_to_sync] from lxml.builder import ElementMaker E = ElementMaker(namespace=COMMTRACK_REPORT_XMLNS) def entry_xml(id, quantity): return E.entry( id=id, quantity=str(int(quantity)), ) def transaction_to_xml(trans): return entry_xml(trans.product_id, trans.stock_on_hand) def consumption_entry(case_id, product_id, section_id): consumption_value = compute_consumption_or_default( case_id, product_id, datetime.utcnow(), section_id, self.stock_settings.consumption_config ) if consumption_value is not None: return entry_xml(product_id, consumption_value) def _unique_products(stock_transaction_queryset): return sorted(stock_transaction_queryset.values_list('product_id', flat=True).distinct()) for commtrack_case in cases: relevant_sections = sorted(StockTransaction.objects.filter( case_id=commtrack_case._id).values_list('section_id', flat=True).distinct()) section_product_map = defaultdict(lambda: []) section_timestamp_map = defaultdict(lambda: json_format_datetime(datetime.utcnow())) for section_id in relevant_sections: relevant_reports = StockTransaction.objects.filter(case_id=commtrack_case._id, section_id=section_id) product_ids = _unique_products(relevant_reports) transactions = [StockTransaction.latest(commtrack_case._id, section_id, p) for p in product_ids] as_of = json_format_datetime(max(txn.report.date for txn in transactions)) section_product_map[section_id] = product_ids section_timestamp_map[section_id] = as_of yield E.balance(*(transaction_to_xml(e) for e in transactions), **{'entity-id': commtrack_case._id, 'date': as_of, 'section-id': section_id}) for section_id, consumption_section_id in self.stock_settings.section_to_consumption_types.items(): if (section_id in relevant_sections or self.stock_settings.force_consumption_case_filter(commtrack_case)): consumption_product_ids = self.stock_settings.default_product_list \ if self.stock_settings.default_product_list \ else section_product_map[section_id] consumption_entries = filter(lambda e: e is not None, [ consumption_entry(commtrack_case._id, p, section_id) for p in consumption_product_ids ]) if consumption_entries: yield E.balance( *consumption_entries, **{ 'entity-id': commtrack_case._id, 'date': section_timestamp_map[section_id], 'section-id': consumption_section_id, } )
def _get_stats(self): stock_state = StockState.objects.get(**self._stock_state_key) latest_txn = StockTransaction.latest(**self._stock_state_key) all_txns = StockTransaction.get_ordered_transactions_for_stock( **self._stock_state_key) return stock_state, latest_txn, all_txns
def get_stock_payload(self, syncop): cases = [e.case for e in syncop.actual_cases_to_sync] from lxml.builder import ElementMaker E = ElementMaker(namespace=COMMTRACK_REPORT_XMLNS) def entry_xml(id, quantity): return E.entry( id=id, quantity=str(int(quantity)), ) def transaction_to_xml(trans): return entry_xml(trans.product_id, trans.stock_on_hand) def consumption_entry(case_id, product_id, section_id): consumption_value = compute_consumption_or_default( case_id, product_id, datetime.utcnow(), section_id, self.stock_settings.consumption_config ) if consumption_value is not None: return entry_xml(product_id, consumption_value) def _unique_products(stock_transaction_queryset): return sorted(stock_transaction_queryset.values_list('product_id', flat=True).distinct()) for commtrack_case in cases: relevant_sections = sorted(StockTransaction.objects.filter( case_id=commtrack_case._id).values_list('section_id', flat=True).distinct()) section_product_map = defaultdict(lambda: []) section_timestamp_map = defaultdict(lambda: json_format_datetime(datetime.utcnow())) for section_id in relevant_sections: relevant_reports = StockTransaction.objects.filter(case_id=commtrack_case._id, section_id=section_id) product_ids = _unique_products(relevant_reports) transactions = [StockTransaction.latest(commtrack_case._id, section_id, p) for p in product_ids] as_of = json_format_datetime(max(txn.report.date for txn in transactions)) section_product_map[section_id] = product_ids section_timestamp_map[section_id] = as_of yield E.balance(*(transaction_to_xml(e) for e in transactions), **{'entity-id': commtrack_case._id, 'date': as_of, 'section-id': section_id}) for section_id, consumption_section_id in self.stock_settings.section_to_consumption_types.items(): if (section_id in relevant_sections or self.stock_settings.force_consumption_case_filter(commtrack_case)): consumption_product_ids = self.stock_settings.default_product_list \ if self.stock_settings.default_product_list \ else section_product_map[section_id] yield E.balance( *filter(lambda e: e is not None, [consumption_entry(commtrack_case._id, p, section_id) for p in consumption_product_ids]), **{'entity-id': commtrack_case._id, 'date': section_timestamp_map[section_id], 'section-id': consumption_section_id} )
def sync_stock_transactions_for_facility(domain, endpoint, facility, xform, checkpoint, date, limit=1000, offset=0): """ Syncs stock data from StockTransaction objects in ILSGateway to StockTransaction objects in HQ """ has_next = True next_url = "" section_id = 'stock' supply_point = facility case = get_supply_point_by_external_id(domain, supply_point) if not case: return save_stock_data_checkpoint(checkpoint, 'stock_transaction', limit, offset, date, facility, True) products_saved = set() while has_next: meta, stocktransactions = endpoint.get_stocktransactions(next_url_params=next_url, limit=limit, offset=offset, filters=(dict(supply_point=supply_point, date__gte=date, order_by='date'))) # set the checkpoint right before the data we are about to process meta_limit = meta.get('limit') or limit meta_offset = meta.get('offset') or offset save_stock_data_checkpoint(checkpoint, 'stock_transaction', meta_limit, meta_offset, date, facility, True) transactions_to_add = [] with transaction.commit_on_success(): for stocktransaction in stocktransactions: params = dict( form_id=xform._id, date=force_to_datetime(stocktransaction.date), type='balance', domain=domain, ) try: report, _ = StockReport.objects.get_or_create(**params) except StockReport.MultipleObjectsReturned: # legacy report = StockReport.objects.filter(**params)[0] sql_product = SQLProduct.objects.get(code=stocktransaction.product, domain=domain) if stocktransaction.quantity != 0: transactions_to_add.append(StockTransaction( case_id=case._id, product_id=sql_product.product_id, sql_product=sql_product, section_id=section_id, type='receipts' if stocktransaction.quantity > 0 else 'consumption', stock_on_hand=Decimal(stocktransaction.ending_balance), quantity=Decimal(stocktransaction.quantity), report=report )) transactions_to_add.append(StockTransaction( case_id=case._id, product_id=sql_product.product_id, sql_product=sql_product, section_id=section_id, type='stockonhand', stock_on_hand=Decimal(stocktransaction.ending_balance), report=report )) products_saved.add(sql_product.product_id) if transactions_to_add: # Doesn't send signal StockTransaction.objects.bulk_create(transactions_to_add) if not meta.get('next', False): has_next = False else: next_url = meta['next'].split('?')[1] for product in products_saved: # if we saved anything rebuild the stock state object by firing the signal # on the last transaction for each product last_st = StockTransaction.latest(case._id, section_id, product) update_stock_state_for_transaction(last_st)
def get_latest_transaction(case_id, section_id, entry_id): return StockTransaction.latest(case_id, section_id, entry_id)
def check_stock_models(self, case, product_id, expected_soh, expected_qty, section_id): latest_trans = StockTransaction.latest(case._id, section_id, product_id) self.assertEqual(section_id, latest_trans.section_id) self.assertEqual(expected_soh, latest_trans.stock_on_hand) self.assertEqual(expected_qty, latest_trans.quantity)