def get_current_ledger_value(self, unique_ledger_reference):
latest_txn = StockTransaction.latest(
case_id=unique_ledger_reference.case_id,
section_id=unique_ledger_reference.section_id,
product_id=unique_ledger_reference.entry_id,
)
return latest_txn.stock_on_hand if latest_txn else 0
def get_current_ledger_state(case_ids, ensure_form_id=False):
"""
Given a list of cases returns a dict of all current ledger data of the following format:
{
"case_id": {
"section_id": {
"product_id": StockState,
"product_id": StockState,
...
},
...
},
...
}
:param ensure_form_id: Set to True to make sure return StockState
have the ``last_modified_form_id`` field populated
"""
from corehq.apps.commtrack.models import StockState
if not case_ids:
return {}
states = StockState.objects.filter(case_id__in=case_ids)
ret = {case_id: {} for case_id in case_ids}
for state in states:
sections = ret[state.case_id].setdefault(state.section_id, {})
sections[state.product_id] = state
if ensure_form_id and not state.last_modified_form_id:
transaction = StockTransaction.latest(state.case_id,
state.section_id,
state.product_id)
if transaction is not None:
state.last_modified_form_id = transaction.report.form_id
state.save()
return ret
def get_valid_reports(self, data):
filtered_transactions = []
excluded_products = []
for product_id, transactions in get_transactions_by_product(
data['transactions']).iteritems():
begin_soh = None
end_soh = None
receipt = 0
for transaction in transactions:
if begin_soh is None:
sql_location = SQLLocation.objects.get(
location_id=transaction.location_id)
latest = StockTransaction.latest(
sql_location.supply_point_id, SECTION_TYPE_STOCK,
transaction.product_id)
begin_soh = 0
if latest:
begin_soh = float(latest.stock_on_hand)
if transaction.action == 'receipts':
receipt += float(transaction.quantity)
elif not end_soh:
end_soh = float(transaction.quantity)
if end_soh > begin_soh + receipt:
excluded_products.append(transaction.product_id)
else:
filtered_transactions.append(transaction)
if excluded_products:
message = ERROR_MESSAGE.format(products_list=', '.join([
SQLProduct.objects.get(product_id=product_id).code
for product_id in set(excluded_products)
]))
self.respond(message)
return filtered_transactions
def get_current_ledger_state(case_ids, ensure_form_id=False):
"""
Given a list of cases returns a dict of all current ledger data of the following format:
{
"case_id": {
"section_id": {
"product_id": StockState,
"product_id": StockState,
...
},
...
},
...
}
:param ensure_form_id: Set to True to make sure return StockState
have the ``last_modified_form_id`` field populated
"""
from corehq.apps.commtrack.models import StockState
if not case_ids:
return {}
states = StockState.objects.filter(
case_id__in=case_ids
)
ret = {case_id: {} for case_id in case_ids}
for state in states:
sections = ret[state.case_id].setdefault(state.section_id, {})
sections[state.product_id] = state
if ensure_form_id and not state.last_modified_form_id:
transaction = StockTransaction.latest(state.case_id, state.section_id, state.product_id)
if transaction is not None:
state.last_modified_form_id = transaction.report.form_id
state.save()
return ret
def get_data(self):
sp_ids = get_relevant_supply_point_ids(
self.domain,
self.active_location
)
products = Product.by_domain(self.domain)
if self.program_id:
products = filter(
lambda product: product.program_id == self.program_id, products
)
for sp_id in sp_ids:
for product in products:
loc = SupplyPointCase.get(sp_id).location
last_transaction = StockTransaction.latest(
sp_id,
STOCK_SECTION_TYPE,
product._id
)
yield {
'loc_id': loc._id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type,
'reporting_status': reporting_status(
last_transaction,
self.start_date,
self.end_date
),
'geo': loc._geopoint,
}
def get_valid_reports(self, data):
filtered_transactions = []
excluded_products = []
for product_id, transactions in get_transactions_by_product(data["transactions"]).iteritems():
begin_soh = None
end_soh = None
receipt = 0
for transaction in transactions:
if begin_soh is None:
sql_location = SQLLocation.objects.get(location_id=transaction.location_id)
latest = StockTransaction.latest(
sql_location.supply_point_id, SECTION_TYPE_STOCK, transaction.product_id
)
begin_soh = 0
if latest:
begin_soh = float(latest.stock_on_hand)
if transaction.action == "receipts":
receipt += float(transaction.quantity)
elif not end_soh:
end_soh = float(transaction.quantity)
if end_soh > begin_soh + receipt:
excluded_products.append(transaction.product_id)
else:
filtered_transactions.append(transaction)
if excluded_products:
message = ERROR_MESSAGE.format(
products_list=", ".join(
[SQLProduct.objects.get(product_id=product_id).code for product_id in set(excluded_products)]
)
)
self.respond(message)
return filtered_transactions
def get_stock_state_for_transaction(transaction):
from corehq.apps.commtrack.models import StockState
from corehq.apps.locations.models import SQLLocation
from corehq.apps.products.models import SQLProduct
# todo: in the worst case, this function makes
# - three calls to couch (for the case, domain, and commtrack config)
# - four postgres queries (transacitons, product, location, and state)
# - one postgres write (to save the state)
# and that doesn't even include the consumption calc, which can do a whole
# bunch more work and hit the database.
sql_product = SQLProduct.objects.get(product_id=transaction.product_id)
try:
domain_name = transaction.__domain
except AttributeError:
domain_name = sql_product.domain
try:
sql_location = SQLLocation.objects.get(supply_point_id=transaction.case_id)
except SQLLocation.DoesNotExist:
sql_location = None
try:
state = StockState.include_archived.get(
section_id=transaction.section_id,
case_id=transaction.case_id,
product_id=transaction.product_id,
)
except StockState.DoesNotExist:
state = StockState(
section_id=transaction.section_id,
case_id=transaction.case_id,
product_id=transaction.product_id,
sql_product=sql_product,
sql_location=sql_location,
)
# we may not be saving the latest transaction so make sure we use that
# todo: this should change to server date
latest_transaction = StockTransaction.latest(
case_id=transaction.case_id,
section_id=transaction.section_id,
product_id=transaction.product_id
)
if latest_transaction != transaction:
logging.warning(
'Just fired signal for a stale stock transaction. Domain: {}, instance: {},latest was {}'.format(
domain_name, transaction, latest_transaction
)
)
transaction = latest_transaction
state.last_modified_date = transaction.report.server_date
state.last_modified_form_id = transaction.report.form_id
state.stock_on_hand = transaction.stock_on_hand
# so you don't have to look it up again in the signal receivers
if domain_name:
state.__domain = domain_name
return state
def _get_ledger(self, unique_ledger_reference):
try:
return StockTransaction.latest(
case_id=unique_ledger_reference.case_id,
section_id=unique_ledger_reference.section_id,
product_id=unique_ledger_reference.entry_id,
)
except StockTransaction.DoesNotExist:
return None
def _get_ledger(self, unique_ledger_reference):
try:
return StockTransaction.latest(
case_id=unique_ledger_reference.case_id,
section_id=unique_ledger_reference.section_id,
product_id=unique_ledger_reference.entry_id,
)
except StockTransaction.DoesNotExist:
return None
def check_stock_models(self, case, product_id, expected_soh, expected_qty, section_id):
if not isinstance(expected_qty, Decimal):
expected_qty = Decimal(str(expected_qty))
if not isinstance(expected_soh, Decimal):
expected_soh = Decimal(str(expected_soh))
latest_trans = StockTransaction.latest(case._id, section_id, product_id)
self.assertIsNotNone(latest_trans)
self.assertEqual(section_id, latest_trans.section_id)
self.assertEqual(expected_soh, latest_trans.stock_on_hand)
self.assertEqual(expected_qty, latest_trans.quantity)
def sync_stock_transactions_for_facility(domain, endpoint, facility, checkpoint,
date, limit=1000, offset=0):
"""
Syncs stock data from StockTransaction objects in ILSGateway to StockTransaction objects in HQ
"""
has_next = True
next_url = ""
section_id = 'stock'
supply_point = facility
case = get_supply_point_case_in_domain_by_id(domain, supply_point)
if not case:
return
location_id = case.location_id
save_stock_data_checkpoint(checkpoint, 'stock_transaction', limit, offset, date, location_id, True)
products_saved = set()
while has_next:
meta, stocktransactions = endpoint.get_stocktransactions(
next_url_params=next_url,
limit=limit,
offset=offset,
filters={
'supply_point': supply_point,
'date__gte': date,
'date__lte': checkpoint.start_date
}
)
# set the checkpoint right before the data we are about to process
meta_limit = meta.get('limit') or limit
meta_offset = meta.get('offset') or offset
save_stock_data_checkpoint(
checkpoint, 'stock_transaction', meta_limit, meta_offset, date, location_id, True
)
transactions_to_add = []
with transaction.atomic():
for stocktransaction in stocktransactions:
transactions = sync_stock_transaction(stocktransaction, domain, case, bulk=True)
transactions_to_add.extend(transactions)
products_saved.update(map(lambda x: x.product_id, transactions))
if transactions_to_add:
# Doesn't send signal
StockTransaction.objects.bulk_create(transactions_to_add)
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
for product in products_saved:
# if we saved anything rebuild the stock state object by firing the signal
# on the last transaction for each product
last_st = StockTransaction.latest(case.get_id, section_id, product)
update_stock_state_for_transaction(last_st)
def get_current_ledger_transactions(case_id):
"""
Given a case returns a dict of all current ledger data.
{
"section_id": {
"product_id": StockTransaction,
"product_id": StockTransaction,
...
},
...
}
"""
from corehq.apps.commtrack.models import StockState
results = StockState.objects.filter(case_id=case_id).values_list('case_id', 'section_id', 'product_id')
ret = {}
for case_id, section_id, product_id in results:
sections = ret.setdefault(section_id, {})
sections[product_id] = StockTransaction.latest(case_id, section_id, product_id)
return ret
def get_current_ledger_transactions(case_id):
"""
Given a case returns a dict of all current ledger data.
{
"section_id": {
"product_id": StockTransaction,
"product_id": StockTransaction,
...
},
...
}
"""
from corehq.apps.commtrack.models import StockState
results = StockState.objects.filter(case_id=case_id).values_list(
'case_id', 'section_id', 'product_id')
ret = {}
for case_id, section_id, product_id in results:
sections = ret.setdefault(section_id, {})
sections[product_id] = StockTransaction.latest(case_id, section_id,
product_id)
return ret
def get_latest_transaction(case_id, section_id, entry_id):
return StockTransaction.latest(case_id, section_id, entry_id)
def _get_stats(self):
stock_state = StockState.objects.get(**self._stock_state_key)
latest_txn = StockTransaction.latest(**self._stock_state_key)
all_txns = StockTransaction.get_ordered_transactions_for_stock(
**self._stock_state_key)
return stock_state, latest_txn, all_txns
def get_stock_payload(self, syncop):
cases = [e.case for e in syncop.actual_cases_to_sync]
from lxml.builder import ElementMaker
E = ElementMaker(namespace=COMMTRACK_REPORT_XMLNS)
def entry_xml(id, quantity):
return E.entry(
id=id,
quantity=str(int(quantity)),
)
def transaction_to_xml(trans):
return entry_xml(trans.product_id, trans.stock_on_hand)
def consumption_entry(case_id, product_id, section_id):
consumption_value = compute_consumption_or_default(
case_id,
product_id,
datetime.utcnow(),
section_id,
self.stock_settings.consumption_config
)
if consumption_value is not None:
return entry_xml(product_id, consumption_value)
def _unique_products(stock_transaction_queryset):
return sorted(stock_transaction_queryset.values_list('product_id', flat=True).distinct())
for commtrack_case in cases:
relevant_sections = sorted(StockTransaction.objects.filter(
case_id=commtrack_case._id).values_list('section_id', flat=True).distinct())
section_product_map = defaultdict(lambda: [])
section_timestamp_map = defaultdict(lambda: json_format_datetime(datetime.utcnow()))
for section_id in relevant_sections:
relevant_reports = StockTransaction.objects.filter(case_id=commtrack_case._id, section_id=section_id)
product_ids = _unique_products(relevant_reports)
transactions = [StockTransaction.latest(commtrack_case._id, section_id, p) for p in product_ids]
as_of = json_format_datetime(max(txn.report.date for txn in transactions))
section_product_map[section_id] = product_ids
section_timestamp_map[section_id] = as_of
yield E.balance(*(transaction_to_xml(e) for e in transactions),
**{'entity-id': commtrack_case._id, 'date': as_of, 'section-id': section_id})
for section_id, consumption_section_id in self.stock_settings.section_to_consumption_types.items():
if (section_id in relevant_sections or
self.stock_settings.force_consumption_case_filter(commtrack_case)):
consumption_product_ids = self.stock_settings.default_product_list \
if self.stock_settings.default_product_list \
else section_product_map[section_id]
consumption_entries = filter(lambda e: e is not None, [
consumption_entry(commtrack_case._id, p, section_id)
for p in consumption_product_ids
])
if consumption_entries:
yield E.balance(
*consumption_entries,
**{
'entity-id': commtrack_case._id,
'date': section_timestamp_map[section_id],
'section-id': consumption_section_id,
}
)
def _get_stats(self):
stock_state = StockState.objects.get(**self._stock_state_key)
latest_txn = StockTransaction.latest(**self._stock_state_key)
all_txns = StockTransaction.get_ordered_transactions_for_stock(
**self._stock_state_key)
return stock_state, latest_txn, all_txns
def get_stock_payload(self, syncop):
cases = [e.case for e in syncop.actual_cases_to_sync]
from lxml.builder import ElementMaker
E = ElementMaker(namespace=COMMTRACK_REPORT_XMLNS)
def entry_xml(id, quantity):
return E.entry(
id=id,
quantity=str(int(quantity)),
)
def transaction_to_xml(trans):
return entry_xml(trans.product_id, trans.stock_on_hand)
def consumption_entry(case_id, product_id, section_id):
consumption_value = compute_consumption_or_default(
case_id,
product_id,
datetime.utcnow(),
section_id,
self.stock_settings.consumption_config
)
if consumption_value is not None:
return entry_xml(product_id, consumption_value)
def _unique_products(stock_transaction_queryset):
return sorted(stock_transaction_queryset.values_list('product_id', flat=True).distinct())
for commtrack_case in cases:
relevant_sections = sorted(StockTransaction.objects.filter(
case_id=commtrack_case._id).values_list('section_id', flat=True).distinct())
section_product_map = defaultdict(lambda: [])
section_timestamp_map = defaultdict(lambda: json_format_datetime(datetime.utcnow()))
for section_id in relevant_sections:
relevant_reports = StockTransaction.objects.filter(case_id=commtrack_case._id, section_id=section_id)
product_ids = _unique_products(relevant_reports)
transactions = [StockTransaction.latest(commtrack_case._id, section_id, p) for p in product_ids]
as_of = json_format_datetime(max(txn.report.date for txn in transactions))
section_product_map[section_id] = product_ids
section_timestamp_map[section_id] = as_of
yield E.balance(*(transaction_to_xml(e) for e in transactions),
**{'entity-id': commtrack_case._id, 'date': as_of, 'section-id': section_id})
for section_id, consumption_section_id in self.stock_settings.section_to_consumption_types.items():
if (section_id in relevant_sections or
self.stock_settings.force_consumption_case_filter(commtrack_case)):
consumption_product_ids = self.stock_settings.default_product_list \
if self.stock_settings.default_product_list \
else section_product_map[section_id]
yield E.balance(
*filter(lambda e: e is not None,
[consumption_entry(commtrack_case._id, p, section_id)
for p in consumption_product_ids]),
**{'entity-id': commtrack_case._id, 'date': section_timestamp_map[section_id],
'section-id': consumption_section_id}
)
def sync_stock_transactions_for_facility(domain, endpoint, facility, xform, checkpoint,
date, limit=1000, offset=0):
"""
Syncs stock data from StockTransaction objects in ILSGateway to StockTransaction objects in HQ
"""
has_next = True
next_url = ""
section_id = 'stock'
supply_point = facility
case = get_supply_point_by_external_id(domain, supply_point)
if not case:
return
save_stock_data_checkpoint(checkpoint, 'stock_transaction', limit, offset, date, facility, True)
products_saved = set()
while has_next:
meta, stocktransactions = endpoint.get_stocktransactions(next_url_params=next_url,
limit=limit,
offset=offset,
filters=(dict(supply_point=supply_point,
date__gte=date,
order_by='date')))
# set the checkpoint right before the data we are about to process
meta_limit = meta.get('limit') or limit
meta_offset = meta.get('offset') or offset
save_stock_data_checkpoint(checkpoint, 'stock_transaction', meta_limit, meta_offset, date, facility, True)
transactions_to_add = []
with transaction.commit_on_success():
for stocktransaction in stocktransactions:
params = dict(
form_id=xform._id,
date=force_to_datetime(stocktransaction.date),
type='balance',
domain=domain,
)
try:
report, _ = StockReport.objects.get_or_create(**params)
except StockReport.MultipleObjectsReturned:
# legacy
report = StockReport.objects.filter(**params)[0]
sql_product = SQLProduct.objects.get(code=stocktransaction.product, domain=domain)
if stocktransaction.quantity != 0:
transactions_to_add.append(StockTransaction(
case_id=case._id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type='receipts' if stocktransaction.quantity > 0 else 'consumption',
stock_on_hand=Decimal(stocktransaction.ending_balance),
quantity=Decimal(stocktransaction.quantity),
report=report
))
transactions_to_add.append(StockTransaction(
case_id=case._id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type='stockonhand',
stock_on_hand=Decimal(stocktransaction.ending_balance),
report=report
))
products_saved.add(sql_product.product_id)
if transactions_to_add:
# Doesn't send signal
StockTransaction.objects.bulk_create(transactions_to_add)
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
for product in products_saved:
# if we saved anything rebuild the stock state object by firing the signal
# on the last transaction for each product
last_st = StockTransaction.latest(case._id, section_id, product)
update_stock_state_for_transaction(last_st)
def get_latest_transaction(case_id, section_id, entry_id):
return StockTransaction.latest(case_id, section_id, entry_id)
def check_stock_models(self, case, product_id, expected_soh, expected_qty, section_id):
latest_trans = StockTransaction.latest(case._id, section_id, product_id)
self.assertEqual(section_id, latest_trans.section_id)
self.assertEqual(expected_soh, latest_trans.stock_on_hand)
self.assertEqual(expected_qty, latest_trans.quantity)
