def _test_subtype(self, initial, final):
case_id = uuid.uuid4().hex
CommCareCase(
_id=case_id,
domain='fakedomain',
).save()
product_id = uuid.uuid4().hex
SQLProduct(product_id=product_id, domain='fakedomain').save()
report = StockReport.objects.create(
form_id=uuid.uuid4().hex,
date=ago(1),
server_date=datetime.utcnow(),
type=const.REPORT_TYPE_BALANCE
)
txn = StockTransaction(
report=report,
section_id=const.SECTION_TYPE_STOCK,
type=const.TRANSACTION_TYPE_STOCKONHAND,
subtype=initial,
case_id=case_id,
product_id=product_id,
stock_on_hand=Decimal(10),
)
txn.save()
saved = StockTransaction.objects.get(id=txn.id)
self.assertEqual(final, saved.subtype)
def _get_model_for_stock_transaction(report, transaction_helper, ledger_db):
assert report.type in const.VALID_REPORT_TYPES
txn = StockTransaction(
report=report,
case_id=transaction_helper.case_id,
section_id=transaction_helper.section_id,
product_id=transaction_helper.product_id,
type=transaction_helper.action,
subtype=transaction_helper.subaction,
)
def lazy_original_balance():
return ledger_db.get_current_balance(_stock_transaction_to_unique_ledger_reference(txn))
new_ledger_values = compute_ledger_values(
lazy_original_balance, report.type,
transaction_helper.relative_quantity)
txn.stock_on_hand = new_ledger_values.balance
txn.quantity = new_ledger_values.delta
if report.domain:
# set this as a shortcut for post save signal receivers
txn.domain = report.domain
# update the ledger DB in case later transactions reference the same ledger item
ledger_db.set_current_balance(_stock_transaction_to_unique_ledger_reference(txn), txn.stock_on_hand)
return txn
def _test_subtype(self, initial, final):
case_id = uuid.uuid4().hex
CommCareCase(
_id=case_id,
domain='fakedomain',
).save()
product_id = uuid.uuid4().hex
SQLProduct(product_id=product_id, domain='fakedomain').save()
report = StockReport.objects.create(form_id=uuid.uuid4().hex,
date=ago(1),
server_date=datetime.utcnow(),
type=const.REPORT_TYPE_BALANCE)
txn = StockTransaction(
report=report,
section_id=const.SECTION_TYPE_STOCK,
type=const.TRANSACTION_TYPE_STOCKONHAND,
subtype=initial,
case_id=case_id,
product_id=product_id,
stock_on_hand=Decimal(10),
)
txn.save()
saved = StockTransaction.objects.get(id=txn.id)
self.assertEqual(final, saved.subtype)
def _get_model_for_stock_transaction(report, transaction_helper, ledger_db):
assert report.type in const.VALID_REPORT_TYPES
txn = StockTransaction(
report=report,
case_id=transaction_helper.case_id,
section_id=transaction_helper.section_id,
product_id=transaction_helper.product_id,
type=transaction_helper.action,
subtype=transaction_helper.subaction,
)
def lazy_original_balance():
return ledger_db.get_current_ledger_value(txn.ledger_reference)
new_ledger_values = compute_ledger_values(
lazy_original_balance, report.type,
transaction_helper.relative_quantity)
txn.stock_on_hand = new_ledger_values.balance
txn.quantity = new_ledger_values.delta
if report.domain:
# set this as a shortcut for post save signal receivers
txn.__domain = report.domain
# update the ledger DB in case later transactions reference the same ledger item
ledger_db.set_ledger(txn)
return txn
def _stock_report(case_id, product_id, amount, days_ago):
report = StockReport.objects.create(form_id=uuid.uuid4().hex, date=ago(days_ago), type=const.REPORT_TYPE_BALANCE)
txn = StockTransaction(
report=report,
section_id=const.SECTION_TYPE_STOCK,
type=const.TRANSACTION_TYPE_STOCKONHAND,
case_id=case_id,
product_id=product_id,
stock_on_hand=Decimal(amount),
)
txn._test_config = ConsumptionConfiguration.test_config()
txn.quantity = 0
txn.save()
def _receipt_report(case_id, product_id, amount, days_ago):
report = StockReport.objects.create(form_id=uuid.uuid4().hex, date=ago(days_ago), type=const.REPORT_TYPE_TRANSFER)
txn = StockTransaction(
report=report,
section_id=const.SECTION_TYPE_STOCK,
type=const.TRANSACTION_TYPE_RECEIPTS,
case_id=case_id,
product_id=product_id,
quantity=amount,
)
previous_transaction = txn.get_previous_transaction()
txn.stock_on_hand = (previous_transaction.stock_on_hand if previous_transaction else 0) + txn.quantity
txn.save()
def _create_stock_state(self, product, consumption):
xform = XFormInstance.get('test-xform')
loc = Location.by_site_code(TEST_DOMAIN, 'garms')
now = datetime.datetime.utcnow()
report = StockReport(form_id=xform._id,
date=(now - datetime.timedelta(days=10)).replace(
second=0, microsecond=0),
type='balance',
domain=TEST_DOMAIN)
report.save()
stock_transaction = StockTransaction(
case_id=loc.linked_supply_point().get_id,
product_id=product.get_id,
sql_product=SQLProduct.objects.get(product_id=product.get_id),
section_id='stock',
type='stockonhand',
stock_on_hand=2 * consumption,
report=report)
stock_transaction.save()
report = StockReport(form_id=xform._id,
date=now.replace(second=0, microsecond=0),
type='balance',
domain=TEST_DOMAIN)
report.save()
stock_transaction = StockTransaction(
case_id=loc.linked_supply_point().get_id,
product_id=product.get_id,
sql_product=SQLProduct.objects.get(product_id=product.get_id),
section_id='stock',
type='stockonhand',
stock_on_hand=consumption,
report=report)
stock_transaction.save()
def create_transactions(self, domain=None):
report = StockReport.objects.create(form_id=uuid.uuid4().hex,
date=ago(2),
type=const.REPORT_TYPE_BALANCE,
domain=domain)
txn = StockTransaction(
report=report,
section_id=const.SECTION_TYPE_STOCK,
type=const.TRANSACTION_TYPE_STOCKONHAND,
case_id=self.case_id,
product_id=self.product_id,
stock_on_hand=Decimal(10),
)
txn.save()
report2 = StockReport.objects.create(form_id=uuid.uuid4().hex,
date=ago(1),
type=const.REPORT_TYPE_BALANCE,
domain=domain)
txn2 = StockTransaction(
report=report2,
section_id=const.SECTION_TYPE_STOCK,
type=const.TRANSACTION_TYPE_STOCKONHAND,
case_id=self.case_id,
product_id=self.product_id,
stock_on_hand=Decimal(30),
)
txn2.save()
def get_valid_reports(self, data):
filtered_transactions = []
excluded_products = []
for product_id, transactions in get_transactions_by_product(data["transactions"]).iteritems():
begin_soh = None
end_soh = None
receipt = 0
for transaction in transactions:
if begin_soh is None:
sql_location = SQLLocation.objects.get(location_id=transaction.location_id)
latest = StockTransaction.latest(
sql_location.supply_point_id, SECTION_TYPE_STOCK, transaction.product_id
)
begin_soh = 0
if latest:
begin_soh = float(latest.stock_on_hand)
if transaction.action == "receipts":
receipt += float(transaction.quantity)
elif not end_soh:
end_soh = float(transaction.quantity)
if end_soh > begin_soh + receipt:
excluded_products.append(transaction.product_id)
else:
filtered_transactions.append(transaction)
if excluded_products:
message = ERROR_MESSAGE.format(
products_list=", ".join(
[SQLProduct.objects.get(product_id=product_id).code for product_id in set(excluded_products)]
)
)
self.respond(message)
return filtered_transactions
def plan_rebuild_stock_state(case_id, section_id, product_id):
"""
planner for rebuild_stock_state
yields actions for rebuild_stock_state to take,
facilitating doing a dry run
Warning: since some important things are still done through signals
rather than here explicitly, there may be some effects that aren't
represented in the plan. For example, inferred transaction creation
will not be represented, nor will updates to the StockState object.
"""
# these come out latest first, so reverse them below
stock_transactions = (
StockTransaction.get_ordered_transactions_for_stock(
case_id=case_id, section_id=section_id, product_id=product_id).
reverse() # we want earliest transactions first
.select_related('report'))
balance = None
for stock_transaction in stock_transactions:
if stock_transaction.subtype == stockconst.TRANSACTION_SUBTYPE_INFERRED:
yield _DeleteStockTransaction(stock_transaction)
else:
before = LedgerValues(balance=stock_transaction.stock_on_hand,
delta=stock_transaction.quantity)
after = _compute_ledger_values(balance, stock_transaction)
# update balance for the next iteration
balance = after.balance
yield _SaveStockTransaction(stock_transaction, before, after)
def get_current_ledger_state(case_ids, ensure_form_id=False):
"""
Given a list of cases returns a dict of all current ledger data of the following format:
{
"case_id": {
"section_id": {
"product_id": StockState,
"product_id": StockState,
...
},
...
},
...
}
:param ensure_form_id: Set to True to make sure return StockState
have the ``last_modified_form_id`` field populated
"""
from corehq.apps.commtrack.models import StockState
if not case_ids:
return {}
states = StockState.objects.filter(
case_id__in=case_ids
)
ret = {case_id: {} for case_id in case_ids}
for state in states:
sections = ret[state.case_id].setdefault(state.section_id, {})
sections[state.product_id] = state
if ensure_form_id and not state.last_modified_form_id:
transaction = StockTransaction.latest(state.case_id, state.section_id, state.product_id)
if transaction is not None:
state.last_modified_form_id = transaction.report.form_id
state.save()
return ret
def get_valid_reports(self, data):
filtered_transactions = []
excluded_products = []
for product_id, transactions in get_transactions_by_product(
data['transactions']).iteritems():
begin_soh = None
end_soh = None
receipt = 0
for transaction in transactions:
if begin_soh is None:
sql_location = SQLLocation.objects.get(
location_id=transaction.location_id)
latest = StockTransaction.latest(
sql_location.supply_point_id, SECTION_TYPE_STOCK,
transaction.product_id)
begin_soh = 0
if latest:
begin_soh = float(latest.stock_on_hand)
if transaction.action == 'receipts':
receipt += float(transaction.quantity)
elif not end_soh:
end_soh = float(transaction.quantity)
if end_soh > begin_soh + receipt:
excluded_products.append(transaction.product_id)
else:
filtered_transactions.append(transaction)
if excluded_products:
message = ERROR_MESSAGE.format(products_list=', '.join([
SQLProduct.objects.get(product_id=product_id).code
for product_id in set(excluded_products)
]))
self.respond(message)
return filtered_transactions
def get_current_ledger_value(self, unique_ledger_reference):
latest_txn = StockTransaction.latest(
case_id=unique_ledger_reference.case_id,
section_id=unique_ledger_reference.section_id,
product_id=unique_ledger_reference.entry_id,
)
return latest_txn.stock_on_hand if latest_txn else 0
def get_data(self):
sp_ids = get_relevant_supply_point_ids(
self.domain,
self.active_location
)
products = Product.by_domain(self.domain)
if self.program_id:
products = filter(
lambda product: product.program_id == self.program_id, products
)
for sp_id in sp_ids:
for product in products:
loc = SupplyPointCase.get(sp_id).location
last_transaction = StockTransaction.latest(
sp_id,
STOCK_SECTION_TYPE,
product._id
)
yield {
'loc_id': loc._id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type,
'reporting_status': reporting_status(
last_transaction,
self.start_date,
self.end_date
),
'geo': loc._geopoint,
}
def get_current_ledger_state(case_ids, ensure_form_id=False):
"""
Given a list of cases returns a dict of all current ledger data of the following format:
{
"case_id": {
"section_id": {
"product_id": StockState,
"product_id": StockState,
...
},
...
},
...
}
:param ensure_form_id: Set to True to make sure return StockState
have the ``last_modified_form_id`` field populated
"""
from corehq.apps.commtrack.models import StockState
if not case_ids:
return {}
states = StockState.objects.filter(case_id__in=case_ids)
ret = {case_id: {} for case_id in case_ids}
for state in states:
sections = ret[state.case_id].setdefault(state.section_id, {})
sections[state.product_id] = state
if ensure_form_id and not state.last_modified_form_id:
transaction = StockTransaction.latest(state.case_id,
state.section_id,
state.product_id)
if transaction is not None:
state.last_modified_form_id = transaction.report.form_id
state.save()
return ret
def get_stock_state_for_transaction(transaction):
from corehq.apps.commtrack.models import StockState
from corehq.apps.locations.models import SQLLocation
from corehq.apps.products.models import SQLProduct
# todo: in the worst case, this function makes
# - three calls to couch (for the case, domain, and commtrack config)
# - four postgres queries (transacitons, product, location, and state)
# - one postgres write (to save the state)
# and that doesn't even include the consumption calc, which can do a whole
# bunch more work and hit the database.
sql_product = SQLProduct.objects.get(product_id=transaction.product_id)
try:
domain_name = transaction.__domain
except AttributeError:
domain_name = sql_product.domain
try:
sql_location = SQLLocation.objects.get(supply_point_id=transaction.case_id)
except SQLLocation.DoesNotExist:
sql_location = None
try:
state = StockState.include_archived.get(
section_id=transaction.section_id,
case_id=transaction.case_id,
product_id=transaction.product_id,
)
except StockState.DoesNotExist:
state = StockState(
section_id=transaction.section_id,
case_id=transaction.case_id,
product_id=transaction.product_id,
sql_product=sql_product,
sql_location=sql_location,
)
# we may not be saving the latest transaction so make sure we use that
# todo: this should change to server date
latest_transaction = StockTransaction.latest(
case_id=transaction.case_id,
section_id=transaction.section_id,
product_id=transaction.product_id
)
if latest_transaction != transaction:
logging.warning(
'Just fired signal for a stale stock transaction. Domain: {}, instance: {},latest was {}'.format(
domain_name, transaction, latest_transaction
)
)
transaction = latest_transaction
state.last_modified_date = transaction.report.server_date
state.last_modified_form_id = transaction.report.form_id
state.stock_on_hand = transaction.stock_on_hand
# so you don't have to look it up again in the signal receivers
if domain_name:
state.__domain = domain_name
return state
def _create_model_for_stock_transaction(report, transaction_helper):
assert report.type in stockconst.VALID_REPORT_TYPES
txn = StockTransaction(
report=report,
case_id=transaction_helper.case_id,
section_id=transaction_helper.section_id,
product_id=transaction_helper.product_id,
type=transaction_helper.action,
subtype=transaction_helper.subaction,
)
def lazy_original_balance():
previous_transaction = txn.get_previous_transaction()
if previous_transaction:
return previous_transaction.stock_on_hand
else:
return None
new_ledger_values = compute_ledger_values(
lazy_original_balance, report.type,
transaction_helper.relative_quantity)
txn.stock_on_hand = new_ledger_values.balance
txn.quantity = new_ledger_values.delta
if report.domain:
# set this as a shortcut for post save signal receivers
txn.domain = report.domain
txn.save()
return txn
def get_couch_transactions(ref):
return [
ledger_transaction_json(tx) for tx in reversed(
StockTransaction.get_ordered_transactions_for_stock(
case_id=ref.case_id,
section_id=ref.section_id,
product_id=ref.entry_id,
).select_related("report"))
]
def _get_ledger(self, unique_ledger_reference):
try:
return StockTransaction.latest(
case_id=unique_ledger_reference.case_id,
section_id=unique_ledger_reference.section_id,
product_id=unique_ledger_reference.entry_id,
)
except StockTransaction.DoesNotExist:
return None
def _get_ledger(self, unique_ledger_reference):
try:
return StockTransaction.latest(
case_id=unique_ledger_reference.case_id,
section_id=unique_ledger_reference.section_id,
product_id=unique_ledger_reference.entry_id,
)
except StockTransaction.DoesNotExist:
return None
def check_stock_models(self, case, product_id, expected_soh, expected_qty, section_id):
if not isinstance(expected_qty, Decimal):
expected_qty = Decimal(str(expected_qty))
if not isinstance(expected_soh, Decimal):
expected_soh = Decimal(str(expected_soh))
latest_trans = StockTransaction.latest(case._id, section_id, product_id)
self.assertIsNotNone(latest_trans)
self.assertEqual(section_id, latest_trans.section_id)
self.assertEqual(expected_soh, latest_trans.stock_on_hand)
self.assertEqual(expected_qty, latest_trans.quantity)
def sync_stock_transactions_for_facility(domain, endpoint, facility, checkpoint,
date, limit=1000, offset=0):
"""
Syncs stock data from StockTransaction objects in ILSGateway to StockTransaction objects in HQ
"""
has_next = True
next_url = ""
section_id = 'stock'
supply_point = facility
case = get_supply_point_case_in_domain_by_id(domain, supply_point)
if not case:
return
location_id = case.location_id
save_stock_data_checkpoint(checkpoint, 'stock_transaction', limit, offset, date, location_id, True)
products_saved = set()
while has_next:
meta, stocktransactions = endpoint.get_stocktransactions(
next_url_params=next_url,
limit=limit,
offset=offset,
filters={
'supply_point': supply_point,
'date__gte': date,
'date__lte': checkpoint.start_date
}
)
# set the checkpoint right before the data we are about to process
meta_limit = meta.get('limit') or limit
meta_offset = meta.get('offset') or offset
save_stock_data_checkpoint(
checkpoint, 'stock_transaction', meta_limit, meta_offset, date, location_id, True
)
transactions_to_add = []
with transaction.atomic():
for stocktransaction in stocktransactions:
transactions = sync_stock_transaction(stocktransaction, domain, case, bulk=True)
transactions_to_add.extend(transactions)
products_saved.update(map(lambda x: x.product_id, transactions))
if transactions_to_add:
# Doesn't send signal
StockTransaction.objects.bulk_create(transactions_to_add)
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
for product in products_saved:
# if we saved anything rebuild the stock state object by firing the signal
# on the last transaction for each product
last_st = StockTransaction.latest(case.get_id, section_id, product)
update_stock_state_for_transaction(last_st)
def _create_stock_state(self, product, consumption):
xform = XFormInstance.get('test-xform')
loc = Location.by_site_code(TEST_DOMAIN, 'garms')
now = datetime.datetime.utcnow()
report = StockReport(
form_id=xform._id,
date=(now - datetime.timedelta(days=10)).replace(second=0, microsecond=0),
type='balance',
domain=TEST_DOMAIN
)
report.save()
stock_transaction = StockTransaction(
case_id=loc.linked_supply_point().get_id,
product_id=product.get_id,
sql_product=SQLProduct.objects.get(product_id=product.get_id),
section_id='stock',
type='stockonhand',
stock_on_hand=2 * consumption,
report=report
)
stock_transaction.save()
def create_transactions(self, domain=None):
report = StockReport.objects.create(
form_id=uuid.uuid4().hex,
date=ago(2),
type=const.REPORT_TYPE_BALANCE,
domain=domain
)
txn = StockTransaction(
report=report,
section_id=const.SECTION_TYPE_STOCK,
type=const.TRANSACTION_TYPE_STOCKONHAND,
case_id=self.case_id,
product_id=self.product_id,
stock_on_hand=Decimal(10),
)
txn.save()
report2 = StockReport.objects.create(
form_id=uuid.uuid4().hex,
date=ago(1),
type=const.REPORT_TYPE_BALANCE,
domain=domain
)
txn2 = StockTransaction(
report=report2,
section_id=const.SECTION_TYPE_STOCK,
type=const.TRANSACTION_TYPE_STOCKONHAND,
case_id=self.case_id,
product_id=self.product_id,
stock_on_hand=Decimal(30),
)
txn2.save()
def create_models(self):
# todo: this function should probably move to somewhere in casexml.apps.stock
if self.tag not in stockconst.VALID_REPORT_TYPES:
return
report = DbStockReport.objects.create(form_id=self.form_id, date=self.timestamp, type=self.tag)
for txn in self.transactions:
db_txn = DbStockTransaction(
report=report,
case_id=txn.case_id,
section_id=txn.section_id,
product_id=txn.product_id,
)
previous_transaction = db_txn.get_previous_transaction()
db_txn.type = txn.action
db_txn.subtype = txn.subaction
if self.tag == stockconst.REPORT_TYPE_BALANCE:
db_txn.stock_on_hand = txn.quantity
db_txn.quantity = 0
else:
assert self.tag == stockconst.REPORT_TYPE_TRANSFER
db_txn.quantity = txn.relative_quantity
db_txn.stock_on_hand = (previous_transaction.stock_on_hand if previous_transaction else 0) + db_txn.quantity
db_txn.save()
def create_stock_report(location, products_quantities, date=None):
date = date or datetime.utcnow()
sql_location = location.sql_location
report = StockReport.objects.create(form_id='ews-reminders-test',
domain=sql_location.domain,
type='balance',
date=date,
server_date=date)
for product_code, quantity in products_quantities.iteritems():
StockTransaction(stock_on_hand=Decimal(quantity),
report=report,
type='stockonhand',
section_id='stock',
case_id=sql_location.supply_point_id,
product_id=SQLProduct.objects.get(
domain=sql_location.domain,
code=product_code).product_id).save()
def recalculate_domain_consumption(domain):
"""
Given a domain, recalculate all saved consumption settings in that domain.
"""
# note: might get slow as this gets huge
found_doc_ids = DocDomainMapping.objects.filter(
domain_name=domain,
doc_type='CommCareCase',
).values_list('doc_id', flat=True)
products = Product.by_domain(domain)
for supply_point_id in found_doc_ids:
for product in products:
filtered_transactions = StockTransaction.get_ordered_transactions_for_stock(
supply_point_id, const.SECTION_TYPE_STOCK, product._id
)
if filtered_transactions:
update_stock_state_for_transaction(filtered_transactions[0])
def _stock_report(case_id, product_id, amount, days_ago):
report = StockReport.objects.create(form_id=uuid.uuid4().hex, date=ago(days_ago),
type=const.REPORT_TYPE_BALANCE)
txn = StockTransaction(
report=report,
section_id=const.SECTION_TYPE_STOCK,
type=const.TRANSACTION_TYPE_STOCKONHAND,
case_id=case_id,
product_id=product_id,
stock_on_hand=Decimal(amount),
)
txn._test_config = ConsumptionConfiguration.test_config()
txn.quantity = 0
txn.save()
def _receipt_report(case_id, product_id, amount, days_ago):
report = StockReport.objects.create(form_id=uuid.uuid4().hex, date=ago(days_ago),
type=const.REPORT_TYPE_TRANSFER)
txn = StockTransaction(
report=report,
section_id=const.SECTION_TYPE_STOCK,
type=const.TRANSACTION_TYPE_RECEIPTS,
case_id=case_id,
product_id=product_id,
quantity=amount,
)
previous_transaction = txn.get_previous_transaction()
txn.stock_on_hand = (previous_transaction.stock_on_hand if previous_transaction else 0) + txn.quantity
txn.save()
def get_current_ledger_transactions(case_id):
"""
Given a case returns a dict of all current ledger data.
{
"section_id": {
"product_id": StockTransaction,
"product_id": StockTransaction,
...
},
...
}
"""
from corehq.apps.commtrack.models import StockState
results = StockState.objects.filter(case_id=case_id).values_list('case_id', 'section_id', 'product_id')
ret = {}
for case_id, section_id, product_id in results:
sections = ret.setdefault(section_id, {})
sections[product_id] = StockTransaction.latest(case_id, section_id, product_id)
return ret
def get_current_ledger_transactions(case_id):
"""
Given a case returns a dict of all current ledger data.
{
"section_id": {
"product_id": StockTransaction,
"product_id": StockTransaction,
...
},
...
}
"""
from corehq.apps.commtrack.models import StockState
results = StockState.objects.filter(case_id=case_id).values_list(
'case_id', 'section_id', 'product_id')
ret = {}
for case_id, section_id, product_id in results:
sections = ret.setdefault(section_id, {})
sections[product_id] = StockTransaction.latest(case_id, section_id,
product_id)
return ret
def recalculate_domain_consumption(domain):
"""
Given a domain, recalculate all saved consumption settings in that domain.
"""
# note: might get slow as this gets huge
found_doc_ids = DocDomainMapping.objects.filter(
domain_name=domain,
doc_type='CommCareCase',
).values_list('doc_id', flat=True)
products = Product.by_domain(domain)
for supply_point_id in found_doc_ids:
for product in products:
try:
latest_transaction = StockTransaction.get_ordered_transactions_for_stock(
supply_point_id, const.SECTION_TYPE_STOCK, product._id)[0]
except IndexError:
pass
else:
state = get_stock_state_for_transaction(latest_transaction)
daily_consumption = get_consumption_for_ledger(state)
state.daily_consumption = daily_consumption
with drop_connected_signals(post_save):
state.save()
def get_stock_state_json(sql_ledger):
"""Build stock state JSON from latest transaction
Returns empty dict if stock transactions do not exist.
"""
# similar to StockTransaction.latest(), but more efficient
transactions = list(StockTransaction.get_ordered_transactions_for_stock(
sql_ledger.case_id,
sql_ledger.section_id,
sql_ledger.product_id,
).select_related("report")[:1])
if not transactions:
return {}
transaction = transactions[0]
return StockState(
case_id=sql_ledger.case_id,
section_id=sql_ledger.section_id,
product_id=sql_ledger.product_id,
sql_location=SQLLocation.objects.get_or_None(supply_point_id=sql_ledger.case_id),
last_modified_date=transaction.report.server_date,
last_modified_form_id=transaction.report.form_id,
stock_on_hand=transaction.stock_on_hand,
).to_json()
def recalculate_domain_consumption(domain):
"""
Given a domain, recalculate all saved consumption settings in that domain.
"""
# note: might get slow as this gets huge
found_doc_ids = DocDomainMapping.objects.filter(
domain_name=domain,
doc_type='CommCareCase',
).values_list('doc_id', flat=True)
products = Product.by_domain(domain)
for supply_point_id in found_doc_ids:
for product in products:
try:
latest_transaction = StockTransaction.get_ordered_transactions_for_stock(
supply_point_id, const.SECTION_TYPE_STOCK, product._id
)[0]
except IndexError:
pass
else:
state = get_stock_state_for_transaction(latest_transaction)
daily_consumption = get_consumption_for_ledger(state)
state.daily_consumption = daily_consumption
with drop_connected_signals(post_save):
state.save()
section_id = 'stock'
params = dict(
form_id='logistics-xform',
date=force_to_datetime(stocktransaction.date),
type='balance',
domain=domain
)
try:
report, _ = StockReport.objects.get_or_create(**params)
except StockReport.MultipleObjectsReturned:
# legacy
report = StockReport.objects.filter(**params)[0]
sql_product = SQLProduct.objects.get(code=stocktransaction.product, domain=domain)
if stocktransaction.report_type.lower() == 'stock received':
transactions.append(StockTransaction(
case_id=case.get_id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type='receipts',
stock_on_hand=Decimal(stocktransaction.ending_balance),
quantity=Decimal(stocktransaction.quantity),
report=report
))
elif stocktransaction.report_type.lower() == 'stock on hand':
if stocktransaction.quantity < 0 and bulk:
transactions.append(StockTransaction(
case_id=case.get_id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
def _get_stats(self):
stock_state = StockState.objects.get(**self._stock_state_key)
latest_txn = StockTransaction.latest(**self._stock_state_key)
all_txns = StockTransaction.get_ordered_transactions_for_stock(
**self._stock_state_key)
return stock_state, latest_txn, all_txns
def get_stock_payload(self, syncop):
cases = [e.case for e in syncop.actual_cases_to_sync]
from lxml.builder import ElementMaker
E = ElementMaker(namespace=COMMTRACK_REPORT_XMLNS)
def entry_xml(id, quantity):
return E.entry(
id=id,
quantity=str(int(quantity)),
)
def transaction_to_xml(trans):
return entry_xml(trans.product_id, trans.stock_on_hand)
def consumption_entry(case_id, product_id, section_id):
consumption_value = compute_consumption_or_default(
case_id,
product_id,
datetime.utcnow(),
section_id,
self.stock_settings.consumption_config
)
if consumption_value is not None:
return entry_xml(product_id, consumption_value)
def _unique_products(stock_transaction_queryset):
return sorted(stock_transaction_queryset.values_list('product_id', flat=True).distinct())
for commtrack_case in cases:
relevant_sections = sorted(StockTransaction.objects.filter(
case_id=commtrack_case._id).values_list('section_id', flat=True).distinct())
section_product_map = defaultdict(lambda: [])
section_timestamp_map = defaultdict(lambda: json_format_datetime(datetime.utcnow()))
for section_id in relevant_sections:
relevant_reports = StockTransaction.objects.filter(case_id=commtrack_case._id, section_id=section_id)
product_ids = _unique_products(relevant_reports)
transactions = [StockTransaction.latest(commtrack_case._id, section_id, p) for p in product_ids]
as_of = json_format_datetime(max(txn.report.date for txn in transactions))
section_product_map[section_id] = product_ids
section_timestamp_map[section_id] = as_of
yield E.balance(*(transaction_to_xml(e) for e in transactions),
**{'entity-id': commtrack_case._id, 'date': as_of, 'section-id': section_id})
for section_id, consumption_section_id in self.stock_settings.section_to_consumption_types.items():
if (section_id in relevant_sections or
self.stock_settings.force_consumption_case_filter(commtrack_case)):
consumption_product_ids = self.stock_settings.default_product_list \
if self.stock_settings.default_product_list \
else section_product_map[section_id]
consumption_entries = filter(lambda e: e is not None, [
consumption_entry(commtrack_case._id, p, section_id)
for p in consumption_product_ids
])
if consumption_entries:
yield E.balance(
*consumption_entries,
**{
'entity-id': commtrack_case._id,
'date': section_timestamp_map[section_id],
'section-id': consumption_section_id,
}
)
def sync_stock_transactions_for_facility(domain, endpoint, facility, xform, checkpoint,
date, limit=1000, offset=0):
"""
Syncs stock data from StockTransaction objects in ILSGateway to StockTransaction objects in HQ
"""
has_next = True
next_url = ""
section_id = 'stock'
supply_point = facility
case = get_supply_point_by_external_id(domain, supply_point)
if not case:
return
save_stock_data_checkpoint(checkpoint, 'stock_transaction', limit, offset, date, facility, True)
products_saved = set()
while has_next:
meta, stocktransactions = endpoint.get_stocktransactions(next_url_params=next_url,
limit=limit,
offset=offset,
filters=(dict(supply_point=supply_point,
date__gte=date,
order_by='date')))
# set the checkpoint right before the data we are about to process
meta_limit = meta.get('limit') or limit
meta_offset = meta.get('offset') or offset
save_stock_data_checkpoint(checkpoint, 'stock_transaction', meta_limit, meta_offset, date, facility, True)
transactions_to_add = []
with transaction.commit_on_success():
for stocktransaction in stocktransactions:
params = dict(
form_id=xform._id,
date=force_to_datetime(stocktransaction.date),
type='balance',
domain=domain,
)
try:
report, _ = StockReport.objects.get_or_create(**params)
except StockReport.MultipleObjectsReturned:
# legacy
report = StockReport.objects.filter(**params)[0]
sql_product = SQLProduct.objects.get(code=stocktransaction.product, domain=domain)
if stocktransaction.quantity != 0:
transactions_to_add.append(StockTransaction(
case_id=case._id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type='receipts' if stocktransaction.quantity > 0 else 'consumption',
stock_on_hand=Decimal(stocktransaction.ending_balance),
quantity=Decimal(stocktransaction.quantity),
report=report
))
transactions_to_add.append(StockTransaction(
case_id=case._id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type='stockonhand',
stock_on_hand=Decimal(stocktransaction.ending_balance),
report=report
))
products_saved.add(sql_product.product_id)
if transactions_to_add:
# Doesn't send signal
StockTransaction.objects.bulk_create(transactions_to_add)
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
for product in products_saved:
# if we saved anything rebuild the stock state object by firing the signal
# on the last transaction for each product
last_st = StockTransaction.latest(case._id, section_id, product)
update_stock_state_for_transaction(last_st)
def create_models_for_stock_report(domain, stock_report_helper):
"""
Save stock report and stock transaction models to the database.
"""
assert stock_report_helper._form.domain == domain
domain = domain
if stock_report_helper.tag not in stockconst.VALID_REPORT_TYPES:
return
report = StockReport.objects.create(
form_id=stock_report_helper.form_id,
date=stock_report_helper.timestamp,
type=stock_report_helper.tag,
domain=domain,
)
for txn in stock_report_helper.transactions:
db_txn = StockTransaction(
report=report,
case_id=txn.case_id,
section_id=txn.section_id,
product_id=txn.product_id,
)
if domain:
# set this as a shortcut for post save signal receivers
db_txn.domain = domain
db_txn.type = txn.action
db_txn.subtype = txn.subaction
if stock_report_helper.tag == stockconst.REPORT_TYPE_BALANCE:
db_txn.stock_on_hand = txn.quantity
db_txn.quantity = 0
else:
assert stock_report_helper.tag == stockconst.REPORT_TYPE_TRANSFER
previous_transaction = db_txn.get_previous_transaction()
db_txn.quantity = txn.relative_quantity
db_txn.stock_on_hand = db_txn.quantity + (
previous_transaction.stock_on_hand
if previous_transaction else 0
)
db_txn.save()
def create_models(self, domain=None):
"""
Save stock report and stock transaction models to the database.
"""
# todo: this function should probably move to somewhere in casexml.apps.stock
if self.tag not in stockconst.VALID_REPORT_TYPES:
return
report = StockReport.objects.create(
form_id=self.form_id,
date=self.timestamp,
type=self.tag,
domain=self._form.domain,
)
for txn in self.transactions:
db_txn = StockTransaction(
report=report,
case_id=txn.case_id,
section_id=txn.section_id,
product_id=txn.product_id,
)
if domain:
# set this as a shortcut for post save signal receivers
db_txn.domain = domain
db_txn.type = txn.action
db_txn.subtype = txn.subaction
if self.tag == stockconst.REPORT_TYPE_BALANCE:
db_txn.stock_on_hand = txn.quantity
db_txn.quantity = 0
else:
assert self.tag == stockconst.REPORT_TYPE_TRANSFER
previous_transaction = db_txn.get_previous_transaction()
db_txn.quantity = txn.relative_quantity
db_txn.stock_on_hand = (previous_transaction.stock_on_hand if previous_transaction else 0) + db_txn.quantity
db_txn.save()
def get_latest_transaction(case_id, section_id, entry_id):
return StockTransaction.latest(case_id, section_id, entry_id)
def get_latest_transaction(case_id, section_id, entry_id):
return StockTransaction.latest(case_id, section_id, entry_id)
def _get_stats(self):
stock_state = StockState.objects.get(**self._stock_state_key)
latest_txn = StockTransaction.latest(**self._stock_state_key)
all_txns = StockTransaction.get_ordered_transactions_for_stock(
**self._stock_state_key)
return stock_state, latest_txn, all_txns
def check_stock_models(self, case, product_id, expected_soh, expected_qty, section_id):
latest_trans = StockTransaction.latest(case._id, section_id, product_id)
self.assertEqual(section_id, latest_trans.section_id)
self.assertEqual(expected_soh, latest_trans.stock_on_hand)
self.assertEqual(expected_qty, latest_trans.quantity)
def get_couch_transactions(case_id, section_id, product_id):
return list(reversed(
StockTransaction.get_ordered_transactions_for_stock(
case_id=case_id, section_id=section_id, product_id=product_id)
.select_related("report")
))
def get_stock_payload(self, syncop):
cases = [e.case for e in syncop.actual_cases_to_sync]
from lxml.builder import ElementMaker
E = ElementMaker(namespace=COMMTRACK_REPORT_XMLNS)
def entry_xml(id, quantity):
return E.entry(
id=id,
quantity=str(int(quantity)),
)
def transaction_to_xml(trans):
return entry_xml(trans.product_id, trans.stock_on_hand)
def consumption_entry(case_id, product_id, section_id):
consumption_value = compute_consumption_or_default(
case_id,
product_id,
datetime.utcnow(),
section_id,
self.stock_settings.consumption_config
)
if consumption_value is not None:
return entry_xml(product_id, consumption_value)
def _unique_products(stock_transaction_queryset):
return sorted(stock_transaction_queryset.values_list('product_id', flat=True).distinct())
for commtrack_case in cases:
relevant_sections = sorted(StockTransaction.objects.filter(
case_id=commtrack_case._id).values_list('section_id', flat=True).distinct())
section_product_map = defaultdict(lambda: [])
section_timestamp_map = defaultdict(lambda: json_format_datetime(datetime.utcnow()))
for section_id in relevant_sections:
relevant_reports = StockTransaction.objects.filter(case_id=commtrack_case._id, section_id=section_id)
product_ids = _unique_products(relevant_reports)
transactions = [StockTransaction.latest(commtrack_case._id, section_id, p) for p in product_ids]
as_of = json_format_datetime(max(txn.report.date for txn in transactions))
section_product_map[section_id] = product_ids
section_timestamp_map[section_id] = as_of
yield E.balance(*(transaction_to_xml(e) for e in transactions),
**{'entity-id': commtrack_case._id, 'date': as_of, 'section-id': section_id})
for section_id, consumption_section_id in self.stock_settings.section_to_consumption_types.items():
if (section_id in relevant_sections or
self.stock_settings.force_consumption_case_filter(commtrack_case)):
consumption_product_ids = self.stock_settings.default_product_list \
if self.stock_settings.default_product_list \
else section_product_map[section_id]
yield E.balance(
*filter(lambda e: e is not None,
[consumption_entry(commtrack_case._id, p, section_id)
for p in consumption_product_ids]),
**{'entity-id': commtrack_case._id, 'date': section_timestamp_map[section_id],
'section-id': consumption_section_id}
)