def sync_stock_transactions_for_facility(domain, endpoint, facility, checkpoint,
date, limit=1000, offset=0):
"""
Syncs stock data from StockTransaction objects in ILSGateway to StockTransaction objects in HQ
"""
has_next = True
next_url = ""
section_id = 'stock'
supply_point = facility
case = get_supply_point_case_in_domain_by_id(domain, supply_point)
if not case:
return
location_id = case.location_id
save_stock_data_checkpoint(checkpoint, 'stock_transaction', limit, offset, date, location_id, True)
products_saved = set()
while has_next:
meta, stocktransactions = endpoint.get_stocktransactions(
next_url_params=next_url,
limit=limit,
offset=offset,
filters={
'supply_point': supply_point,
'date__gte': date,
'date__lte': checkpoint.start_date
}
)
# set the checkpoint right before the data we are about to process
meta_limit = meta.get('limit') or limit
meta_offset = meta.get('offset') or offset
save_stock_data_checkpoint(
checkpoint, 'stock_transaction', meta_limit, meta_offset, date, location_id, True
)
transactions_to_add = []
with transaction.atomic():
for stocktransaction in stocktransactions:
transactions = sync_stock_transaction(stocktransaction, domain, case, bulk=True)
transactions_to_add.extend(transactions)
products_saved.update(map(lambda x: x.product_id, transactions))
if transactions_to_add:
# Doesn't send signal
StockTransaction.objects.bulk_create(transactions_to_add)
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
for product in products_saved:
# if we saved anything rebuild the stock state object by firing the signal
# on the last transaction for each product
last_st = StockTransaction.latest(case.get_id, section_id, product)
update_stock_state_for_transaction(last_st)
def sync_stock_transactions_for_facility(domain, endpoint, facility, checkpoint,
date, limit=1000, offset=0):
"""
Syncs stock data from StockTransaction objects in ILSGateway to StockTransaction objects in HQ
"""
has_next = True
next_url = ""
section_id = 'stock'
supply_point = facility
case = get_supply_point_case_in_domain_by_id(domain, supply_point)
if not case:
return
location_id = case.location_id
save_stock_data_checkpoint(checkpoint, 'stock_transaction', limit, offset, date, location_id, True)
products_saved = set()
while has_next:
meta, stocktransactions = endpoint.get_stocktransactions(
next_url_params=next_url,
limit=limit,
offset=offset,
filters={
'supply_point': supply_point,
'date__gte': date,
'date__lte': checkpoint.start_date
}
)
# set the checkpoint right before the data we are about to process
meta_limit = meta.get('limit') or limit
meta_offset = meta.get('offset') or offset
save_stock_data_checkpoint(
checkpoint, 'stock_transaction', meta_limit, meta_offset, date, location_id, True
)
transactions_to_add = []
with transaction.atomic():
for stocktransaction in stocktransactions:
transactions = sync_stock_transaction(stocktransaction, domain, case, bulk=True)
transactions_to_add.extend(transactions)
products_saved.update(map(lambda x: x.product_id, transactions))
if transactions_to_add:
# Doesn't send signal
StockTransaction.objects.bulk_create(transactions_to_add)
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
for product in products_saved:
# if we saved anything rebuild the stock state object by firing the signal
# on the last transaction for each product
last_st = StockTransaction.latest(case.get_id, section_id, product)
update_stock_state_for_transaction(last_st)
def rows(self):
rows = []
supply_points = []
offset = 1000
meta, chunk = self.endpoint.get_supply_points(limit=1000)
supply_points.extend(chunk)
while meta.get('next', False):
meta, chunk = self.endpoint.get_supply_points(offset=offset, limit=1000)
offset += 1000
supply_points.extend(chunk)
for supply_point in supply_points:
couch_sp = get_supply_point_case_in_domain_by_id(
self.domain, supply_point.id)
if not couch_sp:
rows.append([supply_point.name, supply_point.type,
supply_point.code, supply_point.active, supply_point.last_reported, False])
return rows
def rows(self):
rows = []
supply_points = []
offset = 1000
meta, chunk = self.endpoint.get_supply_points(limit=1000)
supply_points.extend(chunk)
while meta.get('next', False):
meta, chunk = self.endpoint.get_supply_points(offset=offset,
limit=1000)
offset += 1000
supply_points.extend(chunk)
for supply_point in supply_points:
couch_sp = get_supply_point_case_in_domain_by_id(
self.domain, supply_point.id)
if not couch_sp:
rows.append([
supply_point.name, supply_point.type, supply_point.code,
supply_point.active, supply_point.last_reported, False
])
return rows
else:
ilsgateway_id = 1166 # defaults to bondenzi: http://ilsgateway.com/tz/facility/1166/
# cleanup
_cleanup_existing_data(domain, ilsgateway_id)
# migrate
config = ILSGatewayConfig.for_domain(domain)
assert config.enabled, 'ilsgateway sync must be configured for this domain'
endpoint = ILSGatewayEndpoint.from_config(config)
stock_data_task(domain, endpoint, get_ilsgateway_data_migrations(), config,
test_facilities=[ilsgateway_id])
def _cleanup_existing_data(domain, ilsgateway_id):
case = get_supply_point_case_in_domain_by_id(domain, ilsgateway_id)
# delete stock transactions
stock_transactions = StockTransaction.objects.filter(case_id=case._id)
count = stock_transactions.count()
if count:
print 'deleting {} existing StockTransactions'.format(count)
stock_report_ids = stock_transactions.values_list('report_id', flat=True)
stock_transactions.delete()
# and related stock reports
# todo: this may never be necessary due to the stock transactions deletion signal?
stock_reports = StockReport.objects.filter(pk__in=stock_report_ids)
if stock_reports.count():
report_txn_counts = stock_reports.annotate(txn_count=Count('stocktransaction'))
for sr in report_txn_counts:
assert sr.txn_count == 0
def _get_location_id(facility, domain):
supply_point = get_supply_point_case_in_domain_by_id(domain, facility)
return supply_point.location_id if supply_point else None
def sync_stock_transaction(stocktransaction, domain, case=None, bulk=False):
case = case or get_supply_point_case_in_domain_by_id(domain, stocktransaction.supply_point)
if not case:
return
transactions = []
section_id = "stock"
params = dict(
form_id="logistics-xform", date=force_to_datetime(stocktransaction.date), type="balance", domain=domain
)
try:
report, _ = StockReport.objects.get_or_create(**params)
except StockReport.MultipleObjectsReturned:
# legacy
report = StockReport.objects.filter(**params)[0]
sql_product = SQLProduct.objects.get(code=stocktransaction.product, domain=domain)
if stocktransaction.report_type.lower() == "stock received":
transactions.append(
StockTransaction(
case_id=case.get_id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type="receipts",
stock_on_hand=Decimal(stocktransaction.ending_balance),
quantity=Decimal(stocktransaction.quantity),
report=report,
)
)
elif stocktransaction.report_type.lower() == "stock on hand":
if stocktransaction.quantity < 0 and bulk:
transactions.append(
StockTransaction(
case_id=case.get_id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type="consumption",
stock_on_hand=Decimal(stocktransaction.ending_balance),
quantity=Decimal(stocktransaction.quantity),
report=report,
subtype="inferred",
)
)
transactions.append(
StockTransaction(
case_id=case.get_id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type="stockonhand",
stock_on_hand=Decimal(stocktransaction.ending_balance),
report=report,
)
)
elif stocktransaction.report_type.lower() == "loss or adjustment":
transactions.append(
StockTransaction(
case_id=case.get_id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type=TRANSACTION_TYPE_LA,
stock_on_hand=Decimal(stocktransaction.ending_balance),
quantity=stocktransaction.quantity,
report=report,
)
)
if not bulk:
for tx in transactions:
tx.save()
return transactions
def get_supply_point(domain, facility_or_code):
facility_code = facility_or_code if isinstance(facility_or_code, basestring) else facility_or_code.code
return get_supply_point_case_in_domain_by_id(domain, facility_code)
def get_location_id(self, facility):
sp = get_supply_point_case_in_domain_by_id(self.domain, facility)
return sp.location_id
_id=loc.location_id,
name=loc.name,
domain=domain
)
SupplyPointCase.get_or_create_by_location(fake_location)
@celery.task(ignore_result=True)
def add_products_to_loc(api):
endpoint = api.endpoint
synchronization(None, endpoint.get_locations, api.location_sync, None, None, 100, 0,
filters={"is_active": True})
def sync_stock_transaction(stocktransaction, domain, case=None, bulk=False):
case = case or get_supply_point_case_in_domain_by_id(domain, stocktransaction.supply_point)
if not case:
return
transactions = []
section_id = 'stock'
params = dict(
form_id='logistics-xform',
date=force_to_datetime(stocktransaction.date),
type='balance',
domain=domain
)
try:
report, _ = StockReport.objects.get_or_create(**params)
except StockReport.MultipleObjectsReturned:
# legacy
report = StockReport.objects.filter(**params)[0]
def _get_location_id(facility, domain):
supply_point = get_supply_point_case_in_domain_by_id(domain, facility)
return supply_point.location_id if supply_point else None
def sync_stock_transactions_for_facility(domain, endpoint, facility, checkpoint,
date, limit=1000, offset=0):
"""
Syncs stock data from StockTransaction objects in ILSGateway to StockTransaction objects in HQ
"""
has_next = True
next_url = ""
section_id = 'stock'
supply_point = facility
case = get_supply_point_case_in_domain_by_id(domain, supply_point)
if not case:
return
location_id = case.location_id
save_stock_data_checkpoint(checkpoint, 'stock_transaction', limit, offset, date, location_id, True)
products_saved = set()
while has_next:
meta, stocktransactions = endpoint.get_stocktransactions(next_url_params=next_url,
limit=limit,
start_date=date,
end_date=checkpoint.start_date,
offset=offset,
filters=(dict(supply_point=supply_point)))
# set the checkpoint right before the data we are about to process
meta_limit = meta.get('limit') or limit
meta_offset = meta.get('offset') or offset
save_stock_data_checkpoint(
checkpoint, 'stock_transaction', meta_limit, meta_offset, date, location_id, True
)
transactions_to_add = []
with transaction.atomic():
for stocktransaction in stocktransactions:
params = dict(
form_id='logistics-xform',
date=force_to_datetime(stocktransaction.date),
type='balance',
domain=domain,
)
try:
report, _ = StockReport.objects.get_or_create(**params)
except StockReport.MultipleObjectsReturned:
# legacy
report = StockReport.objects.filter(**params)[0]
sql_product = SQLProduct.objects.get(code=stocktransaction.product, domain=domain)
if stocktransaction.report_type.lower() == 'stock received':
transactions_to_add.append(StockTransaction(
case_id=case._id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type='receipts',
stock_on_hand=Decimal(stocktransaction.ending_balance),
quantity=Decimal(stocktransaction.quantity),
report=report
))
products_saved.add(sql_product.product_id)
elif stocktransaction.report_type.lower() == 'stock on hand':
if stocktransaction.quantity < 0:
transactions_to_add.append(StockTransaction(
case_id=case._id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type='consumption',
stock_on_hand=Decimal(stocktransaction.ending_balance),
quantity=Decimal(stocktransaction.quantity),
report=report,
subtype='inferred'
))
transactions_to_add.append(StockTransaction(
case_id=case._id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type='stockonhand',
stock_on_hand=Decimal(stocktransaction.ending_balance),
report=report
))
products_saved.add(sql_product.product_id)
elif stocktransaction.report_type.lower() == 'loss or adjustment':
transactions_to_add.append(StockTransaction(
case_id=case._id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type=TRANSACTION_TYPE_LA,
stock_on_hand=Decimal(stocktransaction.ending_balance),
quantity=stocktransaction.quantity,
report=report
))
products_saved.add(sql_product.product_id)
if transactions_to_add:
# Doesn't send signal
StockTransaction.objects.bulk_create(transactions_to_add)
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
for product in products_saved:
# if we saved anything rebuild the stock state object by firing the signal
# on the last transaction for each product
last_st = StockTransaction.latest(case._id, section_id, product)
update_stock_state_for_transaction(last_st)
def get_supply_point(domain, facility_or_code):
facility_code = facility_or_code if isinstance(
facility_or_code, basestring) else facility_or_code.code
return get_supply_point_case_in_domain_by_id(domain, facility_code)
def get_location_id(self, facility):
sp = get_supply_point_case_in_domain_by_id(self.domain, facility)
return sp.location_id
