def sync_supply_point_status(domain, endpoint, facility, checkpoint, date, limit=100, offset=0):
has_next = True
next_url = ""
while has_next:
meta, supply_point_statuses = endpoint.get_supplypointstatuses(
domain,
limit=limit,
offset=offset,
next_url_params=next_url,
filters=dict(supply_point=facility, status_date__gte=date),
facility=facility
)
# set the checkpoint right before the data we are about to process
if not supply_point_statuses:
return None
location_id = SQLLocation.objects.get(domain=domain, external_id=facility).location_id
save_stock_data_checkpoint(checkpoint,
'supply_point_status',
meta.get('limit') or limit,
meta.get('offset') or offset, date, location_id, True)
for supply_point_status in supply_point_statuses:
process_supply_point_status(supply_point_status, domain, location_id)
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
def sync_supply_point_status(domain, endpoint, facility, checkpoint, date, limit=100, offset=0):
has_next = True
next_url = ""
while has_next:
meta, supply_point_statuses = endpoint.get_supplypointstatuses(
domain,
limit=limit,
offset=offset,
next_url_params=next_url,
filters=dict(supply_point=facility, status_date__gte=date),
facility=facility
)
save_stock_data_checkpoint(checkpoint,
'supply_point_status',
meta.get('limit') or limit,
meta.get('offset') or offset, date, facility, True)
for sps in supply_point_statuses:
try:
SupplyPointStatus.objects.get(external_id=sps.external_id)
except SupplyPointStatus.DoesNotExist:
sps.save()
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
def sync_delivery_group_report(domain, endpoint, facility, checkpoint, date, limit=100, offset=0):
has_next = True
next_url = ""
while has_next:
meta, delivery_group_reports = endpoint.get_deliverygroupreports(
domain,
limit=limit,
offset=offset,
next_url_params=next_url,
filters=dict(supply_point=facility, report_date__gte=date),
facility=facility
)
location_id = SQLLocation.objects.get(domain=domain, external_id=facility).location_id
# set the checkpoint right before the data we are about to process
save_stock_data_checkpoint(checkpoint,
'delivery_group',
meta.get('limit') or limit,
meta.get('offset') or offset,
date, location_id, True)
for dgr in delivery_group_reports:
try:
DeliveryGroupReport.objects.get(external_id=dgr.external_id, location_id=location_id)
except DeliveryGroupReport.DoesNotExist:
dgr.save()
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
def stock_data_task(api_object):
# checkpoint logic
start_date = datetime.today()
default_api = api_object.apis[0][0]
checkpoint, _ = StockDataCheckpoint.objects.get_or_create(
domain=api_object.domain,
defaults={
"api": default_api,
"date": None,
"limit": 1000,
"offset": 0,
"location": None,
"start_date": start_date,
},
)
if not checkpoint.api:
checkpoint.api = default_api
if not checkpoint.start_date:
checkpoint.start_date = start_date
checkpoint.save()
if not api_object.all_stock_data:
facilities = api_object.test_facilities
else:
facilities = api_object.get_ids()
if checkpoint.location:
external_id = api_object.get_last_processed_location(checkpoint)
if external_id:
facilities = list(itertools.dropwhile(lambda x: int(x) != int(external_id), facilities))
process_facility_task(api_object, facilities[0], start_from=checkpoint.api)
facilities = facilities[1:]
if not checkpoint.date or checkpoint.location:
# use subtasks only during initial migration
facilities_chunked_list = chunked(facilities, 5)
for chunk in facilities_chunked_list:
api_object.process_data(process_facility_task, chunk)
else:
offset = checkpoint.offset
for stock_api in itertools.dropwhile(lambda x: x.name != checkpoint.api, api_object.get_stock_apis_objects()):
stock_api.add_date_filter(checkpoint.date, checkpoint.start_date)
synchronization(
stock_api,
checkpoint,
checkpoint.date,
1000,
offset,
params={"domain": api_object.domain},
domain=api_object.domain,
atomic=True,
)
offset = 0
checkpoint = StockDataCheckpoint.objects.get(domain=api_object.domain)
save_stock_data_checkpoint(checkpoint, default_api, 1000, 0, checkpoint.start_date, None, False)
checkpoint.start_date = None
checkpoint.save()
def stock_data_task(api_object):
# checkpoint logic
start_date = datetime.today()
default_api = api_object.apis[0][0]
checkpoint, _ = StockDataCheckpoint.objects.get_or_create(domain=api_object.domain, defaults={
'api': default_api,
'date': None,
'limit': 1000,
'offset': 0,
'location': None,
'start_date': start_date
})
if not checkpoint.api:
checkpoint.api = default_api
if not checkpoint.start_date:
checkpoint.start_date = start_date
checkpoint.save()
if not api_object.all_stock_data:
facilities = api_object.test_facilities
else:
facilities = api_object.get_ids()
if checkpoint.location:
external_id = api_object.get_last_processed_location(checkpoint)
if external_id:
facilities = list(itertools.dropwhile(lambda x: int(x) != int(external_id), facilities))
process_facility_task(api_object, facilities[0], start_from=checkpoint.api)
facilities = facilities[1:]
if not checkpoint.date or checkpoint.location:
# use subtasks only during initial migration
facilities_chunked_list = chunked(facilities, 5)
for chunk in facilities_chunked_list:
api_object.process_data(process_facility_task, chunk)
else:
offset = checkpoint.offset
for stock_api in itertools.dropwhile(
lambda x: x.name != checkpoint.api, api_object.get_stock_apis_objects()
):
stock_api.add_date_filter(checkpoint.date, checkpoint.start_date)
synchronization(
stock_api,
checkpoint,
checkpoint.date,
1000,
offset,
params={'domain': api_object.domain},
domain=api_object.domain
)
offset = 0
checkpoint = StockDataCheckpoint.objects.get(domain=api_object.domain)
save_stock_data_checkpoint(checkpoint, default_api, 1000, 0, checkpoint.start_date, None, False)
checkpoint.start_date = None
checkpoint.save()
def sync_stock_transaction(domain, endpoint, facility, xform, checkpoint,
date, limit=100, offset=0):
has_next = True
next_url = ""
while has_next:
supply_point = facility
case = SupplyPointCase.view('hqcase/by_domain_external_id',
key=[domain, str(supply_point)],
reduce=False,
include_docs=True,
limit=1).first()
if not case:
break
meta, stocktransactions = endpoint.get_stocktransactions(next_url_params=next_url,
limit=limit,
offset=offset,
filters=(dict(supply_point=supply_point,
date__gte=date,
order_by='date')))
save_stock_data_checkpoint(checkpoint,
'stock_transaction',
meta.get('limit') or limit,
meta.get('offset') or offset,
date, facility, True)
transactions_to_add = []
with transaction.commit_on_success():
for stocktransaction in stocktransactions:
if case:
product = Product.get_by_code(domain, stocktransaction.product)
report = StockReport(
form_id=xform._id,
date=force_to_datetime(stocktransaction.date),
type='balance',
domain=domain
)
report.save()
try:
sql_product = SQLProduct.objects.get(product_id=product._id)
except SQLProduct.DoesNotExist:
continue
transactions_to_add.append(StockTransaction(
case_id=case._id,
product_id=product._id,
sql_product=sql_product,
section_id='stock',
type='stockonhand',
stock_on_hand=Decimal(stocktransaction.ending_balance),
report=report
))
# Doesn't send signal
StockTransaction.objects.bulk_create(transactions_to_add)
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
def sync_stock_transactions_for_facility(domain, endpoint, facility, checkpoint,
date, limit=1000, offset=0):
"""
Syncs stock data from StockTransaction objects in ILSGateway to StockTransaction objects in HQ
"""
has_next = True
next_url = ""
section_id = 'stock'
supply_point = facility
case = get_supply_point_case_in_domain_by_id(domain, supply_point)
if not case:
return
location_id = case.location_id
save_stock_data_checkpoint(checkpoint, 'stock_transaction', limit, offset, date, location_id, True)
products_saved = set()
while has_next:
meta, stocktransactions = endpoint.get_stocktransactions(
next_url_params=next_url,
limit=limit,
offset=offset,
filters={
'supply_point': supply_point,
'date__gte': date,
'date__lte': checkpoint.start_date
}
)
# set the checkpoint right before the data we are about to process
meta_limit = meta.get('limit') or limit
meta_offset = meta.get('offset') or offset
save_stock_data_checkpoint(
checkpoint, 'stock_transaction', meta_limit, meta_offset, date, location_id, True
)
transactions_to_add = []
with transaction.atomic():
for stocktransaction in stocktransactions:
transactions = sync_stock_transaction(stocktransaction, domain, case, bulk=True)
transactions_to_add.extend(transactions)
products_saved.update(map(lambda x: x.product_id, transactions))
if transactions_to_add:
# Doesn't send signal
StockTransaction.objects.bulk_create(transactions_to_add)
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
for product in products_saved:
# if we saved anything rebuild the stock state object by firing the signal
# on the last transaction for each product
last_st = StockTransaction.latest(case.get_id, section_id, product)
update_stock_state_for_transaction(last_st)
def sync_stock_transactions_for_facility(domain, endpoint, facility, checkpoint,
date, limit=1000, offset=0):
"""
Syncs stock data from StockTransaction objects in ILSGateway to StockTransaction objects in HQ
"""
has_next = True
next_url = ""
section_id = 'stock'
supply_point = facility
case = get_supply_point_case_in_domain_by_id(domain, supply_point)
if not case:
return
location_id = case.location_id
save_stock_data_checkpoint(checkpoint, 'stock_transaction', limit, offset, date, location_id, True)
products_saved = set()
while has_next:
meta, stocktransactions = endpoint.get_stocktransactions(
next_url_params=next_url,
limit=limit,
offset=offset,
filters={
'supply_point': supply_point,
'date__gte': date,
'date__lte': checkpoint.start_date
}
)
# set the checkpoint right before the data we are about to process
meta_limit = meta.get('limit') or limit
meta_offset = meta.get('offset') or offset
save_stock_data_checkpoint(
checkpoint, 'stock_transaction', meta_limit, meta_offset, date, location_id, True
)
transactions_to_add = []
with transaction.atomic():
for stocktransaction in stocktransactions:
transactions = sync_stock_transaction(stocktransaction, domain, case, bulk=True)
transactions_to_add.extend(transactions)
products_saved.update(map(lambda x: x.product_id, transactions))
if transactions_to_add:
# Doesn't send signal
StockTransaction.objects.bulk_create(transactions_to_add)
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
for product in products_saved:
# if we saved anything rebuild the stock state object by firing the signal
# on the last transaction for each product
last_st = StockTransaction.latest(case.get_id, section_id, product)
update_stock_state_for_transaction(last_st)
def sync_product_stock(domain, endpoint, facility, checkpoint, date, limit=100, offset=0):
has_next = True
next_url = ""
while has_next:
supply_point = facility
case = SupplyPointCase.view('hqcase/by_domain_external_id',
key=[domain, str(supply_point)],
reduce=False,
include_docs=True,
limit=1).first()
meta, product_stocks = endpoint.get_productstocks(
next_url_params=next_url,
limit=limit,
offset=offset,
filters=dict(supply_point=supply_point, last_modified__gte=date)
)
save_stock_data_checkpoint(checkpoint,
'product_stock',
meta.get('limit') or limit,
meta.get('offset') or offset,
date, facility, True)
for product_stock in product_stocks:
if case:
product = Product.get_by_code(domain, product_stock.product)
try:
stock_state = StockState.objects.get(section_id='stock',
case_id=case._id,
product_id=product._id)
stock_state.last_modified_date = product_stock.last_modified
stock_state.stock_on_hand = product_stock.quantity or 0
except StockState.DoesNotExist:
stock_state = StockState(section_id='stock',
case_id=case._id,
product_id=product._id,
stock_on_hand=product_stock.quantity or 0,
last_modified_date=product_stock.last_modified,
sql_product=SQLProduct.objects.get(product_id=product._id))
if product_stock.auto_monthly_consumption:
stock_state.daily_consumption = product_stock.auto_monthly_consumption / DAYS_IN_MONTH
else:
stock_state.daily_consumption = None
stock_state.save()
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
def stock_data_task(api_object):
# checkpoint logic
start_date = datetime.today()
default_api = api_object.apis[0][0]
checkpoint, _ = StockDataCheckpoint.objects.get_or_create(domain=api_object.domain, defaults={
'api': default_api,
'date': None,
'limit': 1000,
'offset': 0,
'location': None,
'start_date': start_date
})
if not checkpoint.start_date:
checkpoint.start_date = start_date
checkpoint.save()
if not api_object.all_stock_data:
facilities = api_object.test_facilities
else:
facilities = api_object.get_ids()
if checkpoint.location:
external_id = api_object.get_last_processed_location(checkpoint)
if external_id:
facilities = list(itertools.dropwhile(lambda x: int(x) != int(external_id), facilities))
process_facility_task(api_object, facilities[0], start_from=checkpoint.api)
facilities = facilities[1:]
if not checkpoint.date:
# use subtasks only during initial migration
facilities_chunked_list = chunked(facilities, 5)
for chunk in facilities_chunked_list:
res = chain(process_facility_task.si(api_object, fac) for fac in chunk)()
res.get()
else:
for facility in facilities:
process_facility_task(api_object, facility)
checkpoint = StockDataCheckpoint.objects.get(domain=api_object.domain)
save_stock_data_checkpoint(checkpoint, default_api, 1000, 0, start_date, None, False)
checkpoint.start_date = None
checkpoint.save()
def process_facility_task(api_object, facility, start_from=None):
checkpoint = StockDataCheckpoint.objects.get(domain=api_object.domain)
limit = checkpoint.limit
offset = checkpoint.offset
apis = api_object.apis
if start_from is not None:
apis = itertools.dropwhile(lambda x: x[0] != checkpoint.api, api_object.apis)
for idx, (api_name, api_function) in enumerate(apis):
api_function(
domain=api_object.domain,
checkpoint=checkpoint,
date=checkpoint.date,
limit=limit,
offset=offset,
endpoint=api_object.endpoint,
facility=facility,
)
limit = 1000
offset = 0
save_stock_data_checkpoint(checkpoint, "", 1000, 0, checkpoint.date, api_object.get_location_id(facility))
def process_facility_task(api_object, facility, start_from=None):
checkpoint = StockDataCheckpoint.objects.get(domain=api_object.domain)
limit = checkpoint.limit
offset = checkpoint.offset
apis = api_object.apis
if start_from is not None:
apis = itertools.dropwhile(lambda x: x[0] != checkpoint.api, api_object.apis)
for idx, (api_name, api_function) in enumerate(apis):
api_function(
domain=api_object.domain,
checkpoint=checkpoint,
date=checkpoint.date,
limit=limit,
offset=offset,
endpoint=api_object.endpoint,
facility=facility
)
limit = 1000
offset = 0
save_stock_data_checkpoint(checkpoint, '', 1000, 0, checkpoint.date, api_object.get_location_id(facility))
def stock_data_task(domain, endpoint, apis, config, test_facilities=None):
# checkpoint logic
start_date = datetime.today()
default_api = apis[0][0]
try:
checkpoint = StockDataCheckpoint.objects.get(domain=domain)
api = checkpoint.api
# legacy
if api == 'product_stock':
api = default_api
date = checkpoint.date
limit = checkpoint.limit
offset = checkpoint.offset
location = checkpoint.location
if not checkpoint.start_date:
checkpoint.start_date = start_date
checkpoint.save()
else:
start_date = checkpoint.start_date
except StockDataCheckpoint.DoesNotExist:
checkpoint = StockDataCheckpoint()
checkpoint.domain = domain
checkpoint.start_date = start_date
api = default_api
date = None
limit = 1000
offset = 0
location = None
if not config.all_stock_data:
facilities = test_facilities
else:
supply_points_ids = SQLLocation.objects.filter(
domain=domain,
location_type__in=get_reporting_types(domain)
).order_by('created_at').values_list('supply_point_id', flat=True)
facilities = [doc['external_id'] for doc in iter_docs(SupplyPointCase.get_db(), supply_points_ids)]
apis_from_checkpoint = itertools.dropwhile(lambda x: x[0] != api, apis)
facilities_copy = list(facilities)
if location:
supply_point = SupplyPointCase.get_by_location_id(domain, location.location_id)
external_id = supply_point.external_id if supply_point else None
if external_id:
facilities = itertools.dropwhile(lambda x: int(x) != int(external_id), facilities)
for idx, (api_name, api_function) in enumerate(apis_from_checkpoint):
api_function(
domain=domain,
checkpoint=checkpoint,
date=date,
limit=limit,
offset=offset,
endpoint=endpoint,
facilities=facilities
)
limit = 1000
offset = 0
# todo: see if we can avoid modifying the list of facilities in place
if idx == 0:
facilities = facilities_copy
save_stock_data_checkpoint(checkpoint, default_api, 1000, 0, start_date, None, False)
checkpoint.start_date = None
checkpoint.save()
def sync_stock_transactions_for_facility(domain, endpoint, facility, xform, checkpoint,
date, limit=1000, offset=0):
"""
Syncs stock data from StockTransaction objects in ILSGateway to StockTransaction objects in HQ
"""
has_next = True
next_url = ""
section_id = 'stock'
supply_point = facility
case = get_supply_point_by_external_id(domain, supply_point)
if not case:
return
save_stock_data_checkpoint(checkpoint, 'stock_transaction', limit, offset, date, facility, True)
products_saved = set()
while has_next:
meta, stocktransactions = endpoint.get_stocktransactions(next_url_params=next_url,
limit=limit,
offset=offset,
filters=(dict(supply_point=supply_point,
date__gte=date,
order_by='date')))
# set the checkpoint right before the data we are about to process
meta_limit = meta.get('limit') or limit
meta_offset = meta.get('offset') or offset
save_stock_data_checkpoint(checkpoint, 'stock_transaction', meta_limit, meta_offset, date, facility, True)
transactions_to_add = []
with transaction.commit_on_success():
for stocktransaction in stocktransactions:
params = dict(
form_id=xform._id,
date=force_to_datetime(stocktransaction.date),
type='balance',
domain=domain,
)
try:
report, _ = StockReport.objects.get_or_create(**params)
except StockReport.MultipleObjectsReturned:
# legacy
report = StockReport.objects.filter(**params)[0]
sql_product = SQLProduct.objects.get(code=stocktransaction.product, domain=domain)
if stocktransaction.quantity != 0:
transactions_to_add.append(StockTransaction(
case_id=case._id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type='receipts' if stocktransaction.quantity > 0 else 'consumption',
stock_on_hand=Decimal(stocktransaction.ending_balance),
quantity=Decimal(stocktransaction.quantity),
report=report
))
transactions_to_add.append(StockTransaction(
case_id=case._id,
product_id=sql_product.product_id,
sql_product=sql_product,
section_id=section_id,
type='stockonhand',
stock_on_hand=Decimal(stocktransaction.ending_balance),
report=report
))
products_saved.add(sql_product.product_id)
if transactions_to_add:
# Doesn't send signal
StockTransaction.objects.bulk_create(transactions_to_add)
if not meta.get('next', False):
has_next = False
else:
next_url = meta['next'].split('?')[1]
for product in products_saved:
# if we saved anything rebuild the stock state object by firing the signal
# on the last transaction for each product
last_st = StockTransaction.latest(case._id, section_id, product)
update_stock_state_for_transaction(last_st)
def stock_data_task(domain, endpoint, apis, test_facilities=None):
start_date = datetime.today()
try:
checkpoint = StockDataCheckpoint.objects.get(domain=domain)
api = checkpoint.api
date = checkpoint.date
limit = checkpoint.limit
offset = checkpoint.offset
location = checkpoint.location
if not checkpoint.start_date:
checkpoint.start_date = start_date
checkpoint.save()
else:
start_date = checkpoint.start_date
except StockDataCheckpoint.DoesNotExist:
checkpoint = StockDataCheckpoint()
checkpoint.domain = domain
checkpoint.start_date = start_date
api = 'product_stock'
date = None
limit = 100
offset = 0
location = None
if TEST:
facilities = test_facilities
else:
facilities = SQLLocation.objects.filter(
domain=domain,
location_type__iexact='FACILITY'
).order_by('created_at').values_list('external_id', flat=True)
apis_from_checkpoint = itertools.dropwhile(lambda x: x[0] != api, apis)
facilities_copy = list(facilities)
if location:
supply_point = SupplyPointCase.view(
'commtrack/supply_point_by_loc',
key=[location.domain, location.location_id],
include_docs=True,
classes={'CommCareCase': SupplyPointCase},
).one()
external_id = supply_point.external_id if supply_point else None
if external_id:
facilities = itertools.dropwhile(lambda x: int(x) != int(external_id), facilities)
for idx, api in enumerate(apis_from_checkpoint):
api[1](
domain=domain,
checkpoint=checkpoint,
date=date,
limit=limit,
offset=offset,
endpoint=endpoint,
facilities=facilities
)
limit = 100
offset = 0
if idx == 0:
facilities = facilities_copy
save_stock_data_checkpoint(checkpoint, 'product_stock', 100, 0, start_date, None, False)
checkpoint.start_date = None
checkpoint.save()
