def handle(self, *args, **options):
ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance')
to_save = []
for doc in iter_docs(XFormInstance.get_db(), ids):
try:
if 'location_id' in doc[
'form'] and not doc['form']['location_id']:
case = SupplyPointCase.get(doc['form']['case']['@case_id'])
if case.type == 'supply-point':
instance = XFormInstance.get(doc['_id'])
# fix the XFormInstance
instance.form['location_id'] = case.location_id
# fix the actual form.xml
xml_object = etree.fromstring(instance.get_xml())
location_id_node = xml_object.find(
re.sub('}.*', '}location_id', xml_object.tag))
location_id_node.text = case.location_id
updated_xml = etree.tostring(xml_object)
attachment_builder = CouchAttachmentsBuilder(
instance._attachments)
attachment_builder.add(
name='form.xml',
content=updated_xml,
content_type=instance._attachments['form.xml']
['content_type'])
instance._attachments = attachment_builder.to_json()
print 'Updating XFormInstance:', doc['_id']
to_save.append(instance)
def test_archive_flips_sp_cases(self):
loc = make_loc('someloc')
sp = make_supply_point(self.domain.name, loc)
self.assertFalse(sp.closed)
loc.archive()
sp = SupplyPointCase.get(sp._id)
self.assertTrue(sp.closed)
loc.unarchive()
sp = SupplyPointCase.get(sp._id)
self.assertFalse(sp.closed)
def handle(self, *args, **options):
ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance')
to_save = []
locations = SQLLocation.objects.filter(
domain='ipm-senegal').values_list('location_id', 'name')
locations_map = {
location_id: name
for (location_id, name) in locations
}
for doc in iter_docs(XFormInstance.get_db(), ids):
try:
if 'PPS_name' in doc['form'] and not doc['form']['PPS_name']:
case = SupplyPointCase.get(doc['form']['case']['@case_id'])
if case.type == 'supply-point':
print 'Updating XFormInstance:', doc['_id']
pps_name = locations_map[case.location_id]
instance = XFormInstance.get(doc['_id'])
# fix the XFormInstance
instance.form['PPS_name'] = pps_name
for instance_prod in instance.form['products']:
instance_prod['PPS_name'] = instance_prod[
'PPS_name'] or pps_name
# fix the actual form.xml
xml_object = etree.fromstring(instance.get_xml())
pps_name_node = xml_object.find(
re.sub('}.*', '}PPS_name', xml_object.tag))
pps_name_node.text = pps_name
products_nodes = xml_object.findall(
re.sub('}.*', '}products', xml_object.tag))
for product_node in products_nodes:
product_pps_name_node = product_node.find(
re.sub('}.*', '}PPS_name', xml_object.tag))
product_pps_name_node.text = pps_name
updated_xml = etree.tostring(xml_object)
attachment_builder = CouchAttachmentsBuilder(
instance._attachments)
attachment_builder.add(
name='form.xml',
content=updated_xml,
content_type=instance._attachments['form.xml']
['content_type'])
instance._attachments = attachment_builder.to_json()
to_save.append(instance)
def test_archive_flips_sp_cases(self):
loc = make_loc('someloc')
sp = loc.linked_supply_point()
self.assertFalse(sp.closed)
loc.archive()
sp = SupplyPointCase.get(sp.case_id)
self.assertTrue(sp.closed)
loc.unarchive()
sp = SupplyPointCase.get(sp.case_id)
self.assertFalse(sp.closed)
def test_archive_flips_sp_cases(self):
loc = make_loc('someloc')
sp = make_supply_point(self.domain.name, loc)
self.assertFalse(sp.closed)
loc.archive()
sp = SupplyPointCase.get(sp._id)
self.assertTrue(sp.closed)
loc.unarchive()
sp = SupplyPointCase.get(sp._id)
self.assertFalse(sp.closed)
def leaf_node_data(self, stock_states):
for state in stock_states:
product = Product.get(state.product_id)
result = {
'product_id': product._id,
'product_name': product.name,
'current_stock': format_decimal(state.stock_on_hand),
}
if self._include_advanced_data():
result.update({
'location_id':
SupplyPointCase.get(state.case_id).location_id,
'location_lineage':
None,
'category':
state.stock_category,
'consumption':
state.get_monthly_consumption(),
'months_remaining':
state.months_remaining,
'resupply_quantity_needed':
state.resupply_quantity_needed
})
yield result
def get_data(self):
sp_ids = get_relevant_supply_point_ids(
self.domain,
self.active_location
)
products = Product.by_domain(self.domain)
if self.program_id:
products = filter(
lambda product: product.program_id == self.program_id, products
)
for sp_id in sp_ids:
for product in products:
loc = SupplyPointCase.get(sp_id).location
last_transaction = StockTransaction.latest(
sp_id,
STOCK_SECTION_TYPE,
product._id
)
yield {
'loc_id': loc._id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type,
'reporting_status': reporting_status(
last_transaction,
self.start_date,
self.end_date
),
'geo': loc._geopoint,
}
def linked_supply_point(self):
from corehq.apps.commtrack.models import SupplyPointCase
if not self.supply_point_id:
return None
try:
return SupplyPointCase.get(self.supply_point_id)
except:
return None
def test_delete_closes_sp_cases(self):
loc = make_loc('test_loc')
sp = loc.linked_supply_point()
self.assertFalse(sp.closed)
loc.full_delete()
sp = SupplyPointCase.get(sp.case_id)
self.assertTrue(sp.closed)
def test_change_to_administrative_and_back(self):
# at first it should have a supply point
self.assertHasSupplyPoint(self.boston)
supply_point_id = self.boston.supply_point_id
self.city_type.administrative = True
self.city_type.save()
# Now that it's administrative, it shouldn't have one
# The case should still exist, but be closed
self.assertHasNoSupplyPoint(self.boston)
self.assertTrue(SupplyPointCase.get(supply_point_id).closed)
self.city_type.administrative = False
self.city_type.save()
# The same supply point case should be reopened
self.assertHasSupplyPoint(self.boston)
self.assertEqual(self.boston.supply_point_id, supply_point_id)
self.assertFalse(SupplyPointCase.get(supply_point_id).closed)
def test_change_to_administrative_and_back(self):
# at first it should have a supply point
self.assertHasSupplyPoint(self.boston)
supply_point_id = self.boston.supply_point_id
self.city_type.administrative = True
self.city_type.save()
# Now that it's administrative, it shouldn't have one
# The case should still exist, but be closed
self.assertHasNoSupplyPoint(self.boston)
self.assertTrue(SupplyPointCase.get(supply_point_id).closed)
self.city_type.administrative = False
self.city_type.save()
# The same supply point case should be reopened
self.assertHasSupplyPoint(self.boston)
self.assertEqual(self.boston.supply_point_id, supply_point_id)
self.assertFalse(SupplyPointCase.get(supply_point_id).closed)
def handle(self, *args, **options):
ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance')
to_save = []
locations = SQLLocation.objects.filter(domain='ipm-senegal').values_list('location_id', 'name')
locations_map = {location_id: name for (location_id, name) in locations}
for doc in iter_docs(XFormInstance.get_db(), ids):
try:
if 'PPS_name' in doc['form'] and not doc['form']['PPS_name']:
case = SupplyPointCase.get(doc['form']['case']['@case_id'])
if case.type == 'supply-point':
print 'Updating XFormInstance:', doc['_id']
pps_name = locations_map[case.location_id]
instance = XFormInstance.get(doc['_id'])
# fix the XFormInstance
instance.form['PPS_name'] = pps_name
for instance_prod in instance.form['products']:
instance_prod['PPS_name'] = instance_prod['PPS_name'] or pps_name
# fix the actual form.xml
xml_object = etree.fromstring(instance.get_xml())
pps_name_node = xml_object.find(re.sub('}.*', '}PPS_name', xml_object.tag))
pps_name_node.text = pps_name
products_nodes = xml_object.findall(re.sub('}.*', '}products', xml_object.tag))
for product_node in products_nodes:
product_pps_name_node = product_node.find(re.sub('}.*', '}PPS_name', xml_object.tag))
product_pps_name_node.text = pps_name
updated_xml = etree.tostring(xml_object)
attachment_builder = CouchAttachmentsBuilder(instance._attachments)
attachment_builder.add(
name='form.xml',
content=updated_xml,
content_type=instance._attachments['form.xml']['content_type']
)
instance._attachments = attachment_builder.to_json()
to_save.append(instance)
except Exception:
print 'Failed to save XFormInstance:', doc['_id']
if len(to_save) > 500:
XFormInstance.get_db().bulk_save(to_save)
to_save = []
if to_save:
XFormInstance.get_db().bulk_save(to_save)
def supply_points_with_latest_status_by_datespan(sps, status_type, status_value, datespan):
"""
This very similar method is used by the reminders.
"""
ids = [sp._id for sp in sps]
inner = SupplyPointStatus.objects.filter(supply_point__in=ids,
status_type=status_type,
status_date__gte=datespan.startdate,
status_date__lte=datespan.enddate).annotate(pk=Max('id'))
ids = SupplyPointStatus.objects.filter(id__in=inner.values('pk').query,
status_type=status_type,
status_value=status_value).distinct().values_list("supply_point", flat=True)
return [SupplyPointCase.get(id) for id in ids]
def leaf_node_data(self, stock_states):
for state in stock_states:
product = Product.get(state.product_id)
yield {
'category': state.stock_category,
'product_id': product._id,
'consumption': state.get_consumption() * 30 if state.get_consumption() is not None else None,
'months_remaining': state.months_remaining,
'location_id': SupplyPointCase.get(state.case_id).location_id,
'product_name': product.name,
'current_stock': self.format_decimal(state.stock_on_hand),
'location_lineage': None,
'resupply_quantity_needed': state.resupply_quantity_needed
}
def leaf_node_data(self, stock_states):
for state in stock_states:
product = Product.get(state.product_id)
yield {
'category': state.stock_category,
'product_id': product._id,
'consumption': state.get_consumption() * Decimal(DAYS_IN_MONTH) if state.get_consumption() is not None else None,
'months_remaining': state.months_remaining,
'location_id': SupplyPointCase.get(state.case_id).location_id,
'product_name': product.name,
'current_stock': self.format_decimal(state.stock_on_hand),
'location_lineage': None,
'resupply_quantity_needed': state.resupply_quantity_needed
}
def leaf_node_data(self, stock_states):
for state in stock_states:
product = Product.get(state.product_id)
yield {
'category': state.stock_category,
'product_id': product._id,
'consumption': state.daily_consumption * 30 if state.daily_consumption else None,
'months_remaining': state.months_remaining,
'location_id': SupplyPointCase.get(state.case_id).location_id,
'product_name': product.name,
'current_stock': state.stock_on_hand,
'location_lineage': None,
'resupply_quantity_needed': state.resupply_quantity_needed
}
def supply_points_with_latest_status_by_datespan(locations, status_type, status_value, datespan):
"""
This very similar method is used by the reminders.
"""
ids = [loc.location_id for loc in locations]
inner = SupplyPointStatus.objects.filter(location_id__in=ids,
status_type=status_type,
status_date__gte=datespan.startdate,
status_date__lte=datespan.enddate).annotate(pk=Max('id'))
ids = SupplyPointStatus.objects.filter(
id__in=inner.values('pk').query,
status_type=status_type,
status_value=status_value).distinct().values_list("location_id", flat=True)
return [SupplyPointCase.get(id) for id in ids]
def handle(self, *args, **options):
startkey = ['ipm-senegal', 'by_type', 'XFormInstance']
endkey = startkey + [{}]
ids = [row['id'] for row in XFormInstance.get_db().view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=False
)]
to_save = []
for doc in iter_docs(XFormInstance.get_db(), ids):
try:
if 'location_id' in doc['form'] and not doc['form']['location_id']:
case = SupplyPointCase.get(doc['form']['case']['@case_id'])
if case.type == 'supply-point':
instance = XFormInstance.get(doc['_id'])
# fix the XFormInstance
instance.form['location_id'] = case.location_id
# fix the actual form.xml
xml_object = etree.fromstring(instance.get_xml())
location_id_node = xml_object.find(re.sub('}.*', '}location_id', xml_object.tag))
location_id_node.text = case.location_id
updated_xml = etree.tostring(xml_object)
attachment_builder = CouchAttachmentsBuilder(instance._attachments)
attachment_builder.add(
name='form.xml',
content=updated_xml,
content_type=instance._attachments['form.xml']['content_type']
)
instance._attachments = attachment_builder.to_json()
print 'Updating XFormInstance:', doc['_id']
to_save.append(instance)
except Exception:
print 'Failed to save XFormInstance:', doc['_id']
if len(to_save) > 500:
XFormInstance.get_db().bulk_save(to_save)
to_save = []
if to_save:
XFormInstance.get_db().bulk_save(to_save)
def make_supply_point(domain, location, owner_id=None):
# a supply point is currently just a case with a special type
id = uuid.uuid4().hex
user_id = get_commtrack_user_id(domain)
owner_id = owner_id or user_id
username = const.COMMTRACK_USERNAME
caseblock = CaseBlock(
case_id=id,
create=True,
version=V2,
case_name=location.name,
user_id=user_id,
owner_id=owner_id,
case_type=const.SUPPLY_POINT_CASE_TYPE,
update={"location_id": location._id},
)
casexml = ElementTree.tostring(caseblock.as_xml())
submit_case_blocks(casexml, domain, username, user_id, xmlns=const.COMMTRACK_SUPPLY_POINT_XMLNS)
return SupplyPointCase.get(id)
def leaf_node_data(self, stock_states):
for state in stock_states:
product = Product.get(state.product_id)
result = {
'product_id': product._id,
'product_name': product.name,
'current_stock': format_decimal(state.stock_on_hand),
}
if self._include_advanced_data():
result.update({
'location_id': SupplyPointCase.get(state.case_id).location_id,
'location_lineage': None,
'category': state.stock_category,
'consumption': state.get_monthly_consumption(),
'months_remaining': state.months_remaining,
'resupply_quantity_needed': state.resupply_quantity_needed
})
yield result
def get_data(self):
sp_ids = get_relevant_supply_point_ids(
self.domain,
self.active_location
)
products = Product.by_domain(self.domain)
if self.program_id:
products = filter(
lambda product: product.program_id == self.program_id, products
)
for sp_id in sp_ids:
loc = SupplyPointCase.get(sp_id).location
transactions = StockTransaction.objects.filter(
case_id=sp_id,
section_id=STOCK_SECTION_TYPE,
)
if transactions:
last_transaction = sorted(
transactions,
key=lambda trans: trans.report.date
)[-1]
else:
last_transaction = None
yield {
'loc_id': loc._id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type,
'reporting_status': reporting_status(
last_transaction,
self.start_date,
self.end_date
),
'geo': loc._geopoint,
}
def make_supply_point(domain, location, owner_id=None):
# a supply point is currently just a case with a special type
id = uuid.uuid4().hex
user_id = get_commtrack_user_id(domain)
owner_id = owner_id or user_id
username = const.COMMTRACK_USERNAME
caseblock = CaseBlock(case_id=id,
create=True,
version=V2,
case_name=location.name,
user_id=user_id,
owner_id=owner_id,
case_type=const.SUPPLY_POINT_CASE_TYPE,
update={
'location_id': location._id,
})
casexml = ElementTree.tostring(caseblock.as_xml())
submit_case_blocks(casexml,
domain,
username,
user_id,
xmlns=const.COMMTRACK_SUPPLY_POINT_XMLNS)
return SupplyPointCase.get(id)
def get_location(self):
try:
return SupplyPointCase.get(self.indices[0].referenced_id).location
except ResourceNotFound:
return None
def get_supply_point(supply_point_id):
return SupplyPointCase.get(supply_point_id)
def supply_point_location(case_id):
return SupplyPointCase.get(case_id).location_id
def get_location(self):
try:
return SupplyPointCase.get(self.indices[0].referenced_id).location
except ResourceNotFound:
return None
def supply_point_location(case_id):
return SupplyPointCase.get(case_id).location_[-1]
def get_data(self):
sp_ids = get_relevant_supply_point_ids(
self.domain,
self.active_location
)
products = Product.by_domain(self.domain)
if self.program_id:
products = filter(
lambda product: product.program_id == self.program_id, products
)
for sp_id in sp_ids:
loc = SupplyPointCase.get(sp_id).location
transactions = StockTransaction.objects.filter(
case_id=sp_id,
).exclude(
report__date__lte=self.start_date
).exclude(
report__date__gte=self.end_date
)
if transactions:
last_transaction = sorted(
transactions,
key=lambda trans: trans.report.date
)[-1]
else:
last_transaction = None
if self.all_relevant_forms:
forms_xmlns = []
for form in self.all_relevant_forms.values():
forms_xmlns.append(form['xmlns'])
if last_transaction:
form = XFormInstance.get(last_transaction.report.form_id)
if form.xmlns in forms_xmlns:
yield {
'loc_id': loc._id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type,
'reporting_status': reporting_status(
last_transaction,
self.start_date,
self.end_date
),
'geo': loc._geopoint,
}
else:
yield {
'loc_id': loc._id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type,
'reporting_status': reporting_status(
None,
self.start_date,
self.end_date
),
'geo': loc._geopoint,
}
class StockStatusDataSource(ReportDataSource, CommtrackDataSourceMixin):
"""
Config:
domain: The domain to report on.
location_id: ID of location to get data for. Omit for all locations.
product_id: ID of product to get data for. Omit for all products.
aggregate: True to aggregate the indicators by product for the current location.
Data Slugs:
product_name: Name of the product
product_id: ID of the product
location_id: The ID of the current location.
location_lineage: The lineage of the current location.
current_stock: The current stock level
consumption: The current monthly consumption rate
months_remaining: The number of months remaining until stock out
category: The status category. See casexml.apps.stock.models.StockState.stock_category
resupply_quantity_needed: Max amount - current amount
"""
slug = 'agg_stock_status'
SLUG_PRODUCT_NAME = 'product_name'
SLUG_PRODUCT_ID = 'product_id'
SLUG_MONTHS_REMAINING = 'months_remaining'
SLUG_CONSUMPTION = 'consumption'
SLUG_CURRENT_STOCK = 'current_stock'
SLUG_LOCATION_ID = 'location_id'
SLUG_LOCATION_LINEAGE = 'location_lineage'
SLUG_STOCKOUT_SINCE = 'stockout_since'
SLUG_STOCKOUT_DURATION = 'stockout_duration'
SLUG_LAST_REPORTED = 'last_reported'
SLUG_CATEGORY = 'category'
SLUG_RESUPPLY_QUANTITY_NEEDED = 'resupply_quantity_needed'
_slug_attrib_map = {
SLUG_PRODUCT_NAME: lambda s: Product.get(s.product_id).name,
SLUG_PRODUCT_ID: lambda s: s.product_id,
SLUG_LOCATION_ID: lambda s: SupplyPointCase.get(s.case_id).location_[-1],
# SLUG_LOCATION_LINEAGE: lambda p: list(reversed(p.location_[:-1])),
SLUG_CURRENT_STOCK: 'stock_on_hand',
SLUG_CONSUMPTION: lambda s: s.get_consumption() * 30 if s.get_consumption() is not None else None,
SLUG_MONTHS_REMAINING: 'months_remaining',
SLUG_CATEGORY: 'stock_category',
# SLUG_STOCKOUT_SINCE: 'stocked_out_since',
# SLUG_STOCKOUT_DURATION: 'stockout_duration_in_months',
SLUG_LAST_REPORTED: 'last_modified_date',
SLUG_RESUPPLY_QUANTITY_NEEDED: 'resupply_quantity_needed',
}
def slugs(self):
return self._slug_attrib_map.keys()
def filter_by_program(self, stock_states):
return stock_states.filter(
product_id__in=product_ids_filtered_by_program(
self.domain,
self.program_id
)
)
def get_data(self, slugs=None):
sp_ids = get_relevant_supply_point_ids(self.domain, self.active_location)
if len(sp_ids) == 1:
stock_states = StockState.objects.filter(
case_id=sp_ids[0],
section_id=STOCK_SECTION_TYPE,
last_modified_date__lte=self.end_date,
last_modified_date__gte=self.start_date,
)
if self.program_id:
stock_states = self.filter_by_program(stock_states)
return self.leaf_node_data(stock_states)
else:
stock_states = StockState.objects.filter(
case_id__in=sp_ids,
section_id=STOCK_SECTION_TYPE,
last_modified_date__lte=self.end_date,
last_modified_date__gte=self.start_date,
)
if self.program_id:
stock_states = self.filter_by_program(stock_states)
if self.config.get('aggregate'):
return self.aggregated_data(stock_states)
else:
return self.raw_product_states(stock_states, slugs)
def format_decimal(self, d):
# https://docs.python.org/2/library/decimal.html#decimal-faq
if d is not None:
return d.quantize(Decimal(1)) if d == d.to_integral() else d.normalize()
else:
return None
def leaf_node_data(self, stock_states):
for state in stock_states:
product = Product.get(state.product_id)
yield {
'category': state.stock_category,
'product_id': product._id,
'consumption': state.get_consumption() * 30 if state.get_consumption() is not None else None,
'months_remaining': state.months_remaining,
'location_id': SupplyPointCase.get(state.case_id).location_id,
'product_name': product.name,
'current_stock': self.format_decimal(state.stock_on_hand),
'location_lineage': None,
'resupply_quantity_needed': state.resupply_quantity_needed
}
def aggregated_data(self, stock_states):
product_aggregation = {}
for state in stock_states:
if state.product_id in product_aggregation:
product = product_aggregation[state.product_id]
product['current_stock'] = self.format_decimal(
product['current_stock'] + state.stock_on_hand
)
if product['total_consumption'] is None:
product['total_consumption'] = state.get_consumption()
elif state.get_consumption() is not None:
product['total_consumption'] += state.get_consumption()
product['count'] += 1
if product['total_consumption'] is not None:
product['consumption'] = product['total_consumption'] / product['count']
else:
product['consumption'] = None
product['category'] = stock_category(
product['current_stock'],
product['consumption'],
Domain.get_by_name(self.domain)
)
product['months_remaining'] = months_of_stock_remaining(
product['current_stock'],
product['consumption']
)
else:
product = Product.get(state.product_id)
consumption = state.get_consumption()
product_aggregation[state.product_id] = {
'product_id': product._id,
'location_id': None,
'product_name': product.name,
'location_lineage': None,
'resupply_quantity_needed': None,
'current_stock': self.format_decimal(state.stock_on_hand),
'total_consumption': consumption,
'count': 1,
'consumption': consumption,
'category': stock_category(
state.stock_on_hand,
consumption,
Domain.get_by_name(self.domain)
),
'months_remaining': months_of_stock_remaining(
state.stock_on_hand,
consumption
)
}
return product_aggregation.values()
def raw_product_states(self, stock_states, slugs):
def _slug_attrib(slug, attrib, product, output):
if not slugs or slug in slugs:
if callable(attrib):
output[slug] = attrib(product)
else:
output[slug] = getattr(product, attrib, '')
for state in stock_states:
out = {}
for slug, attrib in self._slug_attrib_map.items():
_slug_attrib(slug, attrib, state, out)
yield out
def get_supply_point(supply_point_id):
return SupplyPointCase.get(supply_point_id)
