def _aggregate_data(self):
config = {
'domain': self.domain,
'location_id': self.request.GET.get('location_id'),
'startdate': self.datespan.startdate_utc,
'enddate': self.datespan.enddate_utc,
'request': self.request,
}
statuses = list(ReportingStatusDataSource(config).get_data())
def child_loc(path):
root = self.active_location
ix = path.index(root.location_id) if root else -1
try:
return path[ix + 1]
except IndexError:
return None
def case_iter():
for site in statuses:
if child_loc(site['loc_path']) is not None:
yield (site['loc_path'], site['reporting_status'])
status_by_agg_site = map_reduce(
lambda path_status: [(child_loc(path_status[0]), path_status[1])],
data=case_iter())
sites_by_agg_site = map_reduce(lambda path_status1: [(child_loc(
path_status1[0]), path_status1[0][-1])],
data=case_iter())
status_counts = dict(
(loc_id, self.status_tally(statuses))
for loc_id, statuses in six.iteritems(status_by_agg_site))
master_tally = self.status_tally(
[site['reporting_status'] for site in statuses])
locs = (SQLLocation.objects.filter(
is_archived=False,
location_id__in=list(status_counts)).order_by('name'))
def fmt(pct):
return '%.1f%%' % (100. * pct)
def fmt_pct_col(loc_id, col_type):
return fmt(status_counts[loc_id].get(col_type, {'pct': 0.})['pct'])
def fmt_count_col(loc_id, col_type):
return status_counts[loc_id].get(col_type, {'count': 0})['count']
def _rows():
for loc in locs:
row = [loc.name, len(sites_by_agg_site[loc.location_id])]
for k in ('reporting', 'nonreporting'):
row.append(fmt_count_col(loc.location_id, k))
row.append(fmt_pct_col(loc.location_id, k))
yield row
return master_tally, _rows()
def summary_row(site, reports, product_cases, outlets):
all_transactions = list(itertools.chain(*(get_transactions(r) for r in reports)))
tx_by_product = map_reduce(lambda tx: [(tx['product'],)], data=all_transactions, include_docs=True)
cases_by_product = map_reduce(lambda c: [(c.product,)], data=product_cases, include_docs=True)
num_outlets = len(outlets)
product_ids = set(p['_id'] for p in products)
relevant_reports = [r for r in reports if any(tx['product'] in product_ids for tx in get_transactions(r))]
num_active_outlets = len(set(leaf_loc(r) for r in relevant_reports))
data = site_metadata(site, self.ancestry)
data.extend([
num_active_outlets,
num_outlets,
])
for p in products:
tx_by_action = map_reduce(lambda tx: [(tx['action'], int(tx['value']))], data=tx_by_product.get(p['_id'], []))
subcases = cases_by_product.get(p['_id'], [])
stocks = [int(k) for k in (c.get_case_property('current_stock') for c in subcases) if k is not None]
data.append(sum(stocks) if stocks else u'\u2014')
data.append(sum(tx_by_action.get('sales', [])))
data.append(sum(tx_by_action.get('receipts', [])))
data.append(sum(tx_by_action.get('consumption', [])))
return data
def _data(self):
config = {
'domain': self.domain,
'location_id': self.request.GET.get('location_id'),
'startdate': self.datespan.startdate_utc,
'enddate': self.datespan.enddate_utc,
'request': self.request,
}
statuses = list(ReportingStatusDataSource(config).get_data())
def child_loc(path):
root = self.active_location
ix = path.index(root._id) if root else -1
try:
return path[ix + 1]
except IndexError:
return None
def case_iter():
for site in statuses:
if child_loc(site['loc_path']) is not None:
yield (site['loc_path'], site['reporting_status'])
status_by_agg_site = map_reduce(lambda (path, status): [(child_loc(path), status)],
data=case_iter())
sites_by_agg_site = map_reduce(lambda (path, status): [(child_loc(path), path[-1])],
data=case_iter())
def status_tally(statuses):
total = len(statuses)
return map_reduce(lambda s: [(s,)],
lambda v: {'count': len(v), 'pct': len(v) / float(total)},
data=statuses)
status_counts = dict((loc_id, status_tally(statuses))
for loc_id, statuses in status_by_agg_site.iteritems())
master_tally = status_tally([site['reporting_status'] for site in statuses])
locs = sorted(Location.view('_all_docs', keys=status_counts.keys(), include_docs=True),
key=lambda loc: loc.name)
def fmt(pct):
return '%.1f%%' % (100. * pct)
def fmt_pct_col(loc, col_type):
return fmt(status_counts[loc._id].get(col_type, {'pct': 0.})['pct'])
def fmt_count_col(loc, col_type):
return status_counts[loc._id].get(col_type, {'count': 0})['count']
def _rows():
for loc in locs:
row = [loc.name, len(sites_by_agg_site[loc._id])]
for k in ('reporting', 'nonreporting'):
row.append(fmt_count_col(loc, k))
row.append(fmt_pct_col(loc, k))
yield row
return master_tally, _rows()
def _aggregate_data(self):
config = {
'domain': self.domain,
'location_id': self.request.GET.get('location_id'),
'startdate': self.datespan.startdate_utc,
'enddate': self.datespan.enddate_utc,
'request': self.request,
}
statuses = list(ReportingStatusDataSource(config).get_data())
def child_loc(path):
root = self.active_location
ix = path.index(root._id) if root else -1
try:
return path[ix + 1]
except IndexError:
return None
def case_iter():
for site in statuses:
if child_loc(site['loc_path']) is not None:
yield (site['loc_path'], site['reporting_status'])
status_by_agg_site = map_reduce(lambda (path, status): [(child_loc(path), status)],
data=case_iter())
sites_by_agg_site = map_reduce(lambda (path, status): [(child_loc(path), path[-1])],
data=case_iter())
status_counts = dict((loc_id, self.status_tally(statuses))
for loc_id, statuses in status_by_agg_site.iteritems())
master_tally = self.status_tally([site['reporting_status'] for site in statuses])
locs = (SQLLocation.objects
.filter(is_archived=False,
location_id__in=status_counts.keys())
.order_by('name'))
def fmt(pct):
return '%.1f%%' % (100. * pct)
def fmt_pct_col(loc_id, col_type):
return fmt(status_counts[loc_id].get(col_type, {'pct': 0.})['pct'])
def fmt_count_col(loc_id, col_type):
return status_counts[loc_id].get(col_type, {'count': 0})['count']
def _rows():
for loc in locs:
row = [loc.name, len(sites_by_agg_site[loc.location_id])]
for k in ('reporting', 'nonreporting'):
row.append(fmt_count_col(loc.location_id, k))
row.append(fmt_pct_col(loc.location_id, k))
yield row
return master_tally, _rows()
def parent_child(domain):
"""
Returns a dict mapping from a location type to its possible
child types
"""
return map_reduce(lambda (k, v): [(p, k) for p in v],
data=dict(location_hierarchy_config(domain)).iteritems())
def get_data(self):
data = list(super(StockStatusBySupplyPointDataSource, self).get_data())
products = dict((r['product_id'], r['product_name']) for r in data)
product_ids = sorted(products, key=lambda e: products[e])
by_supply_point = map_reduce(lambda e: [(e['location_id'], )],
data=data,
include_docs=True)
locs = _location_map(list(by_supply_point))
for loc_id, subcases in by_supply_point.items():
if loc_id not in locs:
continue # it's archived, skip
loc = locs[loc_id]
by_product = dict((c['product_id'], c) for c in subcases)
rec = {
'name': loc.name,
'type': loc.location_type.name,
'geo': geopoint(loc),
}
for prod in product_ids:
rec.update(
dict(('%s-%s' % (prod, key),
by_product.get(prod, {}).get(key))
for key in ('current_stock', 'consumption',
'months_remaining', 'category')))
yield rec
def parent_child(domain):
"""
Returns a dict mapping from a location type to its possible
child types
"""
return map_reduce(lambda k_v: [(p, k_v[0]) for p in k_v[1]],
data=dict(location_hierarchy_config(domain)).items())
def parent_child(domain):
"""
Returns a dict mapping from a location type to its possible
child types
"""
return map_reduce(lambda k_v: [(p, k_v[0]) for p in k_v[1]],
data=six.iteritems(dict(location_hierarchy_config(domain))))
def get_data(self):
data = list(super(StockStatusBySupplyPointDataSource, self).get_data())
products = dict((r['product_id'], r['product_name']) for r in data)
product_ids = sorted(products.keys(), key=lambda e: products[e])
by_supply_point = map_reduce(lambda e: [(e['location_id'], )],
data=data,
include_docs=True)
locs = dict((loc._id, loc) for loc in Location.view(
'_all_docs', keys=by_supply_point.keys(), include_docs=True))
for loc_id, subcases in by_supply_point.iteritems():
loc = locs[loc_id]
by_product = dict((c['product_id'], c) for c in subcases)
rec = {
'name': loc.name,
'type': loc.location_type,
'geo': loc._geopoint,
}
for prod in product_ids:
rec.update(
dict(('%s-%s' % (prod, key),
by_product.get(prod, {}).get(key))
for key in ('current_stock', 'consumption',
'months_remaining', 'category')))
yield rec
def get_data(self):
startkey = [self.domain, self.active_location._id if self.active_location else None]
product_cases = SPPCase.view('commtrack/product_cases',
startkey=startkey,
endkey=startkey + [{}],
include_docs=True)
if self.program_id:
product_cases = filter(lambda c: Product.get(c.product).program_id == self.program_id, product_cases)
def latest_case(cases):
# getting last report date should probably be moved to a util function in a case wrapper class
return max(cases, key=lambda c: getattr(c, 'last_reported', datetime(2000, 1, 1)).date())
cases_by_site = map_reduce(lambda c: [(tuple(c.location_),)],
lambda v: reporting_status(latest_case(v), self.start_date, self.end_date),
data=product_cases, include_docs=True)
# TODO if aggregating, won't want to fetch all these locs (will only want to fetch aggregation sites)
locs = dict((loc._id, loc) for loc in Location.view(
'_all_docs',
keys=[path[-1] for path in cases_by_site.keys()],
include_docs=True))
for path, status in cases_by_site.iteritems():
loc = locs[path[-1]]
yield {
'loc_id': loc._id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type,
'reporting_status': status,
'geo': loc._geopoint,
}
def process(domain, instance):
"""process an incoming commtrack stock report instance"""
config = CommtrackConfig.for_domain(domain)
root = etree.fromstring(instance)
transactions = unpack_transactions(root, config)
case_ids = [tx["case_id"] for tx in transactions]
cases = dict((c._id, c) for c in CommCareCase.view("_all_docs", keys=case_ids, include_docs=True))
# ensure transaction types are processed in the correct order
def transaction_order(tx):
return [action.action_name for action in config.actions].index(tx["action"])
transactions.sort(key=transaction_order)
# apply all transactions to each product case in bulk
transactions_by_product = map_reduce(lambda tx: [(tx["case_id"],)], data=transactions, include_docs=True)
for product_id, txs in transactions_by_product.iteritems():
product_case = cases[product_id]
case_block, reconciliations = process_product_transactions(product_case, txs)
for recon in reconciliations:
root.append(recon)
root.append(case_block)
submission = etree.tostring(root)
logger.debug("submitting: %s" % submission)
submit_time = root.find(".//%s" % _("timeStart", META_XMLNS)).text
spoof_submission(get_submit_url(domain), submission, headers={"HTTP_X_SUBMIT_TIME": submit_time})
def get_data(self):
data = list(super(StockStatusBySupplyPointDataSource, self).get_data())
products = dict((r['product_id'], r['product_name']) for r in data)
product_ids = sorted(products.keys(), key=lambda e: products[e])
by_supply_point = map_reduce(lambda e: [(e['location_id'],)], data=data, include_docs=True)
locs = dict((loc._id, loc) for loc in Location.view(
'_all_docs',
keys=by_supply_point.keys(),
include_docs=True))
for loc_id, subcases in by_supply_point.iteritems():
loc = locs[loc_id]
by_product = dict((c['product_id'], c) for c in subcases)
rec = {
'name': loc.name,
'type': loc.location_type,
'geo': loc._geopoint,
}
for prod in product_ids:
rec.update(dict(('%s-%s' % (prod, key), by_product.get(prod, {}).get(key)) for key in
('current_stock', 'consumption', 'months_remaining', 'category')))
yield rec
def handle(self, *args, **options):
try:
domain = args[0]
except IndexError:
self.stderr.write('domain required\n')
return
self.println('Migrating...')
for loc_id, case in get_supply_points_json_in_domain_by_location(domain):
loc = Location.get(loc_id)
old_code = case.get('site_code', '')
new_code = getattr(loc, 'site_code', '')
if old_code and not new_code:
loc.site_code = old_code
loc.save()
self.println('migrated %s (%s)' % (loc.name, loc.site_code))
self.println('Verifying code uniqueness...')
all_codes = Location.get_db().view('commtrack/locations_by_code',
startkey=[domain], endkey=[domain, {}])
locs_by_code = map_reduce(lambda e: [(e['key'][-1].lower(), e['id'])], data=all_codes)
for code, loc_ids in locs_by_code.iteritems():
if len(loc_ids) == 1:
continue
self.println('duplicate code [%s]' % code)
locs = Location.view('_all_docs', keys=loc_ids, include_docs=True)
for loc in locs:
self.println(' %s [%s]' % (loc.name, loc._id))
def process_stock(xform, case_db=None):
"""
process the commtrack xml constructs in an incoming submission
"""
case_db = case_db or CaseDbCache()
assert isinstance(case_db, CaseDbCache)
if is_device_report(xform):
return []
domain = xform.domain
config = CommtrackConfig.for_domain(domain)
# these are the raw stock report objects from the xml
stock_reports = list(unpack_commtrack(xform, config))
# flattened transaction list spanning all stock reports in the form
transactions = [t for r in stock_reports for t in r.transactions]
# omitted: normalize_transactions (used for bulk requisitions?)
if not transactions:
return []
# transactions grouped by case/product id
grouped_tx = map_reduce(lambda tx: [((tx.case_id, tx.product_id),)],
lambda v: sorted(v, key=lambda tx: tx.timestamp),
data=transactions,
include_docs=True)
case_ids = list(set(k[0] for k in grouped_tx))
# list of cases that had stock reports in the form
# there is no need to wrap them by case type
relevant_cases = [case_db.get(case_id) for case_id in case_ids]
user_id = xform.form['meta']['userID']
submit_time = xform['received_on']
# touch every case for proper ota restore logic syncing to be preserved
for case in relevant_cases:
case_action = CommCareCaseAction.from_parsed_action(
submit_time, user_id, xform, AbstractAction(CASE_ACTION_COMMTRACK)
)
# hack: clear the sync log id so this modification always counts
# since consumption data could change server-side
case_action.sync_log_id = ''
case.actions.append(case_action)
case_db.mark_changed(case)
# also purge the sync token cache for the same reason
if relevant_cases and xform.get_sync_token():
xform.get_sync_token().invalidate_cached_payloads()
# create the django models
for report in stock_reports:
report.create_models(domain)
# TODO make this a signal
from corehq.apps.commtrack.signals import send_notifications, raise_events
send_notifications(xform, relevant_cases)
raise_events(xform, relevant_cases)
return relevant_cases
def status_tally(statuses):
total = len(statuses)
return map_reduce(lambda s: [(s, )],
lambda v: {
'count': len(v),
'pct': len(v) / float(total)
},
data=statuses)
def _data(self):
startkey = [self.domain, self.active_location._id if self.active_location else None]
product_cases = CommCareCase.view('commtrack/product_cases', startkey=startkey, endkey=startkey + [{}], include_docs=True)
def latest_case(cases):
# getting last report date should probably be moved to a util function in a case wrapper class
return max(cases, key=lambda c: getattr(c, 'last_reported', datetime(2000, 1, 1)).date())
cases_by_site = map_reduce(lambda c: [(tuple(c.location_),)],
lambda v: reporting_status(latest_case(v)),
data=product_cases, include_docs=True)
def child_loc(path):
root = self.active_location
ix = path.index(root._id) if root else -1
try:
return path[ix + 1]
except IndexError:
return None
def case_iter():
for k, v in cases_by_site.iteritems():
if child_loc(k) is not None:
yield (k, v)
status_by_agg_site = map_reduce(lambda (path, status): [(child_loc(path), status)],
data=case_iter())
sites_by_agg_site = map_reduce(lambda (path, status): [(child_loc(path), path[-1])],
data=case_iter())
def status_tally(statuses):
total = len(statuses)
return map_reduce(lambda s: [(s,)], lambda v: {'count': len(v), 'pct': len(v) / float(total)}, data=statuses)
status_counts = dict((loc_id, status_tally(statuses)) for loc_id, statuses in status_by_agg_site.iteritems())
master_tally = status_tally(cases_by_site.values())
locs = sorted(Location.view('_all_docs', keys=status_counts.keys(), include_docs=True), key=lambda loc: loc.name)
def fmt(pct):
return '%.1f%%' % (100. * pct)
def fmt_col(loc, col_type):
return fmt(status_counts[loc._id].get(col_type, {'pct': 0.})['pct'])
def _rows():
for loc in locs:
num_sites = len(sites_by_agg_site[loc._id])
yield [loc.name, len(sites_by_agg_site[loc._id])] + [fmt_col(loc, k) for k in ('ontime', 'late', 'nonreporting')]
return master_tally, _rows()
def get_transactions(all_tx, type_filter):
"""get all the transactions of the relevant type (filtered by type_filter),
grouped by product (returns a dict of 'product subcase id' => list of transactions),
with each set of transactions sorted in the correct order for processing
"""
return map_reduce(lambda tx: [(tx.case_id,)],
lambda v: sorted(v, key=lambda tx: tx.priority_order), # important!
data=filter(type_filter, all_tx),
include_docs=True)
def process_stock(xform):
"""
process the commtrack xml constructs in an incoming submission
"""
if is_device_report(xform):
return
domain = xform.domain
config = CommtrackConfig.for_domain(domain)
# these are the raw stock report objects from the xml
stock_reports = list(unpack_commtrack(xform, config))
# flattened transaction list spanning all stock reports in the form
transactions = [t for r in stock_reports for t in r.transactions]
# omitted: normalize_transactions (used for bulk requisitions?)
if not transactions:
return
# transactions grouped by case/product id
grouped_tx = map_reduce(lambda tx: [((tx.case_id, tx.product_id),)],
lambda v: sorted(v, key=lambda tx: tx.timestamp),
data=transactions,
include_docs=True)
# list of cases that had stock reports in the form, properly wrapped by case type
try:
relevant_cases = [wrap_commtrack_case(result['doc']) for result in
CommCareCase.get_db().view('_all_docs',
keys=list(set(k[0] for k in grouped_tx)),
include_docs=True)]
except KeyError:
raise Exception("Cannot find case matching supplied entity id")
user_id = xform.form['meta']['userID']
submit_time = xform['received_on']
# touch every case for proper ota restore logic syncing to be preserved
for case in relevant_cases:
case_action = CommCareCaseAction.from_parsed_action(
submit_time, user_id, xform, AbstractAction(CASE_ACTION_COMMTRACK)
)
# hack: clear the sync log id so this modification always counts
# since consumption data could change server-side
case_action.sync_log_id = ''
case.actions.append(case_action)
case.save()
# create the django models
for report in stock_reports:
report.create_models()
# TODO make this a signal
from corehq.apps.commtrack.signals import send_notifications, raise_events
send_notifications(xform, relevant_cases)
raise_events(xform, relevant_cases)
def get_prod_data(self):
startkey = [self.domain, self.active_location._id if self.active_location else None]
product_cases = SPPCase.view('commtrack/product_cases', startkey=startkey, endkey=startkey + [{}], include_docs=True)
cases_by_product = map_reduce(lambda c: [(c.product,)], data=product_cases, include_docs=True)
products = Product.view('_all_docs', keys=cases_by_product.keys(), include_docs=True)
def status(case):
return case.current_stock_category if is_timely(case, 1000) else 'nonreporting'
status_by_product = dict((p, map_reduce(lambda c: [(status(c),)], len, data=cases)) for p, cases in cases_by_product.iteritems())
cols = ['stockout', 'understock', 'adequate', 'overstock', 'nodata'] #'nonreporting', 'nodata']
for p in sorted(products, key=lambda p: p.name):
cases = cases_by_product.get(p._id, [])
results = status_by_product.get(p._id, {})
def val(key):
return results.get(key, 0) / float(len(cases))
yield [p.name, len(cases)] + [100. * val(key) for key in cols]
def summary_row(site, reports):
all_transactions = list(itertools.chain(*(get_transactions(r) for r in reports)))
tx_by_product = map_reduce(lambda tx: [(tx['product'],)], data=all_transactions, include_docs=True)
data = outlet_metadata(site, self.ancestry)
stockouts = {}
inactive_site = True
for p in products:
tx_by_action = map_reduce(lambda tx: [(tx['action'], int(tx['value']))], data=tx_by_product.get(p['_id'], []))
product_states = product_state_buckets.get((site._id, p['_id']), [])
stock_update_states = filter(lambda st: 'current_stock' in st['updated_unknown_properties'], product_states)
latest_state = stock_update_states[-1] if stock_update_states else None
if latest_state:
stock = latest_state['updated_unknown_properties']['current_stock']
as_of = dateparse.string_to_datetime(latest_state['server_date']).strftime('%Y-%m-%d')
inactive_site = False
stockout_dates = set()
for state in product_states:
stocked_out_since = state['updated_unknown_properties'].get('stocked_out_since')
if stocked_out_since:
so_start = max(dateparse.string_to_datetime(stocked_out_since).date(), self.datespan.startdate.date())
so_end = dateparse.string_to_datetime(state['server_date']).date() # TODO deal with time zone issues
dt = so_start
while dt < so_end:
stockout_dates.add(dt)
dt += timedelta(days=1)
stockouts[p['_id']] = stockout_dates
data.append('%s (%s)' % (stock, as_of) if latest_state else u'\u2014')
data.append(sum(tx_by_action.get('sales', [])))
data.append(sum(tx_by_action.get('receipts', [])))
data.append(sum(tx_by_action.get('consumption', [])))
combined_stockout_days = len(reduce(lambda a, b: a.intersection(b), stockouts.values()))
data.append(combined_stockout_days)
if self.HIDE_NODATA_LOCS and inactive_site:
return None
return data
def get_transactions(all_tx, type_filter):
"""get all the transactions of the relevant type (filtered by type_filter),
grouped by product (returns a dict of 'product subcase id' => list of transactions),
with each set of transactions sorted in the correct order for processing
"""
return map_reduce(
lambda tx: [(tx.case_id, )],
lambda v: sorted(v, key=lambda tx: tx.priority_order
), # important!
data=filter(type_filter, all_tx),
include_docs=True)
def process_stock(xform, case_db=None):
"""
process the commtrack xml constructs in an incoming submission
"""
case_db = case_db or CaseDbCache()
assert isinstance(case_db, CaseDbCache)
if is_device_report(xform):
return StockProcessingResult(xform)
# these are the raw stock report objects from the xml
stock_reports = list(unpack_commtrack(xform))
# flattened transaction list spanning all stock reports in the form
transactions = [t for r in stock_reports for t in r.transactions]
# omitted: normalize_transactions (used for bulk requisitions?)
if not transactions:
return StockProcessingResult(xform)
# validate product ids
is_empty = lambda product_id: product_id is None or product_id == ''
if any([is_empty(tx.product_id) for tx in transactions]):
raise MissingProductId(
_('Product IDs must be set for all ledger updates!'))
# transactions grouped by case/product id
grouped_tx = map_reduce(lambda tx: [((tx.case_id, tx.product_id),)],
lambda v: sorted(v, key=lambda tx: tx.timestamp),
data=transactions,
include_docs=True)
case_ids = list(set(k[0] for k in grouped_tx))
# list of cases that had stock reports in the form
# there is no need to wrap them by case type
relevant_cases = [case_db.get(case_id) for case_id in case_ids]
user_id = xform.form['meta']['userID']
submit_time = xform['received_on']
# touch every case for proper ota restore logic syncing to be preserved
for case in relevant_cases:
case_action = CommCareCaseAction.from_parsed_action(
submit_time, user_id, xform, AbstractAction(CASE_ACTION_COMMTRACK)
)
# hack: clear the sync log id so this modification always counts
# since consumption data could change server-side
case_action.sync_log_id = ''
case.actions.append(case_action)
case_db.mark_changed(case)
return StockProcessingResult(
xform=xform,
relevant_cases=relevant_cases,
stock_report_helpers=stock_reports,
)
def summary_row(site, reports, product_cases, outlets):
all_transactions = list(
itertools.chain(*(get_transactions(r) for r in reports)))
tx_by_product = map_reduce(lambda tx: [(tx['product'], )],
data=all_transactions,
include_docs=True)
cases_by_product = map_reduce(lambda c: [(c.product, )],
data=product_cases,
include_docs=True)
num_outlets = len(outlets)
product_ids = set(p['_id'] for p in products)
relevant_reports = [
r for r in reports if any(tx['product'] in product_ids
for tx in get_transactions(r))
]
num_active_outlets = len(set(
leaf_loc(r) for r in relevant_reports))
data = site_metadata(site, self.ancestry)
data.extend([
num_active_outlets,
num_outlets,
])
for p in products:
tx_by_action = map_reduce(
lambda tx: [(tx['action'], int(tx['value']))],
data=tx_by_product.get(p['_id'], []))
subcases = cases_by_product.get(p['_id'], [])
stocks = [
int(k) for k in (c.get_case_property('current_stock')
for c in subcases) if k is not None
]
data.append(sum(stocks) if stocks else u'\u2014')
data.extend(
sum(tx_by_action.get(a.action_name, []))
for a in self.incr_actions)
return data
def get_prod_data(self):
startkey = [
self.domain,
self.active_location._id if self.active_location else None
]
product_cases = CommCareCase.view('commtrack/product_cases',
startkey=startkey,
endkey=startkey + [{}],
include_docs=True)
cases_by_product = map_reduce(lambda c: [(c.product, )],
data=product_cases,
include_docs=True)
products = Product.view('_all_docs',
keys=cases_by_product.keys(),
include_docs=True)
def _sum(vals):
return sum(vals) if vals else None
def aggregate_product(cases):
data = [(current_stock(c), monthly_consumption(c)) for c in cases
if is_timely(c, 1000)]
total_stock = _sum([d[0] for d in data if d[0] is not None])
total_consumption = _sum([d[1] for d in data if d[1] is not None])
# exclude stock values w/o corresponding consumption figure from total months left calculation
consumable_stock = _sum(
[d[0] for d in data if d[0] is not None and d[1] is not None])
try:
months_left = consumable_stock / total_consumption
except (TypeError, ZeroDivisionError):
months_left = None
return {
'total_stock': total_stock,
'total_consumption': total_consumption,
'months_left': months_left,
}
status_by_product = dict((p, aggregate_product(cases))
for p, cases in cases_by_product.iteritems())
for p in sorted(products, key=lambda p: p.name):
stats = status_by_product[p._id]
yield [
p.name,
stats['total_stock'],
stats['total_consumption'],
stats['months_left'],
stock_category(stats['total_stock'],
stats['total_consumption'],
stats['months_left']),
]
def get_prod_data(self):
startkey = [
self.domain,
self.active_location._id if self.active_location else None
]
product_cases = CommCareCase.view('commtrack/product_cases',
startkey=startkey,
endkey=startkey + [{}],
include_docs=True)
cases_by_product = map_reduce(lambda c: [(c.product, )],
data=product_cases,
include_docs=True)
products = Product.view('_all_docs',
keys=cases_by_product.keys(),
include_docs=True)
def case_stock_category(case):
return stock_category(current_stock(case),
monthly_consumption(case))
def status(case):
return case_stock_category(case) if is_timely(
case, 1000) else 'nonreporting'
status_by_product = dict(
(p, map_reduce(lambda c: [(status(c), )], len, data=cases))
for p, cases in cases_by_product.iteritems())
cols = ['stockout', 'understock', 'adequate', 'overstock',
'nodata'] #'nonreporting', 'nodata']
for p in sorted(products, key=lambda p: p.name):
cases = cases_by_product.get(p._id, [])
results = status_by_product.get(p._id, {})
def val(key):
return results.get(key, 0) / float(len(cases))
yield [p.name, len(cases)] + [100. * val(key) for key in cols]
def test_raw_cases(self):
config = {"domain": TEST_DOMAIN}
data = list(StockStatusDataSource(config).get_data())
self.assertEqual(len(data), 6)
by_location = map_reduce(lambda row: [(row[LOCATION_ID],)], data=data, include_docs=True)
for site, products in self.sites.values():
site_id = site._id
rows = by_location[site_id]
by_product = dict((row[PRODUCT_ID], row) for row in rows)
for code, level in products.items():
product_id = self.products[code]._id
self.assertEqual(by_product[product_id][CURRENT_STOCK], level)
def test_raw_cases(self):
config = {'domain': TEST_DOMAIN}
data = list(StockStatusDataSource(config).get_data())
self.assertEqual(len(data), 6)
by_location = map_reduce(lambda row: [(row[LOCATION_ID], )],
data=data,
include_docs=True)
for site, products in self.sites.values():
site_id = site._id
rows = by_location[site_id]
by_product = dict((row[PRODUCT_ID], row) for row in rows)
for code, level in products.items():
product_id = self.products[code]._id
self.assertEqual(by_product[product_id][CURRENT_STOCK], level)
def send_confirmation(v, data):
C = CommtrackConfig.for_domain(v.domain)
static_loc = data['location']
location_name = static_loc.name
metadata = MessageMetadata(location_id=static_loc.get_id)
tx_by_action = map_reduce(lambda tx: [(tx.action_config(C).name,)], data=data['transactions'], include_docs=True)
def summarize_action(action, txs):
return '%s %s' % (txs[0].action_config(C).keyword.upper(), ' '.join(sorted(tx.fragment() for tx in txs)))
msg = 'received stock report for %s(%s) %s' % (
static_loc.site_code,
truncate(location_name, 20),
' '.join(sorted(summarize_action(a, txs) for a, txs in tx_by_action.iteritems()))
)
send_sms_to_verified_number(v, msg, metadata=metadata)
def send_confirmation(v, data):
C = CommtrackConfig.for_domain(v.domain)
static_loc = data["location"]
location_name = static_loc.name
tx_by_action = map_reduce(lambda tx: [(tx.action_config(C).name,)], data=data["transactions"], include_docs=True)
def summarize_action(action, txs):
return "%s %s" % (txs[0].action_config(C).keyword.upper(), " ".join(sorted(tx.fragment() for tx in txs)))
msg = "received stock report for %s(%s) %s" % (
static_loc.site_code,
truncate(location_name, 20),
" ".join(sorted(summarize_action(a, txs) for a, txs in tx_by_action.iteritems())),
)
send_sms_to_verified_number(v, msg)
def send_confirmation(v, data):
C = CommtrackConfig.for_domain(v.domain)
static_loc = data['location']
location_name = static_loc.name
metadata = MessageMetadata(location_id=static_loc.get_id)
tx_by_action = map_reduce(lambda tx: [(tx.action_config(C).name,)], data=data['transactions'], include_docs=True)
def summarize_action(action, txs):
return '%s %s' % (txs[0].action_config(C).keyword.upper(), ' '.join(sorted(tx.fragment() for tx in txs)))
msg = 'received stock report for %s(%s) %s' % (
static_loc.site_code,
truncate(location_name, 20),
' '.join(sorted(summarize_action(a, txs) for a, txs in six.iteritems(tx_by_action)))
)
send_sms_to_verified_number(v, msg, metadata=metadata)
def send_confirmation(v, data):
C = CommtrackConfig.for_domain(v.domain)
static_loc = Location.get(data['location'].location_[-1])
location_name = static_loc.name
action_to_code = dict((v, k) for k, v in C.all_keywords().iteritems())
tx_by_action = map_reduce(lambda tx: [(tx.action_name,)], data=data['transactions'], include_docs=True)
def summarize_action(action, txs):
return '%s %s' % (action_to_code[action].upper(), ' '.join(sorted(tx.fragment() for tx in txs)))
msg = 'received stock report for %s(%s) %s' % (
static_loc.site_code,
truncate(location_name, 20),
' '.join(sorted(summarize_action(a, txs) for a, txs in tx_by_action.iteritems()))
)
send_sms_to_verified_number(v, msg)
def handle(self, *args, **options):
try:
domain = args[0]
except IndexError:
self.stderr.write('domain required\n')
return
self.println('Migrating...')
supply_point_cases = CommCareCase.get_db().view(
'commtrack/supply_point_by_loc',
startkey=[domain],
endkey=[domain, {}],
include_docs=True
)
for result in supply_point_cases:
loc_id = result['key'][-1]
loc = Location.get(loc_id)
case = result['doc']
old_code = case.get('site_code', '')
new_code = getattr(loc, 'site_code', '')
if old_code and not new_code:
loc.site_code = old_code
loc.save()
self.println('migrated %s (%s)' % (loc.name, loc.site_code))
self.println('Verifying code uniqueness...')
all_codes = Location.get_db().view('commtrack/locations_by_code',
startkey=[domain], endkey=[domain, {}])
locs_by_code = map_reduce(lambda e: [(e['key'][-1].lower(), e['id'])], data=all_codes)
for code, loc_ids in locs_by_code.iteritems():
if len(loc_ids) == 1:
continue
self.println('duplicate code [%s]' % code)
locs = Location.view('_all_docs', keys=loc_ids, include_docs=True)
for loc in locs:
self.println(' %s [%s]' % (loc.name, loc._id))
def send_confirmation(v, data):
C = CommtrackConfig.for_domain(v.domain)
static_loc = Location.get(data['location'].location_[-1])
location_name = static_loc.name
action_to_code = dict((v, k) for k, v in C.all_keywords().iteritems())
tx_by_action = map_reduce(lambda tx: [(tx.action_name, )],
data=data['transactions'],
include_docs=True)
def summarize_action(action, txs):
return '%s %s' % (action_to_code[action].upper(), ' '.join(
sorted(tx.fragment() for tx in txs)))
msg = 'received stock report for %s(%s) %s' % (
static_loc.site_code, truncate(location_name, 20), ' '.join(
sorted(
summarize_action(a, txs)
for a, txs in tx_by_action.iteritems())))
send_sms_to_verified_number(v, msg)
def aggregate_cases(self, product_cases, slugs):
cases_by_product = map_reduce(lambda c: [(c.product,)], data=product_cases, include_docs=True)
products = Product.view('_all_docs', keys=cases_by_product.keys(), include_docs=True)
def _sum(vals):
return sum(vals) if vals else None
def aggregate_product(cases):
data = [(c.current_stock_level, c.monthly_consumption) for c in cases if is_timely(c, 1000)]
total_stock = _sum([d[0] for d in data if d[0] is not None])
total_consumption = _sum([d[1] for d in data if d[1] is not None])
# exclude stock values w/o corresponding consumption figure from total months left calculation
consumable_stock = _sum([d[0] for d in data if d[0] is not None and d[1] is not None])
return {
'total_stock': total_stock,
'total_consumption': total_consumption,
'consumable_stock': consumable_stock,
}
status_by_product = dict((p, aggregate_product(cases)) for p, cases in cases_by_product.iteritems())
for p in sorted(products, key=lambda p: p.name):
stats = status_by_product[p._id]
months_left = SPPCase.months_of_stock_remaining(stats['consumable_stock'], stats['total_consumption'])
category = SPPCase.stock_category(stats['total_stock'], stats['total_consumption'], stats['consumable_stock'])
full_output = {
self.SLUG_PRODUCT_NAME: p.name,
self.SLUG_PRODUCT_ID: p._id,
self.SLUG_LOCATION_ID: self.active_location._id if self.active_location else None,
self.SLUG_LOCATION_LINEAGE: self.active_location.lineage if self.active_location else None,
self.SLUG_CURRENT_STOCK: stats['total_stock'],
self.SLUG_CONSUMPTION: stats['total_consumption'],
self.SLUG_MONTHS_REMAINING: months_left,
self.SLUG_CATEGORY: category,
}
yield dict((slug, full_output['slug']) for slug in slugs) if slugs else full_output
def get_prod_data(self):
startkey = [self.domain, self.active_location._id if self.active_location else None]
product_cases = CommCareCase.view('commtrack/product_cases', startkey=startkey, endkey=startkey + [{}], include_docs=True)
cases_by_product = map_reduce(lambda c: [(c.product,)], data=product_cases, include_docs=True)
products = Product.view('_all_docs', keys=cases_by_product.keys(), include_docs=True)
def _sum(vals):
return sum(vals) if vals else None
def aggregate_product(cases):
data = [(current_stock(c), monthly_consumption(c)) for c in cases if is_timely(c, 1000)]
total_stock = _sum([d[0] for d in data if d[0] is not None])
total_consumption = _sum([d[1] for d in data if d[1] is not None])
# exclude stock values w/o corresponding consumption figure from total months left calculation
consumable_stock = _sum([d[0] for d in data if d[0] is not None and d[1] is not None])
try:
months_left = consumable_stock / total_consumption
except (TypeError, ZeroDivisionError):
months_left = None
return {
'total_stock': total_stock,
'total_consumption': total_consumption,
'months_left': months_left,
}
status_by_product = dict((p, aggregate_product(cases)) for p, cases in cases_by_product.iteritems())
for p in sorted(products, key=lambda p: p.name):
stats = status_by_product[p._id]
yield [
p.name,
stats['total_stock'],
stats['total_consumption'],
stats['months_left'],
stock_category(stats['total_stock'], stats['total_consumption'], stats['months_left']),
]
def get_data(self):
data = list(super(StockStatusBySupplyPointDataSource, self).get_data())
products = dict((r['product_id'], r['product_name']) for r in data)
product_ids = sorted(products, key=lambda e: products[e])
by_supply_point = map_reduce(lambda e: [(e['location_id'],)], data=data, include_docs=True)
locs = _location_map(list(by_supply_point))
for loc_id, subcases in six.iteritems(by_supply_point):
if loc_id not in locs:
continue # it's archived, skip
loc = locs[loc_id]
by_product = dict((c['product_id'], c) for c in subcases)
rec = {
'name': loc.name,
'type': loc.location_type.name,
'geo': geopoint(loc),
}
for prod in product_ids:
rec.update(dict(('%s-%s' % (prod, key), by_product.get(prod, {}).get(key)) for key in
('current_stock', 'consumption', 'months_remaining', 'category')))
yield rec
def supply_point_categories(self):
return map_reduce(lambda spt: [(category, spt.name)
for category in spt.categories],
data=self.supply_point_types)
def rows(self):
if len(self.outlets) > OUTLETS_LIMIT:
_term = self.get_terminal()
return [[
'This report is limited to <b>%(max)d</b> %(term)ss. Your location filter includes <b>%(count)d</b> %(term)ss. Please make your location filter more specific.' % {
'count': len(self.outlets),
'max': OUTLETS_LIMIT,
'term': _term,
}]]
products = self.active_products
reports = get_stock_reports(self.domain, self.active_location, self.datespan)
reports_by_loc = map_reduce(lambda e: [(leaf_loc(e),)], data=reports, include_docs=True)
mk_key = lambda dt: [self.domain, dateparse.json_format_datetime(dt), self.active_location._id if self.active_location else None]
all_product_states = get_db().view('commtrack/stock_product_state', startkey=mk_key(self.datespan.startdate), endkey=mk_key(self.datespan.end_of_end_day))
def _emit(o):
loc_id, prod_id, state = o['value']
yield ((loc_id, prod_id), state)
product_state_buckets = map_reduce(_emit, lambda v: sorted(v, key=lambda e: e['server_date']), data=all_product_states)
def summary_row(site, reports):
all_transactions = list(itertools.chain(*(get_transactions(r) for r in reports)))
tx_by_product = map_reduce(lambda tx: [(tx['product'],)], data=all_transactions, include_docs=True)
data = self.outlet_metadata(site, self.ancestry)
stockouts = {}
inactive_site = True
for p in products:
tx_by_action = map_reduce(lambda tx: [(tx['action'], int(tx['value']))], data=tx_by_product.get(p['_id'], []))
product_states = product_state_buckets.get((site._id, p['_id']), [])
stock_update_states = filter(lambda st: 'current_stock' in st['updated_unknown_properties'], product_states)
latest_state = stock_update_states[-1] if stock_update_states else None
if latest_state:
stock = latest_state['updated_unknown_properties']['current_stock']
as_of = dateparse.string_to_datetime(latest_state['server_date']).strftime('%Y-%m-%d')
inactive_site = False
stockout_dates = set()
for state in product_states:
stocked_out_since = state['updated_unknown_properties'].get('stocked_out_since')
if stocked_out_since:
so_start = max(dateparse.string_to_datetime(stocked_out_since).date(), self.datespan.startdate.date())
so_end = dateparse.string_to_datetime(state['server_date']).date() # TODO deal with time zone issues
dt = so_start
while dt < so_end:
stockout_dates.add(dt)
dt += timedelta(days=1)
stockouts[p['_id']] = stockout_dates
data.append('%s (%s)' % (stock, as_of) if latest_state else u'\u2014')
data.extend(sum(tx_by_action.get(a.action_name, [])) for a in self.incr_actions)
combined_stockout_days = len(reduce(lambda a, b: a.intersection(b), stockouts.values()))
data.append(combined_stockout_days)
if self.HIDE_NODATA_LOCS and inactive_site:
return None
return data
return filter(None, (summary_row(site, reports_by_loc.get(site._id, [])) for site in self.outlets))
try:
data_cols = validate_headers(domain, set(reader.fieldnames))
except Exception, e:
raise RuntimeError(str(e))
for row in data:
validate_row(row, domain, data_cols)
# abort if any location codes are invalid
if any(not row.get('loc') for row in data):
set_error_bulk(data, 'SKIPPED because some rows could not be assigned to a valid location')
else:
rows_by_loc = map_reduce(lambda row: [(row['loc']._id,)], data=data, include_docs=True)
for loc, rows in rows_by_loc.iteritems():
process_loc(domain, loc, rows, data_cols)
return annotate_csv(data, reader.fieldnames)
def import_products(domain, download, task):
messages = []
products = []
data = download.get_content().split('\n')[1:]
processed = 0
total_rows = len(data)
reader = csv.reader(data)
for row in reader:
try:
def _data(self):
startkey = [
self.domain,
self.active_location._id if self.active_location else None
]
product_cases = CommCareCase.view('commtrack/product_cases',
startkey=startkey,
endkey=startkey + [{}],
include_docs=True)
def latest_case(cases):
# getting last report date should probably be moved to a util function in a case wrapper class
return max(cases,
key=lambda c: getattr(c, 'last_reported',
datetime(2000, 1, 1)).date())
cases_by_site = map_reduce(lambda c: [(tuple(c.location_), )],
lambda v: reporting_status(latest_case(v)),
data=product_cases,
include_docs=True)
def child_loc(path):
root = self.active_location
ix = path.index(root._id) if root else -1
try:
return path[ix + 1]
except IndexError:
return None
def case_iter():
for k, v in cases_by_site.iteritems():
if child_loc(k) is not None:
yield (k, v)
status_by_agg_site = map_reduce(
lambda (path, status): [(child_loc(path), status)],
data=case_iter())
sites_by_agg_site = map_reduce(
lambda (path, status): [(child_loc(path), path[-1])],
data=case_iter())
def status_tally(statuses):
total = len(statuses)
return map_reduce(lambda s: [(s, )],
lambda v: {
'count': len(v),
'pct': len(v) / float(total)
},
data=statuses)
status_counts = dict(
(loc_id, status_tally(statuses))
for loc_id, statuses in status_by_agg_site.iteritems())
master_tally = status_tally(cases_by_site.values())
locs = sorted(Location.view('_all_docs',
keys=status_counts.keys(),
include_docs=True),
key=lambda loc: loc.name)
def fmt(pct):
return '%.1f%%' % (100. * pct)
def fmt_col(loc, col_type):
return fmt(status_counts[loc._id].get(col_type,
{'pct': 0.})['pct'])
def _rows():
for loc in locs:
num_sites = len(sites_by_agg_site[loc._id])
yield [loc.name, len(sites_by_agg_site[loc._id])] + [
fmt_col(loc, k) for k in ('ontime', 'late', 'nonreporting')
]
return master_tally, _rows()
def parent_child(domain):
return map_reduce(lambda (k, v): [(p, k) for p in v], data=dict(location_hierarchy_config(domain)).iteritems())
def summary_row(site, reports):
all_transactions = list(
itertools.chain(*(get_transactions(r) for r in reports)))
tx_by_product = map_reduce(lambda tx: [(tx['product'], )],
data=all_transactions,
include_docs=True)
data = outlet_metadata(site, self.ancestry)
stockouts = {}
inactive_site = True
for p in products:
tx_by_action = map_reduce(
lambda tx: [(tx['action'], int(tx['value']))],
data=tx_by_product.get(p['_id'], []))
product_states = product_state_buckets.get(
(site._id, p['_id']), [])
stock_update_states = filter(
lambda st: 'current_stock' in st[
'updated_unknown_properties'], product_states)
latest_state = stock_update_states[
-1] if stock_update_states else None
if latest_state:
stock = latest_state['updated_unknown_properties'][
'current_stock']
as_of = dateparse.string_to_datetime(
latest_state['server_date']).strftime('%Y-%m-%d')
inactive_site = False
stockout_dates = set()
for state in product_states:
stocked_out_since = state[
'updated_unknown_properties'].get('stocked_out_since')
if stocked_out_since:
so_start = max(
dateparse.string_to_datetime(
stocked_out_since).date(),
self.datespan.startdate.date())
so_end = dateparse.string_to_datetime(
state['server_date']).date(
) # TODO deal with time zone issues
dt = so_start
while dt < so_end:
stockout_dates.add(dt)
dt += timedelta(days=1)
stockouts[p['_id']] = stockout_dates
data.append('%s (%s)' %
(stock, as_of) if latest_state else u'\u2014')
data.extend(
sum(tx_by_action.get(a.action_name, []))
for a in self.incr_actions)
combined_stockout_days = len(
reduce(lambda a, b: a.intersection(b), stockouts.values()))
data.append(combined_stockout_days)
if self.HIDE_NODATA_LOCS and inactive_site:
return None
return data
def rows(self):
if not self.aggregate_by:
return [['Choose a location type to aggregate by.']]
if not self.aggregation_locs:
return [['There are no locations of type "%s" inside the selected location. Choose an administrative location higher up in the hierarchy.' % self.aggregate_by]]
products = self.active_products
locs = self.aggregation_locs
active_outlets = [loc for loc in self.leaf_locs if self.outlet_type_filter(loc)]
active_outlet_ids = set(loc._id for loc in active_outlets)
aggregation_sites = set(loc._id for loc in locs)
def get_aggregation_site(outlet):
for k in outlet.path:
if k in aggregation_sites:
return k
outlets_by_aggregation_site = map_reduce(lambda e: [(get_aggregation_site(e),)], data=active_outlets)
reports = filter(lambda r: leaf_loc(r) in active_outlet_ids, get_stock_reports(self.domain, self.active_location, self.datespan))
def get_aggregators(report):
for k in report['location_']:
if k in aggregation_sites:
yield (k,)
reports_by_loc = map_reduce(get_aggregators, data=reports, include_docs=True)
startkey = [self.domain, self.active_location._id if self.active_location else None, 'CommCareCase']
product_cases = [c for c in CommCareCase.view('locations/linked_docs', startkey=startkey, endkey=startkey + [{}], include_docs=True)
if c.type == const.SUPPLY_POINT_PRODUCT_CASE_TYPE and leaf_loc(c) in active_outlet_ids]
product_cases_by_parent = map_reduce(get_aggregators, data=product_cases, include_docs=True)
def summary_row(site, reports, product_cases, outlets):
all_transactions = list(itertools.chain(*(get_transactions(r) for r in reports)))
tx_by_product = map_reduce(lambda tx: [(tx['product'],)], data=all_transactions, include_docs=True)
cases_by_product = map_reduce(lambda c: [(c.product,)], data=product_cases, include_docs=True)
num_outlets = len(outlets)
product_ids = set(p['_id'] for p in products)
relevant_reports = [r for r in reports if any(tx['product'] in product_ids for tx in get_transactions(r))]
num_active_outlets = len(set(leaf_loc(r) for r in relevant_reports))
data = site_metadata(self, site, self.ancestry)
data.extend([
num_active_outlets,
num_outlets,
])
for p in products:
tx_by_action = map_reduce(lambda tx: [(tx['action'], int(tx['value']))], data=tx_by_product.get(p['_id'], []))
subcases = cases_by_product.get(p['_id'], [])
stocks = [int(k) for k in (c.get_case_property('current_stock') for c in subcases) if k is not None]
data.append(sum(stocks) if stocks else u'\u2014')
data.extend(sum(tx_by_action.get(a.action_name, [])) for a in self.incr_actions)
return data
return [summary_row(site,
reports_by_loc.get(site._id, []),
product_cases_by_parent.get(site._id, []),
outlets_by_aggregation_site.get(site._id, []),
) for site in locs]
def rows(self):
if len(self.outlets) > OUTLETS_LIMIT:
_term = get_terminal(self.domain)
return [[
'This report is limited to <b>%(max)d</b> %(term)ss. Your location filter includes <b>%(count)d</b> %(term)ss. Please make your location filter more specific.'
% {
'count': len(self.outlets),
'max': OUTLETS_LIMIT,
'term': _term,
}
]]
products = self.active_products
reports = get_stock_reports(self.domain, self.active_location,
self.datespan)
reports_by_loc = map_reduce(lambda e: [(leaf_loc(e), )],
data=reports,
include_docs=True)
mk_key = lambda dt: [
self.domain,
dateparse.json_format_datetime(dt), self.active_location._id
if self.active_location else None
]
all_product_states = get_db().view(
'commtrack/stock_product_state',
startkey=mk_key(self.datespan.startdate),
endkey=mk_key(self.datespan.end_of_end_day))
def _emit(o):
loc_id, prod_id, state = o['value']
yield ((loc_id, prod_id), state)
product_state_buckets = map_reduce(
_emit,
lambda v: sorted(v, key=lambda e: e['server_date']),
data=all_product_states)
def summary_row(site, reports):
all_transactions = list(
itertools.chain(*(get_transactions(r) for r in reports)))
tx_by_product = map_reduce(lambda tx: [(tx['product'], )],
data=all_transactions,
include_docs=True)
data = outlet_metadata(site, self.ancestry)
stockouts = {}
inactive_site = True
for p in products:
tx_by_action = map_reduce(
lambda tx: [(tx['action'], int(tx['value']))],
data=tx_by_product.get(p['_id'], []))
product_states = product_state_buckets.get(
(site._id, p['_id']), [])
stock_update_states = filter(
lambda st: 'current_stock' in st[
'updated_unknown_properties'], product_states)
latest_state = stock_update_states[
-1] if stock_update_states else None
if latest_state:
stock = latest_state['updated_unknown_properties'][
'current_stock']
as_of = dateparse.string_to_datetime(
latest_state['server_date']).strftime('%Y-%m-%d')
inactive_site = False
stockout_dates = set()
for state in product_states:
stocked_out_since = state[
'updated_unknown_properties'].get('stocked_out_since')
if stocked_out_since:
so_start = max(
dateparse.string_to_datetime(
stocked_out_since).date(),
self.datespan.startdate.date())
so_end = dateparse.string_to_datetime(
state['server_date']).date(
) # TODO deal with time zone issues
dt = so_start
while dt < so_end:
stockout_dates.add(dt)
dt += timedelta(days=1)
stockouts[p['_id']] = stockout_dates
data.append('%s (%s)' %
(stock, as_of) if latest_state else u'\u2014')
data.extend(
sum(tx_by_action.get(a.action_name, []))
for a in self.incr_actions)
combined_stockout_days = len(
reduce(lambda a, b: a.intersection(b), stockouts.values()))
data.append(combined_stockout_days)
if self.HIDE_NODATA_LOCS and inactive_site:
return None
return data
return filter(None,
(summary_row(site, reports_by_loc.get(site._id, []))
for site in self.outlets))
raise RuntimeError(str(e))
for row in data:
validate_row(row, domain, data_cols)
# abort if any location codes are invalid
if any(not row.get('loc') for row in data):
set_error_bulk(
data,
'SKIPPED because some rows could not be assigned to a valid location'
)
else:
rows_by_loc = map_reduce(lambda row: [(row['loc']._id, )],
data=data,
include_docs=True)
for loc, rows in rows_by_loc.iteritems():
process_loc(domain, loc, rows, data_cols)
return annotate_csv(data, reader.fieldnames)
def import_products(domain, importer):
messages = []
to_save = []
product_count = 0
for row in importer.worksheet:
try:
p = Product.from_excel(row)
def rows(self):
rows = list(super(HealthCenter, self).rows)
rows_by_hc = map_reduce(emitfunc=lambda r: [(r[0],)], reducefunc=self.reduce, data=rows, include_docs=True)
return rows_by_hc.values()
def supply_point_categories(self):
return map_reduce(lambda spt: [(category, spt.name) for category in spt.categories], data=self.supply_point_types)
def status_tally(self, statuses):
total = len(statuses)
return map_reduce(lambda s: [(s,)],
lambda v: {'count': len(v), 'pct': len(v) / float(total)},
data=statuses)
def rows(self):
if not self.aggregate_by:
return [['Choose a location type to aggregate by.']]
if not self.aggregation_locs:
return [[
'There are no locations of type "%s" inside the selected location. Choose an administrative location higher up in the hierarchy.'
% self.aggregate_by
]]
products = self.active_products
locs = self.aggregation_locs
active_outlets = [
loc for loc in self.leaf_locs if self.outlet_type_filter(loc)
]
active_outlet_ids = set(loc._id for loc in active_outlets)
aggregation_sites = set(loc._id for loc in locs)
def get_aggregation_site(outlet):
for k in outlet.path:
if k in aggregation_sites:
return k
outlets_by_aggregation_site = map_reduce(
lambda e: [(get_aggregation_site(e), )], data=active_outlets)
reports = filter(
lambda r: leaf_loc(r) in active_outlet_ids,
get_stock_reports(self.domain, self.active_location,
self.datespan))
def get_aggregators(report):
for k in report['location_']:
if k in aggregation_sites:
yield (k, )
reports_by_loc = map_reduce(get_aggregators,
data=reports,
include_docs=True)
startkey = [
self.domain,
self.active_location._id if self.active_location else None,
'CommCareCase'
]
product_cases = [
c for c in CommCareCase.view('locations/linked_docs',
startkey=startkey,
endkey=startkey + [{}],
include_docs=True)
if c.type == const.SUPPLY_POINT_PRODUCT_CASE_TYPE
and leaf_loc(c) in active_outlet_ids
]
product_cases_by_parent = map_reduce(get_aggregators,
data=product_cases,
include_docs=True)
def summary_row(site, reports, product_cases, outlets):
all_transactions = list(
itertools.chain(*(get_transactions(r) for r in reports)))
tx_by_product = map_reduce(lambda tx: [(tx['product'], )],
data=all_transactions,
include_docs=True)
cases_by_product = map_reduce(lambda c: [(c.product, )],
data=product_cases,
include_docs=True)
num_outlets = len(outlets)
product_ids = set(p['_id'] for p in products)
relevant_reports = [
r for r in reports if any(tx['product'] in product_ids
for tx in get_transactions(r))
]
num_active_outlets = len(set(
leaf_loc(r) for r in relevant_reports))
data = site_metadata(site, self.ancestry)
data.extend([
num_active_outlets,
num_outlets,
])
for p in products:
tx_by_action = map_reduce(
lambda tx: [(tx['action'], int(tx['value']))],
data=tx_by_product.get(p['_id'], []))
subcases = cases_by_product.get(p['_id'], [])
stocks = [
int(k) for k in (c.get_case_property('current_stock')
for c in subcases) if k is not None
]
data.append(sum(stocks) if stocks else u'\u2014')
data.extend(
sum(tx_by_action.get(a.action_name, []))
for a in self.incr_actions)
return data
return [
summary_row(
site,
reports_by_loc.get(site._id, []),
product_cases_by_parent.get(site._id, []),
outlets_by_aggregation_site.get(site._id, []),
) for site in locs
]
