def recalculate_on_group_change(location, last_run):
OrganizationSummary.objects.filter(location_id=location.get_id).delete()
process_facility_warehouse_data(location, default_start_date(), last_run.end)
for parent in location.sql_location.get_ancestors(ascending=True):
process_non_facility_warehouse_data(parent.couch_location,
default_start_date(), last_run.end, strict=False)
def recalculate_non_facilities_task(domain):
task_progress = OneOffTaskProgress.objects.get_or_create(
domain=domain, task_name=TASK_NAME)[0]
if task_progress.complete:
return
start_date = default_start_date()
end_date = datetime(2016, 3, 1) - timedelta(microseconds=.1)
districts = list(
SQLLocation.active_objects.filter(
domain='ils-gateway',
location_type__name__iexact='DISTRICT').order_by('pk'))
regions = list(
SQLLocation.active_objects.filter(
domain='ils-gateway',
location_type__name__iexact='REGION').order_by('pk'))
msd_zones = list(
SQLLocation.active_objects.filter(
domain='ils-gateway',
location_type__name__iexact='MSDZONE').order_by('pk'))
mohsw = list(
SQLLocation.active_objects.filter(
domain='ils-gateway',
location_type__name__iexact='MOHSW').order_by('pk'))
non_facilities = districts + regions + msd_zones + mohsw
task_progress.total = len(non_facilities)
task_progress.save()
if task_progress.last_synced_object_id:
non_facilities = list(
itertools.dropwhile(
lambda x: x.location_id != task_progress.last_synced_object_id,
non_facilities))
last_processed = non_facilities[0]
non_facilities = non_facilities[1:]
process_non_facility_warehouse_data(last_processed.couch_location,
start_date,
end_date,
strict=False)
for non_facility in non_facilities:
task_progress.last_synced_object_id = non_facility.location_id
task_progress.progress += 1
task_progress.save()
process_non_facility_warehouse_data(non_facility.couch_location,
default_start_date(),
end_date,
strict=False)
task_progress.complete = True
task_progress.save()
def location_edited_receiver(sender, loc, moved, **kwargs):
from custom.ilsgateway.tanzania.warehouse.updater import default_start_date, \
process_non_facility_warehouse_data
config = ILSGatewayConfig.for_domain(loc.domain)
if not config or not config.enabled or not moved or not loc.previous_parents:
return
last_run = ReportRun.last_success(loc.domain)
if not last_run:
return
previous_parent = SQLLocation.objects.get(location_id=loc.previous_parents[-1])
type_location_map = defaultdict(set)
previous_ancestors = list(previous_parent.get_ancestors(include_self=True))
actual_ancestors = list(loc.sql_location.get_ancestors())
for sql_location in previous_ancestors + actual_ancestors:
type_location_map[sql_location.location_type.name].add(sql_location)
for location_type in ["DISTRICT", "REGION", "MSDZONE"]:
for sql_location in type_location_map[location_type]:
process_non_facility_warehouse_data.delay(
sql_location.couch_location, default_start_date(), last_run.end, last_run, strict=False
)
def recalculation_on_location_change(domain, last_run):
if not last_run:
PendingReportingDataRecalculation.objects.filter(
domain=domain).delete()
return
pending_recalculations = PendingReportingDataRecalculation.objects.filter(
domain=domain).order_by('pk')
recalcs_dict = defaultdict(list)
for pending_recalculation in pending_recalculations:
key = (pending_recalculation.sql_location, pending_recalculation.type)
recalcs_dict[key].append(pending_recalculation.data)
non_facilities_to_recalculate = defaultdict(set)
recalculated = set()
for (sql_location,
recalculation_type), data_list in six.iteritems(recalcs_dict):
# If there are more changes, consider earliest and latest change.
# Thanks to this we avoid recalculations when in fact group/parent wasn't changed.
# E.g Group is changed from A -> B and later from B -> A.
# In this situation there is no need to recalculate data.
if not OrganizationSummary.objects.filter(
location_id=sql_location.location_id).exists():
# There are no data for that location so there is no need to recalculate
PendingReportingDataRecalculation.objects.filter(
sql_location=sql_location,
type=recalculation_type,
domain=domain).delete()
continue
if recalculation_type == 'group_change'\
and data_list[0]['previous_group'] != data_list[-1]['current_group']\
and not sql_location.location_type.administrative:
to_recalculate = recalculate_on_group_change(
sql_location, last_run)
elif recalculation_type == 'parent_change' \
and data_list[0]['previous_parent'] != data_list[-1]['current_parent']:
to_recalculate = recalculate_on_parent_change(
sql_location, data_list[0]['previous_parent'], last_run)
else:
to_recalculate = {}
recalculated.add(sql_location)
for location_type, sql_locations_to_recalculate in six.iteritems(
to_recalculate):
for sql_location_to_recalculate in sql_locations_to_recalculate:
non_facilities_to_recalculate[location_type].add(
sql_location_to_recalculate)
for location_type in ["DISTRICT", "REGION", "MSDZONE", "MOHSW"]:
for sql_location in non_facilities_to_recalculate.get(
location_type, []):
process_non_facility_warehouse_data(sql_location,
default_start_date(),
last_run.end,
strict=False)
PendingReportingDataRecalculation.objects.filter(
sql_location__in=recalculated, domain=domain).delete()
def recalculate_on_group_change(location, last_run):
OrganizationSummary.objects.filter(location_id=location.get_id).delete()
process_facility_warehouse_data(location, default_start_date(),
last_run.end)
return {
ancestor.location_type.name: {ancestor}
for ancestor in location.sql_location.get_ancestors(ascending=True)
}
def recalculate_non_facilities_task(domain):
task_progress = OneOffTaskProgress.objects.get_or_create(domain=domain, task_name=TASK_NAME)[0]
if task_progress.complete:
return
start_date = default_start_date()
end_date = datetime(2016, 3, 1) - timedelta(microseconds=.1)
districts = list(SQLLocation.active_objects.filter(
domain='ils-gateway', location_type__name__iexact='DISTRICT'
).order_by('pk'))
regions = list(SQLLocation.active_objects.filter(
domain='ils-gateway', location_type__name__iexact='REGION'
).order_by('pk'))
msd_zones = list(SQLLocation.active_objects.filter(
domain='ils-gateway', location_type__name__iexact='MSDZONE'
).order_by('pk'))
mohsw = list(SQLLocation.active_objects.filter(
domain='ils-gateway', location_type__name__iexact='MOHSW'
).order_by('pk'))
non_facilities = districts + regions + msd_zones + mohsw
task_progress.total = len(non_facilities)
task_progress.save()
if task_progress.last_synced_object_id:
non_facilities = list(itertools.dropwhile(
lambda x: x.location_id != task_progress.last_synced_object_id,
non_facilities
))
last_processed = non_facilities[0]
non_facilities = non_facilities[1:]
process_non_facility_warehouse_data(last_processed, start_date, end_date, strict=False)
for non_facility in non_facilities:
task_progress.last_synced_object_id = non_facility.location_id
task_progress.progress += 1
task_progress.save()
process_non_facility_warehouse_data(
non_facility, default_start_date(), end_date, strict=False
)
task_progress.complete = True
task_progress.save()
def recalculate_moshi_rural_task():
site_code = 'district-moshi-rural'
moshi = SQLLocation.objects.get(domain='ils-gateway',
site_code__iexact=site_code)
end_date = datetime(2016, 3, 1) - timedelta(microseconds=.1)
process_non_facility_warehouse_data(moshi.couch_location,
default_start_date(),
end_date,
strict=False)
def recalculation_on_location_change(domain, last_run):
if not last_run:
PendingReportingDataRecalculation.objects.filter(domain=domain).delete()
return
pending_recalculations = PendingReportingDataRecalculation.objects.filter(domain=domain).order_by('pk')
recalcs_dict = defaultdict(list)
for pending_recalculation in pending_recalculations:
key = (pending_recalculation.sql_location, pending_recalculation.type)
recalcs_dict[key].append(pending_recalculation.data)
non_facilities_to_recalculate = defaultdict(set)
recalculated = set()
for (sql_location, recalculation_type), data_list in recalcs_dict.iteritems():
# If there are more changes, consider earliest and latest change.
# Thanks to this we avoid recalculations when in fact group/parent wasn't changed.
# E.g Group is changed from A -> B and later from B -> A.
# In this situation there is no need to recalculate data.
if not OrganizationSummary.objects.filter(location_id=sql_location.location_id).exists():
# There are no data for that location so there is no need to recalculate
PendingReportingDataRecalculation.objects.filter(
sql_location=sql_location, type=recalculation_type, domain=domain
).delete()
continue
if recalculation_type == 'group_change'\
and data_list[0]['previous_group'] != data_list[-1]['current_group']\
and not sql_location.location_type.administrative:
to_recalculate = recalculate_on_group_change(sql_location.couch_location, last_run)
elif recalculation_type == 'parent_change' \
and data_list[0]['previous_parent'] != data_list[-1]['current_parent']:
to_recalculate = recalculate_on_parent_change(
sql_location.couch_location, data_list[0]['previous_parent'], last_run
)
else:
to_recalculate = {}
recalculated.add(sql_location)
for location_type, sql_locations_to_recalculate in to_recalculate.iteritems():
for sql_location_to_recalculate in sql_locations_to_recalculate:
non_facilities_to_recalculate[location_type].add(sql_location_to_recalculate)
for location_type in ["DISTRICT", "REGION", "MSDZONE", "MOHSW"]:
for sql_location in non_facilities_to_recalculate.get(location_type, []):
process_non_facility_warehouse_data(
sql_location.couch_location, default_start_date(), last_run.end, strict=False
)
PendingReportingDataRecalculation.objects.filter(
sql_location__in=recalculated, domain=domain
).delete()
def recalculate_on_parent_change(location, previous_parent_id, last_run):
previous_parent = SQLLocation.objects.get(location_id=previous_parent_id)
type_location_map = defaultdict(set)
previous_ancestors = list(previous_parent.get_ancestors(include_self=True, ascending=True))
actual_ancestors = list(location.sql_location.get_ancestors(ascending=True))
locations_to_recalculate = set()
i = 0
while previous_ancestors[i] != actual_ancestors[i] and i < len(previous_ancestors):
locations_to_recalculate.add(previous_ancestors[i])
locations_to_recalculate.add(actual_ancestors[i])
i += 1
for sql_location in locations_to_recalculate:
type_location_map[sql_location.location_type.name].add(sql_location)
for location_type in ["DISTRICT", "REGION", "MSDZONE"]:
for sql_location in type_location_map[location_type]:
process_non_facility_warehouse_data(
sql_location.couch_location, default_start_date(), last_run.end, strict=False
)
def recalculate_moshi_rural_task():
site_code = 'district-moshi-rural'
moshi = SQLLocation.objects.get(domain='ils-gateway', site_code__iexact=site_code)
end_date = datetime(2016, 3, 1) - timedelta(microseconds=.1)
process_non_facility_warehouse_data(moshi, default_start_date(), end_date, strict=False)
def recalculate_on_group_change(location, last_run):
OrganizationSummary.objects.filter(location_id=location.get_id).delete()
process_facility_warehouse_data(location, default_start_date(), last_run.end)
return {
ancestor.location_type.name: {ancestor} for ancestor in location.sql_location.get_ancestors(ascending=True)
}