def recalculate_on_group_change(location, last_run): OrganizationSummary.objects.filter(location_id=location.get_id).delete() process_facility_warehouse_data(location, default_start_date(), last_run.end) for parent in location.sql_location.get_ancestors(ascending=True): process_non_facility_warehouse_data(parent.couch_location, default_start_date(), last_run.end, strict=False)
def recalculate_non_facilities_task(domain): task_progress = OneOffTaskProgress.objects.get_or_create( domain=domain, task_name=TASK_NAME)[0] if task_progress.complete: return start_date = default_start_date() end_date = datetime(2016, 3, 1) - timedelta(microseconds=.1) districts = list( SQLLocation.active_objects.filter( domain='ils-gateway', location_type__name__iexact='DISTRICT').order_by('pk')) regions = list( SQLLocation.active_objects.filter( domain='ils-gateway', location_type__name__iexact='REGION').order_by('pk')) msd_zones = list( SQLLocation.active_objects.filter( domain='ils-gateway', location_type__name__iexact='MSDZONE').order_by('pk')) mohsw = list( SQLLocation.active_objects.filter( domain='ils-gateway', location_type__name__iexact='MOHSW').order_by('pk')) non_facilities = districts + regions + msd_zones + mohsw task_progress.total = len(non_facilities) task_progress.save() if task_progress.last_synced_object_id: non_facilities = list( itertools.dropwhile( lambda x: x.location_id != task_progress.last_synced_object_id, non_facilities)) last_processed = non_facilities[0] non_facilities = non_facilities[1:] process_non_facility_warehouse_data(last_processed.couch_location, start_date, end_date, strict=False) for non_facility in non_facilities: task_progress.last_synced_object_id = non_facility.location_id task_progress.progress += 1 task_progress.save() process_non_facility_warehouse_data(non_facility.couch_location, default_start_date(), end_date, strict=False) task_progress.complete = True task_progress.save()
def location_edited_receiver(sender, loc, moved, **kwargs): from custom.ilsgateway.tanzania.warehouse.updater import default_start_date, \ process_non_facility_warehouse_data config = ILSGatewayConfig.for_domain(loc.domain) if not config or not config.enabled or not moved or not loc.previous_parents: return last_run = ReportRun.last_success(loc.domain) if not last_run: return previous_parent = SQLLocation.objects.get(location_id=loc.previous_parents[-1]) type_location_map = defaultdict(set) previous_ancestors = list(previous_parent.get_ancestors(include_self=True)) actual_ancestors = list(loc.sql_location.get_ancestors()) for sql_location in previous_ancestors + actual_ancestors: type_location_map[sql_location.location_type.name].add(sql_location) for location_type in ["DISTRICT", "REGION", "MSDZONE"]: for sql_location in type_location_map[location_type]: process_non_facility_warehouse_data.delay( sql_location.couch_location, default_start_date(), last_run.end, last_run, strict=False )
def recalculation_on_location_change(domain, last_run): if not last_run: PendingReportingDataRecalculation.objects.filter( domain=domain).delete() return pending_recalculations = PendingReportingDataRecalculation.objects.filter( domain=domain).order_by('pk') recalcs_dict = defaultdict(list) for pending_recalculation in pending_recalculations: key = (pending_recalculation.sql_location, pending_recalculation.type) recalcs_dict[key].append(pending_recalculation.data) non_facilities_to_recalculate = defaultdict(set) recalculated = set() for (sql_location, recalculation_type), data_list in six.iteritems(recalcs_dict): # If there are more changes, consider earliest and latest change. # Thanks to this we avoid recalculations when in fact group/parent wasn't changed. # E.g Group is changed from A -> B and later from B -> A. # In this situation there is no need to recalculate data. if not OrganizationSummary.objects.filter( location_id=sql_location.location_id).exists(): # There are no data for that location so there is no need to recalculate PendingReportingDataRecalculation.objects.filter( sql_location=sql_location, type=recalculation_type, domain=domain).delete() continue if recalculation_type == 'group_change'\ and data_list[0]['previous_group'] != data_list[-1]['current_group']\ and not sql_location.location_type.administrative: to_recalculate = recalculate_on_group_change( sql_location, last_run) elif recalculation_type == 'parent_change' \ and data_list[0]['previous_parent'] != data_list[-1]['current_parent']: to_recalculate = recalculate_on_parent_change( sql_location, data_list[0]['previous_parent'], last_run) else: to_recalculate = {} recalculated.add(sql_location) for location_type, sql_locations_to_recalculate in six.iteritems( to_recalculate): for sql_location_to_recalculate in sql_locations_to_recalculate: non_facilities_to_recalculate[location_type].add( sql_location_to_recalculate) for location_type in ["DISTRICT", "REGION", "MSDZONE", "MOHSW"]: for sql_location in non_facilities_to_recalculate.get( location_type, []): process_non_facility_warehouse_data(sql_location, default_start_date(), last_run.end, strict=False) PendingReportingDataRecalculation.objects.filter( sql_location__in=recalculated, domain=domain).delete()
def recalculate_on_group_change(location, last_run): OrganizationSummary.objects.filter(location_id=location.get_id).delete() process_facility_warehouse_data(location, default_start_date(), last_run.end) return { ancestor.location_type.name: {ancestor} for ancestor in location.sql_location.get_ancestors(ascending=True) }
def recalculate_non_facilities_task(domain): task_progress = OneOffTaskProgress.objects.get_or_create(domain=domain, task_name=TASK_NAME)[0] if task_progress.complete: return start_date = default_start_date() end_date = datetime(2016, 3, 1) - timedelta(microseconds=.1) districts = list(SQLLocation.active_objects.filter( domain='ils-gateway', location_type__name__iexact='DISTRICT' ).order_by('pk')) regions = list(SQLLocation.active_objects.filter( domain='ils-gateway', location_type__name__iexact='REGION' ).order_by('pk')) msd_zones = list(SQLLocation.active_objects.filter( domain='ils-gateway', location_type__name__iexact='MSDZONE' ).order_by('pk')) mohsw = list(SQLLocation.active_objects.filter( domain='ils-gateway', location_type__name__iexact='MOHSW' ).order_by('pk')) non_facilities = districts + regions + msd_zones + mohsw task_progress.total = len(non_facilities) task_progress.save() if task_progress.last_synced_object_id: non_facilities = list(itertools.dropwhile( lambda x: x.location_id != task_progress.last_synced_object_id, non_facilities )) last_processed = non_facilities[0] non_facilities = non_facilities[1:] process_non_facility_warehouse_data(last_processed, start_date, end_date, strict=False) for non_facility in non_facilities: task_progress.last_synced_object_id = non_facility.location_id task_progress.progress += 1 task_progress.save() process_non_facility_warehouse_data( non_facility, default_start_date(), end_date, strict=False ) task_progress.complete = True task_progress.save()
def recalculate_moshi_rural_task(): site_code = 'district-moshi-rural' moshi = SQLLocation.objects.get(domain='ils-gateway', site_code__iexact=site_code) end_date = datetime(2016, 3, 1) - timedelta(microseconds=.1) process_non_facility_warehouse_data(moshi.couch_location, default_start_date(), end_date, strict=False)
def recalculation_on_location_change(domain, last_run): if not last_run: PendingReportingDataRecalculation.objects.filter(domain=domain).delete() return pending_recalculations = PendingReportingDataRecalculation.objects.filter(domain=domain).order_by('pk') recalcs_dict = defaultdict(list) for pending_recalculation in pending_recalculations: key = (pending_recalculation.sql_location, pending_recalculation.type) recalcs_dict[key].append(pending_recalculation.data) non_facilities_to_recalculate = defaultdict(set) recalculated = set() for (sql_location, recalculation_type), data_list in recalcs_dict.iteritems(): # If there are more changes, consider earliest and latest change. # Thanks to this we avoid recalculations when in fact group/parent wasn't changed. # E.g Group is changed from A -> B and later from B -> A. # In this situation there is no need to recalculate data. if not OrganizationSummary.objects.filter(location_id=sql_location.location_id).exists(): # There are no data for that location so there is no need to recalculate PendingReportingDataRecalculation.objects.filter( sql_location=sql_location, type=recalculation_type, domain=domain ).delete() continue if recalculation_type == 'group_change'\ and data_list[0]['previous_group'] != data_list[-1]['current_group']\ and not sql_location.location_type.administrative: to_recalculate = recalculate_on_group_change(sql_location.couch_location, last_run) elif recalculation_type == 'parent_change' \ and data_list[0]['previous_parent'] != data_list[-1]['current_parent']: to_recalculate = recalculate_on_parent_change( sql_location.couch_location, data_list[0]['previous_parent'], last_run ) else: to_recalculate = {} recalculated.add(sql_location) for location_type, sql_locations_to_recalculate in to_recalculate.iteritems(): for sql_location_to_recalculate in sql_locations_to_recalculate: non_facilities_to_recalculate[location_type].add(sql_location_to_recalculate) for location_type in ["DISTRICT", "REGION", "MSDZONE", "MOHSW"]: for sql_location in non_facilities_to_recalculate.get(location_type, []): process_non_facility_warehouse_data( sql_location.couch_location, default_start_date(), last_run.end, strict=False ) PendingReportingDataRecalculation.objects.filter( sql_location__in=recalculated, domain=domain ).delete()
def recalculate_on_parent_change(location, previous_parent_id, last_run): previous_parent = SQLLocation.objects.get(location_id=previous_parent_id) type_location_map = defaultdict(set) previous_ancestors = list(previous_parent.get_ancestors(include_self=True, ascending=True)) actual_ancestors = list(location.sql_location.get_ancestors(ascending=True)) locations_to_recalculate = set() i = 0 while previous_ancestors[i] != actual_ancestors[i] and i < len(previous_ancestors): locations_to_recalculate.add(previous_ancestors[i]) locations_to_recalculate.add(actual_ancestors[i]) i += 1 for sql_location in locations_to_recalculate: type_location_map[sql_location.location_type.name].add(sql_location) for location_type in ["DISTRICT", "REGION", "MSDZONE"]: for sql_location in type_location_map[location_type]: process_non_facility_warehouse_data( sql_location.couch_location, default_start_date(), last_run.end, strict=False )
def recalculate_moshi_rural_task(): site_code = 'district-moshi-rural' moshi = SQLLocation.objects.get(domain='ils-gateway', site_code__iexact=site_code) end_date = datetime(2016, 3, 1) - timedelta(microseconds=.1) process_non_facility_warehouse_data(moshi, default_start_date(), end_date, strict=False)