def icds_state_aggregation_task(self, state_id, date, func):
if six.PY2 and isinstance(date, bytes):
date = date.decode('utf-8')
db_alias = get_icds_ucr_db_alias()
if not db_alias:
return
celery_task_logger.info("Starting icds reports {} {} {}".format(state_id, date, func.__name__))
try:
func(state_id, date)
except Error as exc:
notify_exception(
None, message="Error occurred during ICDS aggregation",
details={'func': func.__name__, 'date': date, 'state_id': state_id, 'error': exc}
)
_dashboard_team_soft_assert(
False,
"{} aggregation failed on {} for {} on {}. This task will be retried in 15 minutes".format(
func.__name__, settings.SERVER_ENVIRONMENT, state_id, date
)
)
self.retry(exc=exc)
celery_task_logger.info("Ended icds reports {} {} {}".format(state_id, date, func.__name__))
def allow_migrate(db, app_label):
"""
Return ``True`` if a app's migrations should be applied to the specified database otherwise
return ``False``.
Note: returning ``None`` is tantamount to returning ``True``
:return: Must return a boolean value, not None.
"""
if app_label == ICDS_REPORTS_APP:
db_alias = get_icds_ucr_db_alias()
return bool(db_alias and db_alias == db)
elif app_label == SYNCLOGS_APP:
return db == settings.SYNCLOGS_SQL_DB_ALIAS
elif app_label == WAREHOUSE_APP:
return db == settings.WAREHOUSE_DATABASE_ALIAS
if not settings.USE_PARTITIONED_DATABASE:
return app_label != PROXY_APP and db in ('default', None)
if app_label == PROXY_APP:
return db == partition_config.get_proxy_db()
elif app_label == BLOB_DB_APP and db == 'default':
return True
elif app_label in (FORM_PROCESSOR_APP, SCHEDULING_PARTITIONED_APP,
BLOB_DB_APP):
return (db == partition_config.get_proxy_db()
or db in partition_config.get_form_processing_dbs())
elif app_label == SQL_ACCESSORS_APP:
return db in partition_config.get_form_processing_dbs()
else:
return db == partition_config.get_main_db()
def move_ucr_data_into_aggregation_tables(date=None, intervals=2):
date = date or datetime.utcnow().date()
monthly_dates = []
first_day_of_month = date.replace(day=1)
for interval in range(intervals - 1, 0, -1):
# calculate the last day of the previous months to send to the aggregation script
first_day_next_month = first_day_of_month - relativedelta(
months=interval - 1)
monthly_dates.append(first_day_next_month - relativedelta(days=1))
monthly_dates.append(date)
db_alias = get_icds_ucr_db_alias()
if db_alias:
with connections[db_alias].cursor() as cursor:
_create_aggregate_functions(cursor)
_update_aggregate_locations_tables(cursor)
aggregation_tasks = []
for monthly_date in monthly_dates:
calculation_date = monthly_date.strftime('%Y-%m-%d')
aggregation_tasks.append(
UCRAggregationTask('monthly', calculation_date))
aggregation_tasks.append(
UCRAggregationTask('daily', date.strftime('%Y-%m-%d')))
aggregate_tables.delay(aggregation_tasks[0], aggregation_tasks[1:])
def _run_custom_sql_script(command, day=None):
db_alias = get_icds_ucr_db_alias()
if not db_alias:
return
with connections[db_alias].cursor() as cursor:
cursor.execute(command, [day])
def _run_custom_sql_script(commands, day=None):
db_alias = get_icds_ucr_db_alias()
if not db_alias:
return
with transaction.atomic(using=db_alias):
with connections[db_alias].cursor() as cursor:
for command in commands:
cursor.execute(command, [day])
def _run_custom_sql_script(commands, day=None):
db_alias = get_icds_ucr_db_alias()
if not db_alias:
return
with transaction.atomic(using=db_alias):
with connections[db_alias].cursor() as cursor:
for command in commands:
cursor.execute(command, [day])
def aggregate_tables(self, current_task, future_tasks):
aggregation_type = current_task.type
aggregation_date = current_task.date
if aggregation_type == 'monthly':
path = os.path.join(os.path.dirname(__file__), 'sql_templates',
'update_monthly_aggregate_tables.sql')
elif aggregation_type == 'daily':
path = os.path.join(os.path.dirname(__file__), 'sql_templates',
'update_daily_aggregate_table.sql')
else:
raise ValueError(
"Invalid aggregation type {}".format(aggregation_type))
db_alias = get_icds_ucr_db_alias()
if db_alias:
with connections[db_alias].cursor() as cursor:
with open(path, "r") as sql_file:
sql_to_execute = sql_file.read()
celery_task_logger.info(
"Starting icds reports {} update_{}_aggregate_tables".
format(aggregation_date, aggregation_type))
try:
cursor.execute(sql_to_execute, {"date": aggregation_date})
except Error as exc:
_dashboard_team_soft_assert(
False,
"{} aggregation failed on {} for {}. This task will be retried in 15 minutes"
.format(aggregation_type, settings.SERVER_ENVIRONMENT,
aggregation_date))
notify_exception(
None,
message="Error occurred during ICDS aggregation",
details={
'type': aggregation_type,
'date': aggregation_date,
'error': exc,
})
self.retry(exc=exc)
celery_task_logger.info(
"Ended icds reports {} update_{}_aggregate_tables".format(
aggregation_date, aggregation_type))
if future_tasks:
aggregate_tables.delay(future_tasks[0], future_tasks[1:])
else:
# temporary soft assert to verify it's completing
_dashboard_team_soft_assert(
False,
"Aggregation completed on {}".format(settings.SERVER_ENVIRONMENT))
celery_task_logger.info("Aggregation has completed")
icds_data_validation.delay(aggregation_date)
def icds_state_aggregation_task(self, state_id, date, func_name):
func = {
'_aggregate_gm_forms': _aggregate_gm_forms,
'_aggregate_df_forms': _aggregate_df_forms,
'_aggregate_cf_forms': _aggregate_cf_forms,
'_aggregate_ccs_cf_forms': _aggregate_ccs_cf_forms,
'_aggregate_child_health_thr_forms': _aggregate_child_health_thr_forms,
'_aggregate_ccs_record_thr_forms': _aggregate_ccs_record_thr_forms,
'_aggregate_child_health_pnc_forms': _aggregate_child_health_pnc_forms,
'_aggregate_ccs_record_pnc_forms': _aggregate_ccs_record_pnc_forms,
'_aggregate_delivery_forms': _aggregate_delivery_forms,
'_aggregate_bp_forms': _aggregate_bp_forms,
'_aggregate_awc_infra_forms': _aggregate_awc_infra_forms,
'_child_health_monthly_table': _child_health_monthly_table,
'_agg_ls_awc_mgt_form': _agg_ls_awc_mgt_form,
'_agg_ls_vhnd_form': _agg_ls_vhnd_form,
'_agg_beneficiary_form': _agg_beneficiary_form,
}[func_name]
if six.PY2 and isinstance(date, bytes):
date = date.decode('utf-8')
db_alias = get_icds_ucr_db_alias()
if not db_alias:
return
celery_task_logger.info("Starting icds reports {} {} {}".format(
state_id, date, func.__name__))
try:
func(state_id, date)
except Error as exc:
notify_exception(None,
message="Error occurred during ICDS aggregation",
details={
'func': func.__name__,
'date': date,
'state_id': state_id,
'error': exc
})
_dashboard_team_soft_assert(
False,
"{} aggregation failed on {} for {} on {}. This task will be retried in 15 minutes"
.format(func.__name__, settings.SERVER_ENVIRONMENT, state_id,
date))
self.retry(exc=exc)
celery_task_logger.info("Ended icds reports {} {} {}".format(
state_id, date, func.__name__))
def allow_migrate(db, app_label):
if app_label == ICDS_REPORTS_APP:
db_alias = get_icds_ucr_db_alias()
return db_alias and db_alias == db
if not settings.USE_PARTITIONED_DATABASE:
return app_label != PROXY_APP
if app_label == PROXY_APP:
return db == partition_config.get_proxy_db()
elif app_label in (FORM_PROCESSOR_APP, SCHEDULING_PARTITIONED_APP):
return (db == partition_config.get_proxy_db()
or db in partition_config.get_form_processing_dbs())
elif app_label == SQL_ACCESSORS_APP:
return db in partition_config.get_form_processing_dbs()
elif app_label == WAREHOUSE_APP:
return hasattr(settings, "WAREHOUSE_DATABASE_ALIAS"
) and db == settings.WAREHOUSE_DATABASE_ALIAS
else:
return db == partition_config.get_main_db()
def icds_aggregation_task(self, date, func_name):
func = {
'_agg_ls_table': _agg_ls_table,
'_update_months_table': _update_months_table,
'_daily_attendance_table': _daily_attendance_table,
'_agg_child_health_table': _agg_child_health_table,
'_ccs_record_monthly_table': _ccs_record_monthly_table,
'_agg_ccs_record_table': _agg_ccs_record_table,
'_agg_awc_table': _agg_awc_table,
'_agg_awc_table_weekly': _agg_awc_table_weekly,
'aggregate_awc_daily': aggregate_awc_daily,
}[func_name]
if six.PY2 and isinstance(date, bytes):
date = date.decode('utf-8')
db_alias = get_icds_ucr_db_alias()
if not db_alias:
return
celery_task_logger.info("Starting icds reports {} {}".format(
date, func.__name__))
try:
func(date)
except Error as exc:
notify_exception(None,
message="Error occurred during ICDS aggregation",
details={
'func': func.__name__,
'date': date,
'error': exc
})
_dashboard_team_soft_assert(
False,
"{} aggregation failed on {} for {}. This task will be retried in 15 minutes"
.format(func.__name__, settings.SERVER_ENVIRONMENT, date))
self.retry(exc=exc)
celery_task_logger.info("Ended icds reports {} {}".format(
date, func.__name__))
def icds_aggregation_task(self, date, func):
db_alias = get_icds_ucr_db_alias()
if not db_alias:
return
celery_task_logger.info("Starting icds reports {} {}".format(date, func.__name__))
try:
func(date)
except Error as exc:
notify_exception(
None, message="Error occurred during ICDS aggregation",
details={'func': func.__name__, 'date': date, 'error': exc}
)
_dashboard_team_soft_assert(
False,
"{} aggregation failed on {} for {}. This task will be retried in 15 minutes".format(
func.__name__, settings.SERVER_ENVIRONMENT, date
)
)
self.retry(exc=exc)
celery_task_logger.info("Ended icds reports {} {}".format(date, func.__name__))
def allow_migrate(db, app_label):
"""
Return ``True`` if a app's migrations should be applied to the specified database otherwise
return ``False``.
Note: returning ``None`` is tantamount to returning ``True``
:return: Must return a boolean value, not None.
"""
if app_label == ICDS_REPORTS_APP:
db_alias = get_icds_ucr_db_alias()
return bool(db_alias and db_alias == db)
elif app_label == AAA_APP:
db_alias = get_aaa_db_alias()
return bool(db_alias and db_alias == db)
elif app_label == SYNCLOGS_APP:
return db == settings.SYNCLOGS_SQL_DB_ALIAS
elif app_label == WAREHOUSE_APP:
return db == settings.WAREHOUSE_DATABASE_ALIAS
if not settings.USE_PARTITIONED_DATABASE:
return app_label != PROXY_APP and db in ('default', None)
if app_label == PROXY_APP:
return db == partition_config.get_proxy_db()
elif app_label == BLOB_DB_APP and db == 'default':
return True
elif app_label in (FORM_PROCESSOR_APP, SCHEDULING_PARTITIONED_APP, BLOB_DB_APP):
return (
db == partition_config.get_proxy_db() or
db in partition_config.get_form_processing_dbs()
)
elif app_label == SQL_ACCESSORS_APP:
return db in partition_config.get_form_processing_dbs()
else:
return db == partition_config.get_main_db()
def move_ucr_data_into_aggregation_tables(date=None, intervals=2):
date = date or datetime.utcnow().date()
monthly_dates = []
# probably this should be run one time, for now I leave this in aggregations script (not a big cost)
# but remove issues when someone add new table to mapping, also we don't need to add new rows manually
# on production servers
_update_ucr_table_mapping()
first_day_of_month = date.replace(day=1)
for interval in range(intervals - 1, 0, -1):
# calculate the last day of the previous months to send to the aggregation script
first_day_next_month = first_day_of_month - relativedelta(months=interval - 1)
monthly_dates.append(first_day_next_month - relativedelta(days=1))
monthly_dates.append(date)
db_alias = get_icds_ucr_db_alias()
if db_alias:
with connections[db_alias].cursor() as cursor:
_create_aggregate_functions(cursor)
_update_aggregate_locations_tables()
state_ids = list(SQLLocation.objects
.filter(domain=DASHBOARD_DOMAIN, location_type__name='state')
.values_list('location_id', flat=True))
for monthly_date in monthly_dates:
calculation_date = monthly_date.strftime('%Y-%m-%d')
stage_1_tasks = [
icds_state_aggregation_task.si(state_id=state_id, date=monthly_date, func=_aggregate_gm_forms)
for state_id in state_ids
]
stage_1_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id, date=monthly_date, func=_aggregate_df_forms)
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id, date=monthly_date, func=_aggregate_cf_forms)
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id, date=monthly_date, func=_aggregate_ccs_cf_forms)
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id, date=monthly_date, func=_aggregate_child_health_thr_forms)
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id, date=monthly_date, func=_aggregate_ccs_record_thr_forms)
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id, date=monthly_date, func=_aggregate_child_health_pnc_forms
) for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id, date=monthly_date, func=_aggregate_ccs_record_pnc_forms
) for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id, date=monthly_date, func=_aggregate_delivery_forms
) for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id, date=monthly_date, func=_aggregate_bp_forms
) for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id, date=monthly_date, func=_aggregate_awc_infra_forms)
for state_id in state_ids
])
stage_1_tasks.append(icds_aggregation_task.si(date=calculation_date, func=_update_months_table))
res = group(*stage_1_tasks).apply_async()
res_daily = icds_aggregation_task.delay(date=calculation_date, func=_daily_attendance_table)
res.get()
res_child = chain(
icds_state_aggregation_task.si(
state_id=state_ids, date=calculation_date, func=_child_health_monthly_table
),
icds_aggregation_task.si(date=calculation_date, func=_agg_child_health_table),
).apply_async()
res_ccs = chain(
icds_aggregation_task.si(date=calculation_date, func=_ccs_record_monthly_table),
icds_aggregation_task.si(date=calculation_date, func=_agg_ccs_record_table),
).apply_async()
res_daily.get()
res_ccs.get()
res_child.get()
res_ls_tasks = list()
res_ls_tasks.extend([icds_state_aggregation_task.si(state_id=state_id, date=calculation_date,
func=_agg_ls_awc_mgt_form)
for state_id in state_ids
])
res_ls_tasks.extend([icds_state_aggregation_task.si(state_id=state_id, date=calculation_date,
func=_agg_ls_vhnd_form)
for state_id in state_ids
])
res_ls_tasks.extend([icds_state_aggregation_task.si(state_id=state_id, date=calculation_date,
func=_agg_beneficiary_form)
for state_id in state_ids
])
res_ls_tasks.append(icds_aggregation_task.si(date=calculation_date, func=_agg_ls_table))
res_awc = chain(icds_aggregation_task.si(date=calculation_date, func=_agg_awc_table),
*res_ls_tasks
).apply_async()
res_awc.get()
first_of_month_string = monthly_date.strftime('%Y-%m-01')
for state_id in state_ids:
create_mbt_for_month.delay(state_id, first_of_month_string)
if date.weekday() == 5:
icds_aggregation_task.delay(date=date.strftime('%Y-%m-%d'), func=_agg_awc_table_weekly)
chain(
icds_aggregation_task.si(date=date.strftime('%Y-%m-%d'), func=aggregate_awc_daily),
email_dashboad_team.si(aggregation_date=date.strftime('%Y-%m-%d'))
).delay()
def move_ucr_data_into_aggregation_tables(date=None, intervals=2):
date = date or datetime.utcnow().date()
monthly_dates = []
# probably this should be run one time, for now I leave this in aggregations script (not a big cost)
# but remove issues when someone add new table to mapping, also we don't need to add new rows manually
# on production servers
_update_ucr_table_mapping()
first_day_of_month = date.replace(day=1)
for interval in range(intervals - 1, 0, -1):
# calculate the last day of the previous months to send to the aggregation script
first_day_next_month = first_day_of_month - relativedelta(months=interval - 1)
monthly_dates.append(first_day_next_month - relativedelta(days=1))
monthly_dates.append(date)
db_alias = get_icds_ucr_db_alias()
if db_alias:
with connections[db_alias].cursor() as cursor:
_create_aggregate_functions(cursor)
_update_aggregate_locations_tables(cursor)
state_ids = (SQLLocation.objects
.filter(domain=DASHBOARD_DOMAIN, location_type__name='state')
.values_list('location_id', flat=True))
for monthly_date in monthly_dates:
calculation_date = monthly_date.strftime('%Y-%m-%d')
stage_1_tasks = [
icds_state_aggregation_task.si(state_id=state_id, date=monthly_date, func=_aggregate_gm_forms)
for state_id in state_ids
]
stage_1_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id, date=monthly_date, func=_aggregate_df_forms)
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id, date=monthly_date, func=_aggregate_cf_forms)
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id, date=monthly_date, func=_aggregate_child_health_thr_forms)
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id, date=monthly_date, func=_aggregate_ccs_record_thr_forms)
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id, date=monthly_date, func=_aggregate_child_health_pnc_forms
) for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id, date=monthly_date, func=_aggregate_ccs_record_pnc_forms
) for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id, date=monthly_date, func=_aggregate_delivery_forms
) for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id, date=monthly_date, func=_aggregate_bp_forms
) for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id, date=monthly_date, func=_aggregate_awc_infra_forms)
for state_id in state_ids
])
stage_1_tasks.append(icds_aggregation_task.si(date=calculation_date, func=_update_months_table))
res = group(*stage_1_tasks).apply_async()
res_daily = icds_aggregation_task.delay(date=calculation_date, func=_daily_attendance_table)
res.get()
res_child = chain(
icds_aggregation_task.si(date=calculation_date, func=_child_health_monthly_table),
icds_aggregation_task.si(date=calculation_date, func=_agg_child_health_table),
).apply_async()
res_ccs = chain(
icds_aggregation_task.si(date=calculation_date, func=_ccs_record_monthly_table),
icds_aggregation_task.si(date=calculation_date, func=_agg_ccs_record_table),
).apply_async()
res_daily.get()
res_ccs.get()
res_child.get()
res_awc = icds_aggregation_task.delay(date=calculation_date, func=_agg_awc_table)
res_awc.get()
chain(
icds_aggregation_task.si(date=date.strftime('%Y-%m-%d'), func=aggregate_awc_daily),
email_dashboad_team.si(aggregation_date=date.strftime('%Y-%m-%d'))
).delay()
def execute_query(self, query):
db_alias = get_icds_ucr_db_alias()
with connections[db_alias].cursor() as cursor:
cursor.execute(query)
return cursor.fetchall()[0]
def execute_query(self, query):
db_alias = get_icds_ucr_db_alias()
with connections[db_alias].cursor() as cursor:
cursor.execute(query)
return cursor.fetchall()[0]
def move_ucr_data_into_aggregation_tables(date=None, intervals=2):
date = date or datetime.utcnow().date()
monthly_dates = []
# probably this should be run one time, for now I leave this in aggregations script (not a big cost)
# but remove issues when someone add new table to mapping, also we don't need to add new rows manually
# on production servers
_update_ucr_table_mapping()
first_day_of_month = date.replace(day=1)
for interval in range(intervals - 1, 0, -1):
# calculate the last day of the previous months to send to the aggregation script
first_day_next_month = first_day_of_month - relativedelta(
months=interval - 1)
monthly_dates.append(first_day_next_month - relativedelta(days=1))
monthly_dates.append(date)
db_alias = get_icds_ucr_db_alias()
if db_alias:
with connections[db_alias].cursor() as cursor:
_create_aggregate_functions(cursor)
_update_aggregate_locations_tables()
state_ids = list(
SQLLocation.objects.filter(
domain=DASHBOARD_DOMAIN,
location_type__name='state').values_list('location_id',
flat=True))
for monthly_date in monthly_dates:
calculation_date = monthly_date.strftime('%Y-%m-%d')
stage_1_tasks = [
icds_state_aggregation_task.si(state_id=state_id,
date=monthly_date,
func_name='_aggregate_gm_forms')
for state_id in state_ids
]
stage_1_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id,
date=monthly_date,
func_name='_aggregate_df_forms')
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id,
date=monthly_date,
func_name='_aggregate_cf_forms')
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id,
date=monthly_date,
func_name='_aggregate_ccs_cf_forms')
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id,
date=monthly_date,
func_name='_aggregate_child_health_thr_forms')
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id,
date=monthly_date,
func_name='_aggregate_ccs_record_thr_forms')
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id,
date=monthly_date,
func_name='_aggregate_child_health_pnc_forms')
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id,
date=monthly_date,
func_name='_aggregate_ccs_record_pnc_forms')
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id,
date=monthly_date,
func_name='_aggregate_delivery_forms')
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id,
date=monthly_date,
func_name='_aggregate_bp_forms')
for state_id in state_ids
])
stage_1_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id,
date=monthly_date,
func_name='_aggregate_awc_infra_forms')
for state_id in state_ids
])
stage_1_tasks.append(
icds_aggregation_task.si(date=calculation_date,
func_name='_update_months_table'))
res_daily = icds_aggregation_task.delay(
date=calculation_date, func_name='_daily_attendance_table')
# https://github.com/celery/celery/issues/4274
stage_1_task_results = [
stage_1_task.delay() for stage_1_task in stage_1_tasks
]
for stage_1_task_result in stage_1_task_results:
stage_1_task_result.get(disable_sync_subtasks=False)
res_child = chain(
icds_state_aggregation_task.si(
state_id=state_ids,
date=calculation_date,
func_name='_child_health_monthly_table'),
icds_aggregation_task.si(date=calculation_date,
func_name='_agg_child_health_table'),
).apply_async()
res_ccs = chain(
icds_aggregation_task.si(
date=calculation_date,
func_name='_ccs_record_monthly_table'),
icds_aggregation_task.si(date=calculation_date,
func_name='_agg_ccs_record_table'),
).apply_async()
res_daily.get(disable_sync_subtasks=False)
res_ccs.get(disable_sync_subtasks=False)
res_child.get(disable_sync_subtasks=False)
res_ls_tasks = list()
res_ls_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id,
date=calculation_date,
func_name='_agg_ls_awc_mgt_form') for state_id in state_ids
])
res_ls_tasks.extend([
icds_state_aggregation_task.si(state_id=state_id,
date=calculation_date,
func_name='_agg_ls_vhnd_form')
for state_id in state_ids
])
res_ls_tasks.extend([
icds_state_aggregation_task.si(
state_id=state_id,
date=calculation_date,
func_name='_agg_beneficiary_form')
for state_id in state_ids
])
res_ls_tasks.append(
icds_aggregation_task.si(date=calculation_date,
func_name='_agg_ls_table'))
res_awc = chain(
icds_aggregation_task.si(date=calculation_date,
func_name='_agg_awc_table'),
*res_ls_tasks).apply_async()
res_awc.get(disable_sync_subtasks=False)
first_of_month_string = monthly_date.strftime('%Y-%m-01')
for state_id in state_ids:
create_mbt_for_month.delay(state_id, first_of_month_string)
if date.weekday() == 5:
icds_aggregation_task.delay(date=date.strftime('%Y-%m-%d'),
func_name='_agg_awc_table_weekly')
chain(
icds_aggregation_task.si(date=date.strftime('%Y-%m-%d'),
func_name='aggregate_awc_daily'),
_bust_awc_cache.si(),
email_dashboad_team.si(
aggregation_date=date.strftime('%Y-%m-%d'))).delay()
def move_ucr_data_into_aggregation_tables(date=None, intervals=2):
date = date or datetime.utcnow().date()
monthly_dates = []
# probably this should be run one time, for now I leave this in aggregations script (not a big cost)
# but remove issues when someone add new table to mapping, also we don't need to add new rows manually
# on production servers
_update_ucr_table_mapping()
first_day_of_month = date.replace(day=1)
for interval in range(intervals - 1, 0, -1):
# calculate the last day of the previous months to send to the aggregation script
first_day_next_month = first_day_of_month - relativedelta(
months=interval - 1)
monthly_dates.append(first_day_next_month - relativedelta(days=1))
monthly_dates.append(date)
db_alias = get_icds_ucr_db_alias()
if db_alias:
with connections[db_alias].cursor() as cursor:
_create_aggregate_functions(cursor)
_update_aggregate_locations_tables(cursor)
_create_child_health_monthly_view()
tasks = []
for monthly_date in monthly_dates:
calculation_date = monthly_date.strftime('%Y-%m-%d')
tasks.extend([
group(
icds_aggregation_task.si(date=calculation_date,
func=_update_months_table),
icds_aggregation_task.si(date=calculation_date,
func=_aggregate_cf_forms),
icds_aggregation_task.si(date=calculation_date,
func=_aggregate_thr_forms),
icds_aggregation_task.si(date=calculation_date,
func=_aggregate_gm_forms),
icds_aggregation_task.si(
date=calculation_date,
func=_aggregate_child_health_pnc_forms),
),
group(
icds_aggregation_task.si(date=calculation_date,
func=_child_health_monthly_table),
icds_aggregation_task.si(date=calculation_date,
func=_ccs_record_monthly_table),
icds_aggregation_task.si(date=calculation_date,
func=_daily_attendance_table),
),
group(
icds_aggregation_task.si(date=calculation_date,
func=_agg_child_health_table),
icds_aggregation_task.si(date=calculation_date,
func=_agg_ccs_record_table),
),
group(
icds_aggregation_task.si(date=calculation_date,
func=_agg_awc_table),
no_op_task_for_celery_bug.si(),
)
])
tasks.append(
group(
icds_aggregation_task.si(date=date.strftime('%Y-%m-%d'),
func=aggregate_awc_daily),
no_op_task_for_celery_bug.si(),
))
tasks.append(
group(
email_dashboad_team.si(
aggregation_date=date.strftime('%Y-%m-%d')),
no_op_task_for_celery_bug.si(),
))
chain(*tasks).delay()
