def populate_daily_metrics(date_for=None, force_update=False):
'''Populates the daily metrics models for the given date
This method populates CourseDailyMetrics for all the courses in the site,
then populates SiteDailyMetrics
It calls the individual tasks, ``populate_single_cdm`` and
``populate_site_daily_metrics`` as immediate calls so that no courses are
missed when the site daily metrics record is populated.
NOTE: We have an experimental task that runs the course populators in
parallel, then when they are all done, populates the site metrics. See the
function ``experimental_populate_daily_metrics`` docstring for details
TODO: Add error handling and error logging
TODO: Create and add decorator to assign 'date_for' if None
'''
if date_for:
date_for = as_date(date_for)
else:
date_for = datetime.datetime.utcnow().replace(tzinfo=utc).date()
logger.info(
'Starting task "figures.populate_daily_metrics" for date "{}"'.format(
date_for))
for site in Site.objects.all():
for course in figures.sites.get_courses_for_site(site):
try:
populate_single_cdm(course_id=course.id,
date_for=date_for,
force_update=force_update)
except Exception as e:
logger.exception('figures.tasks.populate_daily_metrics failed')
# Always capture CDM load exceptions to the Figures pipeline
# error table
error_data = dict(
date_for=date_for,
msg='figures.tasks.populate_daily_metrics failed',
exception_class=e.__class__.__name__,
)
if hasattr(e, 'message_dict'):
error_data['message_dict'] = e.message_dict
log_error_to_db(
error_data=error_data,
error_type=PipelineError.COURSE_DATA,
course_id=str(course.id),
site=site,
logger=logger,
log_pipeline_errors_to_db=True,
)
populate_site_daily_metrics(site_id=site.id,
date_for=date_for,
force_update=force_update)
logger.info(
'Finished task "figures.populate_daily_metrics" for date "{}"'.format(
date_for))
def populate_daily_metrics(date_for=None, force_update=False):
'''Populates the daily metrics models for the given date
This method populates CourseDailyMetrics for all the courses in the site,
then populates SiteDailyMetrics
It calls the individual tasks, ``populate_single_cdm`` and
``populate_site_daily_metrics`` as immediate calls so that no courses are
missed when the site daily metrics record is populated.
NOTE: We have an experimental task that runs the course populators in
parallel, then when they are all done, populates the site metrics. See the
function ``experimental_populate_daily_metrics`` docstring for details
TODO: Add error handling and error logging
TODO: Create and add decorator to assign 'date_for' if None
'''
if date_for:
date_for = as_date(date_for)
else:
date_for = datetime.datetime.utcnow().replace(tzinfo=utc).date()
logger.info(
'Starting task "figures.populate_daily_metrics" for date "{}"'.format(
date_for))
sites_count = Site.objects.count()
for i, site in enumerate(Site.objects.all()):
try:
for course in figures.sites.get_courses_for_site(site):
try:
populate_single_cdm(course_id=course.id,
date_for=date_for,
force_update=force_update)
except Exception as e: # pylint: disable=broad-except
logger.exception(
'figures.tasks.populate_daily_metrics failed')
# Always capture CDM load exceptions to the Figures pipeline
# error table
error_data = dict(
date_for=date_for,
msg='figures.tasks.populate_daily_metrics failed',
exception_class=e.__class__.__name__,
)
if hasattr(e, 'message_dict'):
error_data['message_dict'] = e.message_dict # pylint: disable=no-member
log_error_to_db(
error_data=error_data,
error_type=PipelineError.COURSE_DATA,
course_id=str(course.id),
site=site,
logger=logger,
log_pipeline_errors_to_db=True,
)
populate_site_daily_metrics(site_id=site.id,
date_for=date_for,
force_update=force_update)
# Until we implement signal triggers
try:
update_enrollment_data(site_id=site.id)
except Exception: # pylint: disable=broad-except
msg = ('FIGURES:FAIL figures.tasks update_enrollment_data '
' unhandled exception. site[{}]:{}')
logger.exception(msg.format(site.id, site.domain))
except Exception: # pylint: disable=broad-except
msg = ('FIGURES:FAIL populate_daily_metrics unhandled site level'
' exception for site[{}]={}')
logger.exception(msg.format(site.id, site.domain))
logger.info(
"figures.populate_daily_metrics: finished Site {:04d} of {:04d}".
format(i, sites_count))
logger.info(
'Finished task "figures.populate_daily_metrics" for date "{}"'.format(
date_for))