def days_missed(cls, report, session):
"""
Examine the past runs of a recurring report, for up to 30 days.
Find any missed runs, including today's run. Raise an exception if there
are more runs than expected.
Parameters:
report : the recurring report to examine
session : session to the database
Returns:
An array of datetimes representing the days when the report did not run
"""
search_from = strip_time(report.created)
look_at_most_this_far = to_datetime(thirty_days_ago())
if search_from < look_at_most_this_far:
search_from = look_at_most_this_far
completed_days = [pr[0] for pr in session.query(PersistentReport.created)
.filter(PersistentReport.recurrent_parent_id == report.id)
.filter(PersistentReport.created >= search_from)
.filter(PersistentReport.status != celery.states.FAILURE)
.all()]
expected_days = timestamps_to_now(search_from, timedelta(days=1))
missed_days, unexpected_days = diff_datewise(expected_days, completed_days)
if len(unexpected_days) > 0:
task_logger.warn('Problem with recurrent report id {}'.format(report.id))
task_logger.warn('Completed runs: {}'.format(sorted(completed_days)))
task_logger.warn('Unexpected runs: {}'.format(sorted(unexpected_days)))
raise Exception('More reports ran than were supposed to')
return missed_days
def days_missed(cls, report, session):
"""
Examine the past runs of a recurring report
Find any missed runs, including today's run. Raise an exception if there
are more runs than expected.
Parameters:
report : the recurring report to examine
session : session to the database
Returns:
An array of datetimes representing the days when the report did not run
"""
search_from = strip_time(report.created)
# if a report is pending by this point, it means that it should be re-tried
session.query(ReportStore) \
.filter(ReportStore.recurrent_parent_id == report.id) \
.filter(ReportStore.created >= search_from) \
.filter(ReportStore.status == celery.states.PENDING) \
.delete()
completed_days = [
pr[0] for pr in session.query(ReportStore.created).filter(
ReportStore.recurrent_parent_id == report.id).filter(
ReportStore.created >= search_from).filter(
ReportStore.status != celery.states.FAILURE).all()
]
expected_days = timestamps_to_now(search_from, timedelta(days=1))
missed_days, unexpected_days = diff_datewise(expected_days,
completed_days)
if len(unexpected_days) > 0:
task_logger.warn('Problem with recurrent report id {}'.format(
report.id))
task_logger.warn('Completed runs: {}'.format(
sorted(completed_days)))
task_logger.warn('Unexpected runs: {}'.format(
sorted(unexpected_days)))
raise Exception('More reports ran than were supposed to')
return sorted(missed_days)
def days_missed(cls, report, session):
"""
Examine the past runs of a recurring report
Find any missed runs, including today's run. Raise an exception if there
are more runs than expected.
Parameters:
report : the recurring report to examine
session : session to the database
Returns:
An array of datetimes representing the days when the report did not run
"""
search_from = strip_time(report.created)
# if a report is pending by this point, it means that it should be re-tried
session.query(ReportStore) \
.filter(ReportStore.recurrent_parent_id == report.id) \
.filter(ReportStore.created >= search_from) \
.filter(ReportStore.status == celery.states.PENDING) \
.delete()
completed_days = [pr[0] for pr in session.query(ReportStore.created)
.filter(ReportStore.recurrent_parent_id == report.id)
.filter(ReportStore.created >= search_from)
.filter(ReportStore.status != celery.states.FAILURE)
.all()]
expected_days = timestamps_to_now(search_from, timedelta(days=1))
missed_days, unexpected_days = diff_datewise(expected_days, completed_days)
if len(unexpected_days) > 0:
task_logger.warn('Problem with recurrent report id {}'.format(report.id))
task_logger.warn('Completed runs: {}'.format(sorted(completed_days)))
task_logger.warn('Unexpected runs: {}'.format(sorted(unexpected_days)))
raise Exception('More reports ran than were supposed to')
return sorted(missed_days)
def setUp(self):
DatabaseTest.setUp(self)
# turn off the scheduler for this test
self.save_schedule = queue.conf['CELERYBEAT_SCHEDULE']
queue.conf['CELERYBEAT_SCHEDULE'] = {}
self.common_cohort_1()
self.uid = self.owner_user_id
self.today = strip_time(datetime.today())
# NOTE: running with 20 just makes the tests run faster, but any value > 11 works
self.no_more_than = 20
self.missed_by_index = {
0: [1, 2, 11],
1: [1, 2, 11, self.no_more_than + 1, self.no_more_than + 3],
2: [1, 2],
3: range(0, self.no_more_than + 10),
}
self.ago_by_index = {
0: 26,
1: self.no_more_than + 10,
2: 6,
3: self.no_more_than + 10
}
age = {
i: self.today - timedelta(days=v - 1)
for i, v in self.ago_by_index.items()
}
self.p = {
'metric': {
'start_date': age[0],
'end_date': self.today,
'name': 'NamespaceEdits',
},
'recurrent': True,
'cohort': {
'id': self.cohort.id,
'name': self.cohort.name
},
'name': 'test-recurrent-reports',
}
ps = stringify(self.p)
self.reports = [
ReportStore(recurrent=True,
created=age[0],
parameters=ps,
user_id=self.uid),
ReportStore(recurrent=True,
created=age[1],
parameters=ps,
user_id=self.uid),
ReportStore(recurrent=True,
created=age[2],
parameters=ps,
user_id=self.uid),
ReportStore(recurrent=True,
created=age[3],
parameters=ps,
user_id=self.uid),
]
self.session.add_all(self.reports)
self.session.commit()
def setUp(self):
DatabaseTest.setUp(self)
# turn off the scheduler for this test
self.save_schedule = queue.conf['CELERYBEAT_SCHEDULE']
queue.conf['CELERYBEAT_SCHEDULE'] = {}
self.common_cohort_1()
uid = self.owner_user_id
self.today = strip_time(datetime.today())
ago_25 = self.today - timedelta(days=25)
ago_35 = self.today - timedelta(days=35)
ago_05 = self.today - timedelta(days=5)
p = {
'metric': {
'start_date': ago_05, 'end_date': self.today, 'name': 'NamespaceEdits',
},
'recurrent': True,
'cohort': {'id': self.cohort.id, 'name': self.cohort.name},
'name': 'test-recurrent-reports',
}
ps = stringify(p)
self.reports = [
PersistentReport(recurrent=True, created=ago_25, parameters=ps, user_id=uid),
PersistentReport(recurrent=True, created=ago_35, parameters=ps, user_id=uid),
PersistentReport(recurrent=True, created=ago_05, parameters=ps, user_id=uid),
]
self.session.add_all(self.reports)
self.session.commit()
self.report_runs = []
for d in range(0, 35):
day = self.today - timedelta(days=d)
p['metric']['start_date'] = day - timedelta(days=1)
p['metric']['end_date'] = day
p['recurrent'] = False
ps = stringify(p)
if d not in [1, 2, 11] and d < 26:
self.report_runs.append(PersistentReport(
recurrent_parent_id=self.reports[0].id,
created=day,
status='SUCCESS',
parameters=ps,
user_id=uid,
))
if d not in [1, 2, 11, 31, 33]:
self.report_runs.append(PersistentReport(
recurrent_parent_id=self.reports[1].id,
created=day,
status='SUCCESS',
parameters=ps,
user_id=uid,
))
if d not in [1, 2] and d < 6:
self.report_runs.append(PersistentReport(
recurrent_parent_id=self.reports[2].id,
created=day,
status='SUCCESS',
parameters=ps,
user_id=uid,
))
self.session.add_all(self.report_runs)
self.session.commit()
