def test_diff_datewise(self):
l = []
l_just_dates = []
r = []
r_just_dates = []
lp = 'blah%Y...%m...%d...%Hblahblah'
rp = 'neenee%Y%m%d%Hneenee'
expect0 = set([datetime(2012, 6, 14, 13), datetime(2012, 11, 9, 3)])
expect1 = set([datetime(2012, 6, 14, 14), datetime(2013, 11, 10, 22)])
for y in range(2012, 2014):
for m in range(1, 13):
# we're just diffing so we don't care about getting all days
for d in range(1, 28):
for h in range(0, 24):
x = datetime(y, m, d, h)
if x not in expect1:
l.append(datetime.strftime(x, lp))
l_just_dates.append(x)
if x not in expect0:
r.append(datetime.strftime(x, rp))
r_just_dates.append(x)
result = diff_datewise(l, r, left_parse=lp, right_parse=rp)
self.assertEqual(result[0], expect0)
self.assertEqual(result[1], expect1)
result = diff_datewise(l_just_dates, r, right_parse=rp)
self.assertEqual(result[0], expect0)
self.assertEqual(result[1], expect1)
result = diff_datewise(l_just_dates, r_just_dates)
self.assertEqual(result[0], expect0)
self.assertEqual(result[1], expect1)
def test_diff_datewise(self):
l = []
l_just_dates = []
r = []
r_just_dates = []
lp = 'blah%Y...%m...%d...%Hblahblah'
rp = 'neenee%Y%m%d%Hneenee'
expect0 = set([datetime(2012, 6, 14, 13), datetime(2012, 11, 9, 3)])
expect1 = set([datetime(2012, 6, 14, 14), datetime(2013, 11, 10, 22)])
for y in range(2012, 2014):
for m in range(1, 13):
# we're just diffing so we don't care about getting all days
for d in range(1, 28):
for h in range(0, 24):
x = datetime(y, m, d, h)
if x not in expect1:
l.append(datetime.strftime(x, lp))
l_just_dates.append(x)
if x not in expect0:
r.append(datetime.strftime(x, rp))
r_just_dates.append(x)
result = diff_datewise(l, r, left_parse=lp, right_parse=rp)
self.assertEqual(result[0], expect0)
self.assertEqual(result[1], expect1)
result = diff_datewise(l_just_dates, r, right_parse=rp)
self.assertEqual(result[0], expect0)
self.assertEqual(result[1], expect1)
result = diff_datewise(l_just_dates, r_just_dates)
self.assertEqual(result[0], expect0)
self.assertEqual(result[1], expect1)
def days_missed(cls, report, session):
"""
Examine the past runs of a recurring report, for up to 30 days.
Find any missed runs, including today's run. Raise an exception if there
are more runs than expected.
Parameters:
report : the recurring report to examine
session : session to the database
Returns:
An array of datetimes representing the days when the report did not run
"""
search_from = strip_time(report.created)
look_at_most_this_far = to_datetime(thirty_days_ago())
if search_from < look_at_most_this_far:
search_from = look_at_most_this_far
completed_days = [pr[0] for pr in session.query(PersistentReport.created)
.filter(PersistentReport.recurrent_parent_id == report.id)
.filter(PersistentReport.created >= search_from)
.filter(PersistentReport.status != celery.states.FAILURE)
.all()]
expected_days = timestamps_to_now(search_from, timedelta(days=1))
missed_days, unexpected_days = diff_datewise(expected_days, completed_days)
if len(unexpected_days) > 0:
task_logger.warn('Problem with recurrent report id {}'.format(report.id))
task_logger.warn('Completed runs: {}'.format(sorted(completed_days)))
task_logger.warn('Unexpected runs: {}'.format(sorted(unexpected_days)))
raise Exception('More reports ran than were supposed to')
return missed_days
def days_missed(cls, report, session):
"""
Examine the past runs of a recurring report
Find any missed runs, including today's run. Raise an exception if there
are more runs than expected.
Parameters:
report : the recurring report to examine
session : session to the database
Returns:
An array of datetimes representing the days when the report did not run
"""
search_from = strip_time(report.created)
# if a report is pending by this point, it means that it should be re-tried
session.query(ReportStore) \
.filter(ReportStore.recurrent_parent_id == report.id) \
.filter(ReportStore.created >= search_from) \
.filter(ReportStore.status == celery.states.PENDING) \
.delete()
completed_days = [
pr[0] for pr in session.query(ReportStore.created).filter(
ReportStore.recurrent_parent_id == report.id).filter(
ReportStore.created >= search_from).filter(
ReportStore.status != celery.states.FAILURE).all()
]
expected_days = timestamps_to_now(search_from, timedelta(days=1))
missed_days, unexpected_days = diff_datewise(expected_days,
completed_days)
if len(unexpected_days) > 0:
task_logger.warn('Problem with recurrent report id {}'.format(
report.id))
task_logger.warn('Completed runs: {}'.format(
sorted(completed_days)))
task_logger.warn('Unexpected runs: {}'.format(
sorted(unexpected_days)))
raise Exception('More reports ran than were supposed to')
return sorted(missed_days)
def days_missed(cls, report, session):
"""
Examine the past runs of a recurring report
Find any missed runs, including today's run. Raise an exception if there
are more runs than expected.
Parameters:
report : the recurring report to examine
session : session to the database
Returns:
An array of datetimes representing the days when the report did not run
"""
search_from = strip_time(report.created)
# if a report is pending by this point, it means that it should be re-tried
session.query(ReportStore) \
.filter(ReportStore.recurrent_parent_id == report.id) \
.filter(ReportStore.created >= search_from) \
.filter(ReportStore.status == celery.states.PENDING) \
.delete()
completed_days = [pr[0] for pr in session.query(ReportStore.created)
.filter(ReportStore.recurrent_parent_id == report.id)
.filter(ReportStore.created >= search_from)
.filter(ReportStore.status != celery.states.FAILURE)
.all()]
expected_days = timestamps_to_now(search_from, timedelta(days=1))
missed_days, unexpected_days = diff_datewise(expected_days, completed_days)
if len(unexpected_days) > 0:
task_logger.warn('Problem with recurrent report id {}'.format(report.id))
task_logger.warn('Completed runs: {}'.format(sorted(completed_days)))
task_logger.warn('Unexpected runs: {}'.format(sorted(unexpected_days)))
raise Exception('More reports ran than were supposed to')
return sorted(missed_days)
