def days_missed(cls, report, session): """ Examine the past runs of a recurring report, for up to 30 days. Find any missed runs, including today's run. Raise an exception if there are more runs than expected. Parameters: report : the recurring report to examine session : session to the database Returns: An array of datetimes representing the days when the report did not run """ search_from = strip_time(report.created) look_at_most_this_far = to_datetime(thirty_days_ago()) if search_from < look_at_most_this_far: search_from = look_at_most_this_far completed_days = [pr[0] for pr in session.query(PersistentReport.created) .filter(PersistentReport.recurrent_parent_id == report.id) .filter(PersistentReport.created >= search_from) .filter(PersistentReport.status != celery.states.FAILURE) .all()] expected_days = timestamps_to_now(search_from, timedelta(days=1)) missed_days, unexpected_days = diff_datewise(expected_days, completed_days) if len(unexpected_days) > 0: task_logger.warn('Problem with recurrent report id {}'.format(report.id)) task_logger.warn('Completed runs: {}'.format(sorted(completed_days))) task_logger.warn('Unexpected runs: {}'.format(sorted(unexpected_days))) raise Exception('More reports ran than were supposed to') return missed_days
def days_missed(cls, report, session): """ Examine the past runs of a recurring report Find any missed runs, including today's run. Raise an exception if there are more runs than expected. Parameters: report : the recurring report to examine session : session to the database Returns: An array of datetimes representing the days when the report did not run """ search_from = strip_time(report.created) # if a report is pending by this point, it means that it should be re-tried session.query(ReportStore) \ .filter(ReportStore.recurrent_parent_id == report.id) \ .filter(ReportStore.created >= search_from) \ .filter(ReportStore.status == celery.states.PENDING) \ .delete() completed_days = [ pr[0] for pr in session.query(ReportStore.created).filter( ReportStore.recurrent_parent_id == report.id).filter( ReportStore.created >= search_from).filter( ReportStore.status != celery.states.FAILURE).all() ] expected_days = timestamps_to_now(search_from, timedelta(days=1)) missed_days, unexpected_days = diff_datewise(expected_days, completed_days) if len(unexpected_days) > 0: task_logger.warn('Problem with recurrent report id {}'.format( report.id)) task_logger.warn('Completed runs: {}'.format( sorted(completed_days))) task_logger.warn('Unexpected runs: {}'.format( sorted(unexpected_days))) raise Exception('More reports ran than were supposed to') return sorted(missed_days)
def days_missed(cls, report, session): """ Examine the past runs of a recurring report Find any missed runs, including today's run. Raise an exception if there are more runs than expected. Parameters: report : the recurring report to examine session : session to the database Returns: An array of datetimes representing the days when the report did not run """ search_from = strip_time(report.created) # if a report is pending by this point, it means that it should be re-tried session.query(ReportStore) \ .filter(ReportStore.recurrent_parent_id == report.id) \ .filter(ReportStore.created >= search_from) \ .filter(ReportStore.status == celery.states.PENDING) \ .delete() completed_days = [pr[0] for pr in session.query(ReportStore.created) .filter(ReportStore.recurrent_parent_id == report.id) .filter(ReportStore.created >= search_from) .filter(ReportStore.status != celery.states.FAILURE) .all()] expected_days = timestamps_to_now(search_from, timedelta(days=1)) missed_days, unexpected_days = diff_datewise(expected_days, completed_days) if len(unexpected_days) > 0: task_logger.warn('Problem with recurrent report id {}'.format(report.id)) task_logger.warn('Completed runs: {}'.format(sorted(completed_days))) task_logger.warn('Unexpected runs: {}'.format(sorted(unexpected_days))) raise Exception('More reports ran than were supposed to') return sorted(missed_days)
def setUp(self): DatabaseTest.setUp(self) # turn off the scheduler for this test self.save_schedule = queue.conf['CELERYBEAT_SCHEDULE'] queue.conf['CELERYBEAT_SCHEDULE'] = {} self.common_cohort_1() self.uid = self.owner_user_id self.today = strip_time(datetime.today()) # NOTE: running with 20 just makes the tests run faster, but any value > 11 works self.no_more_than = 20 self.missed_by_index = { 0: [1, 2, 11], 1: [1, 2, 11, self.no_more_than + 1, self.no_more_than + 3], 2: [1, 2], 3: range(0, self.no_more_than + 10), } self.ago_by_index = { 0: 26, 1: self.no_more_than + 10, 2: 6, 3: self.no_more_than + 10 } age = { i: self.today - timedelta(days=v - 1) for i, v in self.ago_by_index.items() } self.p = { 'metric': { 'start_date': age[0], 'end_date': self.today, 'name': 'NamespaceEdits', }, 'recurrent': True, 'cohort': { 'id': self.cohort.id, 'name': self.cohort.name }, 'name': 'test-recurrent-reports', } ps = stringify(self.p) self.reports = [ ReportStore(recurrent=True, created=age[0], parameters=ps, user_id=self.uid), ReportStore(recurrent=True, created=age[1], parameters=ps, user_id=self.uid), ReportStore(recurrent=True, created=age[2], parameters=ps, user_id=self.uid), ReportStore(recurrent=True, created=age[3], parameters=ps, user_id=self.uid), ] self.session.add_all(self.reports) self.session.commit()
def setUp(self): DatabaseTest.setUp(self) # turn off the scheduler for this test self.save_schedule = queue.conf['CELERYBEAT_SCHEDULE'] queue.conf['CELERYBEAT_SCHEDULE'] = {} self.common_cohort_1() uid = self.owner_user_id self.today = strip_time(datetime.today()) ago_25 = self.today - timedelta(days=25) ago_35 = self.today - timedelta(days=35) ago_05 = self.today - timedelta(days=5) p = { 'metric': { 'start_date': ago_05, 'end_date': self.today, 'name': 'NamespaceEdits', }, 'recurrent': True, 'cohort': {'id': self.cohort.id, 'name': self.cohort.name}, 'name': 'test-recurrent-reports', } ps = stringify(p) self.reports = [ PersistentReport(recurrent=True, created=ago_25, parameters=ps, user_id=uid), PersistentReport(recurrent=True, created=ago_35, parameters=ps, user_id=uid), PersistentReport(recurrent=True, created=ago_05, parameters=ps, user_id=uid), ] self.session.add_all(self.reports) self.session.commit() self.report_runs = [] for d in range(0, 35): day = self.today - timedelta(days=d) p['metric']['start_date'] = day - timedelta(days=1) p['metric']['end_date'] = day p['recurrent'] = False ps = stringify(p) if d not in [1, 2, 11] and d < 26: self.report_runs.append(PersistentReport( recurrent_parent_id=self.reports[0].id, created=day, status='SUCCESS', parameters=ps, user_id=uid, )) if d not in [1, 2, 11, 31, 33]: self.report_runs.append(PersistentReport( recurrent_parent_id=self.reports[1].id, created=day, status='SUCCESS', parameters=ps, user_id=uid, )) if d not in [1, 2] and d < 6: self.report_runs.append(PersistentReport( recurrent_parent_id=self.reports[2].id, created=day, status='SUCCESS', parameters=ps, user_id=uid, )) self.session.add_all(self.report_runs) self.session.commit()