def _backfill_record(self, record: BackfillRecord,
left: int) -> Tuple[int, int]:
consumed = 0
try:
context = record.get_context()
except JSONDecodeError:
logger.warning(
f'Failed to backfill record {record.alert.id}: invalid JSON context.'
)
record.status = BackfillRecord.FAILED
record.save()
else:
for data_point in context:
if left <= 0 or self.runtime_exceeded():
break
try:
using_job_id = data_point['job_id']
self.backfill_tool.backfill_job(using_job_id)
left, consumed = left - 1, consumed + 1
except (KeyError, CannotBackfill, Exception) as ex:
logger.debug(
f'Failed to backfill record {record.alert.id}: {ex}')
else:
self.__try_setting_job_type_of(record, using_job_id)
success, outcome = self._note_backfill_outcome(
record, len(context), consumed)
log_level = INFO if success else WARNING
logger.log(log_level,
f'{outcome} (for backfill record {record.alert.id})')
return left, consumed
def check(self, record: BackfillRecord) -> OutcomeStatus:
if record.job_type is None:
raise ValueError(f"No job_type for record {record.alert.id}.")
of_type = record.job_type
with_successful_results = 'success' # state is "completed"
with_unknown_results = 'unknown' # state is "running" or "pending"
total_backfills_in_progress = 0
total_backfills_failed = 0
total_backfills_successful = 0
pushes_in_range = record.get_pushes_in_context_range()
for push in pushes_in_range:
# make sure it has at least one successful job of job type
if push.total_jobs(of_type, with_successful_results) == 0:
# either (at least) one job is in progress or it failed
if push.total_jobs(of_type, with_unknown_results) > 0:
total_backfills_in_progress += 1
else:
total_backfills_failed += 1
else:
total_backfills_successful += 1
record.total_backfills_failed = total_backfills_failed
record.total_backfills_successful = total_backfills_successful
record.total_backfills_in_progress = total_backfills_in_progress
record.save()
if total_backfills_in_progress > 0:
return OutcomeStatus.IN_PROGRESS
elif total_backfills_failed > 0:
return OutcomeStatus.FAILED
elif total_backfills_successful > 0:
return OutcomeStatus.SUCCESSFUL
def _note_backfill_outcome(
record: BackfillRecord, to_backfill: int, actually_backfilled: int
) -> Tuple[bool, str]:
success = False
record.total_backfills_triggered = actually_backfilled
if actually_backfilled == to_backfill:
record.status = BackfillRecord.BACKFILLED
success = True
outcome = 'Backfilled all data points'
else:
record.status = BackfillRecord.FAILED
if actually_backfilled == 0:
outcome = 'Backfill attempts on all data points failed right upon request.'
elif actually_backfilled < to_backfill:
outcome = 'Backfill attempts on some data points failed right upon request.'
else:
raise ValueError(
f'Cannot have backfilled more than available attempts ({actually_backfilled} out of {to_backfill}).'
)
record.set_log_details({'action': 'BACKFILL', 'outcome': outcome})
record.save()
return success, outcome