def _note_backfill_outcome(
record: BackfillRecord, to_backfill: int, actually_backfilled: int
) -> Tuple[bool, str]:
success = False
record.total_backfills_triggered = actually_backfilled
if actually_backfilled == to_backfill:
record.status = BackfillRecord.BACKFILLED
success = True
outcome = 'Backfilled all data points'
else:
record.status = BackfillRecord.FAILED
if actually_backfilled == 0:
outcome = 'Backfill attempts on all data points failed right upon request.'
elif actually_backfilled < to_backfill:
outcome = 'Backfill attempts on some data points failed right upon request.'
else:
raise ValueError(
f'Cannot have backfilled more than available attempts ({actually_backfilled} out of {to_backfill}).'
)
record.set_log_details({'action': 'BACKFILL', 'outcome': outcome})
record.save()
return success, outcome
def _backfill_record(self, record: BackfillRecord,
left: int) -> Tuple[int, int]:
consumed = 0
try:
context = record.get_context()
except JSONDecodeError:
logger.warning(
f'Failed to backfill record {record.alert.id}: invalid JSON context.'
)
record.status = BackfillRecord.FAILED
record.save()
else:
for data_point in context:
if left <= 0 or self.runtime_exceeded():
break
try:
using_job_id = data_point['job_id']
self.backfill_tool.backfill_job(using_job_id)
left, consumed = left - 1, consumed + 1
except (KeyError, CannotBackfill, Exception) as ex:
logger.debug(
f'Failed to backfill record {record.alert.id}: {ex}')
else:
self.__try_setting_job_type_of(record, using_job_id)
success, outcome = self._note_backfill_outcome(
record, len(context), consumed)
log_level = INFO if success else WARNING
logger.log(log_level,
f'{outcome} (for backfill record {record.alert.id})')
return left, consumed