def _backfill_record(self, record: BackfillRecord,
left: int) -> Tuple[int, int]:
consumed = 0
try:
context = record.get_context()
except JSONDecodeError:
logger.warning(
f'Failed to backfill record {record.alert.id}: invalid JSON context.'
)
record.status = BackfillRecord.FAILED
record.save()
else:
for data_point in context:
if left <= 0 or self.runtime_exceeded():
break
try:
using_job_id = data_point['job_id']
self.backfill_tool.backfill_job(using_job_id)
left, consumed = left - 1, consumed + 1
except (KeyError, CannotBackfill, Exception) as ex:
logger.debug(
f'Failed to backfill record {record.alert.id}: {ex}')
else:
self.__try_setting_job_type_of(record, using_job_id)
success, outcome = self._note_backfill_outcome(
record, len(context), consumed)
log_level = INFO if success else WARNING
logger.log(log_level,
f'{outcome} (for backfill record {record.alert.id})')
return left, consumed
def __build_push_range_link(self, record: BackfillRecord) -> str:
repo = record.repository.name
from_change, to_change = record.get_context_border_info("push__revision")
query_params = f"repo={repo}&fromchange={from_change}&tochange={to_change}"
query_params = self.__try_embed_search_str(query_params, using_record=record)
return f"{settings.SITE_URL}/jobs?{query_params}"
def _note_backfill_outcome(
record: BackfillRecord, to_backfill: int, actually_backfilled: int
) -> Tuple[bool, str]:
success = False
record.total_backfills_triggered = actually_backfilled
if actually_backfilled == to_backfill:
record.status = BackfillRecord.BACKFILLED
success = True
outcome = 'Backfilled all data points'
else:
record.status = BackfillRecord.FAILED
if actually_backfilled == 0:
outcome = 'Backfill attempts on all data points failed right upon request.'
elif actually_backfilled < to_backfill:
outcome = 'Backfill attempts on some data points failed right upon request.'
else:
raise ValueError(
f'Cannot have backfilled more than available attempts ({actually_backfilled} out of {to_backfill}).'
)
record.set_log_details({'action': 'BACKFILL', 'outcome': outcome})
record.save()
return success, outcome
def __build_push_range(self, record: BackfillRecord) -> str:
"""
Provides link to Treeherder' s Job view
"""
try:
from_change, to_change = record.get_context_border_info(
"push__revision")
return f"{settings.SITE_URL}/jobs?repo={record.repository}&fromchange={from_change}&tochange={to_change}"
except Exception:
return 'N/A'
def check(self, record: BackfillRecord) -> OutcomeStatus:
if record.job_type is None:
raise ValueError(f"No job_type for record {record.alert.id}.")
of_type = record.job_type
with_successful_results = 'success' # state is "completed"
with_unknown_results = 'unknown' # state is "running" or "pending"
total_backfills_in_progress = 0
total_backfills_failed = 0
total_backfills_successful = 0
pushes_in_range = record.get_pushes_in_context_range()
for push in pushes_in_range:
# make sure it has at least one successful job of job type
if push.total_jobs(of_type, with_successful_results) == 0:
# either (at least) one job is in progress or it failed
if push.total_jobs(of_type, with_unknown_results) > 0:
total_backfills_in_progress += 1
else:
total_backfills_failed += 1
else:
total_backfills_successful += 1
record.total_backfills_failed = total_backfills_failed
record.total_backfills_successful = total_backfills_successful
record.total_backfills_in_progress = total_backfills_in_progress
record.save()
if total_backfills_in_progress > 0:
return OutcomeStatus.IN_PROGRESS
elif total_backfills_failed > 0:
return OutcomeStatus.FAILED
elif total_backfills_successful > 0:
return OutcomeStatus.SUCCESSFUL
def _build_table_row(self, record: BackfillRecord) -> str:
alert_summary = record.alert.summary
alert = record.alert
job_type = record.job_type
total_backfills = record.total_backfills_triggered
push_range = self.__build_push_range(record.get_context())
# some fields require adjustments
summary_id = alert_summary.id
alert_id = alert.id
job_symbol = str(job_type)
return f"| {summary_id} | {alert_id} | {job_symbol} | {total_backfills} | {push_range} |"
def __try_embed_search_str(
self,
query_params: str,
using_record: BackfillRecord,
) -> str:
search_str = using_record.get_job_search_str()
if search_str:
search_str = urllib.parse.quote_plus(search_str)
query_params = f"{query_params}&searchStr={search_str}"
else:
logger.warning(
f"Failed to enrich push range URL using record with ID {using_record.alert_id}."
)
return query_params
def check(self, record: BackfillRecord) -> OutcomeStatus:
if record.job_type is None:
raise ValueError(f"No job_type for record {record.alert.id}.")
of_type = record.job_type
with_successful_results = 'success'
with_unknown_results = 'unknown'
pushes_in_range = record.get_pushes_in_context_range()
for push in pushes_in_range:
# make sure it has at least one successful job of job type
if push.total_jobs(of_type, with_successful_results) == 0:
# either (at least) one job is in progress or it failed
if push.total_jobs(of_type, with_unknown_results) > 0:
return OutcomeStatus.IN_PROGRESS
else:
return OutcomeStatus.FAILED
return OutcomeStatus.SUCCESSFUL
def check(self, record: BackfillRecord) -> OutcomeStatus:
# TODO: get job_type from record when soft launch lands ---> job_type = record.job_type
job_type = get_job_type(record)
from_time, to_time = self._get_push_timestamp_range(
record.get_context())
repository_id = record.alert.summary.repository.id
pushes_in_range = self._get_pushes_in_range(from_time, to_time,
repository_id)
for push in pushes_in_range:
# make sure it has at least one successful job of job type
if push.jobs.filter(job_type=job_type,
result="success").count() == 0:
# if there is no successful job of job_type in push, we check for jobs of job_type that are still running
if push.jobs.filter(job_type=job_type,
result="unknown").count() == 0:
return OutcomeStatus.FAILED
else:
return OutcomeStatus.IN_PROGRESS
return OutcomeStatus.SUCCESSFUL