def _backfill_record(self, record: BackfillRecord, left: int) -> Tuple[int, int]: consumed = 0 try: context = record.get_context() except JSONDecodeError: logger.warning( f'Failed to backfill record {record.alert.id}: invalid JSON context.' ) record.status = BackfillRecord.FAILED record.save() else: for data_point in context: if left <= 0 or self.runtime_exceeded(): break try: using_job_id = data_point['job_id'] self.backfill_tool.backfill_job(using_job_id) left, consumed = left - 1, consumed + 1 except (KeyError, CannotBackfill, Exception) as ex: logger.debug( f'Failed to backfill record {record.alert.id}: {ex}') else: self.__try_setting_job_type_of(record, using_job_id) success, outcome = self._note_backfill_outcome( record, len(context), consumed) log_level = INFO if success else WARNING logger.log(log_level, f'{outcome} (for backfill record {record.alert.id})') return left, consumed
def __build_push_range_link(self, record: BackfillRecord) -> str: repo = record.repository.name from_change, to_change = record.get_context_border_info("push__revision") query_params = f"repo={repo}&fromchange={from_change}&tochange={to_change}" query_params = self.__try_embed_search_str(query_params, using_record=record) return f"{settings.SITE_URL}/jobs?{query_params}"
def _note_backfill_outcome( record: BackfillRecord, to_backfill: int, actually_backfilled: int ) -> Tuple[bool, str]: success = False record.total_backfills_triggered = actually_backfilled if actually_backfilled == to_backfill: record.status = BackfillRecord.BACKFILLED success = True outcome = 'Backfilled all data points' else: record.status = BackfillRecord.FAILED if actually_backfilled == 0: outcome = 'Backfill attempts on all data points failed right upon request.' elif actually_backfilled < to_backfill: outcome = 'Backfill attempts on some data points failed right upon request.' else: raise ValueError( f'Cannot have backfilled more than available attempts ({actually_backfilled} out of {to_backfill}).' ) record.set_log_details({'action': 'BACKFILL', 'outcome': outcome}) record.save() return success, outcome
def __build_push_range(self, record: BackfillRecord) -> str: """ Provides link to Treeherder' s Job view """ try: from_change, to_change = record.get_context_border_info( "push__revision") return f"{settings.SITE_URL}/jobs?repo={record.repository}&fromchange={from_change}&tochange={to_change}" except Exception: return 'N/A'
def check(self, record: BackfillRecord) -> OutcomeStatus: if record.job_type is None: raise ValueError(f"No job_type for record {record.alert.id}.") of_type = record.job_type with_successful_results = 'success' # state is "completed" with_unknown_results = 'unknown' # state is "running" or "pending" total_backfills_in_progress = 0 total_backfills_failed = 0 total_backfills_successful = 0 pushes_in_range = record.get_pushes_in_context_range() for push in pushes_in_range: # make sure it has at least one successful job of job type if push.total_jobs(of_type, with_successful_results) == 0: # either (at least) one job is in progress or it failed if push.total_jobs(of_type, with_unknown_results) > 0: total_backfills_in_progress += 1 else: total_backfills_failed += 1 else: total_backfills_successful += 1 record.total_backfills_failed = total_backfills_failed record.total_backfills_successful = total_backfills_successful record.total_backfills_in_progress = total_backfills_in_progress record.save() if total_backfills_in_progress > 0: return OutcomeStatus.IN_PROGRESS elif total_backfills_failed > 0: return OutcomeStatus.FAILED elif total_backfills_successful > 0: return OutcomeStatus.SUCCESSFUL
def _build_table_row(self, record: BackfillRecord) -> str: alert_summary = record.alert.summary alert = record.alert job_type = record.job_type total_backfills = record.total_backfills_triggered push_range = self.__build_push_range(record.get_context()) # some fields require adjustments summary_id = alert_summary.id alert_id = alert.id job_symbol = str(job_type) return f"| {summary_id} | {alert_id} | {job_symbol} | {total_backfills} | {push_range} |"
def __try_embed_search_str( self, query_params: str, using_record: BackfillRecord, ) -> str: search_str = using_record.get_job_search_str() if search_str: search_str = urllib.parse.quote_plus(search_str) query_params = f"{query_params}&searchStr={search_str}" else: logger.warning( f"Failed to enrich push range URL using record with ID {using_record.alert_id}." ) return query_params
def check(self, record: BackfillRecord) -> OutcomeStatus: if record.job_type is None: raise ValueError(f"No job_type for record {record.alert.id}.") of_type = record.job_type with_successful_results = 'success' with_unknown_results = 'unknown' pushes_in_range = record.get_pushes_in_context_range() for push in pushes_in_range: # make sure it has at least one successful job of job type if push.total_jobs(of_type, with_successful_results) == 0: # either (at least) one job is in progress or it failed if push.total_jobs(of_type, with_unknown_results) > 0: return OutcomeStatus.IN_PROGRESS else: return OutcomeStatus.FAILED return OutcomeStatus.SUCCESSFUL
def check(self, record: BackfillRecord) -> OutcomeStatus: # TODO: get job_type from record when soft launch lands ---> job_type = record.job_type job_type = get_job_type(record) from_time, to_time = self._get_push_timestamp_range( record.get_context()) repository_id = record.alert.summary.repository.id pushes_in_range = self._get_pushes_in_range(from_time, to_time, repository_id) for push in pushes_in_range: # make sure it has at least one successful job of job type if push.jobs.filter(job_type=job_type, result="success").count() == 0: # if there is no successful job of job_type in push, we check for jobs of job_type that are still running if push.jobs.filter(job_type=job_type, result="unknown").count() == 0: return OutcomeStatus.FAILED else: return OutcomeStatus.IN_PROGRESS return OutcomeStatus.SUCCESSFUL