def execute_jobs( verb, jobs, locks, event_queue, log_path, max_toplevel_jobs=None, continue_on_failure=False, continue_without_deps=False): """Process a number of jobs asynchronously. :param jobs: A list of topologically-sorted Jobs with no circular dependencies. :param event_queue: A python queue for reporting events. :param log_path: The path in which logfiles can be written :param max_toplevel_jobs: Max number of top-level jobs :param continue_on_failure: Keep running jobs even if one fails. :param continue_without_deps: Run jobs even if their dependencies fail. """ # Map of jid -> job job_map = dict([(j.jid, j) for j in jobs]) # Jobs which are not ready to be executed pending_jobs = [] # Jobs which are ready to be executed once workers are available queued_jobs = [] # List of active jobs active_jobs = [] # Set of active job futures active_job_fs = set() # Dict of completd jobs job_id -> succeeded completed_jobs = {} # List of jobs whose deps failed abandoned_jobs = [] # Make sure job server has been initialized if not job_server.initialized(): raise RuntimeError('JobServer has not been initialized.') # Create a thread pool executor for blocking python stages in the asynchronous jobs threadpool = ThreadPoolExecutor(max_workers=job_server.max_jobs()) # Immediately abandon jobs with bad dependencies pending_jobs, new_abandoned_jobs = split(jobs, lambda j: all([d in job_map for d in j.deps])) for abandoned_job in new_abandoned_jobs: abandoned_jobs.append(abandoned_job) event_queue.put(ExecutionEvent( 'ABANDONED_JOB', job_id=abandoned_job.jid, reason='MISSING_DEPS', dep_ids=[d for d in abandoned_job.deps if d not in job_map])) # Initialize list of ready and pending jobs (jobs not ready to be executed) queued_jobs, pending_jobs = split(pending_jobs, lambda j: len(j.deps) == 0) # Process all jobs asynchronously until there are none left while len(active_job_fs) + len(queued_jobs) + len(pending_jobs) > 0: # Activate jobs while the jobserver dispenses tokens while ((len(queued_jobs) > 0) and ((max_toplevel_jobs is None) or (len(active_jobs) < max_toplevel_jobs)) and (job_server.try_acquire() is not None)): # Pop a job off of the job queue job = queued_jobs.pop(0) # Label it (for debugging) job_server.add_label(job.jid) # Notify that the job is being started event_queue.put(ExecutionEvent( 'STARTED_JOB', job_id=job.jid)) # Start the job coroutine active_jobs.append(job) active_job_fs.add(async_job(verb, job, threadpool, locks, event_queue, log_path)) # Report running jobs event_queue.put(ExecutionEvent( 'JOB_STATUS', pending=[j.jid for j in pending_jobs], queued=[j.jid for j in queued_jobs], active=[j.jid for j in active_jobs], abandoned=[j.jid for j in abandoned_jobs], completed=completed_jobs )) # Process jobs as they complete asynchronously done_job_fs, active_job_fs = yield asyncio.From(asyncio.wait( active_job_fs, timeout=0.10, return_when=FIRST_COMPLETED)) for done_job_f in done_job_fs: # Capture a result once the job has finished job_id, succeeded = yield asyncio.From(done_job_f) # Release a jobserver token now that this job has succeeded job_server.release(job_id) active_jobs = [j for j in active_jobs if j.jid != job_id] # Generate event with the results of this job event_queue.put(ExecutionEvent( 'FINISHED_JOB', job_id=job_id, succeeded=succeeded)) # Add the job to the completed list completed_jobs[job_id] = succeeded # Handle failure modes if not succeeded: # Handle different abandoning policies if not continue_on_failure: # Abort all pending jobs if any job fails new_abandoned_jobs = queued_jobs + pending_jobs queued_jobs = [] pending_jobs = [] # Notify that jobs have been abandoned for abandoned_job in new_abandoned_jobs: abandoned_jobs.append(abandoned_job) event_queue.put(ExecutionEvent( 'ABANDONED_JOB', job_id=abandoned_job.jid, reason='PEER_FAILED', peer_job_id=job_id)) elif not continue_without_deps: unhandled_abandoned_job_ids = [job_id] # Abandon jobs which depend on abandoned jobs while len(unhandled_abandoned_job_ids) > 0: # Get the abandoned job abandoned_job_id = unhandled_abandoned_job_ids.pop(0) # Abandon all pending jobs which depend on this job_id unhandled_abandoned_jobs, pending_jobs = split( pending_jobs, lambda j: abandoned_job_id in j.deps) # Handle each new abandoned job for abandoned_job in unhandled_abandoned_jobs: abandoned_jobs.append(abandoned_job) # Notify if any jobs have been abandoned event_queue.put(ExecutionEvent( 'ABANDONED_JOB', job_id=abandoned_job.jid, reason='DEP_FAILED', direct_dep_job_id=abandoned_job_id, dep_job_id=job_id)) # Add additional job ids to check unhandled_abandoned_job_ids.extend( [j.jid for j in unhandled_abandoned_jobs]) # Update the list of ready jobs (based on completed job dependencies) new_queued_jobs, pending_jobs = split( pending_jobs, lambda j: j.all_deps_completed(completed_jobs)) queued_jobs.extend(new_queued_jobs) # Notify of newly queued jobs for queued_job in new_queued_jobs: event_queue.put(ExecutionEvent( 'QUEUED_JOB', job_id=queued_job.jid)) # Report running jobs event_queue.put(ExecutionEvent( 'JOB_STATUS', pending=[j.jid for j in pending_jobs], queued=[j.jid for j in queued_jobs], active=[j.jid for j in active_jobs], abandoned=[j.jid for j in abandoned_jobs], completed=completed_jobs )) raise asyncio.Return(all(completed_jobs.values()))
def execute_jobs(verb, jobs, locks, event_queue, log_path, max_toplevel_jobs=None, continue_on_failure=False, continue_without_deps=False): """Process a number of jobs asynchronously. :param jobs: A list of topologically-sorted Jobs with no circular dependencies. :param event_queue: A python queue for reporting events. :param log_path: The path in which logfiles can be written :param max_toplevel_jobs: Max number of top-level jobs :param continue_on_failure: Keep running jobs even if one fails. :param continue_without_deps: Run jobs even if their dependencies fail. """ # Map of jid -> job job_map = dict([(j.jid, j) for j in jobs]) # Jobs which are not ready to be executed pending_jobs = [] # Jobs which are ready to be executed once workers are available queued_jobs = [] # List of active jobs active_jobs = [] # Set of active job futures active_job_fs = set() # Dict of completd jobs job_id -> succeeded completed_jobs = {} # List of jobs whose deps failed abandoned_jobs = [] # Make sure job server has been initialized if not job_server.initialized(): raise RuntimeError('JobServer has not been initialized.') # Create a thread pool executor for blocking python stages in the asynchronous jobs threadpool = ThreadPoolExecutor(max_workers=job_server.max_jobs()) # Immediately abandon jobs with bad dependencies pending_jobs, new_abandoned_jobs = split( jobs, lambda j: all([d in job_map for d in j.deps])) for abandoned_job in new_abandoned_jobs: abandoned_jobs.append(abandoned_job) event_queue.put( ExecutionEvent( 'ABANDONED_JOB', job_id=abandoned_job.jid, reason='MISSING_DEPS', dep_ids=[d for d in abandoned_job.deps if d not in job_map])) # Initialize list of ready and pending jobs (jobs not ready to be executed) queued_jobs, pending_jobs = split(pending_jobs, lambda j: len(j.deps) == 0) # Process all jobs asynchronously until there are none left while len(active_job_fs) + len(queued_jobs) + len(pending_jobs) > 0: # Activate jobs while the jobserver dispenses tokens while ((len(queued_jobs) > 0) and ((max_toplevel_jobs is None) or (len(active_jobs) < max_toplevel_jobs)) and (job_server.try_acquire() is not None)): # Pop a job off of the job queue job = queued_jobs.pop(0) # Label it (for debugging) job_server.add_label(job.jid) # Notify that the job is being started event_queue.put(ExecutionEvent('STARTED_JOB', job_id=job.jid)) # Start the job coroutine active_jobs.append(job) active_job_fs.add( async_job(verb, job, threadpool, locks, event_queue, log_path)) # Report running jobs event_queue.put( ExecutionEvent('JOB_STATUS', pending=[j.jid for j in pending_jobs], queued=[j.jid for j in queued_jobs], active=[j.jid for j in active_jobs], abandoned=[j.jid for j in abandoned_jobs], completed=completed_jobs)) # Process jobs as they complete asynchronously done_job_fs, active_job_fs = yield from asyncio.wait( active_job_fs, timeout=0.10, return_when=FIRST_COMPLETED) for done_job_f in done_job_fs: # Capture a result once the job has finished job_id, succeeded = yield from done_job_f # Release a jobserver token now that this job has succeeded job_server.release(job_id) active_jobs = [j for j in active_jobs if j.jid != job_id] # Generate event with the results of this job event_queue.put( ExecutionEvent('FINISHED_JOB', job_id=job_id, succeeded=succeeded)) # Add the job to the completed list completed_jobs[job_id] = succeeded # Handle failure modes if not succeeded: # Handle different abandoning policies if not continue_on_failure: # Abort all pending jobs if any job fails new_abandoned_jobs = queued_jobs + pending_jobs queued_jobs = [] pending_jobs = [] # Notify that jobs have been abandoned for abandoned_job in new_abandoned_jobs: abandoned_jobs.append(abandoned_job) event_queue.put( ExecutionEvent('ABANDONED_JOB', job_id=abandoned_job.jid, reason='PEER_FAILED', peer_job_id=job_id)) elif not continue_without_deps: unhandled_abandoned_job_ids = [job_id] # Abandon jobs which depend on abandoned jobs while len(unhandled_abandoned_job_ids) > 0: # Get the abandoned job abandoned_job_id = unhandled_abandoned_job_ids.pop(0) # Abandon all pending jobs which depend on this job_id unhandled_abandoned_jobs, pending_jobs = split( pending_jobs, lambda j: abandoned_job_id in j.deps) # Handle each new abandoned job for abandoned_job in unhandled_abandoned_jobs: abandoned_jobs.append(abandoned_job) # Notify if any jobs have been abandoned event_queue.put( ExecutionEvent( 'ABANDONED_JOB', job_id=abandoned_job.jid, reason='DEP_FAILED', direct_dep_job_id=abandoned_job_id, dep_job_id=job_id)) # Add additional job ids to check unhandled_abandoned_job_ids.extend( [j.jid for j in unhandled_abandoned_jobs]) # Update the list of ready jobs (based on completed job dependencies) new_queued_jobs, pending_jobs = split( pending_jobs, lambda j: j.all_deps_completed(completed_jobs)) queued_jobs.extend(new_queued_jobs) # Notify of newly queued jobs for queued_job in new_queued_jobs: event_queue.put( ExecutionEvent('QUEUED_JOB', job_id=queued_job.jid)) # Report running jobs event_queue.put( ExecutionEvent('JOB_STATUS', pending=[j.jid for j in pending_jobs], queued=[j.jid for j in queued_jobs], active=[j.jid for j in active_jobs], abandoned=[j.jid for j in abandoned_jobs], completed=completed_jobs)) return all(completed_jobs.values())
def run(self): queued_jobs = [] active_jobs = [] completed_jobs = {} failed_jobs = [] warned_jobs = [] cumulative_times = dict() start_times = dict() active_stages = dict() start_time = self.pre_start_time or time.time() last_update_time = time.time() # If the status rate is too low, just disable it if self.active_status_rate < 1E-3: self.show_active_status = False else: update_duration = 1.0 / self.active_status_rate # Disable the wide log padding if the status is disabled if not self.show_active_status: disable_wide_log() while True: # Check if we should stop if not self.keep_running: wide_log( clr('[{}] An internal error occurred!').format(self.label)) return # Write a continuously-updated status line if self.show_active_status: # Try to get an event from the queue (non-blocking) try: event = self.event_queue.get(False) except Empty: # Determine if the status should be shown based on the desired # status rate elapsed_time = time.time() - last_update_time show_status_now = elapsed_time > update_duration if show_status_now: # Print live status (overwrites last line) status_line = clr( '[{} {} s] [{}/{} complete] [{}/{} jobs] [{} queued]' ).format( self.label, format_time_delta_short(time.time() - start_time), len(completed_jobs), len(self.jobs), job_server.running_jobs(), job_server.max_jobs(), len(queued_jobs) + len(active_jobs) - len(active_stages)) # Show failed jobs if len(failed_jobs) > 0: status_line += clr( ' [@!@{rf}{}@| @{rf}failed@|]').format( len(failed_jobs)) # Check load / mem if not job_server.load_ok(): status_line += clr(' [@!@{rf}High Load@|]') if not job_server.mem_ok(): status_line += clr(' [@!@{rf}Low Memory@|]') # Add active jobs if len(active_jobs) == 0: status_line += clr( ' @/@!@{kf}Waiting for jobs...@|') else: active_labels = [] for j, (s, t, p) in active_stages.items(): d = format_time_delta_short( cumulative_times[j] + time.time() - t) if p == '': active_labels.append( clr('[{}:{} - {}]').format(j, s, d)) else: active_labels.append( clr('[{}:{} ({}%) - {}]').format( j, s, p, d)) status_line += ' ' + ' '.join(active_labels) # Print the status line # wide_log(status_line) wide_log(status_line, rhs='', end='\r') sys.stdout.flush() # Store this update time last_update_time = time.time() else: time.sleep( max(0.0, min(update_duration - elapsed_time, 0.01))) # Only continue when no event was received continue else: # Try to get an event from the queue (blocking) try: event = self.event_queue.get(True) except Empty: break # A `None` event is a signal to terminate if event is None: break # Handle the received events eid = event.event_id if 'JOB_STATUS' == eid: queued_jobs = event.data['queued'] active_jobs = event.data['active'] completed_jobs = event.data['completed'] # Check if all jobs have finished in some way if all([ len(event.data[t]) == 0 for t in ['pending', 'queued', 'active'] ]): break elif 'STARTED_JOB' == eid: cumulative_times[event.data['job_id']] = 0.0 wide_log( clr('Starting >>> {:<{}}').format(event.data['job_id'], self.max_jid_length)) elif 'FINISHED_JOB' == eid: duration = format_time_delta( cumulative_times[event.data['job_id']]) if event.data['succeeded']: wide_log( clr('Finished <<< {:<{}} [ {} ]').format( event.data['job_id'], self.max_jid_length, duration)) else: failed_jobs.append(event.data['job_id']) wide_log( clr('Failed <<< {:<{}} [ {} ]').format( event.data['job_id'], self.max_jid_length, duration)) elif 'ABANDONED_JOB' == eid: # Create a human-readable reason string if 'DEP_FAILED' == event.data['reason']: direct = event.data['dep_job_id'] == event.data[ 'direct_dep_job_id'] if direct: reason = clr('Depends on failed job {}').format( event.data['dep_job_id']) else: reason = clr('Depends on failed job {} via {}').format( event.data['dep_job_id'], event.data['direct_dep_job_id']) elif 'PEER_FAILED' == event.data['reason']: reason = clr('Unrelated job failed') elif 'MISSING_DEPS' == event.data['reason']: reason = clr('Depends on unknown jobs: {}').format( ', '.join([ clr('@!{}@|').format(jid) for jid in event.data['dep_ids'] ])) wide_log( clr('Abandoned <<< {:<{}} [ {} ]').format( event.data['job_id'], self.max_jid_length, reason)) elif 'STARTED_STAGE' == eid: active_stages[event.data['job_id']] = [ event.data['stage_label'], event.time, '' ] start_times[event.data['job_id']] = event.time if self.show_stage_events: wide_log( clr('Starting >> {}:{}').format( event.data['job_id'], event.data['stage_label'])) elif 'STAGE_PROGRESS' == eid: active_stages[event.data['job_id']][2] = event.data['percent'] elif 'SUBPROCESS' == eid: if self.show_stage_events: wide_log( clr('Subprocess > {}:{} `{}`').format( event.data['job_id'], event.data['stage_label'], event.data['stage_repro'])) elif 'FINISHED_STAGE' == eid: # Get the stage duration duration = event.time - start_times[event.data['job_id']] cumulative_times[event.data['job_id']] += duration # This is no longer the active stage for this job del active_stages[event.data['job_id']] header_border = None header_border_file = sys.stdout header_title = None header_title_file = sys.stdout lines = [] footer_title = None footer_title_file = sys.stdout footer_border = None footer_border_file = sys.stdout # Generate headers / borders for output if event.data['succeeded']: footer_title = clr('Finished << {}:{}').format( event.data['job_id'], event.data['stage_label']) if len(event.data['stderr']) > 0: # Mark that this job warned about something if event.data['job_id'] not in warned_jobs: warned_jobs.append(event.data['job_id']) # Output contains warnings header_border = clr('@!@{yf}' + '_' * (terminal_width() - 1) + '@|') header_border_file = sys.stderr header_title = clr('Warnings << {}:{} {}').format( event.data['job_id'], event.data['stage_label'], event.data['logfile_filename']) header_title_file = sys.stderr footer_border = clr('@{yf}' + '.' * (terminal_width() - 1) + '@|') footer_border_file = sys.stderr else: # Normal output, no warnings header_title = clr('Output << {}:{} {}').format( event.data['job_id'], event.data['stage_label'], event.data['logfile_filename']) # Don't print footer title if not self.show_stage_events: footer_title = None else: # Output contains errors header_border = clr('@!@{rf}' + '_' * (terminal_width() - 1) + '@|') header_border_file = sys.stderr header_title = clr('Errors << {}:{} {}').format( event.data['job_id'], event.data['stage_label'], event.data['logfile_filename']) header_title_file = sys.stderr footer_border = clr('@{rf}' + '.' * (terminal_width() - 1) + '@|') footer_border_file = sys.stderr footer_title = clr( 'Failed << {}:{:<{}} [ Exited with code {} ]').format( event.data['job_id'], event.data['stage_label'], max( 0, self.max_jid_length - len(event.data['job_id'])), event.data['retcode']) footer_title_file = sys.stderr lines_target = sys.stdout if self.show_buffered_stdout: if len(event.data['interleaved']) > 0: lines = [ line for line in event.data['interleaved'].splitlines(True) if (self.show_compact_io is False or len(line.strip()) > 0) ] else: header_border = None header_title = None footer_border = None elif self.show_buffered_stderr: if len(event.data['stderr']) > 0: lines = [ line for line in event.data['stderr'].splitlines(True) if (self.show_compact_io is False or len(line.strip()) > 0) ] lines_target = sys.stderr else: header_border = None header_title = None footer_border = None if len(lines) > 0: if self.show_repro_cmd: if event.data['repro'] is not None: lines.append( clr('@!@{kf}{}@|\n').format( event.data['repro'])) # Print the output if header_border: wide_log(header_border, file=header_border_file) if header_title: wide_log(header_title, file=header_title_file) if len(lines) > 0: wide_log(''.join(lines), end='\r', file=lines_target) if footer_border: wide_log(footer_border, file=footer_border_file) if footer_title: wide_log(footer_title, file=footer_title_file) elif 'STDERR' == eid: if self.show_live_stderr and len(event.data['data']) > 0: wide_log(self.format_interleaved_lines(event.data), end='\r', file=sys.stderr) elif 'STDOUT' == eid: if self.show_live_stdout and len(event.data['data']) > 0: wide_log(self.format_interleaved_lines(event.data), end='\r') elif 'MESSAGE' == eid: wide_log(event.data['msg']) # Print the full summary if self.show_full_summary: self.print_exec_summary(completed_jobs, warned_jobs, failed_jobs) # Print a compact summary if self.show_summary or self.show_full_summary: self.print_compact_summary(completed_jobs, warned_jobs, failed_jobs) # Print final runtime wide_log( clr('[{}] Runtime: {} total.').format( self.label, format_time_delta(time.time() - start_time)))
def run(self): queued_jobs = [] active_jobs = [] completed_jobs = {} failed_jobs = [] warned_jobs = [] cumulative_times = dict() start_times = dict() active_stages = dict() start_time = self.pre_start_time or time.time() last_update_time = time.time() # If the status rate is too low, just disable it if self.active_status_rate < 1E-3: self.show_active_status = False else: update_duration = 1.0 / self.active_status_rate # Disable the wide log padding if the status is disabled if not self.show_active_status: disable_wide_log() while True: # Check if we should stop if not self.keep_running: wide_log(clr('[{}] An internal error occurred!').format(self.label)) return # Write a continuously-updated status line if self.show_active_status: # Try to get an event from the queue (non-blocking) try: event = self.event_queue.get(False) except Empty: # Determine if the status should be shown based on the desired # status rate elapsed_time = time.time() - last_update_time show_status_now = elapsed_time > update_duration if show_status_now: # Print live status (overwrites last line) status_line = clr('[{} {} s] [{}/{} complete] [{}/{} jobs] [{} queued]').format( self.label, format_time_delta_short(time.time() - start_time), len(completed_jobs), len(self.jobs), job_server.running_jobs(), job_server.max_jobs(), len(queued_jobs) + len(active_jobs) - len(active_stages) ) # Show failed jobs if len(failed_jobs) > 0: status_line += clr(' [@!@{rf}{}@| @{rf}failed@|]').format(len(failed_jobs)) # Check load / mem if not job_server.load_ok(): status_line += clr(' [@!@{rf}High Load@|]') if not job_server.mem_ok(): status_line += clr(' [@!@{rf}Low Memory@|]') # Add active jobs if len(active_jobs) == 0: status_line += clr(' @/@!@{kf}Waiting for jobs...@|') else: active_labels = [] for j, (s, t, p) in active_stages.items(): d = format_time_delta_short(cumulative_times[j] + time.time() - t) if p == '': active_labels.append(clr('[{}:{} - {}]').format(j, s, d)) else: active_labels.append(clr('[{}:{} ({}%) - {}]').format(j, s, p, d)) status_line += ' ' + ' '.join(active_labels) # Print the status line # wide_log(status_line) wide_log(status_line, rhs='', end='\r') sys.stdout.flush() # Store this update time last_update_time = time.time() else: time.sleep(max(0.0, min(update_duration - elapsed_time, 0.01))) # Only continue when no event was received continue else: # Try to get an event from the queue (blocking) try: event = self.event_queue.get(True) except Empty: break # A `None` event is a signal to terminate if event is None: break # Handle the received events eid = event.event_id if 'JOB_STATUS' == eid: queued_jobs = event.data['queued'] active_jobs = event.data['active'] completed_jobs = event.data['completed'] # Check if all jobs have finished in some way if all([len(event.data[t]) == 0 for t in ['pending', 'queued', 'active']]): break elif 'STARTED_JOB' == eid: cumulative_times[event.data['job_id']] = 0.0 wide_log(clr('Starting >>> {:<{}}').format( event.data['job_id'], self.max_jid_length)) elif 'FINISHED_JOB' == eid: duration = format_time_delta(cumulative_times[event.data['job_id']]) if event.data['succeeded']: wide_log(clr('Finished <<< {:<{}} [ {} ]').format( event.data['job_id'], self.max_jid_length, duration)) else: failed_jobs.append(event.data['job_id']) wide_log(clr('Failed <<< {:<{}} [ {} ]').format( event.data['job_id'], self.max_jid_length, duration)) elif 'ABANDONED_JOB' == eid: # Create a human-readable reason string if 'DEP_FAILED' == event.data['reason']: direct = event.data['dep_job_id'] == event.data['direct_dep_job_id'] if direct: reason = clr('Depends on failed job {}').format(event.data['dep_job_id']) else: reason = clr('Depends on failed job {} via {}').format( event.data['dep_job_id'], event.data['direct_dep_job_id']) elif 'PEER_FAILED' == event.data['reason']: reason = clr('Unrelated job failed') elif 'MISSING_DEPS' == event.data['reason']: reason = clr('Depends on unknown jobs: {}').format( ', '.join([clr('@!{}@|').format(jid) for jid in event.data['dep_ids']])) wide_log(clr('Abandoned <<< {:<{}} [ {} ]').format( event.data['job_id'], self.max_jid_length, reason)) elif 'STARTED_STAGE' == eid: active_stages[event.data['job_id']] = [event.data['stage_label'], event.time, ''] start_times[event.data['job_id']] = event.time if self.show_stage_events: wide_log(clr('Starting >> {}:{}').format( event.data['job_id'], event.data['stage_label'])) elif 'STAGE_PROGRESS' == eid: active_stages[event.data['job_id']][2] = event.data['percent'] elif 'SUBPROCESS' == eid: if self.show_stage_events: wide_log(clr('Subprocess > {}:{} `{}`').format( event.data['job_id'], event.data['stage_label'], event.data['stage_repro'])) elif 'FINISHED_STAGE' == eid: # Get the stage duration duration = event.time - start_times[event.data['job_id']] cumulative_times[event.data['job_id']] += duration # This is no longer the active stage for this job del active_stages[event.data['job_id']] header_border = None header_border_file = sys.stdout header_title = None header_title_file = sys.stdout lines = [] footer_title = None footer_title_file = sys.stdout footer_border = None footer_border_file = sys.stdout # Generate headers / borders for output if event.data['succeeded']: footer_title = clr( 'Finished << {}:{}').format( event.data['job_id'], event.data['stage_label']) if len(event.data['stderr']) > 0: # Mark that this job warned about something if event.data['job_id'] not in warned_jobs: warned_jobs.append(event.data['job_id']) # Output contains warnings header_border = clr('@!@{yf}' + '_' * (terminal_width() - 1) + '@|') header_border_file = sys.stderr header_title = clr( 'Warnings << {}:{} {}').format( event.data['job_id'], event.data['stage_label'], event.data['logfile_filename']) header_title_file = sys.stderr footer_border = clr('@{yf}' + '.' * (terminal_width() - 1) + '@|') footer_border_file = sys.stderr else: # Normal output, no warnings header_title = clr( 'Output << {}:{} {}').format( event.data['job_id'], event.data['stage_label'], event.data['logfile_filename']) # Don't print footer title if not self.show_stage_events: footer_title = None else: # Output contains errors header_border = clr('@!@{rf}' + '_' * (terminal_width() - 1) + '@|') header_border_file = sys.stderr header_title = clr( 'Errors << {}:{} {}').format( event.data['job_id'], event.data['stage_label'], event.data['logfile_filename']) header_title_file = sys.stderr footer_border = clr('@{rf}' + '.' * (terminal_width() - 1) + '@|') footer_border_file = sys.stderr footer_title = clr( 'Failed << {}:{:<{}} [ Exited with code {} ]').format( event.data['job_id'], event.data['stage_label'], max(0, self.max_jid_length - len(event.data['job_id'])), event.data['retcode']) footer_title_file = sys.stderr lines_target = sys.stdout if self.show_buffered_stdout: if len(event.data['interleaved']) > 0: lines = [ l for l in event.data['interleaved'].splitlines(True) if (self.show_compact_io is False or len(l.strip()) > 0) ] else: header_border = None header_title = None footer_border = None elif self.show_buffered_stderr: if len(event.data['stderr']) > 0: lines = [ l for l in event.data['stderr'].splitlines(True) if (self.show_compact_io is False or len(l.strip()) > 0) ] lines_target = sys.stderr else: header_border = None header_title = None footer_border = None if len(lines) > 0: if self.show_repro_cmd: if event.data['repro'] is not None: lines.append(clr('@!@{kf}{}@|\n').format(event.data['repro'])) # Print the output if header_border: wide_log(header_border, file=header_border_file) if header_title: wide_log(header_title, file=header_title_file) if len(lines) > 0: wide_log(''.join(lines), end='\r', file=lines_target) if footer_border: wide_log(footer_border, file=footer_border_file) if footer_title: wide_log(footer_title, file=footer_title_file) elif 'STDERR' == eid: if self.show_live_stderr and len(event.data['data']) > 0: wide_log(self.format_interleaved_lines(event.data), end='\r', file=sys.stderr) elif 'STDOUT' == eid: if self.show_live_stdout and len(event.data['data']) > 0: wide_log(self.format_interleaved_lines(event.data), end='\r') elif 'MESSAGE' == eid: wide_log(event.data['msg']) # Print the full summary if self.show_full_summary: self.print_exec_summary(completed_jobs, warned_jobs, failed_jobs) # Print a compact summary if self.show_summary or self.show_full_summary: self.print_compact_summary(completed_jobs, warned_jobs, failed_jobs) # Print final runtime wide_log(clr('[{}] Runtime: {} total.').format( self.label, format_time_delta(time.time() - start_time)))