def Run(cls):
"""Runs the command handler.
Return:
int: Exit status. 0 means no error.
"""
tasks = cls._ComputeTasks(Flags.ARGS.task, Flags.ARGS.ignore_tasks)
# TermColor.Info('Tasks: %s' % tasks)
# TermColor.Info('')
# for key in tasks.iterkeys():
# TermColor.Info('%s: %s' % (key, tasks[key]))
if not tasks:
TermColor.Warning('Could not find any tasks.')
return 101
(successful_tasks, failed_tasks) = cls.WorkHorse(tasks)
if successful_tasks:
TermColor.Info('')
TermColor.Success('No. of tasks: %d' % len(successful_tasks))
TermColor.VInfo(
1, 'Successful tasks: %s' %
json.dumps(PipelineUtils.TasksDisplayNames(successful_tasks),
indent=2))
if failed_tasks:
TermColor.Info('')
TermColor.Failure('No. of tasks: %d' % len(failed_tasks))
TermColor.Failure('tasks: %s' % json.dumps(
PipelineUtils.TasksDisplayNames(failed_tasks), indent=2))
return 102
return 0
def WorkHorse(cls, tasks):
"""Runs the workhorse for the command.
Args:
tasks: OrderedDict {int, set(string)}: Dict from priority to set of tasks to execute at the
priority. Note: the dict is ordered by priority.
Return:
(list, list): Returns a tuple of list in the form
(successful_tasks, failed_tasks) specifying tasks that succeeded and
ones that failed.
"""
all_tasks = []
dirs_to_import = {}
dir_to_task_map = {}
for set_tasks in tasks.values():
for task in set_tasks:
all_tasks += [task]
out_dir = PipelineUtils.GetOutDirForTask(task)
publish_dir = PipelineUtils.GetPublishCurrentDirForTask(task)
if not out_dir or not publish_dir: continue
dirs_to_import[publish_dir] = out_dir
dir_to_task_map[publish_dir] = (dir_to_task_map.get(publish_dir, []) + [publish_dir])
# Check if there are any directories to publish.
if not dirs_to_import:
TermColor.Error('Did not find any dirs to import. Do not forget to specify publish root '
'using --publish_root')
return ([], all_tasks)
# Create all the target dirs to import to.
for dir in dirs_to_import.values():
FileUtils.MakeDirs(dir)
# Run all the copy tasks.
successful_dirs = []; failed_dirs = []
args = zip(itertools.repeat(cls), itertools.repeat('_RunSingeTask'),
list(dirs_to_import), list(dirs_to_import.values()))
dir_res = ExecUtils.ExecuteParallel(args, Flags.ARGS.pool_size)
if not dir_res:
TermColor.Error('Could not process: %s' % all_tasks)
return ([], all_tasks)
for (res, dir) in dir_res:
if res == Importer.EXITCODE['SUCCESS']:
successful_dirs += [dir]
elif res == Importer.EXITCODE['FAILURE']:
failed_dirs += [dir]
else:
TermColor.Fatal('Invalid return %d code for %s' % (res, dir))
# Get the reverse mapping from dirs to tasks.
successful_tasks = []; failed_tasks = []
for i in successful_dirs:
successful_tasks += dir_to_task_map.get(i, [])
for i in failed_dirs:
failed_tasks += dir_to_task_map.get(i, [])
return (successful_tasks, failed_tasks)
def WorkHorse(cls, tasks):
"""Runs the workhorse for the command.
Args:
tasks: OrderedDict {int, set(string)}: Dict from priority to set of tasks to execute at the
priority. Note: the dict is ordered by priority.
Return:
(list, list): Returns a tuple of list in the form
(successful_tasks, failed_tasks) specifying tasks that succeeded and
ones that failed.
"""
success_tasks = []
paths_to_clean = set()
for set_tasks in tasks.values():
for task in set_tasks:
paths = PipelineConfig.Instance().GetAllSubDirsForPath(
PipelineUtils.GetTaskOutputRelativeDir(task))
paths_to_clean |= set(paths.values())
success_tasks += [task]
TermColor.VInfo(1, 'Cleaning %d' % len(paths_to_clean))
for i in paths_to_clean:
TermColor.VInfo(3, 'Cleaning %s' % i)
shutil.rmtree(i, True)
return (success_tasks, [])
def __GetTimeOutForTask(cls, task):
"""Returns the timeout for the task.
Args:
task: string: The task for which the timeout should be prepared.
Returns:
int: The timeout in seconds.
"""
timeout = FileUtils.FileContents(task + '.timeout')
if not timeout:
timeout = FileUtils.FileContents(os.path.join(PipelineUtils.TaskDirName(task), 'timeout'))
if not timeout: return Flags.ARGS.timeout
timeout = re.sub('\s*', '', timeout)
timeout_parts = re.split('(\d+)', timeout)
if len(timeout_parts) < 3:
TermColor.Warning('Ignoring invalid timeout [%s] for task: %s' % (timeout, task))
return Flags.ARGS.timeout
timeout = float(timeout_parts[1])
annotation = timeout_parts[2]
if not annotation: return timeout
elif annotation == 'd': timeout *= 86400
elif annotation == 'h': timeout *= 3600
elif annotation == 'm': timeout *= 60
elif annotation == 'ms': timeout *= 0.001
elif annotation == 'us': timeout *= 0.000001
return timeout
def __GetEnvVarsForTask(cls, task):
"""Returns the env vars for the task.
Args:
task: string: The task for which the envvar should be prepared.
Returns:
dict {string, string}: The dictionary of IDS to values.
"""
rel_path = PipelineUtils.GetTaskOutputRelativeDir(task)
vars = {}
for k, v in PipelineConfig.Instance().GetAllSubDirsForPath(rel_path).items():
vars[k] = v
prev_dir = FileUtils.GetPreviousDatedDir(v)
if not prev_dir: prev_dir = v
vars[k + '_PREV'] = prev_dir
vars.update(PipelineConfig.Instance().GetAllENVVars())
# Check if the task is critical or not.
task_options = cls.__GetTaskOptions(task)
if task_options[Runner.TASK_OPTIONS['ABORT_FAIL']]:
vars['PIPELINE_TASK_ABORT_FAIL'] = '1'
if task_options[Runner.TASK_OPTIONS['ALLOW_FAIL']]:
vars['PIPELINE_TASK_ALLOW_FAIL'] = '1'
return vars
def _CreateDirsForTasks(cls, tasks):
"""Creates the relevant dirs for tasks.
Args:
tasks: OrderedDict {int, set(string)}: Dict from priority to set of tasks to execute at the
priority. Note: the dict is ordered by priority.
"""
for set_tasks in tasks.values():
for task in set_tasks:
rel_path = PipelineUtils.GetTaskOutputRelativeDir(task)
PipelineConfig.Instance().CreateAllSubDirsForPath(rel_path)
def _RunSingeTask(cls, task):
"""Runs a Single Task.
Args:
task: string: The task to run.
Return:
(EXITCODE, string): Returns a tuple of the result status and the task.
"""
TermColor.Info('Executing %s' % PipelineUtils.TaskDisplayName(task))
task_vars = cls.__GetEnvVarsForTask(task)
TermColor.VInfo(4, 'VARS: \n%s' % task_vars)
task_cmd = task
pipe_output = True
log_file = PipelineUtils.GetLogFileForTask(task)
if log_file:
task_cmd += ' > ' + PipelineUtils.GetLogFileForTask(task) + ' 2>&1'
pipe_output = False
timeout = cls.__GetTimeOutForTask(task)
start = time.time()
(status, out) = ExecUtils.RunCmd(task_cmd, timeout, pipe_output, task_vars)
time_taken = time.time() - start
TermColor.Info('Executed %s. Took %.2fs' % (PipelineUtils.TaskDisplayName(task), time_taken))
if status:
TermColor.Failure('Failed Task: %s' % PipelineUtils.TaskDisplayName(task))
if task_vars.get('PIPELINE_TASK_ABORT_FAIL', None):
status_code = Runner.EXITCODE['ABORT_FAIL']
elif task_vars.get('PIPELINE_TASK_ALLOW_FAIL', None):
status_code = Runner.EXITCODE['ALLOW_FAIL']
else:
status_code = Runner.EXITCODE['FAILURE']
else:
status_code = Runner.EXITCODE['SUCCESS']
cls._SendMailForTask(task, status_code, time_taken, log_file, out)
# Everything done. Mark the task as successful.
return (status_code, task)
def WorkHorse(cls, tasks):
"""Runs the workhorse for the command.
Args:
tasks: OrderedDict {int, set(string)}: Dict from priority to set of tasks to execute at the
priority. Note: the dict is ordered by priority.
Return:
(list, list): Returns a tuple of list in the form
(successful_tasks, failed_tasks) specifying tasks that succeeded and
ones that failed.
"""
all_tasks = []
dirs_to_publish = set()
publish_dir_to_task_map = {}
for set_tasks in tasks.values():
for task in set_tasks:
all_tasks += [task]
publish_dir = PipelineUtils.GetPublishDirForTask(task)
if not publish_dir: continue
dirs_to_publish |= set([publish_dir])
publish_dir_to_task_map[publish_dir] = (publish_dir_to_task_map.get(publish_dir, []) +
[publish_dir])
# Check if there are any directories to publish.
if not dirs_to_publish:
TermColor.Error('Did not find any dirs to publish. Do not forget to specify publish root '
'using --publish_root')
return ([], all_tasks)
# Run all the copy tasks.
successful_dirs = []; failed_dirs = []
for dir in dirs_to_publish:
publish_dir = cls._GetActualPublishDir(dir)
if not publish_dir:
failed_dirs += [publish_dir]
continue
(parent, name) = os.path.split(publish_dir)
TermColor.Info('Making current: %s' % publish_dir)
with FileUtils.PushDir(parent):
FileUtils.CreateLink('current', name)
successful_dirs += [publish_dir]
# Get the reverse mapping from dirs to tasks.
successful_tasks = []; failed_tasks = []
for i in successful_dirs:
successful_tasks += publish_dir_to_task_map.get(i, [])
for i in failed_dirs:
failed_tasks += publish_dir_to_task_map.get(i, [])
return (successful_tasks, failed_tasks)
def _GetActualPublishDir(cls, publish_dir_hint):
"""Returns the actual publish dir statring at the hinted dir. If the directory does not contain
'SUCCESS', previous siblings are checked to get the publishable directory.
Args:
publish_dir_hint: string: The publish dir hint.
Return:
string: Actual publish dir.
"""
if not publish_dir_hint: return None
if os.path.exists(os.path.join(publish_dir_hint, 'SUCCESS')): return publish_dir_hint
return PipelineUtils.GetPrevDatedDirCotainingPattern(publish_dir_hint, 'SUCCESS')
def __AddFileToTasks(cls, tasks, target):
"""Adds a file to the tasks.
Args:
tasks: dict{int, set(string)}: Dict from priority to set of tasks to execute at the priority.
target: string: File to consider.
"""
priority = PipelineUtils.GetTaskPriority(target)
if not priority:
TermColor.Warning('Ignored target %s as it has no priority info.' %
target)
else:
tasks.update(
{priority: tasks.get(priority, set()) | set([target])})
def WorkHorse(cls, tasks):
"""Runs the workhorse for the command.
Args:
tasks: OrderedDict {int, set(string)}: Dict from priority to set of tasks to execute at the
priority. Note: the dict is ordered by priority.
Return:
(list, list): Returns a tuple of list in the form
(successful_tasks, failed_tasks) specifying tasks that succeeded and
ones that failed.
"""
already_successful = []
tasks_to_run = OrderedDict()
for priority, set_tasks in list(tasks.items()):
new_task_set = set()
for task in set_tasks:
out_dir = PipelineUtils.GetOutDirForTask(task)
if out_dir and os.path.exists(os.path.join(out_dir,
'SUCCESS')):
already_successful += [task]
continue
new_task_set |= set([task])
if new_task_set:
tasks_to_run[priority] = new_task_set
TermColor.Info(
'Tasks already successful: %d\n%s' %
(len(already_successful),
json.dumps(PipelineUtils.TasksDisplayNames(already_successful),
indent=2)))
if not tasks_to_run: return (already_successful, [])
# Run the remaining tasks.
(successful_run, failed_run) = super(Continuer,
cls).WorkHorse(tasks_to_run)
return (successful_run + already_successful, failed_run)
def _SendMailForTask(cls, task, status_code, time_taken, log_file, msg):
"""Sends the mail if required for the task.
Args:
task: string: The task for which the envvar should be prepared.
status_code: EXITCODE: The exit code for the task.
time_taken: float: Time taken in seconds.
log_file: string: The log file containing the output of the task.
msg: string: The output message piped directly. Note only one of msg or log_file will be
present at any time.
Returns:
dict {string, string}: The dictionary of IDS to values.
"""
if status_code == Runner.EXITCODE['SUCCESS']:
if not Flags.ARGS.detailed_success_mail: return
receiver = Flags.ARGS.success_mail
else: receiver = Flags.ARGS.failure_mail
# Check if there is no receiver for the mail.
if not receiver: return
mail_domain = Flags.ARGS.mail_domain
status_description = Runner.EXITCODE_DESCRIPTION[status_code]
subject = "[%s:%s] %s : %s" % (PipelineConfig.Instance().pipeline_id(),
PipelineConfig.Instance().pipeline_date(),
status_description, PipelineUtils.TaskDisplayName(task))
body = 'Executed task: %s. \nStatus:%s \nTime: %.2fs.' % (task, status_description, time_taken)
if msg:
body += '\n%s' % msg
Mailer().send_simple_message(
PipelineUtils.ZeusEmailId(mail_domain), [receiver], subject, body)
else:
Mailer().send_message_from_files(
PipelineUtils.ZeusEmailId(mail_domain), [receiver], subject, [log_file],
body)
def __GetTaskOptions(cls, task):
rel_task = PipelineUtils.TaskRelativeName(task)
options = {
v: False
for v in Runner.TASK_OPTIONS.values()
}
if '.abort_fail' in rel_task:
options[Runner.TASK_OPTIONS['ABORT_FAIL']] = True
if '.allow_fail' in rel_task:
options[Runner.TASK_OPTIONS['ALLOW_FAIL']] = True
if '.require_dir_success' in rel_task:
options[Runner.TASK_OPTIONS['REQUIRE_DIR_SUCCESS']] = True
if not options:
options[Runner.TASK_OPTIONS['NORMAL']] = True
return options
def _SendFinalStatusMail(cls, successful_run, failed_run, aborted_task, time_taken):
"""Sends the final status mail if required.
Args:
(list, list): Returns a tuple of list in the form
(successful_tasks, failed_tasks) specifying tasks that succeeded and
ones that failed.
aborted_task: string: True if the pipeline was aborted.
time_taken: float: Time taken in seconds.
"""
if not successful_run and not failed_run: return
if not failed_run:
receiver = Flags.ARGS.success_mail
status_description = 'SUCCESS'
else:
receiver = Flags.ARGS.failure_mail
status_description = 'ABORT FAIL' if aborted_task else 'FAIL'
# Check if there is no receiver for the mail.
if not receiver: return
mail_domain = Flags.ARGS.mail_domain
subject = "[%s:%s] Final Status: %s" % (PipelineConfig.Instance().pipeline_id(),
PipelineConfig.Instance().pipeline_date(),
status_description)
body = 'Aborted by: %s\n\n' % aborted_task if aborted_task else ''
body += ('Successful tasks: %d\n%s\n\n'
'Failed tasks: %d\n%s\n\n'
'Total Time: %.2fs.\n'
'\n%s\n\n' %
(len(successful_run), json.dumps(successful_run, indent=2),
len(failed_run), json.dumps(failed_run, indent=2),
time_taken,
PipelineConfig.Instance().GetConfigString()))
Mailer().send_simple_message(
PipelineUtils.ZeusEmailId(mail_domain), [receiver], subject, body)
def WorkHorse(cls, tasks):
"""Runs the workhorse for the command.
Args:
tasks: OrderedDict {int, set(string)}: Dict from priority to set of tasks to execute at the
priority. Note: the dict is ordered by priority.
Return:
(list, list): Returns a tuple of list in the form
(successful_tasks, failed_tasks) specifying tasks that succeeded and
ones that failed.
"""
# All our binaries assume they will be run from the source root.
start = time.time()
os.chdir(FileUtils.GetSrcRoot())
cls._CreateDirsForTasks(tasks)
successful_run = []; failed_run = []
aborted_task = None
# NOTE(stephen): Storing task dir status and task out dir status separately since
# pipelines do not always have an out dir defined.
dirs_status = {}
out_dirs_status = {}
for set_tasks in tasks.values():
if aborted_task:
failed_run += set_tasks
continue
tasks_to_run = []
for task in set_tasks:
task_options = cls.__GetTaskOptions(task)
# Check if this task requires all previous tasks in the same directory to be
# successful.
if task_options[Runner.TASK_OPTIONS['REQUIRE_DIR_SUCCESS']]:
task_dir = PipelineUtils.TaskDirName(task)
cur_dir_status = dirs_status.get(task_dir)
# If any previous tasks have been run in this directory, check to ensure all
# of them were successful.
if cur_dir_status and cur_dir_status != Runner.EXITCODE['SUCCESS']:
failed_run += [task]
task_display_name = PipelineUtils.TaskDisplayName(task)
TermColor.Info('Skipped %s' % task_display_name)
TermColor.Failure(
'Skipped Task: %s due to earlier failures in task dir' % task_display_name
)
continue
tasks_to_run.append(task)
# It is possible for all steps at this priority level to be skipped due to the
# task options selected.
if set_tasks and not tasks_to_run:
continue
# Run all the tasks at the same priority in parallel.
args = zip(itertools.repeat(cls), itertools.repeat('_RunSingeTask'),
tasks_to_run)
task_res = ExecUtils.ExecuteParallel(args, Flags.ARGS.pool_size)
# task_res = []
# for task in tasks_to_run: task_res += [cls._RunSingeTask(task)]
if not task_res:
TermColor.Error('Could not process: %s' % tasks_to_run)
failed_run += tasks_to_run
continue
for (res, task) in task_res:
if res == Runner.EXITCODE['SUCCESS']:
successful_run += [task]
elif res == Runner.EXITCODE['FAILURE']:
failed_run += [task]
elif res == Runner.EXITCODE['ALLOW_FAIL']:
failed_run += [task]
elif res == Runner.EXITCODE['ABORT_FAIL']:
failed_run += [task]
aborted_task = task
else:
TermColor.Fatal('Invalid return %d code for %s' % (res, task))
# Update the current status of all tasks in the same directory.
task_dir = PipelineUtils.TaskDirName(task)
dirs_status[task_dir] = max(
dirs_status.get(task_dir, Runner.EXITCODE['_LOWEST']), res,
)
# Update the out dir status.
out_dir = PipelineUtils.GetOutDirForTask(task)
if out_dir:
out_dirs_status[out_dir] = max(
out_dirs_status.get(out_dir, Runner.EXITCODE['_LOWEST']), res,
)
# Write the status files to the dirs.
cls._WriteOutDirsStatus(out_dirs_status)
# Send the final status mail.
time_taken = time.time() - start
cls._SendFinalStatusMail(successful_run, failed_run, aborted_task, time_taken)
if aborted_task:
TermColor.Failure('Aborted by task: %s' % aborted_task)
return (successful_run, failed_run)