def MakeRules(cls, rules, makefile): """Makes all the rules in the give list. Args: rules: list: List of rules by type_base to make. makefile: string: The *main* makefile name. Return: (list, list): Returns a tuple of list in the form (successful_rules, failed_rules) specifying rules for which the make rules were successfully generated and for which it failed. """ if not rules: TermColor.Warning('No rules to build.') return ([], rules) args = zip(itertools.repeat(cls), itertools.repeat('_WorkHorse'), rules, itertools.repeat(makefile)) rule_res = ExecUtils.ExecuteParallel(args, Flags.ARGS.pool_size) successful_rules = []; failed_rules = [] for (res, rule) in rule_res: if res == 1: successful_rules += [rule] elif res == -1: failed_rules += [rule] return (successful_rules, failed_rules)
def WorkHorse(cls, rules): """Runs the workhorse for the command. Args: rules: list: List of rules to be handled. Return: (list, list): Returns a tuple of list in the form (successful_rules, failed_rules) specifying rules that succeeded and ones that failed. """ (successful_build, failed_build) = Builder.WorkHorse(rules) # All our binaries assume they will be run from the source root. os.chdir(FileUtils.GetSrcRoot()) pipe_output = len(successful_build) > 1 args = zip(itertools.repeat(cls), itertools.repeat('_RunSingeRule'), successful_build, itertools.repeat(pipe_output)) rule_res = ExecUtils.ExecuteParallel(args, Flags.ARGS.pool_size) successful_run = []; failed_run = [] for (res, rule) in rule_res: if res == 1: successful_run += [rule] elif res == -1: failed_run += [rule] return (successful_run, failed_build + failed_run)
def WorkHorse(cls, rules): """Runs the workhorse for the command. Args: rules: list: List of rules to be handled. Return: (list, list): Returns a tuple of list in the form (successful_rules, failed_rules) specifying rules that succeeded and ones that failed. """ (successful_expand, failed_expand) = Rules.GetExpandedRules(rules, Flags.ARGS.allowed_rule_types) args = zip(itertools.repeat(cls), itertools.repeat('_RunSingeRule'), successful_expand) rule_res = ExecUtils.ExecuteParallel(args, Flags.ARGS.pool_size) successful_deps = [] failed_deps = [] for (res, rule) in rule_res: if res == 1: successful_deps += [rule] elif res == -1: failed_deps += [rule] return (successful_deps, failed_expand + failed_deps)
def WorkHorse(cls, tasks): """Runs the workhorse for the command. Args: tasks: OrderedDict {int, set(string)}: Dict from priority to set of tasks to execute at the priority. Note: the dict is ordered by priority. Return: (list, list): Returns a tuple of list in the form (successful_tasks, failed_tasks) specifying tasks that succeeded and ones that failed. """ all_tasks = [] dirs_to_import = {} dir_to_task_map = {} for set_tasks in tasks.values(): for task in set_tasks: all_tasks += [task] out_dir = PipelineUtils.GetOutDirForTask(task) publish_dir = PipelineUtils.GetPublishCurrentDirForTask(task) if not out_dir or not publish_dir: continue dirs_to_import[publish_dir] = out_dir dir_to_task_map[publish_dir] = (dir_to_task_map.get(publish_dir, []) + [publish_dir]) # Check if there are any directories to publish. if not dirs_to_import: TermColor.Error('Did not find any dirs to import. Do not forget to specify publish root ' 'using --publish_root') return ([], all_tasks) # Create all the target dirs to import to. for dir in dirs_to_import.values(): FileUtils.MakeDirs(dir) # Run all the copy tasks. successful_dirs = []; failed_dirs = [] args = zip(itertools.repeat(cls), itertools.repeat('_RunSingeTask'), list(dirs_to_import), list(dirs_to_import.values())) dir_res = ExecUtils.ExecuteParallel(args, Flags.ARGS.pool_size) if not dir_res: TermColor.Error('Could not process: %s' % all_tasks) return ([], all_tasks) for (res, dir) in dir_res: if res == Importer.EXITCODE['SUCCESS']: successful_dirs += [dir] elif res == Importer.EXITCODE['FAILURE']: failed_dirs += [dir] else: TermColor.Fatal('Invalid return %d code for %s' % (res, dir)) # Get the reverse mapping from dirs to tasks. successful_tasks = []; failed_tasks = [] for i in successful_dirs: successful_tasks += dir_to_task_map.get(i, []) for i in failed_dirs: failed_tasks += dir_to_task_map.get(i, []) return (successful_tasks, failed_tasks)
def WorkHorse(cls, tasks): """Runs the workhorse for the command. Args: tasks: OrderedDict {int, set(string)}: Dict from priority to set of tasks to execute at the priority. Note: the dict is ordered by priority. Return: (list, list): Returns a tuple of list in the form (successful_tasks, failed_tasks) specifying tasks that succeeded and ones that failed. """ # All our binaries assume they will be run from the source root. start = time.time() os.chdir(FileUtils.GetSrcRoot()) cls._CreateDirsForTasks(tasks) successful_run = []; failed_run = [] aborted_task = None # NOTE(stephen): Storing task dir status and task out dir status separately since # pipelines do not always have an out dir defined. dirs_status = {} out_dirs_status = {} for set_tasks in tasks.values(): if aborted_task: failed_run += set_tasks continue tasks_to_run = [] for task in set_tasks: task_options = cls.__GetTaskOptions(task) # Check if this task requires all previous tasks in the same directory to be # successful. if task_options[Runner.TASK_OPTIONS['REQUIRE_DIR_SUCCESS']]: task_dir = PipelineUtils.TaskDirName(task) cur_dir_status = dirs_status.get(task_dir) # If any previous tasks have been run in this directory, check to ensure all # of them were successful. if cur_dir_status and cur_dir_status != Runner.EXITCODE['SUCCESS']: failed_run += [task] task_display_name = PipelineUtils.TaskDisplayName(task) TermColor.Info('Skipped %s' % task_display_name) TermColor.Failure( 'Skipped Task: %s due to earlier failures in task dir' % task_display_name ) continue tasks_to_run.append(task) # It is possible for all steps at this priority level to be skipped due to the # task options selected. if set_tasks and not tasks_to_run: continue # Run all the tasks at the same priority in parallel. args = zip(itertools.repeat(cls), itertools.repeat('_RunSingeTask'), tasks_to_run) task_res = ExecUtils.ExecuteParallel(args, Flags.ARGS.pool_size) # task_res = [] # for task in tasks_to_run: task_res += [cls._RunSingeTask(task)] if not task_res: TermColor.Error('Could not process: %s' % tasks_to_run) failed_run += tasks_to_run continue for (res, task) in task_res: if res == Runner.EXITCODE['SUCCESS']: successful_run += [task] elif res == Runner.EXITCODE['FAILURE']: failed_run += [task] elif res == Runner.EXITCODE['ALLOW_FAIL']: failed_run += [task] elif res == Runner.EXITCODE['ABORT_FAIL']: failed_run += [task] aborted_task = task else: TermColor.Fatal('Invalid return %d code for %s' % (res, task)) # Update the current status of all tasks in the same directory. task_dir = PipelineUtils.TaskDirName(task) dirs_status[task_dir] = max( dirs_status.get(task_dir, Runner.EXITCODE['_LOWEST']), res, ) # Update the out dir status. out_dir = PipelineUtils.GetOutDirForTask(task) if out_dir: out_dirs_status[out_dir] = max( out_dirs_status.get(out_dir, Runner.EXITCODE['_LOWEST']), res, ) # Write the status files to the dirs. cls._WriteOutDirsStatus(out_dirs_status) # Send the final status mail. time_taken = time.time() - start cls._SendFinalStatusMail(successful_run, failed_run, aborted_task, time_taken) if aborted_task: TermColor.Failure('Aborted by task: %s' % aborted_task) return (successful_run, failed_run)