def __CreateInitialSubDirs(self): """Creates all necessary directories.""" # Check if we have to create any output directory if not Flags.ARGS.out_dirs and Flags.ARGS.nolog_output: return # Create the user friendly link to pipeline dir if it doesn't already exist. FileUtils.CreateLink(FileUtils.GetPipelineLinkDir(), FileUtils.GetPipelineDir()) # Create the output directory. if Flags.ARGS.out_dirs or not Flags.ARGS.log_to_tmp: self._pipeline_output_dir = os.path.join( FileUtils.GetPipelineDir(), self._id) FileUtils.MakeDirs(self._pipeline_output_dir) # Create the log dir if required. if not Flags.ARGS.nolog_output: if Flags.ARGS.log_to_tmp: log_root = os.path.join('/tmp', 'pipeline', self.pipeline_id()) else: log_root = self._pipeline_output_dir self._pipeline_log_dir = os.path.join(log_root, 'log', self.pipeline_date()) FileUtils.MakeDirs(self._pipeline_log_dir) # Create all the subdirs. self._subdirs = {} for i in Flags.ARGS.out_dirs: subdir = os.path.join(self.pipeline_output_dir(), i) FileUtils.MakeDirs(subdir) self._subdirs['PIPELINE_' + i.upper() + '_DIR'] = subdir
def WorkHorse(cls, tasks): """Runs the workhorse for the command. Args: tasks: OrderedDict {int, set(string)}: Dict from priority to set of tasks to execute at the priority. Note: the dict is ordered by priority. Return: (list, list): Returns a tuple of list in the form (successful_tasks, failed_tasks) specifying tasks that succeeded and ones that failed. """ all_tasks = [] dirs_to_import = {} dir_to_task_map = {} for set_tasks in tasks.values(): for task in set_tasks: all_tasks += [task] out_dir = PipelineUtils.GetOutDirForTask(task) publish_dir = PipelineUtils.GetPublishCurrentDirForTask(task) if not out_dir or not publish_dir: continue dirs_to_import[publish_dir] = out_dir dir_to_task_map[publish_dir] = (dir_to_task_map.get(publish_dir, []) + [publish_dir]) # Check if there are any directories to publish. if not dirs_to_import: TermColor.Error('Did not find any dirs to import. Do not forget to specify publish root ' 'using --publish_root') return ([], all_tasks) # Create all the target dirs to import to. for dir in dirs_to_import.values(): FileUtils.MakeDirs(dir) # Run all the copy tasks. successful_dirs = []; failed_dirs = [] args = zip(itertools.repeat(cls), itertools.repeat('_RunSingeTask'), list(dirs_to_import), list(dirs_to_import.values())) dir_res = ExecUtils.ExecuteParallel(args, Flags.ARGS.pool_size) if not dir_res: TermColor.Error('Could not process: %s' % all_tasks) return ([], all_tasks) for (res, dir) in dir_res: if res == Importer.EXITCODE['SUCCESS']: successful_dirs += [dir] elif res == Importer.EXITCODE['FAILURE']: failed_dirs += [dir] else: TermColor.Fatal('Invalid return %d code for %s' % (res, dir)) # Get the reverse mapping from dirs to tasks. successful_tasks = []; failed_tasks = [] for i in successful_dirs: successful_tasks += dir_to_task_map.get(i, []) for i in failed_dirs: failed_tasks += dir_to_task_map.get(i, []) return (successful_tasks, failed_tasks)
def CreateAllSubDirsForPath(self, path): """Creates all the subdirs for the given node. Args: node: string: The node for which the path needs to be created. Returns: dict {string, string}: The dictionary of SUBDIR IDS to actual paths. """ for k, v in self.GetAllSubDirsForPath(path).items(): FileUtils.MakeDirs(v)