def _run_tasks(self, task_list, interactive): """ Runs the tasks given on the command line. :param task_list: a task name or a list of task names to run :type task_list: str|list :param interactive: if asserted do not run the tasks in parallel :type interactive: bool :return: None """ if not is_sequence(task_list): task_list = [task_list] verified_task_list = self._verified_tasks(task_list) debug("task_list: {tasks}".format(tasks=task_list)) debug("verified_task_list: {tasks}".format(tasks=verified_task_list)) if not verified_task_list: raise ValueError('No tasks given. Run "herring -T" to see available tasks.') TaskWithArgs.argv = list([arg for arg in task_list if arg not in verified_task_list]) def task_lookup(task_name_): info("Running: {name} ({description})".format(name=task_name_, description=HerringTasks[task_name_]['description'])) TaskWithArgs.arg_prompt = HerringTasks[task_name_]['arg_prompt'] try: return HerringTasks[task_name_]['task'] except Exception as ex: error(str(ex)) for task_name_list in self._resolve_dependencies(verified_task_list, HerringTasks): if interactive: for task_name in task_name_list: task_lookup(task_name)() else: parallelize_process(*[task_lookup(task_name) for task_name in task_name_list])
def _load_plugin(self, plugin, paths): """load a plugin module if we haven't yet loaded it :param plugin: the herringlib plugin to load :param paths: the herringlib path """ # check if we haven't loaded it already try: return sys.modules[plugin] except KeyError: pass # ok not found so load it debug("_load_plugin({plugin}, {paths})".format(plugin=plugin, paths=paths)) try: # python3 # noinspection PyUnresolvedReferences,PyCompatibility from importlib import import_module package = 'herringlib' import_module(package) mod = import_module(plugin, package) except ImportError: # python2 from imp import load_module, PY_SOURCE filename = os.path.join(paths, plugin) extension = os.path.splitext(filename)[1] mode = 'r' desc = (extension, mode, PY_SOURCE) debug(repr(desc)) with open(filename, mode) as fp: mod = load_module(plugin, fp, filename, desc) return mod
def _load_file(self, file_name): """ Loads the tasks from the herringfile populating the HerringApp.HerringTasks structure. :param file_name: the herringfile :type file_name: str :return: None """ plugin = os.path.basename(file_name) path = os.path.dirname(file_name) debug("plugin: {plugin}, path: {path}".format(plugin=plugin, path=path)) self._load_plugin(plugin, path)
def library_files(library_paths=None, pattern='*.py'): """ Yield any .py files located in herringlib subdirectory in the same directory as the given herringfile. Ignore package __init__.py files, .svn and templates sub-directories. :param library_paths: the path to the herringlib directory :type library_paths: list(Path) :param pattern: the file pattern (glob) to select :type pattern: str :return: iterator for path to a library herring file :rtype: iterator[str] """ for lib_path in library_paths or []: debug("lib_path: {path}".format(path=lib_path)) parent_path = lib_path.parent if lib_path.is_dir(): files = find_files(str(lib_path), excludes=['*/templates/*', '.svn'], includes=[pattern]) for file_path in [Path(file_name) for file_name in files]: if file_path.name == '__init__.py': continue debug("parent_path: %s" % str(parent_path)) debug("loading from herringlib: %s" % file_path) rel_path = file_path.relative_to(parent_path) debug("relative path: %s" % str(rel_path)) yield rel_path
def wrapper(function_, queue__): """ Wraps the function to capture the ReportService logging which is returned via the queue. If the function raises an exception, return 1 :param function_: function to execute :type function_: function :param queue__: queue used to return a string containing the ReportService log filled by the function :type queue__: multiprocessing.Queue :return: exit the processes :rtype: zero to indicate pass or positive integer to indicate error """ # print("wrapper({name})".format(name=function.__name__)) sys.stdout = sys.stderr = StringIO() try: debug("Starting process wrapped function") result = function_() debug("Finished process wrapped function") except Exception as ex: error("parallelize_process error: " + str(ex)) result = 1 messages = sys.stdout.getvalue() or "" debug("messages: " + messages) queue__.put(messages) # HACK: documentation says to use sys.exit() but that blows on python 2.6, # so using return which works on python 2.6, 2.7, 3.4 return result
def _import(self, mod_name): try: __import__(mod_name) debug('imported {name}'.format(name=mod_name)) except ImportError as ex: debug(str(ex)) debug('failed to import {name}'.format(name=mod_name))
def run_tasks(task_list): """ Runs the tasks given on the command line. :param task_list: a task name or a list of task names to run :type task_list: str|list :return: None """ if not is_sequence(task_list): task_list = [task_list] verified_task_list = HerringApp._verified_tasks(task_list) debug("task_list: {tasks}".format(tasks=task_list)) debug("verified_task_list: {tasks}".format(tasks=verified_task_list)) if not verified_task_list: raise ValueError('No tasks given. Run "herring -T" to see available tasks.') TaskWithArgs.argv = list([arg for arg in task_list if arg not in verified_task_list]) for task_name in HerringApp._resolve_dependencies(verified_task_list, HerringTasks): info("Running: {name} ({description})".format(name=task_name, description=HerringTasks[task_name]['description'])) TaskWithArgs.arg_prompt = HerringTasks[task_name]['arg_prompt'] HerringTasks[task_name]['task']()
def _load_modules(self, herringfile, library_paths): """ :param herringfile: :type herringfile: :param library_paths: :type library_paths: list[Path] :return: :rtype: """ herringfile_path = Path(herringfile).parent debug("library_paths: %s" % repr(library_paths)) HerringFile.herringlib_paths = [str(path.parent) for path in library_paths if path.parent != herringfile_path] + [str(herringfile_path)] sys.path = unique_list(HerringFile.herringlib_paths + self.__sys_path[:]) for path in HerringFile.herringlib_paths: debug("herringlib path: %s" % path) debug(pformat("sys.path: %s" % repr(sys.path))) try: self._load_file(herringfile) except ImportError as ex: debug(str(ex)) debug('failed to import herringfile') self._import('herringlib') for lib_path in library_paths: sys.path = [lib_path] + self.__sys_path debug("sys.path: %s" % repr(sys.path)) for file_name in self.library_files(library_paths=[lib_path]): self._import(mod_name='herringlib.' + Path(file_name).stem) sys.path = self.__sys_path[:]