def _run_tasks(self, task_list, interactive):
"""
Runs the tasks given on the command line.
:param task_list: a task name or a list of task names to run
:type task_list: str|list
:param interactive: if asserted do not run the tasks in parallel
:type interactive: bool
:return: None
"""
if not is_sequence(task_list):
task_list = [task_list]
verified_task_list = self._verified_tasks(task_list)
debug("task_list: {tasks}".format(tasks=task_list))
debug("verified_task_list: {tasks}".format(tasks=verified_task_list))
if not verified_task_list:
raise ValueError('No tasks given. Run "herring -T" to see available tasks.')
TaskWithArgs.argv = list([arg for arg in task_list if arg not in verified_task_list])
def task_lookup(task_name_):
info("Running: {name} ({description})".format(name=task_name_,
description=HerringTasks[task_name_]['description']))
TaskWithArgs.arg_prompt = HerringTasks[task_name_]['arg_prompt']
try:
return HerringTasks[task_name_]['task']
except Exception as ex:
error(str(ex))
for task_name_list in self._resolve_dependencies(verified_task_list, HerringTasks):
if interactive:
for task_name in task_name_list:
task_lookup(task_name)()
else:
parallelize_process(*[task_lookup(task_name) for task_name in task_name_list])
def _load_plugin(self, plugin, paths):
"""load a plugin module if we haven't yet loaded it
:param plugin: the herringlib plugin to load
:param paths: the herringlib path
"""
# check if we haven't loaded it already
try:
return sys.modules[plugin]
except KeyError:
pass
# ok not found so load it
debug("_load_plugin({plugin}, {paths})".format(plugin=plugin, paths=paths))
try:
# python3
# noinspection PyUnresolvedReferences,PyCompatibility
from importlib import import_module
package = 'herringlib'
import_module(package)
mod = import_module(plugin, package)
except ImportError:
# python2
from imp import load_module, PY_SOURCE
filename = os.path.join(paths, plugin)
extension = os.path.splitext(filename)[1]
mode = 'r'
desc = (extension, mode, PY_SOURCE)
debug(repr(desc))
with open(filename, mode) as fp:
mod = load_module(plugin, fp, filename, desc)
return mod
def _load_file(self, file_name):
"""
Loads the tasks from the herringfile populating the
HerringApp.HerringTasks structure.
:param file_name: the herringfile
:type file_name: str
:return: None
"""
plugin = os.path.basename(file_name)
path = os.path.dirname(file_name)
debug("plugin: {plugin}, path: {path}".format(plugin=plugin, path=path))
self._load_plugin(plugin, path)
def library_files(library_paths=None, pattern='*.py'):
"""
Yield any .py files located in herringlib subdirectory in the
same directory as the given herringfile. Ignore package __init__.py
files, .svn and templates sub-directories.
:param library_paths: the path to the herringlib directory
:type library_paths: list(Path)
:param pattern: the file pattern (glob) to select
:type pattern: str
:return: iterator for path to a library herring file
:rtype: iterator[str]
"""
for lib_path in library_paths or []:
debug("lib_path: {path}".format(path=lib_path))
parent_path = lib_path.parent
if lib_path.is_dir():
files = find_files(str(lib_path), excludes=['*/templates/*', '.svn'], includes=[pattern])
for file_path in [Path(file_name) for file_name in files]:
if file_path.name == '__init__.py':
continue
debug("parent_path: %s" % str(parent_path))
debug("loading from herringlib: %s" % file_path)
rel_path = file_path.relative_to(parent_path)
debug("relative path: %s" % str(rel_path))
yield rel_path
def wrapper(function_, queue__):
"""
Wraps the function to capture the ReportService logging which is returned via the queue.
If the function raises an exception, return 1
:param function_: function to execute
:type function_: function
:param queue__: queue used to return a string containing the ReportService log filled by the function
:type queue__: multiprocessing.Queue
:return: exit the processes
:rtype: zero to indicate pass or positive integer to indicate error
"""
# print("wrapper({name})".format(name=function.__name__))
sys.stdout = sys.stderr = StringIO()
try:
debug("Starting process wrapped function")
result = function_()
debug("Finished process wrapped function")
except Exception as ex:
error("parallelize_process error: " + str(ex))
result = 1
messages = sys.stdout.getvalue() or ""
debug("messages: " + messages)
queue__.put(messages)
# HACK: documentation says to use sys.exit() but that blows on python 2.6,
# so using return which works on python 2.6, 2.7, 3.4
return result
def _import(self, mod_name):
try:
__import__(mod_name)
debug('imported {name}'.format(name=mod_name))
except ImportError as ex:
debug(str(ex))
debug('failed to import {name}'.format(name=mod_name))
def run_tasks(task_list):
"""
Runs the tasks given on the command line.
:param task_list: a task name or a list of task names to run
:type task_list: str|list
:return: None
"""
if not is_sequence(task_list):
task_list = [task_list]
verified_task_list = HerringApp._verified_tasks(task_list)
debug("task_list: {tasks}".format(tasks=task_list))
debug("verified_task_list: {tasks}".format(tasks=verified_task_list))
if not verified_task_list:
raise ValueError('No tasks given. Run "herring -T" to see available tasks.')
TaskWithArgs.argv = list([arg for arg in task_list if arg not in verified_task_list])
for task_name in HerringApp._resolve_dependencies(verified_task_list, HerringTasks):
info("Running: {name} ({description})".format(name=task_name,
description=HerringTasks[task_name]['description']))
TaskWithArgs.arg_prompt = HerringTasks[task_name]['arg_prompt']
HerringTasks[task_name]['task']()
def _load_modules(self, herringfile, library_paths):
"""
:param herringfile:
:type herringfile:
:param library_paths:
:type library_paths: list[Path]
:return:
:rtype:
"""
herringfile_path = Path(herringfile).parent
debug("library_paths: %s" % repr(library_paths))
HerringFile.herringlib_paths = [str(path.parent) for path in library_paths
if path.parent != herringfile_path] + [str(herringfile_path)]
sys.path = unique_list(HerringFile.herringlib_paths + self.__sys_path[:])
for path in HerringFile.herringlib_paths:
debug("herringlib path: %s" % path)
debug(pformat("sys.path: %s" % repr(sys.path)))
try:
self._load_file(herringfile)
except ImportError as ex:
debug(str(ex))
debug('failed to import herringfile')
self._import('herringlib')
for lib_path in library_paths:
sys.path = [lib_path] + self.__sys_path
debug("sys.path: %s" % repr(sys.path))
for file_name in self.library_files(library_paths=[lib_path]):
self._import(mod_name='herringlib.' + Path(file_name).stem)
sys.path = self.__sys_path[:]