def check_options(self):
Glue = self.Glue
self.pipeline_desc = self._deduce_pipeline_desc(Glue.option('pipeline'), Glue.module_list())
log_dict(Glue.debug, 'pipeline description', self.pipeline_desc)
# list modules
groups = Glue.option('list-modules')
if groups == [True]:
sys.stdout.write('%s\n' % Glue.module_list_usage([]))
sys.exit(0)
elif groups:
sys.stdout.write('%s\n' % Glue.module_list_usage(groups))
sys.exit(0)
if Glue.option('list-shared'):
import tabulate
functions = []
for mod_name in Glue.module_list():
# pylint: disable=line-too-long
functions += [[func_name, mod_name] for func_name in Glue.modules[mod_name]['class'].shared_functions] # Ignore PEP8Bear
functions = sorted(functions, key=lambda row: row[0])
sys.stdout.write("""Available shared functions
{}
""".format(tabulate.tabulate(functions, ['Shared function', 'Module name'], tablefmt='simple')))
sys.exit(0)
def policy_least_crowded(logger: gluetool.log.ContextAdapter,
session: sqlalchemy.orm.session.Session,
pools: List[PoolDriver],
guest_request: GuestRequest) -> PolicyReturnType:
"""
Pick the least crowded pools, i.e. pools with the lowest absolute usage.
"""
if len(pools) <= 1:
return Ok(PolicyRuling(allowed_pools=pools))
r_pool_metrics = collect_pool_metrics(pools)
if r_pool_metrics.is_error:
return Error(r_pool_metrics.unwrap_error())
pool_metrics = r_pool_metrics.unwrap()
log_dict(logger.debug, 'pool metrics', pool_metrics)
min_usage = min(
[metrics.current_guest_request_count for _, metrics in pool_metrics])
return Ok(
PolicyRuling(allowed_pools=[
pool for pool, metrics in pool_metrics
if metrics.current_guest_request_count == min_usage
]))
def wrapper(logger: gluetool.log.ContextAdapter,
session: sqlalchemy.orm.session.Session,
pools: List[PoolDriver],
guest_request: GuestRequest) -> PolicyReturnType:
try:
policy_logger = PolicyLogger(logger, policy_name)
log_dict(policy_logger.debug, 'input pools', pools)
r_enabled = knob_enabled.get_value(session=session)
if r_enabled.is_error:
return Error(
Failure.from_failure('failed to test policy enablement',
r_enabled.unwrap_error()))
if r_enabled.unwrap() is not True:
policy_logger.debug('policy disabled, skipping')
return Ok(PolicyRuling(allowed_pools=pools))
r = fn(policy_logger, session, pools, guest_request)
if r.is_error:
return r
policy_logger.debug(f'ruling: {r.unwrap()}')
return r
except Exception as exc:
return Error(
Failure.from_exc('routing policy crashed',
exc,
routing_policy=policy_name))
def _render(template):
log_blob(logger.debug, 'rendering template', template.source)
log_dict(logger.debug, 'context', kwargs)
return str(template.render(**kwargs).strip())
def run(self, inspect=False, inspect_callback=None, **kwargs):
"""
Run the command, wait for it to finish and return the output.
:param bool inspect: If set, ``inspect_callback`` will receive the output of command in "real-time".
:param callable inspect_callback: callable that will receive command output. If not set, default
"write to ``sys.stdout``" is used.
:rtype: gluetool.utils.ProcessOutput instance
:returns: :py:class:`gluetool.utils.ProcessOutput` instance whose attributes contain data returned
by the child process.
:raises gluetool.glue.GlueError: When somethign went wrong.
:raises gluetool.glue.GlueCommandError: When command exited with non-zero exit code.
"""
# pylint: disable=too-many-branches
def _check_types(items):
if not isinstance(items, list):
raise GlueError('Only list of strings is accepted')
if not all((isinstance(s, str) for s in items)):
raise GlueError(
'Only list of strings is accepted, {} found'.format(
[type(s) for s in items]))
_check_types(self.executable)
_check_types(self.options)
self._command = self._apply_quotes()
if self.use_shell is True:
self._command = [' '.join(self._command)]
# Set default stdout/stderr, unless told otherwise
if 'stdout' not in kwargs:
kwargs['stdout'] = subprocess.PIPE
if 'stderr' not in kwargs:
kwargs['stderr'] = subprocess.PIPE
if self.use_shell:
kwargs['shell'] = True
self._popen_kwargs = kwargs
def _format_stream(stream):
if stream == subprocess.PIPE:
return 'PIPE'
if stream == DEVNULL:
return 'DEVNULL'
if stream == subprocess.STDOUT:
return 'STDOUT'
return stream
printable_kwargs = kwargs.copy()
for stream in ('stdout', 'stderr'):
if stream in printable_kwargs:
printable_kwargs[stream] = _format_stream(
printable_kwargs[stream])
log_dict(self.debug, 'command', self._command)
log_dict(self.debug, 'kwargs', printable_kwargs)
log_blob(self.debug, 'runnable (copy & paste)',
format_command_line([self._command]))
try:
self._process = subprocess.Popen(self._command,
**self._popen_kwargs)
if inspect is True:
self._communicate_inspect(inspect_callback)
else:
self._communicate_batch()
except OSError as e:
if e.errno == errno.ENOENT:
raise GlueError("Command '{}' not found".format(
self._command[0]))
raise e
self._exit_code = self._process.poll()
output = self._construct_output()
if self._exit_code != 0:
raise GlueCommandError(self._command, output)
return output
def __init__(self, filepath, spices=None, logger=None):
self.logger = logger or Logging.get_logger()
logger.connect(self)
spices = spices or {}
pattern_map = load_yaml(filepath, logger=self.logger)
if pattern_map is None:
raise GlueError(
"pattern map '{}' does not contain any patterns".format(
filepath))
def _create_simple_repl(repl):
def _replace(pattern, target):
"""
Use `repl` to construct image from `target`, honoring all backreferences made by `pattern`.
"""
self.debug("pattern '{}', repl '{}', target '{}'".format(
pattern.pattern, repl, target))
try:
return pattern.sub(repl, target)
except re.error as e:
raise GlueError(
"Cannot transform pattern '{}' with target '{}', repl '{}': {}"
.format(pattern.pattern, target, repl, str(e)))
return _replace
self._compiled_map = []
for pattern_dict in pattern_map:
log_dict(logger.debug, 'pattern dict', pattern_dict)
if not isinstance(pattern_dict, dict):
raise GlueError(
"Invalid format: '- <pattern>: <transform>' expected, '{}' found"
.format(pattern_dict))
pattern = pattern_dict.keys()[0]
converter_chains = pattern_dict[pattern]
if isinstance(converter_chains, str):
converter_chains = [converter_chains]
try:
pattern = re.compile(pattern)
except re.error as e:
raise GlueError("Pattern '{}' is not valid: {}".format(
pattern, str(e)))
compiled_chains = []
for chain in converter_chains:
converters = [s.strip() for s in chain.split(',')]
# first item in `converters` is always a simple string used by `pattern.sub()` call
converter = _create_simple_repl(converters.pop(0))
# if there any any items left, they name "spices" to apply, one by one,
# on the result of the first operation
for spice in converters:
if spice not in spices:
raise GlueError(
"Unknown 'spice' function '{}'".format(spice))
converter = spices[spice](converter)
compiled_chains.append(converter)
self._compiled_map.append((pattern, compiled_chains))
def run_command(cmd, logger=None, inspect=False, inspect_callback=None, **kwargs):
"""
Run external command, and return it's exit code and output.
This is a very thin and simple wrapper above :py:class:`subprocess.Popen`,
and its main purpose is to log everything that happens before and after
execution. All additional arguments are passed directly to `Popen` constructor.
If ``stdout`` or ``stderr`` keyword arguments are not specified, function
will set them to :py:const:`subprocess.PIPE`, to capture both output streams
in separate strings.
By default, output of the process is captured for both ``stdout`` and ``stderr``,
and returned back to the caller. Under some conditions, caller might want to see
the output in "real-time". For that purpose, it can pass callable via ``inspect_callback``
parameter - such callable will be called for every received bit of input on both
``stdout`` and ``stderr``. E.g.
.. code-block:: python
def foo(stream, s, flush=False):
if s is not None and 'a' in s:
print s
run_command(['/bin/foo'], inspect=foo)
This example will print all substrings containing letter `a`. Strings passed to ``foo``
may be of arbitrary lengths, and may change between subsequent calls of ``run_command``.
:param list cmd: command to execute.
:param gluetool.log.ContextAdapter logger: parent logger whose methods will be used for logging.
:param bool inspect: if set, ``inspect_callback`` will receive the output of command in "real-time".
:param callable inspect_callback: callable that will receive command output. If not set,
default "write to ``sys.stdout``" is used.
:rtype: gluetool.utils.ProcessOutput instance
:returns: :py:class:`gluetool.utils.ProcessOutput` instance whose attributes contain
data returned by the process.
:raises gluetool.glue.GlueError: when command was not found.
:raises gluetool.glue.GlueCommandError: when command exited with non-zero exit code.
:raises Exception: when anything else breaks.
"""
assert isinstance(cmd, list), 'Only list of strings accepted as a command'
if not all((isinstance(s, str) for s in cmd)):
raise GlueError('Only list of strings accepted as a command, {} found'.format([type(s) for s in cmd]))
logger = logger or Logging.get_logger()
stdout, stderr = None, None
# Set default stdout/stderr, unless told otherwise
if 'stdout' not in kwargs:
kwargs['stdout'] = subprocess.PIPE
if 'stderr' not in kwargs:
kwargs['stderr'] = subprocess.PIPE
def _format_stream(stream):
if stream == subprocess.PIPE:
return 'PIPE'
if stream == DEVNULL:
return 'DEVNULL'
if stream == subprocess.STDOUT:
return 'STDOUT'
return stream
printable_kwargs = kwargs.copy()
for stream in ('stdout', 'stderr'):
if stream in printable_kwargs:
printable_kwargs[stream] = _format_stream(printable_kwargs[stream])
# Make tests happy by sorting kwargs - it's a dictionary, therefore
# unpredictable from the observer's point of view. Can print its entries
# in different order with different Pythons, making tests a mess.
# sorted_kwargs = ', '.join(["'%s': '%s'" % (k, printable_kwargs[k]) for k in sorted(printable_kwargs.iterkeys())])
log_dict(logger.debug, 'command', cmd)
log_dict(logger.debug, 'kwargs', printable_kwargs)
log_blob(logger.debug, 'runnable (copy & paste)', format_command_line([cmd]))
try:
p = subprocess.Popen(cmd, **kwargs)
if inspect is True:
# let's capture *both* streams - capturing just a single one leads to so many ifs
# and elses and messy code
p_stdout = StreamReader(p.stdout, name='<stdout>')
p_stderr = StreamReader(p.stderr, name='<stderr>')
if inspect_callback is None:
def stdout_write(stream, data, flush=False):
# pylint: disable=unused-argument
if data is None:
return
# Not suitable for multiple simultaneous commands. Shuffled output will
# ruin your day. And night. And few following weeks, full of debugging, as well.
sys.stdout.write(data)
sys.stdout.flush()
inspect_callback = stdout_write
inputs = (p_stdout, p_stderr)
with BlobLogger('Output of command: {}'.format(format_command_line([cmd])), outro='End of command output',
writer=logger.info):
logger.debug("output of command is inspected by the caller")
logger.debug('following blob-like header and footer are expected to be empty')
logger.debug('the captured output will follow them')
# As long as process runs, keep calling callbacks with incoming data
while True:
for stream in inputs:
inspect_callback(stream, stream.read())
if p.poll() is not None:
break
# give up OS' attention and let others run
# time.sleep(0) is a Python synonym for "thread yields the rest of its quantum"
time.sleep(0.1)
# OK, process finished but we have to wait for our readers to finish as well
p_stdout.wait()
p_stderr.wait()
for stream in inputs:
while True:
data = stream.read()
if data in ('', None):
break
inspect_callback(stream, data)
inspect_callback(stream, None, flush=True)
stdout, stderr = p_stdout.content, p_stderr.content
else:
stdout, stderr = p.communicate()
except OSError as e:
if e.errno == errno.ENOENT:
raise GlueError("Command '{}' not found".format(cmd[0]))
raise e
exit_code = p.poll()
output = ProcessOutput(cmd, exit_code, stdout, stderr, kwargs)
output.log(logger.debug)
if exit_code != 0:
raise GlueCommandError(cmd, output)
return output
def check_options(self):
# type: () -> None
Glue = self.Glue
assert Glue is not None
self.pipeline_desc = self._deduce_pipeline_desc(
Glue.option('pipeline'), Glue.module_list())
log_dict(Glue.debug, 'pipeline description', self.pipeline_desc)
# list modules
groups = Glue.option('list-modules')
if groups == [True]:
sys.stdout.write('%s\n' % Glue.module_list_usage([]))
sys.exit(0)
elif groups:
sys.stdout.write('%s\n' % Glue.module_list_usage(groups))
sys.exit(0)
if Glue.option('list-shared'):
functions = [] # type: List[List[str]]
for mod_name in Glue.module_list():
# pylint: disable=line-too-long
functions += [[func_name, mod_name]
for func_name in Glue.modules[mod_name]
['class'].shared_functions] # Ignore PEP8Bear
if functions:
functions = sorted(functions, key=lambda row: row[0])
else:
functions = [['-- no shared functions available --', '']]
sys.stdout.write("""Available shared functions
{}
""".format(
tabulate.tabulate(functions,
['Shared function', 'Module name'],
tablefmt='simple')))
sys.exit(0)
if Glue.option('list-eval-context'):
variables = []
def _add_variables(source):
# type: (gluetool.glue.Configurable) -> None
info = extract_eval_context_info(source)
for name, description in info.iteritems():
variables.append([
name, source.name,
docstring_to_help(description, line_prefix='')
])
for mod_name in Glue.module_list():
_add_variables(Glue.init_module(mod_name))
_add_variables(Glue)
if variables:
variables = sorted(variables, key=lambda row: row[0])
else:
variables = [['-- no variables available --', '', '']]
table = tabulate.tabulate(
variables, ['Variable', 'Module name', 'Description'],
tablefmt='simple')
print render_template("""
{{ '** Variables available in eval context **' | style(fg='yellow') }}
{{ TABLE }}
""",
TABLE=table)
sys.exit(0)
def execute(self):
# we must fix "type" keys in pipeline options: it's supposed to be a callable
# but we cannot store callable in YAML, therefore let's convert from strings,
# using builtins.
#
# Also, find required options/
required_options = []
for name, properties in self.pipeline['options'].iteritems():
if 'required' in properties:
if properties['required'] is True:
required_options.append(name)
del properties['required']
option_type = properties.get('type', None)
if option_type is None:
continue
if option_type not in __builtins__:
raise gluetool.GlueError("Cannot find option type '{}'".format(option_type))
properties['type'] = __builtins__[option_type]
# our custom "pipeline" module
class Pipeline(gluetool.Module):
name = self.pipeline['name']
desc = self.pipeline['description']
options = self.pipeline['options']
# cannot assign local name to Pipeline's class property while delcaring it, therefore setting it now
Pipeline.required_options = required_options
log_dict(self.debug, 'pipeline options', Pipeline.options)
pipeline_module = Pipeline(self.glue, self.pipeline['name'])
pipeline_module.parse_args(self.option('pipeline_options')[1:]) # skip leading '--'
pipeline_module.check_dryrun()
pipeline_module.check_required_options()
# Run each module, one by one - we cannot construct a pipeline, because options
# of a module might depend on state of previous modules - available via
# PIPELINE.shared, for example.
run_module = functools.partial(self.glue.run_module, register=True)
def evaluate_value(value):
# If the template is not a string type, just return it as a string. This helps
# simplifyusers of this method: *every* value is treated by this method, no
# exceptions. User gets new one, and is not concerned whether the original was
# a template or boolean or whatever.
if not isinstance(value, str):
return str(value)
return jinja2.Template(value).render(PIPELINE=pipeline_module, ENV=os.environ)
for module in self.pipeline['pipeline']:
log_dict(self.debug, 'module', module)
# just a module name
if isinstance(module, str):
run_module(module, [])
continue
if not isinstance(module, dict):
raise gluetool.GlueError('Unexpected module syntax: {}'.format(module))
# Check 'when' - if it's set and evaluates as false-ish, skip the module
when = module.pop('when', None)
if when is not None:
# If "when" is a string, expect it's an expression - wrap it with {{ }}
# to form a Jinja template, and evaluate it.
if isinstance(when, str):
when = evaluate_value('{{ ' + when + ' }}')
self.debug("evalued when: '{}' ({})".format(when, type(when)))
# Works for both Python false-ish values and strings, coming from template evaluation
# If it's false-ish by nature, or its string representation is false-ish, skip the module.
if not when or when.lower() in ('no', 'off', '0', 'false', 'none'):
self.debug('skipping module')
continue
# remaining key is the module name
module_name = module.keys()[0]
# empty options
if module[module_name] is None:
run_module(module_name, [])
continue
module_argv = []
for option, value in module[module_name].iteritems():
value = evaluate_value(value)
if value is None:
module_argv.append('--{}'.format(option))
else:
module_argv.append('--{}={}'.format(option, value))
run_module(module_name, module_argv)