def load_yaml(filepath, logger=None):
"""
Load data stored in YAML file, and return their Python representation.
:param str filepath: Path to a file. ``~`` or ``~<username>`` are expanded before using.
:param gluetool.log.ContextLogger logger: Logger used for logging.
:rtype: object
:returns: structures representing data in the file.
:raises gluetool.glue.GlueError: if it was not possible to successfully load content of the file.
"""
if not filepath:
raise GlueError('File path is not valid: {}'.format(filepath))
logger = logger or Logging.get_logger()
real_filepath = normalize_path(filepath)
logger.debug("attempt to load YAML from '{}' (maps to '{}')".format(
filepath, real_filepath))
if not os.path.exists(real_filepath):
raise GlueError("File '{}' does not exist".format(filepath))
try:
with open(real_filepath, 'r') as f:
data = YAML.load(f)
logger.debug("loaded YAML data from '{}':\n{}".format(
filepath, format_dict(data)))
return data
except ruamel.yaml.YAMLError as e:
raise GlueError("Unable to load YAML file '{}': {}".format(
filepath, str(e)))
def dump_yaml(data, filepath, logger=None):
"""
Save data stored in variable to YAML file.
:param object data: Data to store in YAML file
:param str filepath: Path to an output file.
:raises gluetool.glue.GlueError: if it was not possible to successfully save data to file.
"""
if not filepath:
raise GlueError("File path is not valid: '{}'".format(filepath))
logger = logger or Logging.get_logger()
real_filepath = normalize_path(filepath)
dirpath = os.path.dirname(real_filepath)
if not os.path.exists(dirpath):
raise GlueError(
"Cannot save file in nonexistent directory '{}'".format(dirpath))
try:
with open(real_filepath, 'w') as f:
YAML.dump(data, f)
f.flush()
except ruamel.yaml.YAMLError as e:
raise GlueError("Unable to save YAML file '{}': {}".format(
filepath, str(e)))
def __init__(self, filepath, logger=None):
self.logger = logger or Logging.get_logger()
logger.connect(self)
pattern_map = load_yaml(filepath, logger=self.logger)
if pattern_map is None:
raise GlueError(
"pattern map '{}' does not contain any patterns".format(
filepath))
self._compiled_map = []
for pattern_dict in pattern_map:
if not isinstance(pattern_dict, dict):
raise GlueError(
"Invalid format: '- <pattern>: <result>' expected, '{}' found"
.format(pattern_dict))
pattern = pattern_dict.keys()[0]
result = pattern_dict[pattern].strip()
try:
pattern = re.compile(pattern)
except re.error as exc:
raise GlueError("Pattern '{}' is not valid: {}".format(
pattern, str(exc)))
self._compiled_map.append((pattern, result))
def fetch_url(url, logger=None, success_codes=(200, )):
"""
"Get me content of this URL" helper.
Very thin wrapper around urllib. Added value is logging, and converting
possible errors to :py:class:`gluetool.glue.GlueError` exception.
:param str url: URL to get.
:param gluetool.log.ContextLogger logger: Logger used for logging.
:param tuple success_codes: tuple of HTTP response codes representing successfull request.
:returns: tuple ``(response, content)`` where ``response`` is what :py:func:`urllib2.urlopen`
returns, and ``content`` is the payload of the response.
"""
logger = logger or Logging.get_logger()
logger.debug("opening URL '{}'".format(url))
try:
response = urllib2.urlopen(url)
code, content = response.getcode(), response.read()
except urllib2.HTTPError as exc:
logger.exception(exc)
raise GlueError("Failed to fetch URL '{}': {}".format(
url, exc.message))
log_blob(logger.debug, '{}: {}'.format(url, code), content)
if code not in success_codes:
raise GlueError("Unsuccessfull response from '{}'".format(url))
return response, content
def _check_types(items):
if not isinstance(items, list):
raise GlueError('Only list of strings is accepted')
if not all((isinstance(s, str) for s in items)):
raise GlueError(
'Only list of strings is accepted, {} found'.format(
[type(s) for s in items]))
def test_sentry_fingerprint(current, explicit):
if not explicit:
assert GlueError(
'',
sentry_fingerprint=explicit).sentry_fingerprint(current) is current
else:
assert GlueError('', sentry_fingerprint=explicit).sentry_fingerprint(
current) == ['GlueError'] + explicit
def run_pipeline(self):
Glue = self.Glue
# no modules
if not self.pipeline_desc:
raise GlueError('No module specified, use -l to list available')
# command-line info
if Glue.option('info'):
self.log_cmdline(self.argv, self.pipeline_desc)
# actually the execution loop is retries+1
# there is always one execution
retries = Glue.option('retries')
for loop_number in range(retries + 1):
try:
# Reset pipeline - destroy all modules that exist so far
Glue.destroy_modules()
# Print retry info
if loop_number:
Glue.warn('retrying execution (attempt #{} out of {})'.format(loop_number, retries))
# Run the pipeline
Glue.run_modules(self.pipeline_desc, register=True)
except GlueRetryError as e:
Glue.error(e)
continue
break
def match(self, s, multiple=False):
"""
Try to match ``s`` by the map. If the match is found - the first one wins - then its
conversions are applied to the ``s``.
There can be multiple conversions for a pattern, by default only the product of
the first one is returned. If ``multiple`` is set to ``True``, list of all products
is returned instead.
:rtype: str
:returns: if matched, output of the corresponding transformation.
"""
self.debug(
"trying to match string '{}' with patterns in the map".format(s))
for pattern, converters in self._compiled_map:
self.debug("testing pattern '{}'".format(pattern.pattern))
match = pattern.match(s)
if match is None:
continue
self.debug(' matched!')
if multiple is not True:
return converters[0](pattern, s)
return [converter(pattern, s) for converter in converters]
raise GlueError(
"Could not match string '{}' with any pattern".format(s))
def wait(label, check, timeout=None, tick=30, logger=None):
"""
Wait for a condition to be true.
:param str label: printable label used for logging.
:param callable check: called to test the condition. If its return value evaluates as ``True``,
the condition is assumed to pass the test and waiting ends.
:param int timeout: fail after this many seconds. ``None`` means test forever.
:param int tick: test condition every ``tick`` seconds.
:param gluetool.log.ContextAdapter logger: parent logger whose methods will be used for logging.
:raises gluetool.glue.GlueError: when ``timeout`` elapses while condition did not pass the check.
"""
if not isinstance(tick, int):
raise GlueError('Tick must be an integer')
if tick < 0:
raise GlueError('Tick must be a positive integer')
logger = logger or Logging.get_logger()
if timeout is not None:
end_time = time.time() + timeout
def _timeout():
return '{} seconds'.format(
int(end_time - time.time())) if timeout is not None else 'infinite'
logger.debug(
"waiting for condition '{}', timeout {}, check every {} seconds".
format(label, _timeout(), tick))
while timeout is None or time.time() < end_time:
logger.debug("calling callback function")
ret = check()
if ret:
logger.debug('check passed, assuming success')
return ret
logger.debug("check failed with {}, assuming failure".format(ret))
logger.debug('{} left, sleeping for {} seconds'.format(
_timeout(), tick))
time.sleep(tick)
raise GlueError(
"Condition '{}' failed to pass within given time".format(label))
def check_for_commands(cmds):
""" Checks if all commands in list cmds are valid """
for cmd in cmds:
try:
run_command(['/bin/bash', '-c', 'command -v {}'.format(cmd)], stdout=DEVNULL)
except GlueError:
raise GlueError("Command '{}' not found on the system".format(cmd))
def render_template(template, logger=None, **kwargs):
"""
Render Jinja2 template. Logs errors, and raises an exception when it's not possible
to correctly render the template.
:param template: Template to render. It can be either :py:class:`jinja2.environment.Template` instance,
or a string.
:param dict kwargs: Keyword arguments passed to render process.
:rtype: str
:returns: Rendered template.
:raises gluetool.glue.GlueError: when the rednering failed.
"""
logger = logger or Logging.get_logger()
try:
def _render(template):
log_blob(logger.debug, 'rendering template', template.source)
log_dict(logger.debug, 'context', kwargs)
return str(template.render(**kwargs).strip())
if isinstance(template, (str, unicode)):
source, template = template, jinja2.Template(template)
template.source = source
return _render(template)
if isinstance(template, jinja2.environment.Template):
if template.filename != '<template>':
with open(template.filename, 'r') as f:
template.source = f.read()
else:
template.source = '<unknown template source>'
return _render(template)
raise GlueError('Unhandled template type {}'.format(type(template)))
except Exception as exc:
raise GlueError('Cannot render template: {}'.format(exc))
def _replace(pattern, target):
"""
Use `repl` to construct image from `target`, honoring all backreferences made by `pattern`.
"""
self.debug("pattern '{}', repl '{}', target '{}'".format(pattern.pattern, repl, target))
try:
return pattern.sub(repl, target)
except re.error as e:
raise GlueError("Cannot transform pattern '{}' with target '{}', repl '{}': {}".format(
pattern.pattern, target, repl, str(e)))
def _deduce_pipeline_desc(self, argv, modules):
# type: (List[Any], List[str]) -> List[gluetool.glue.PipelineStep]
# pylint: disable=no-self-use
"""
Split command-line arguments, left by ``gluetool``, into a pipeline description, splitting them
by modules and their options.
:param list argv: Remainder of :py:data:`sys.argv` after removing ``gluetool``'s own options.
:param list(str) modules: List of known module names.
:returns: Pipeline description in a form of a list of :py:class:`gluetool.glue.PipelineStep` instances.
"""
alias_pattern = re.compile(r'^([a-z\-]*):([a-z\-]*)$', re.I)
pipeline_desc = []
step = None
while argv:
arg = argv.pop(0)
# is the "arg" a module name? If so, add new step to the pipeline
if arg in modules:
step = PipelineStep(arg)
pipeline_desc.append(step)
continue
# is the "arg" a module with an alias? If so, add a new step to the pipeline, and note the alias
match = alias_pattern.match(arg)
if match is not None:
module, actual_module = match.groups()
step = PipelineStep(module, actual_module=actual_module)
pipeline_desc.append(step)
continue
if step is None:
raise GlueError(
"Cannot parse module argument: '{}'".format(arg))
step.argv.append(arg)
return pipeline_desc
def test_caused_by_detect():
mock_exc = ValueError('dummy error')
with pytest.raises(GlueError, match=r'dummy message') as excinfo:
try:
raise mock_exc
except ValueError:
raise GlueError('dummy message')
exc = excinfo.value
assert isinstance(exc.caused_by, tuple)
assert len(exc.caused_by) == 3
klass, error, tb = exc.caused_by
assert klass is mock_exc.__class__
assert error is mock_exc
assert isinstance(tb, types.TracebackType)
def match(self, s):
"""
Try to match ``s`` by the map. If the match is found - the first one wins - then its
transformation is applied to the ``s``.
:rtype: str
:returns: if matched, output of the corresponding transformation.
"""
self.debug("trying to match string '{}' with patterns in the map".format(s))
for pattern, result in self._compiled_map:
self.debug("testing pattern '{}'".format(pattern.pattern))
match = pattern.match(s)
if match is None:
continue
self.debug(' matched!')
return result
raise GlueError("Could not match string '{}' with any pattern".format(s))
def test_sentry_fingerprint(current):
assert GlueError('').sentry_fingerprint(current) == current
def test_caused_by_detect_empty():
with pytest.raises(GlueError) as excinfo:
raise GlueError('')
assert excinfo.value.caused_by is None
def test_caused_by_explicit(caused_by):
expected = None if caused_by == (None, None, None) else caused_by
assert GlueError('dummy message',
caused_by=caused_by).caused_by == expected
def test_message(message):
with pytest.raises(GlueError) as excinfo:
raise GlueError(message)
assert excinfo.value.message == message
def _sigusr1_handler(signum, frame):
# type: (int, FrameType) -> None
# pylint: disable=unused-argument
raise GlueError('Pipeline timeout expired.')
def run(self, inspect=False, inspect_callback=None, **kwargs):
"""
Run the command, wait for it to finish and return the output.
:param bool inspect: If set, ``inspect_callback`` will receive the output of command in "real-time".
:param callable inspect_callback: callable that will receive command output. If not set, default
"write to ``sys.stdout``" is used.
:rtype: gluetool.utils.ProcessOutput instance
:returns: :py:class:`gluetool.utils.ProcessOutput` instance whose attributes contain data returned
by the child process.
:raises gluetool.glue.GlueError: When somethign went wrong.
:raises gluetool.glue.GlueCommandError: When command exited with non-zero exit code.
"""
# pylint: disable=too-many-branches
def _check_types(items):
if not isinstance(items, list):
raise GlueError('Only list of strings is accepted')
if not all((isinstance(s, str) for s in items)):
raise GlueError(
'Only list of strings is accepted, {} found'.format(
[type(s) for s in items]))
_check_types(self.executable)
_check_types(self.options)
self._command = self._apply_quotes()
if self.use_shell is True:
self._command = [' '.join(self._command)]
# Set default stdout/stderr, unless told otherwise
if 'stdout' not in kwargs:
kwargs['stdout'] = subprocess.PIPE
if 'stderr' not in kwargs:
kwargs['stderr'] = subprocess.PIPE
if self.use_shell:
kwargs['shell'] = True
self._popen_kwargs = kwargs
def _format_stream(stream):
if stream == subprocess.PIPE:
return 'PIPE'
if stream == DEVNULL:
return 'DEVNULL'
if stream == subprocess.STDOUT:
return 'STDOUT'
return stream
printable_kwargs = kwargs.copy()
for stream in ('stdout', 'stderr'):
if stream in printable_kwargs:
printable_kwargs[stream] = _format_stream(
printable_kwargs[stream])
log_dict(self.debug, 'command', self._command)
log_dict(self.debug, 'kwargs', printable_kwargs)
log_blob(self.debug, 'runnable (copy & paste)',
format_command_line([self._command]))
try:
self._process = subprocess.Popen(self._command,
**self._popen_kwargs)
if inspect is True:
self._communicate_inspect(inspect_callback)
else:
self._communicate_batch()
except OSError as e:
if e.errno == errno.ENOENT:
raise GlueError("Command '{}' not found".format(
self._command[0]))
raise e
self._exit_code = self._process.poll()
output = self._construct_output()
if self._exit_code != 0:
raise GlueCommandError(self._command, output)
return output
def test_sentry_tags(current):
assert GlueError('').sentry_tags(current) == current
def run_command(cmd, logger=None, inspect=False, inspect_callback=None, **kwargs):
"""
Run external command, and return it's exit code and output.
This is a very thin and simple wrapper above :py:class:`subprocess.Popen`,
and its main purpose is to log everything that happens before and after
execution. All additional arguments are passed directly to `Popen` constructor.
If ``stdout`` or ``stderr`` keyword arguments are not specified, function
will set them to :py:const:`subprocess.PIPE`, to capture both output streams
in separate strings.
By default, output of the process is captured for both ``stdout`` and ``stderr``,
and returned back to the caller. Under some conditions, caller might want to see
the output in "real-time". For that purpose, it can pass callable via ``inspect_callback``
parameter - such callable will be called for every received bit of input on both
``stdout`` and ``stderr``. E.g.
.. code-block:: python
def foo(stream, s, flush=False):
if s is not None and 'a' in s:
print s
run_command(['/bin/foo'], inspect=foo)
This example will print all substrings containing letter `a`. Strings passed to ``foo``
may be of arbitrary lengths, and may change between subsequent calls of ``run_command``.
:param list cmd: command to execute.
:param gluetool.log.ContextAdapter logger: parent logger whose methods will be used for logging.
:param bool inspect: if set, ``inspect_callback`` will receive the output of command in "real-time".
:param callable inspect_callback: callable that will receive command output. If not set,
default "write to ``sys.stdout``" is used.
:rtype: gluetool.utils.ProcessOutput instance
:returns: :py:class:`gluetool.utils.ProcessOutput` instance whose attributes contain
data returned by the process.
:raises gluetool.glue.GlueError: when command was not found.
:raises gluetool.glue.GlueCommandError: when command exited with non-zero exit code.
:raises Exception: when anything else breaks.
"""
assert isinstance(cmd, list), 'Only list of strings accepted as a command'
if not all((isinstance(s, str) for s in cmd)):
raise GlueError('Only list of strings accepted as a command, {} found'.format([type(s) for s in cmd]))
logger = logger or Logging.get_logger()
stdout, stderr = None, None
# Set default stdout/stderr, unless told otherwise
if 'stdout' not in kwargs:
kwargs['stdout'] = subprocess.PIPE
if 'stderr' not in kwargs:
kwargs['stderr'] = subprocess.PIPE
def _format_stream(stream):
if stream == subprocess.PIPE:
return 'PIPE'
if stream == DEVNULL:
return 'DEVNULL'
if stream == subprocess.STDOUT:
return 'STDOUT'
return stream
printable_kwargs = kwargs.copy()
for stream in ('stdout', 'stderr'):
if stream in printable_kwargs:
printable_kwargs[stream] = _format_stream(printable_kwargs[stream])
# Make tests happy by sorting kwargs - it's a dictionary, therefore
# unpredictable from the observer's point of view. Can print its entries
# in different order with different Pythons, making tests a mess.
# sorted_kwargs = ', '.join(["'%s': '%s'" % (k, printable_kwargs[k]) for k in sorted(printable_kwargs.iterkeys())])
log_dict(logger.debug, 'command', cmd)
log_dict(logger.debug, 'kwargs', printable_kwargs)
log_blob(logger.debug, 'runnable (copy & paste)', format_command_line([cmd]))
try:
p = subprocess.Popen(cmd, **kwargs)
if inspect is True:
# let's capture *both* streams - capturing just a single one leads to so many ifs
# and elses and messy code
p_stdout = StreamReader(p.stdout, name='<stdout>')
p_stderr = StreamReader(p.stderr, name='<stderr>')
if inspect_callback is None:
def stdout_write(stream, data, flush=False):
# pylint: disable=unused-argument
if data is None:
return
# Not suitable for multiple simultaneous commands. Shuffled output will
# ruin your day. And night. And few following weeks, full of debugging, as well.
sys.stdout.write(data)
sys.stdout.flush()
inspect_callback = stdout_write
inputs = (p_stdout, p_stderr)
with BlobLogger('Output of command: {}'.format(format_command_line([cmd])), outro='End of command output',
writer=logger.info):
logger.debug("output of command is inspected by the caller")
logger.debug('following blob-like header and footer are expected to be empty')
logger.debug('the captured output will follow them')
# As long as process runs, keep calling callbacks with incoming data
while True:
for stream in inputs:
inspect_callback(stream, stream.read())
if p.poll() is not None:
break
# give up OS' attention and let others run
# time.sleep(0) is a Python synonym for "thread yields the rest of its quantum"
time.sleep(0.1)
# OK, process finished but we have to wait for our readers to finish as well
p_stdout.wait()
p_stderr.wait()
for stream in inputs:
while True:
data = stream.read()
if data in ('', None):
break
inspect_callback(stream, data)
inspect_callback(stream, None, flush=True)
stdout, stderr = p_stdout.content, p_stderr.content
else:
stdout, stderr = p.communicate()
except OSError as e:
if e.errno == errno.ENOENT:
raise GlueError("Command '{}' not found".format(cmd[0]))
raise e
exit_code = p.poll()
output = ProcessOutput(cmd, exit_code, stdout, stderr, kwargs)
output.log(logger.debug)
if exit_code != 0:
raise GlueCommandError(cmd, output)
return output
def test_sentry_tags(current, explicit):
expected = current.copy()
if explicit:
expected.update(explicit)
assert GlueError('', sentry_tags=explicit).sentry_tags(current) == expected
def __init__(self, filepath, spices=None, logger=None):
self.logger = logger or Logging.get_logger()
logger.connect(self)
spices = spices or {}
pattern_map = load_yaml(filepath, logger=self.logger)
if pattern_map is None:
raise GlueError(
"pattern map '{}' does not contain any patterns".format(
filepath))
def _create_simple_repl(repl):
def _replace(pattern, target):
"""
Use `repl` to construct image from `target`, honoring all backreferences made by `pattern`.
"""
self.debug("pattern '{}', repl '{}', target '{}'".format(
pattern.pattern, repl, target))
try:
return pattern.sub(repl, target)
except re.error as e:
raise GlueError(
"Cannot transform pattern '{}' with target '{}', repl '{}': {}"
.format(pattern.pattern, target, repl, str(e)))
return _replace
self._compiled_map = []
for pattern_dict in pattern_map:
log_dict(logger.debug, 'pattern dict', pattern_dict)
if not isinstance(pattern_dict, dict):
raise GlueError(
"Invalid format: '- <pattern>: <transform>' expected, '{}' found"
.format(pattern_dict))
pattern = pattern_dict.keys()[0]
converter_chains = pattern_dict[pattern]
if isinstance(converter_chains, str):
converter_chains = [converter_chains]
try:
pattern = re.compile(pattern)
except re.error as e:
raise GlueError("Pattern '{}' is not valid: {}".format(
pattern, str(e)))
compiled_chains = []
for chain in converter_chains:
converters = [s.strip() for s in chain.split(',')]
# first item in `converters` is always a simple string used by `pattern.sub()` call
converter = _create_simple_repl(converters.pop(0))
# if there any any items left, they name "spices" to apply, one by one,
# on the result of the first operation
for spice in converters:
if spice not in spices:
raise GlueError(
"Unknown 'spice' function '{}'".format(spice))
converter = spices[spice](converter)
compiled_chains.append(converter)
self._compiled_map.append((pattern, compiled_chains))