def __init__(self, filepath, logger=None):
self.logger = logger or Logging.get_logger()
logger.connect(self)
pattern_map = load_yaml(filepath, logger=self.logger)
if pattern_map is None:
raise GlueError(
"pattern map '{}' does not contain any patterns".format(
filepath))
self._compiled_map = []
for pattern_dict in pattern_map:
if not isinstance(pattern_dict, dict):
raise GlueError(
"Invalid format: '- <pattern>: <result>' expected, '{}' found"
.format(pattern_dict))
pattern = pattern_dict.keys()[0]
result = pattern_dict[pattern].strip()
try:
pattern = re.compile(pattern)
except re.error as exc:
raise GlueError("Pattern '{}' is not valid: {}".format(
pattern, str(exc)))
self._compiled_map.append((pattern, result))
def load_yaml(filepath, logger=None):
"""
Load data stored in YAML file, and return their Python representation.
:param str filepath: Path to a file. ``~`` or ``~<username>`` are expanded before using.
:param gluetool.log.ContextLogger logger: Logger used for logging.
:rtype: object
:returns: structures representing data in the file.
:raises gluetool.glue.GlueError: if it was not possible to successfully load content of the file.
"""
if not filepath:
raise GlueError('File path is not valid: {}'.format(filepath))
logger = logger or Logging.get_logger()
real_filepath = normalize_path(filepath)
logger.debug("attempt to load YAML from '{}' (maps to '{}')".format(
filepath, real_filepath))
if not os.path.exists(real_filepath):
raise GlueError("File '{}' does not exist".format(filepath))
try:
with open(real_filepath, 'r') as f:
data = YAML.load(f)
logger.debug("loaded YAML data from '{}':\n{}".format(
filepath, format_dict(data)))
return data
except ruamel.yaml.YAMLError as e:
raise GlueError("Unable to load YAML file '{}': {}".format(
filepath, str(e)))
def dump_yaml(data, filepath, logger=None):
"""
Save data stored in variable to YAML file.
:param object data: Data to store in YAML file
:param str filepath: Path to an output file.
:raises gluetool.glue.GlueError: if it was not possible to successfully save data to file.
"""
if not filepath:
raise GlueError("File path is not valid: '{}'".format(filepath))
logger = logger or Logging.get_logger()
real_filepath = normalize_path(filepath)
dirpath = os.path.dirname(real_filepath)
if not os.path.exists(dirpath):
raise GlueError(
"Cannot save file in nonexistent directory '{}'".format(dirpath))
try:
with open(real_filepath, 'w') as f:
YAML.dump(data, f)
f.flush()
except ruamel.yaml.YAMLError as e:
raise GlueError("Unable to save YAML file '{}': {}".format(
filepath, str(e)))
def treat_url(url, logger=None):
"""
Remove "weird" artifacts from the given URL. Collapse adjacent '.'s, apply '..', etc.
:param str url: URL to clear.
:param gluetool.log.ContextAdapter logger: logger to use for logging.
:rtype: str
:returns: Treated URL.
"""
logger = logger or Logging.get_logger()
logger.debug("treating a URL '{}'".format(url))
try:
url = str(urlnorm.norm(url))
except urlnorm.InvalidUrl as exc:
# urlnorm cannot handle localhost: https://github.com/jehiah/urlnorm/issues/3
if exc.message == "host u'localhost' is not valid":
pass
else:
raise exc
return url.strip()
def fetch_url(url, logger=None, success_codes=(200, )):
"""
"Get me content of this URL" helper.
Very thin wrapper around urllib. Added value is logging, and converting
possible errors to :py:class:`gluetool.glue.GlueError` exception.
:param str url: URL to get.
:param gluetool.log.ContextLogger logger: Logger used for logging.
:param tuple success_codes: tuple of HTTP response codes representing successfull request.
:returns: tuple ``(response, content)`` where ``response`` is what :py:func:`urllib2.urlopen`
returns, and ``content`` is the payload of the response.
"""
logger = logger or Logging.get_logger()
logger.debug("opening URL '{}'".format(url))
try:
response = urllib2.urlopen(url)
code, content = response.getcode(), response.read()
except urllib2.HTTPError as exc:
logger.exception(exc)
raise GlueError("Failed to fetch URL '{}': {}".format(
url, exc.message))
log_blob(logger.debug, '{}: {}'.format(url, code), content)
if code not in success_codes:
raise GlueError("Unsuccessfull response from '{}'".format(url))
return response, content
def wait(label, check, timeout=None, tick=30, logger=None):
"""
Wait for a condition to be true.
:param str label: printable label used for logging.
:param callable check: called to test the condition. If its return value evaluates as ``True``,
the condition is assumed to pass the test and waiting ends.
:param int timeout: fail after this many seconds. ``None`` means test forever.
:param int tick: test condition every ``tick`` seconds.
:param gluetool.log.ContextAdapter logger: parent logger whose methods will be used for logging.
:raises gluetool.glue.GlueError: when ``timeout`` elapses while condition did not pass the check.
"""
if not isinstance(tick, int):
raise GlueError('Tick must be an integer')
if tick < 0:
raise GlueError('Tick must be a positive integer')
logger = logger or Logging.get_logger()
if timeout is not None:
end_time = time.time() + timeout
def _timeout():
return '{} seconds'.format(
int(end_time - time.time())) if timeout is not None else 'infinite'
logger.debug(
"waiting for condition '{}', timeout {}, check every {} seconds".
format(label, _timeout(), tick))
while timeout is None or time.time() < end_time:
logger.debug("calling callback function")
ret = check()
if ret:
logger.debug('check passed, assuming success')
return ret
logger.debug("check failed with {}, assuming failure".format(ret))
logger.debug('{} left, sleeping for {} seconds'.format(
_timeout(), tick))
time.sleep(tick)
raise GlueError(
"Condition '{}' failed to pass within given time".format(label))
def __init__(self, executable, options=None, logger=None):
self.logger = logger or ContextAdapter(Logging.get_logger())
self.logger.connect(self)
self.executable = executable
self.options = options or []
self.use_shell = False
self.quote_args = False
self._command = None
self._popen_kwargs = None
self._process = None
self._exit_code = None
self._stdout = None
self._stderr = None
def render_template(template, logger=None, **kwargs):
"""
Render Jinja2 template. Logs errors, and raises an exception when it's not possible
to correctly render the template.
:param template: Template to render. It can be either :py:class:`jinja2.environment.Template` instance,
or a string.
:param dict kwargs: Keyword arguments passed to render process.
:rtype: str
:returns: Rendered template.
:raises gluetool.glue.GlueError: when the rednering failed.
"""
logger = logger or Logging.get_logger()
try:
def _render(template):
log_blob(logger.debug, 'rendering template', template.source)
log_dict(logger.debug, 'context', kwargs)
return str(template.render(**kwargs).strip())
if isinstance(template, (str, unicode)):
source, template = template, jinja2.Template(template)
template.source = source
return _render(template)
if isinstance(template, jinja2.environment.Template):
if template.filename != '<template>':
with open(template.filename, 'r') as f:
template.source = f.read()
else:
template.source = '<unknown template source>'
return _render(template)
raise GlueError('Unhandled template type {}'.format(type(template)))
except Exception as exc:
raise GlueError('Cannot render template: {}'.format(exc))
def __init__(self, filepath, spices=None, logger=None):
self.logger = logger or Logging.get_logger()
logger.connect(self)
spices = spices or {}
pattern_map = load_yaml(filepath, logger=self.logger)
if pattern_map is None:
raise GlueError(
"pattern map '{}' does not contain any patterns".format(
filepath))
def _create_simple_repl(repl):
def _replace(pattern, target):
"""
Use `repl` to construct image from `target`, honoring all backreferences made by `pattern`.
"""
self.debug("pattern '{}', repl '{}', target '{}'".format(
pattern.pattern, repl, target))
try:
return pattern.sub(repl, target)
except re.error as e:
raise GlueError(
"Cannot transform pattern '{}' with target '{}', repl '{}': {}"
.format(pattern.pattern, target, repl, str(e)))
return _replace
self._compiled_map = []
for pattern_dict in pattern_map:
log_dict(logger.debug, 'pattern dict', pattern_dict)
if not isinstance(pattern_dict, dict):
raise GlueError(
"Invalid format: '- <pattern>: <transform>' expected, '{}' found"
.format(pattern_dict))
pattern = pattern_dict.keys()[0]
converter_chains = pattern_dict[pattern]
if isinstance(converter_chains, str):
converter_chains = [converter_chains]
try:
pattern = re.compile(pattern)
except re.error as e:
raise GlueError("Pattern '{}' is not valid: {}".format(
pattern, str(e)))
compiled_chains = []
for chain in converter_chains:
converters = [s.strip() for s in chain.split(',')]
# first item in `converters` is always a simple string used by `pattern.sub()` call
converter = _create_simple_repl(converters.pop(0))
# if there any any items left, they name "spices" to apply, one by one,
# on the result of the first operation
for spice in converters:
if spice not in spices:
raise GlueError(
"Unknown 'spice' function '{}'".format(spice))
converter = spices[spice](converter)
compiled_chains.append(converter)
self._compiled_map.append((pattern, compiled_chains))
def run_command(cmd, logger=None, inspect=False, inspect_callback=None, **kwargs):
"""
Run external command, and return it's exit code and output.
This is a very thin and simple wrapper above :py:class:`subprocess.Popen`,
and its main purpose is to log everything that happens before and after
execution. All additional arguments are passed directly to `Popen` constructor.
If ``stdout`` or ``stderr`` keyword arguments are not specified, function
will set them to :py:const:`subprocess.PIPE`, to capture both output streams
in separate strings.
By default, output of the process is captured for both ``stdout`` and ``stderr``,
and returned back to the caller. Under some conditions, caller might want to see
the output in "real-time". For that purpose, it can pass callable via ``inspect_callback``
parameter - such callable will be called for every received bit of input on both
``stdout`` and ``stderr``. E.g.
.. code-block:: python
def foo(stream, s, flush=False):
if s is not None and 'a' in s:
print s
run_command(['/bin/foo'], inspect=foo)
This example will print all substrings containing letter `a`. Strings passed to ``foo``
may be of arbitrary lengths, and may change between subsequent calls of ``run_command``.
:param list cmd: command to execute.
:param gluetool.log.ContextAdapter logger: parent logger whose methods will be used for logging.
:param bool inspect: if set, ``inspect_callback`` will receive the output of command in "real-time".
:param callable inspect_callback: callable that will receive command output. If not set,
default "write to ``sys.stdout``" is used.
:rtype: gluetool.utils.ProcessOutput instance
:returns: :py:class:`gluetool.utils.ProcessOutput` instance whose attributes contain
data returned by the process.
:raises gluetool.glue.GlueError: when command was not found.
:raises gluetool.glue.GlueCommandError: when command exited with non-zero exit code.
:raises Exception: when anything else breaks.
"""
assert isinstance(cmd, list), 'Only list of strings accepted as a command'
if not all((isinstance(s, str) for s in cmd)):
raise GlueError('Only list of strings accepted as a command, {} found'.format([type(s) for s in cmd]))
logger = logger or Logging.get_logger()
stdout, stderr = None, None
# Set default stdout/stderr, unless told otherwise
if 'stdout' not in kwargs:
kwargs['stdout'] = subprocess.PIPE
if 'stderr' not in kwargs:
kwargs['stderr'] = subprocess.PIPE
def _format_stream(stream):
if stream == subprocess.PIPE:
return 'PIPE'
if stream == DEVNULL:
return 'DEVNULL'
if stream == subprocess.STDOUT:
return 'STDOUT'
return stream
printable_kwargs = kwargs.copy()
for stream in ('stdout', 'stderr'):
if stream in printable_kwargs:
printable_kwargs[stream] = _format_stream(printable_kwargs[stream])
# Make tests happy by sorting kwargs - it's a dictionary, therefore
# unpredictable from the observer's point of view. Can print its entries
# in different order with different Pythons, making tests a mess.
# sorted_kwargs = ', '.join(["'%s': '%s'" % (k, printable_kwargs[k]) for k in sorted(printable_kwargs.iterkeys())])
log_dict(logger.debug, 'command', cmd)
log_dict(logger.debug, 'kwargs', printable_kwargs)
log_blob(logger.debug, 'runnable (copy & paste)', format_command_line([cmd]))
try:
p = subprocess.Popen(cmd, **kwargs)
if inspect is True:
# let's capture *both* streams - capturing just a single one leads to so many ifs
# and elses and messy code
p_stdout = StreamReader(p.stdout, name='<stdout>')
p_stderr = StreamReader(p.stderr, name='<stderr>')
if inspect_callback is None:
def stdout_write(stream, data, flush=False):
# pylint: disable=unused-argument
if data is None:
return
# Not suitable for multiple simultaneous commands. Shuffled output will
# ruin your day. And night. And few following weeks, full of debugging, as well.
sys.stdout.write(data)
sys.stdout.flush()
inspect_callback = stdout_write
inputs = (p_stdout, p_stderr)
with BlobLogger('Output of command: {}'.format(format_command_line([cmd])), outro='End of command output',
writer=logger.info):
logger.debug("output of command is inspected by the caller")
logger.debug('following blob-like header and footer are expected to be empty')
logger.debug('the captured output will follow them')
# As long as process runs, keep calling callbacks with incoming data
while True:
for stream in inputs:
inspect_callback(stream, stream.read())
if p.poll() is not None:
break
# give up OS' attention and let others run
# time.sleep(0) is a Python synonym for "thread yields the rest of its quantum"
time.sleep(0.1)
# OK, process finished but we have to wait for our readers to finish as well
p_stdout.wait()
p_stderr.wait()
for stream in inputs:
while True:
data = stream.read()
if data in ('', None):
break
inspect_callback(stream, data)
inspect_callback(stream, None, flush=True)
stdout, stderr = p_stdout.content, p_stderr.content
else:
stdout, stderr = p.communicate()
except OSError as e:
if e.errno == errno.ENOENT:
raise GlueError("Command '{}' not found".format(cmd[0]))
raise e
exit_code = p.poll()
output = ProcessOutput(cmd, exit_code, stdout, stderr, kwargs)
output.log(logger.debug)
if exit_code != 0:
raise GlueCommandError(cmd, output)
return output