def load_files(paths):
"""
Recursively load a list of paths and merge their content left to right.
That is, last to load will override last.
:param list paths: List of Path objects pointing to files to load.
:return: Merged content of all files.
:rtype: dict
"""
bundle = {}
# Load files
# The returned dict of a parsed file cannot be guaranteed consistently
# ordered, so sadly here we loose sequentially of declaration in files.
for file in paths:
log.info('Loading file {} ...'.format(file))
content = load_file(file)
log.debug('Content loaded:\n{}'.format(pformat(content)))
# Update the general bundle
update(bundle, content)
if bundle:
log.debug('Final bundle:\n{}'.format(pformat(bundle)))
return bundle
def print_progress(self, message):
try:
data = json.loads(message)
if not isinstance(data, dict):
raise ValueError
if 'status' in data and 'progress' in data:
sys.stdout.write('\r[%s] %s: %s' % (data['id'], data['status'], data['progress']))
elif 'status' in data and 'id' in data:
sys.stdout.write('\n[%s] %s' % (data['id'], data['status']))
elif 'status' in data:
sys.stdout.write('\n%s' % (data['status']))
else:
if isinstance(data, basestring):
sys.stdout.write(data)
else:
print pprintpp.pformat(data)
except ValueError as e:
message = message.encode('utf-8')
sys.stdout.write(message)
def load_values(values_files, values):
"""
:param list values_files: List of Path objects pointing to files with
values for the rendering.
:param OrderedDict values: Dictionary with keys in dot-notation and its
associated values.
:return: Normalized values loaded from all files and overrode with the
values dictionary, if any.
:rtype: dict
"""
bundle = load_files(values_files)
if values:
log.debug('Expanding dot-notation dictionary:\n{}'.format(
pformat(values)))
expanded = expand_dotdict(values)
log.debug('Expanded dot-notation dictionary:\n{}'.format(
pformat(expanded)))
update(bundle, expanded)
if bundle:
log.debug('Final values bundle:\n{}'.format(pformat(bundle)))
return bundle
def pytest_assertrepr_compare(config, op, left, right):
if op != '==':
return
return ['equals failed'] + list(icdiff.ConsoleDiff(tabsize=2).make_table(
pformat(left, width=40).splitlines(),
pformat(right, width=40).splitlines(),
))
async def wrapper(request):
# Check media type
if request.content_type != 'application/json':
raise web.HTTPUnsupportedMediaType(
text=(
'Invalid Content-Type "{}". '
'Only "application/json" is supported.'
).format(request.content_type)
)
# Parse JSON request
body = await request.text()
try:
payload = loads(body)
except ValueError:
log.error('Invalid JSON payload:\n{}'.format(body))
raise web.HTTPBadRequest(
text='Invalid JSON payload'
)
# Log request and responses
log.info('Request:\n{}'.format(pformat(payload)))
response = await handler(request, payload)
log.info('Response:\n{}'.format(pformat(response)))
# Convert dictionaries to JSON responses
if isinstance(response, dict):
return web.json_response(response)
return response
def __init__(self, lhs, rhs, width, show_symbols=False) -> None:
self.width = width
self.lhs = lhs if isinstance(lhs, str) else pprintpp.pformat(
lhs, width=width)
self.rhs = rhs if isinstance(rhs, str) else pprintpp.pformat(
rhs, width=width)
self.show_symbols = show_symbols
def print_progress(self, message):
try:
data = json.loads(message)
if not isinstance(data, dict):
raise ValueError
if 'status' in data and 'progress' in data:
sys.stdout.write(
'\r[%s] %s: %s' %
(data['id'], data['status'], data['progress']))
elif 'status' in data and 'id' in data:
sys.stdout.write('\n[%s] %s' % (data['id'], data['status']))
elif 'status' in data:
sys.stdout.write('\n%s' % (data['status']))
else:
if isinstance(data, basestring):
sys.stdout.write(data)
else:
print pprintpp.pformat(data)
except ValueError as e:
message = message.encode('utf-8')
sys.stdout.write(message)
def distribute(self, data):
from influxdb import InfluxDBClient
# Allow to filter data
super().distribute(data)
# Get key
if self.config.key.value is None:
key = datetime.now().isoformat()
else:
current = data
for part in self.config.key.value.split('.'):
current = current[part]
key = current
# Connect to database
if self.config.uri.value is None:
client = InfluxDBClient(
host=self.config.host.value,
path=self.config.path.value,
port=self.config.port.value,
username=self.config.username.value,
password=self.config.password.value,
database=self.config.database.value,
ssl=self.config.ssl.value,
verify_ssl=self.config.verify_ssl.value,
)
else:
client = InfluxDBClient.from_dsn(
self.config.uri.value,
path=self.config.path.value,
database=self.config.database.value,
verify_ssl=self.config.verify_ssl.value,
)
# Flat data
log.info('Flattening data for InfluxDB')
flat = transform_to_flat(
data,
self.config.keysjoiner.value,
self.config.keysjoinerreplace.value,
)
del data # Release a bit of ram here
log.debug('Flat data:\n{}'.format(pformat(flat)))
# Insert points
log.info('Creating points using timestamp {}'.format(key))
points = transform_to_points(key, flat)
del flat # Release a bit of ram here
log.debug('Points data:\n{}'.format(pformat(points)))
log.info('Inserting points to InfluxDB: {}'.format([
point['measurement'] for point in points
]))
client.write_points(points)
def test_long_dict(testdir):
one = {
'currency': 'USD',
'default_UK_warehouse': 'xforce',
'default_incoterm': 'EXW',
'name': 'John Doe',
'payment_term': '30% deposit, 70% balance',
'reference': '42551456-a1b3-49bd-beed-b168d9a5ac83',
'website': 'http://megasofas.example.com',
'main_contact': {
'city': 'Madeira',
'country': 'PT',
'email': '*****@*****.**',
'fax': '012356 789039',
'mobile': '012356 789039',
'name': 'Almeida & Filhos - Example, S.A.',
'phone': '253444802010',
'postcode': '4815-123',
'street': "Senhora Test D'Ajuda, 432",
'street2': 'Moreira de Conegos'
},
}
two = {
'currency': 'USD',
'default_UK_warehouse': 'iforce',
'default_incoterm': 'EXW',
'freight_forwarder': 'flexport',
'name': 'John Doe',
'payment_term': '30% deposit, 70% balance',
'reference': '42551456-a1b3-49bd-beed-b168d9a5ac83',
'website': 'http://megasofas.example.com',
'main_contact': {
'name': 'Almeida & Filhos - Example, S.A.',
'email': '*****@*****.**',
'street': "Senhora Test D'Ajuda, 432",
'street2': 'Moreira de Conegos',
'postcode': '4815-123',
'city': 'Madeira',
'country': 'PT',
'phone': '253444802010',
'fax': '012356 789039',
'mobile': '012356 789039'
}
}
testdir.makepyfile(f"""
def test_two():
assert {one!r} == {two!r}
""")
output = testdir.runpytest('-vv', '--color=yes').stdout.str()
expected_lines = icdiff.ConsoleDiff().make_table(
pformat(one).splitlines(),
pformat(two).splitlines(),
)
for l in expected_lines:
assert l.strip() in output
def pytest_assertrepr_compare(config, op, left, right):
if op != '==':
return
icdiff_lines = list(icdiff.ConsoleDiff(tabsize=2).make_table(
pformat(left, width=40).splitlines(),
pformat(right, width=40).splitlines(),
))
if len(icdiff_lines) == 1:
icdiff_lines[0] = hacky_whitespace_reduce(icdiff_lines[0])
return ['equals failed'] + [f'{COLOR_OFF}{l}' for l in icdiff_lines]
def make_diff(lhs, rhs, width=60) -> str:
"""Transform input into best format for diffing, then return output diff."""
if isinstance(lhs, str):
lhs_repr = lhs
else:
lhs_repr = pprintpp.pformat(lhs, width=width)
if isinstance(rhs, str):
rhs_repr = rhs
else:
rhs_repr = pprintpp.pformat(rhs, width=width)
return build_unified_diff(lhs_repr, rhs_repr)
def __init__(
self,
lhs: object,
rhs: object,
width: int,
show_symbols: bool = False,
) -> None:
self.width = width
self.lhs = lhs if isinstance(lhs, str) else pprintpp.pformat(
lhs, width=width)
self.rhs = rhs if isinstance(rhs, str) else pprintpp.pformat(
rhs, width=width)
self.show_symbols = show_symbols
def test_prepends_icdiff_output_lines_with_color_off(testdir):
one = ['hello', 'hello']
two = ['bello', 'hella']
testdir.makepyfile(f"""
def test_thing():
assert {one!r} == {two!r}
""")
output = testdir.runpytest('--color=yes').stdout.str()
expected = list(icdiff.ConsoleDiff().make_table(
pformat(one, width=1).splitlines(),
pformat(two, width=1).splitlines(),
))
print('\n'.join(repr(l) for l in output.splitlines()))
_assert_line_in_ignoring_whitespace(expected[0], output)
def make_diff(lhs, rhs, width=80, show_symbols=False):
"""Transform input into best format for diffing, then return output diff."""
if isinstance(lhs, str):
lhs_repr = lhs
else:
lhs_repr = pprintpp.pformat(lhs, width=width)
if isinstance(rhs, str):
rhs_repr = rhs
else:
rhs_repr = pprintpp.pformat(rhs, width=width)
if show_symbols:
return build_symbolic_unified_diff(lhs_repr, rhs_repr)
return build_unified_diff(lhs_repr, rhs_repr)
def __console__(
self, console: "Console", options: "ConsoleOptions"
) -> "RenderResult":
pretty_str = pformat(self._object, width=options.max_width)
pretty_str = pretty_str.replace("\r", "")
pretty_text = self.highlighter(pretty_str)
yield pretty_text
def load_pipeline(path):
"""
Load, replace and validate the pipeline definition file.
- Path can be in any of the supported file formats.
- Replacement values will be performed with the available namespaces.
- Schema validation will be performed.
:param Path path: Path to the pipeline definition file.
:return: A dictionary data structure with the pipeline definition.
:rtype: dict
"""
try:
definition = load_file(path)
except Exception as e:
log.critical('Unable to parse pipeline definition {}'.format(path))
raise e
# Validate data structure
validated = validate_definition(definition)
log.info('Pipeline definition loaded, realized and validated.')
log.debug(pformat(validated))
return validated
def distribute(self, data):
from pprintpp import pformat
# Allow to filter data
super().distribute(data)
print(pformat(data))
async def wrapper(request):
# Log connection
metadata = {
'remote': request.remote,
'agent': request.headers['User-Agent'],
'content_type': request.content_type,
}
message = (
'Connection from {remote} using {content_type} '
'with user agent {agent}'
).format(**metadata)
log.info(message)
try:
return await handler(request)
except web.HTTPException as e:
return web.json_response(
{
'error': e.reason
},
status=e.status,
)
except Exception as e:
response = {
'error': ' '.join(str(arg) for arg in e.args),
}
log.exception('Unexpected server exception:\n{}'.format(
pformat(response)
))
return web.json_response(response, status=500)
def html(self, code):
bot = Bot()
html = bot.define('html')
p1 = bot.define('p1')
with (here / 'fixtures/sample.html').open('rb') as f:
content = f.read()
html.append([('https://example.com/', {
'headers': {},
'cookies': {},
'status_code': 200,
'encoding': 'utf-8',
'content': content,
})])
eval(code, {}, {
'bot': bot,
'html': html,
'p1': p1(html),
'this': this,
'int': int,
'select': select,
})
return pformat(list(p1.items()), width=42)
def __rich_console__(self, console: "Console",
options: "ConsoleOptions") -> "RenderResult":
# TODO: pformat tends to render a smaller width than it needs to, investigate why
pretty_str = pformat(self._object, width=options.max_width)
pretty_str = pretty_str.replace("\r", "")
pretty_text = self.highlighter(pretty_str)
yield pretty_text
def __rich_measure__(self, console: "Console",
max_width: int) -> "Measurement":
pretty_str = pformat(self._object, width=max_width)
pretty_str = pretty_str.replace("\r", "")
text = Text(pretty_str)
measurement = Measurement.get(console, text, max_width)
return measurement
def pp(obj):
if isinstance(obj, Iterator):
out = list(islice(obj, 10))
out = '<generator> ' + pprintpp.pformat(out)
obj = chain(out, obj)
else:
out = pprintpp.pformat(obj)
frame = inspect.currentframe()
frame = inspect.getouterframes(frame)[1]
line = inspect.getframeinfo(frame[0]).code_context[0].strip()
if line.endswith(')'):
arg = line[line.find('(') + 1:-1]
out = f'{arg} = {out}'
out = highlight(out, PythonLexer(), formatter())
print()
print(out, file=sys.__stderr__)
return obj
def _log_format(self, *args):
s = ""
for a in args:
if not isinstance(a, str):
a = pprintpp.pformat(a)
s += a
s += " "
return s
def run(self, dry_run=False, override=False, levels=None):
log.info('{} -> {}'.format(self._source, self._destination))
log.info('With:\n{}'.format(self._values))
log.info('Using filters:\n{}'.format(pformat(self._filters)))
log.info('Using namespaces:\n{}'.format(pformat(self._namespaces)))
self._dry_run = dry_run
self._override = override
self._levels = levels
return self.process_file(
self._source,
self._destination,
self._filename,
self._levels,
)
def print_msg(options, msg, prefix='', indent=0):
indent = ' ' * indent if indent else ''
if options.show_header and msg.header:
header = '\n%sheader: %s' % (
indent, pprintpp.pformat(msg.header, indent=2, width=80))
else:
header = ''
if options.show_data and msg.data:
data = '\n%sdata: %s' % (
indent, pprintpp.pformat(msg.data, indent=2, width=80))
else:
data = ''
now = datetime.fromtimestamp(msg.timestamp).strftime('%H:%M:%S:%f')
return '%s[%s] %s -> %s: %s%s%s' % (prefix, now, msg.src, msg.dst,
msg.info, header, data)
def start(self, ref, init=False, env=None, deployment=None, **kwargs):
app, service = self.parse_app_ref(ref, kwargs)
if init:
app_instance = yield self.get_app(app)
if not app_instance:
yield self.init(app, os.getcwd(), env=env, deployment=deployment, sync=True)
data = yield self._remote_exec('start', self.format_app_srv(app, service))
print 'result: %s' % pprintpp.pformat(data)
def diff_dicts(left, right, use_markup) -> Optional[List[str]]:
half_cols = MAX_COLS / 2 - MARGINS
pretty_left = pformat(left, indent=1, width=half_cols).splitlines()
pretty_right = pformat(right, indent=1, width=half_cols).splitlines()
diff_cols = MAX_COLS - MARGINS
if len(pretty_left) < 3 or len(pretty_right) < 3:
# avoid small diffs far apart by smooshing them up to the left
smallest_left = pformat(left, indent=2, width=1).splitlines()
smallest_right = pformat(right, indent=2, width=1).splitlines()
max_side = max(len(line) + 1 for line in smallest_left + smallest_right)
if (max_side * 2 + MARGIN_LEFT) < MAX_COLS:
diff_cols = max_side * 2 + GUTTER
pretty_left = pformat(left, indent=2, width=max_side).splitlines()
pretty_right = pformat(right, indent=2, width=max_side).splitlines()
differ = icdiff.ConsoleDiff(cols=diff_cols, tabsize=2)
if not use_markup:
# colorization is disabled in Pytest - either due to the terminal not
# supporting it or the user disabling it. We should obey, but there is
# no option in icdiff to disable it, so we replace its colorization
# function with a no-op
differ.colorize = lambda string: string
color_off = ""
else:
color_off = icdiff.color_codes["none"]
icdiff_lines = list(differ.make_table(pretty_left, pretty_right, context=True))
return ["equals failed"] + [color_off + line for line in icdiff_lines]
def __repr__(self) -> str:
"""Provide a full representation of the model and its content."""
try:
from pprintpp import pformat
except ImportError:
from pprint import pformat # type: ignore
return "%s(sync_id=%s, %s)" % (
type(self).__name__,
self.sync_id,
pformat(self._data),
)
def pformat(obj, indent=4, width=80, depth=None):
"""Returns a pretty string representation of the Python object.
Args:
obj: the Python object
indent (4): the number of spaces to use when indenting
width (80): the max width of each line in the pretty representation
depth (None): the maximum depth at which to pretty render nested dicts
Returns:
the pretty-formatted string
"""
return _pprint.pformat(obj, indent=indent, width=width, depth=depth)
def argrepr(value, nested=False):
if isinstance(value, list):
value = [argrepr(x, nested=True) for x in value]
elif isinstance(value, tuple):
value = tuple([argrepr(x, nested=True) for x in value])
elif isinstance(value, dict):
value = {k: argrepr(v, nested=True) for k, v in sorted(value.items())}
elif inspect.isclass(value):
value = value.__name__
if nested:
return value
else:
return pprintpp.pformat(value)
def expressions(self, code):
result = eval(code, {}, {
'this': this,
'int': int,
})
return pformat(result._eval({
'key': 1,
'value': {
'title': ' Foo bar "1000" ',
'link': 'https://www.example.com/?q=42',
'list': [1, 2, 3],
},
}),
width=42)
def __init__(self, port=None, logfile='coral.log'):
self.root = Path(__file__).resolve().parent.parent
self.port = port or self._find_free_tcp_port()
self.workdir = Path.cwd()
self.logfile = logfile
self._bootstrap = self.workdir / 'bootstrap.sh'
self._logs = self.workdir / logfile
self._process = None
# Log basic information
log.info('Project root: {}'.format(self.root, ))
log.info('Environment:\n{}'.format(pformat(dict(environ)), ))
def start(self, ref, init=False, env=None, deployment=None, **kwargs):
app, service = self.parse_app_ref(ref, kwargs)
if init:
app_instance = yield self.get_app(app)
if not app_instance:
yield self.init(app,
os.getcwd(),
env=env,
deployment=deployment,
sync=True)
data = yield self._remote_exec('start',
self.format_app_srv(app, service))
print 'result: %s' % pprintpp.pformat(data)
def test_unhashable_repr(self):
# In Python 3, C extensions can define a __repr__ method which is an
# instance of `instancemethod`, which is unhashable. It turns out to be
# spectacularly difficult to create an `instancemethod` and attach it to
# a type without using C... so we'll simulate it using a more explicitly
# unhashable type.
# See also: http://stackoverflow.com/q/40876368/71522
class UnhashableCallable(object):
__hash__ = None
def __call__(self):
return "some-repr"
class MyCls(object):
__repr__ = UnhashableCallable()
obj = MyCls()
assert_equal(p.pformat(obj), "some-repr")
def inspect(self, ref, **kwargs):
app, service = self.parse_app_ref(ref, kwargs, require_service=True)
data = yield self._remote_exec('inspect', app, service)
if not isinstance(data, dict):
print data
else:
table = Texttable(max_width=120)
table.set_cols_dtype(['t', 'a'])
table.set_cols_width([20, 100])
rows = [["Name", "Value"]]
for name, val in data.items():
rows.append([name, pprintpp.pformat(val)])
table.add_rows(rows)
print table.draw() + "\\n"
def remove(self, ref, **kwargs):
app, service = self.parse_app_ref(ref, kwargs, app_only=True)
data = yield self._remote_exec('remove', self.format_app_srv(app, service))
print 'result: %s' % pprintpp.pformat(data)
def repr(o: object) -> str:
return '%s.%s %s' % (
type(o).__module__, type(o).__qualname__, pprintpp.pformat(o.__dict__))
def __repr__(self):
return 'Buffer(%s)' % pformat(self._buffer)
def deployment_remove(self, deployment, **kwargs):
data = yield self._remote_exec('deployment_remove', name=deployment)
print 'result: %s' % pprintpp.pformat(data)
yield self.deployments()
def stop(self, ref, **kwargs):
app, service = self.parse_app_ref(ref, kwargs)
data = yield self._remote_exec('stop', self.format_app_srv(app, service))
print 'result: %s' % pprintpp.pformat(data)
def rebuild(self, ref, scrub_data, **kwargs):
app, service = self.parse_app_ref(ref, kwargs)
data = yield self._remote_exec('rebuild', self.format_app_srv(app, service), scrub_data)
print 'result: %s' % pprintpp.pformat(data)
def _eval(self, value, base=None):
"""Evaluate expression with given value and base.
Parameters
----------
value : object
Value used to evaluate this expression.
base : object
Base value used as value for expressions passed as arguments. If not given, base will be equal to value.
"""
self._evals += 1
base = value if base is None else base
try:
for i, item in enumerate(self._stack):
if isinstance(item, Func):
for handler in HANDLERS[item.name]:
conditions = (
(handler.items is None or 'func' in handler.items) and
(handler.types is None or isinstance(value, handler.types))
)
if conditions:
args, kwargs = self._eval_args(base, item, handler.eval_args)
logger.debug('eval: %s', _HandlerRepr(handler.handler, (value,) + args, kwargs))
try:
value = handler.handler(self, i, value, *args, **kwargs)
except StopEval as e:
logger.debug('eval: StopEval')
return e.value
break
else:
raise ExpressionError("Unknown function %r for value %r." % (item.name, value))
elif isinstance(item, Method):
for handler in HANDLERS[item.name]:
conditions = (
(handler.items is None or 'method' in handler.items) and
(handler.types is None or isinstance(value, handler.types))
)
if conditions:
args, kwargs = self._eval_args(base, item, handler.eval_args)
logger.debug('eval: %s', _HandlerRepr(handler.handler, (value,) + args, kwargs))
try:
value = handler.handler(self, i, value, *args, **kwargs)
except StopEval as e:
logger.debug('eval: StopEval')
return e.value
break
else:
method = getattr(value, item.name)
args, kwargs = self._eval_args(base, item)
logger.debug('eval: %s', _HandlerRepr(method, args, kwargs))
value = method(*args, **kwargs)
elif isinstance(item, Attr):
for handler in HANDLERS[item.key]:
conditions = (
(handler.items is None or 'attr' in handler.items) and
(handler.types is None or isinstance(value, handler.types))
)
if conditions:
logger.debug('eval: %s', _HandlerRepr(handler.handler, (value,), {}))
try:
value = handler.handler(value)
except StopEval as e:
logger.debug('eval: StopEval')
return e.value
break
else:
if isinstance(value, dict):
logger.debug('eval: [%r]', item.key)
value = value[item.key]
else:
logger.debug('eval: %r.%s', value, item.key)
value = getattr(value, item.key)
elif isinstance(item, Item):
logger.debug('eval: [%r]', item.key)
value = value[item.key]
else:
raise ExpressionError("Unknown item type %r." % item)
except:
raise ExpressionError("error while processing expression:\n%s\nevaluated with:\n%s" % (
textwrap.indent(str(self), ' '),
textwrap.indent(pprintpp.pformat(base), ' '),
))
return value