def test_result_type(self):
"""
Test that the expressions return the right value type.
"""
for computable_expr, expr_val_list in self.execute():
for expr_val in expr_val_list:
TestResult.fail_if(
not isinstance(expr_val.value, Final),
'Wrong value type: expected {} but got {}'.format(
utils.get_name(Final),
utils.get_name(type(expr_val.value))),
[computable_expr])
def handle_cycle(path):
error('Cyclic dependency detected: {path}'.format(
path=' -> '.join(
utils.get_name(callable_)
for callable_ in path
)
))
def handle_non_produced(cls_name, consumer_name, param_name, callable_path):
info('Nothing can produce instances of {cls} needed for {consumer} (parameter "{param}", along path {path})'.format(
cls=cls_name,
consumer=consumer_name,
param=param_name,
path=' -> '.join(utils.get_name(callable_) for callable_ in callable_path)
))
def test_excep(self):
for computable_expr, expr_val_list in self.execute(check_excep=False):
for expr_val in expr_val_list:
failed_parent_list = expr_val.get_excep()
TestResult.fail_if(not failed_parent_list,
'No exception detected', [computable_expr])
for failed_froz_val in failed_parent_list:
excep = failed_froz_val.excep
TestResult.fail_if(
not isinstance(excep, ValueError),
'Wrong exception type: expected {} but got {}'.format(
utils.get_name(ValueError),
repr(excep),
), [computable_expr])
def format_result(self, expr_val):
"""
Format an :class:`exekall.engine.ExprVal` that is the result of an
expression. It should return a (short) string that will be displayed at
the end of the computation.
"""
val = expr_val.value
if val is NoValue or val is None:
for failed_parent in expr_val.get_excep():
excep = failed_parent.excep
return 'EXCEPTION ({type}): {msg}'.format(type=get_name(
type(excep), full_qual=False),
msg=excep)
return 'No value computed'
else:
return str(val)
def build_patch_map(sweep_spec_list, param_spec_list, op_set):
sweep_spec_list = copy.copy(sweep_spec_list)
# Make a sweep spec from a simpler param spec
sweep_spec_list.extend(
(callable_pattern, param, value, value, 1)
for callable_pattern, param, value in param_spec_list)
patch_map = dict()
for callable_pattern, param, start, stop, step in sweep_spec_list:
for op in op_set:
callable_ = op.callable_
callable_name = utils.get_name(callable_, full_qual=True)
if not utils.match_name(callable_name, [callable_pattern]):
continue
patch_map.setdefault(op, dict()).setdefault(param, []).extend(
utils.sweep_param(callable_, param, start, stop, step))
return patch_map
def from_expr_data(cls, data: ExprData, consumer: Consumer) -> 'ExekallArtifactPath':
"""
Factory used when running under `exekall`
"""
artifact_dir = Path(data['expr_artifact_dir']).resolve()
consumer_name = get_name(consumer)
# Find a non-used directory
for i in itertools.count(1):
artifact_dir_ = Path(artifact_dir, consumer_name, str(i))
if not artifact_dir_.exists():
artifact_dir = artifact_dir_
break
cls.get_logger().info(f'Creating {consumer_name} artifact storage: {artifact_dir}')
artifact_dir.mkdir(parents=True)
# Get canonical absolute paths
artifact_dir = artifact_dir.resolve()
root = data['artifact_dir']
relative = artifact_dir.relative_to(root)
return cls(root, relative)
def exec_expr_list(iteration_expr_list, adaptor, artifact_dir,
testsession_uuid, hidden_callable_set,
only_template_scripts, adaptor_cls, verbose, save_db,
use_pdb):
if not only_template_scripts:
with (artifact_dir / 'UUID').open('wt') as f:
f.write(testsession_uuid + '\n')
(artifact_dir / 'BY_UUID').mkdir()
out('\nArtifacts dir: {}\n'.format(artifact_dir))
for expr in utils.flatten_seq(iteration_expr_list):
expr_short_id = expr.get_id(
hidden_callable_set=hidden_callable_set,
with_tags=False,
full_qual=False,
qual=False,
)
data = expr.data
data['id'] = expr_short_id
data['uuid'] = expr.uuid
expr_artifact_dir = pathlib.Path(artifact_dir, expr_short_id,
expr.uuid)
expr_artifact_dir.mkdir(parents=True)
expr_artifact_dir = expr_artifact_dir.resolve()
data['artifact_dir'] = artifact_dir
data['expr_artifact_dir'] = expr_artifact_dir
with (expr_artifact_dir / 'UUID').open('wt') as f:
f.write(expr.uuid + '\n')
with (expr_artifact_dir / 'ID').open('wt') as f:
f.write(expr_short_id + '\n')
with (expr_artifact_dir / 'STRUCTURE').open('wt') as f:
f.write(
expr.get_id(
hidden_callable_set=hidden_callable_set,
with_tags=False,
full_qual=True,
) + '\n\n')
f.write(expr.format_structure() + '\n')
is_svg, dot_output = utils.render_graphviz(expr)
graphviz_path = expr_artifact_dir / 'STRUCTURE.{}'.format(
'svg' if is_svg else 'dot')
with graphviz_path.open('wt', encoding='utf-8') as f:
f.write(dot_output)
with (expr_artifact_dir / 'EXPRESSION_TEMPLATE.py').open(
'wt', encoding='utf-8') as f:
f.write(
expr.get_script(
prefix='expr',
db_path=os.path.join('..', utils.DB_FILENAME),
db_relative_to='__file__',
)[1] + '\n', )
if only_template_scripts:
return 0
# Preserve the execution order, so the summary is displayed in the same
# order
result_map = collections.OrderedDict()
for i, expr_list in enumerate(iteration_expr_list):
i += 1
info('Iteration #{}\n'.format(i))
for expr in expr_list:
exec_start_msg = 'Executing: {short_id}\n\nID: {full_id}\nArtifacts: {folder}\nUUID: {uuid_}'.format(
short_id=expr.get_id(
hidden_callable_set=hidden_callable_set,
full_qual=False,
qual=False,
),
full_id=expr.get_id(
hidden_callable_set=hidden_callable_set
if not verbose else None,
full_qual=True,
),
folder=expr.data['expr_artifact_dir'],
uuid_=expr.uuid).replace('\n', '\n# ')
delim = '#' * (len(exec_start_msg.splitlines()[0]) + 2)
out(delim + '\n# ' + exec_start_msg + '\n' + delim)
result_list = list()
result_map[expr] = result_list
def pre_line():
out('-' * 40)
# Make sure that all the output of the expression is flushed to ensure
# there won't be any buffered stderr output being displayed after the
# "official" end of the Expression's execution.
def flush_std_streams():
sys.stdout.flush()
sys.stderr.flush()
def get_uuid_str(expr_val):
return 'UUID={}'.format(expr_val.uuid)
computed_expr_val_set = set()
reused_expr_val_set = set()
def log_expr_val(expr_val, reused):
# Consider that PrebuiltOperator reuse values instead of
# actually computing them.
if isinstance(expr_val.expr.op, engine.PrebuiltOperator):
reused = True
if reused:
msg = 'Reusing already computed {id} {uuid}'
reused_expr_val_set.add(expr_val)
else:
msg = 'Computed {id} {uuid}'
computed_expr_val_set.add(expr_val)
op = expr_val.expr.op
if (op.callable_ not in hidden_callable_set and
not issubclass(op.value_type, engine.ForcedParamType)):
log_f = info
else:
log_f = debug
log_f(
msg.format(
id=expr_val.get_id(
full_qual=False,
with_tags=True,
hidden_callable_set=hidden_callable_set,
),
uuid=get_uuid_str(expr_val),
))
# Drop into the debugger if we got an exception
excep = expr_val.excep
if use_pdb and excep is not NoValue:
error(utils.format_exception(excep))
pdb.post_mortem(excep.__traceback__)
def get_duration_str(expr_val):
if expr_val.duration is None:
duration = ''
else:
duration = '{:.2f}s'.format(expr_val.duration)
cumulative = expr_val.cumulative_duration
cumulative = ' (cumulative: {:.2f}s)'.format(
cumulative) if cumulative else ''
return '{}{}'.format(duration, cumulative)
# This returns an iterator
executor = expr.execute(log_expr_val)
out('')
for result in utils.iterate_cb(executor, pre_line,
flush_std_streams):
for excep_val in result.get_excep():
excep = excep_val.excep
tb = utils.format_exception(excep)
error(
'{e_name}: {e}\nID: {id}\n{tb}'.format(
id=excep_val.get_id(),
e_name=utils.get_name(type(excep)),
e=excep,
tb=tb,
), )
prefix = 'Finished {uuid} in {duration} '.format(
uuid=get_uuid_str(result),
duration=get_duration_str(result),
)
out('{prefix}{id}'.format(
id=result.get_id(
full_qual=False,
qual=False,
mark_excep=True,
with_tags=True,
hidden_callable_set=hidden_callable_set,
).strip().replace('\n', '\n' + len(prefix) * ' '),
prefix=prefix,
))
out(adaptor.format_result(result))
result_list.append(result)
out('')
expr_artifact_dir = expr.data['expr_artifact_dir']
# Finalize the computation
adaptor.finalize_expr(expr)
# Dump the reproducer script
with (expr_artifact_dir / 'EXPRESSION.py').open(
'wt', encoding='utf-8') as f:
f.write(
expr.get_script(
prefix='expr',
db_path=os.path.join('..', '..', utils.DB_FILENAME),
db_relative_to='__file__',
)[1] + '\n', )
def format_uuid(expr_val_list):
uuid_list = sorted(
{expr_val.uuid
for expr_val in expr_val_list})
return '\n'.join(uuid_list)
def write_uuid(path, *args):
with path.open('wt') as f:
f.write(format_uuid(*args) + '\n')
write_uuid(expr_artifact_dir / 'VALUES_UUID', result_list)
write_uuid(expr_artifact_dir / 'REUSED_VALUES_UUID',
reused_expr_val_set)
write_uuid(expr_artifact_dir / 'COMPUTED_VALUES_UUID',
computed_expr_val_set)
# From there, use a relative path for symlinks
expr_artifact_dir = pathlib.Path(
'..', expr_artifact_dir.relative_to(artifact_dir))
computed_uuid_set = {
expr_val.uuid
for expr_val in computed_expr_val_set
}
computed_uuid_set.add(expr.uuid)
for uuid_ in computed_uuid_set:
(artifact_dir / 'BY_UUID' /
uuid_).symlink_to(expr_artifact_dir)
if save_db:
db = engine.ValueDB(
engine.FrozenExprValSeq.from_expr_list(
utils.flatten_seq(iteration_expr_list),
hidden_callable_set=hidden_callable_set,
),
adaptor_cls=adaptor_cls,
)
db_path = artifact_dir / utils.DB_FILENAME
db.to_path(db_path)
relative_db_path = db_path.relative_to(artifact_dir)
else:
relative_db_path = None
db = None
out('#' * 80)
info('Artifacts dir: {}'.format(artifact_dir))
info('Result summary:')
# Display the results summary
summary = adaptor.get_summary(result_map)
out(summary)
with (artifact_dir / 'SUMMARY').open('wt', encoding='utf-8') as f:
f.write(summary + '\n')
# Output the merged script with all subscripts
script_path = artifact_dir / 'ALL_SCRIPTS.py'
result_name_map, all_scripts = engine.Expression.get_all_script(
utils.flatten_seq(iteration_expr_list),
prefix='expr',
db_path=relative_db_path,
db_relative_to='__file__',
db=db,
adaptor_cls=adaptor_cls,
)
with script_path.open('wt', encoding='utf-8') as f:
f.write(all_scripts + '\n')
return adaptor.get_run_exit_code(result_map)
def show_db(self, db):
parse_uuid_attr = self._parse_uuid_attr
def indent(s):
idt = ' ' * 4
return idt + s.replace('\n', '\n' + idt)
def get_uuid(uuid):
try:
froz_val = db.get_by_uuid(uuid)
except KeyError:
raise KeyError('UUID={} not found in the database'.format(uuid))
else:
return froz_val
def get_attr_key(obj, attr_key):
# parse "attr[key1][key2][...]"
attr = attr_key.split('[', 1)[0]
keys = re.findall(r'\[(.*?)\]', attr_key)
if attr:
obj = getattr(obj, attr)
return get_nested_key(obj, keys)
def resolve_attr(obj, attr_key):
if attr_key is None:
return obj
try:
attr_key, remainder = attr_key.split('.', 1)
except ValueError:
return get_attr_key(obj, attr_key)
else:
obj = get_attr_key(obj, attr_key)
return resolve_attr(obj, remainder)
args = self.args
if not (args.show or args.show_yaml):
super().show_db(db)
attr_map = {}
for uuid, attr in args.show:
attr_map.setdefault(uuid, set()).add(attr)
if len(args.show) == 1:
show_format = '{val}'
else:
show_format = 'UUID={uuid} {type}{attr}{eq}{val}'
serialize_spec_list = args.serialize
yaml_show_spec_list = args.show_yaml
for uuid, attr_set in attr_map.items():
attr_list = sorted(attr_set)
froz_val = get_uuid(uuid)
value = froz_val.value
for attr in attr_list:
attr_value = resolve_attr(value, attr)
attr_str = str(attr_value)
if '\n' in attr_str:
attr_str = '\n' + indent(attr_str)
eq = ':'
else:
eq = '='
print(show_format.format(
uuid=froz_val.uuid,
type=get_name(type(value)),
attr='.' + attr if attr else '',
val=attr_str,
eq=eq,
))
if len(yaml_show_spec_list) == 1:
yaml_show_format = '{yaml}'
yaml_indent = lambda x: x
else:
yaml_show_format = 'UUID={uuid} {type}:\n\n{yaml}'
yaml_indent = indent
for uuid, attr in yaml_show_spec_list:
froz_val = get_uuid(uuid)
value = froz_val.value
value = resolve_attr(value, attr)
if isinstance(value, Serializable):
yaml_str = value.to_yaml()
else:
yaml_str = Serializable._to_yaml(value)
print(yaml_show_format.format(
uuid=uuid,
type=get_name(type(value)),
yaml=yaml_indent(yaml_str),
))
for uuid_attr, path in serialize_spec_list:
uuid, attr = parse_uuid_attr(uuid_attr)
froz_val = get_uuid(uuid)
value = froz_val.value
value = resolve_attr(value, attr)
if isinstance(value, Serializable):
value.to_path(path)
else:
Serializable._to_path(value, path, fmt='yaml')
return 0
