def test_wire_inputs_index_lookup():
inputs = {'a': '^first.second'}
og = deepcopy(inputs)
results = {'first': {'second': rand_str()}}
inputs = wire_inputs(inputs, results, lambda a: None)
for k, v in inputs.items():
assert v == results['first']['second']
def test_wire_inputs_missing_param():
inputs = {}
results = {'first': 'b'}
inputs = wire_inputs(inputs, results, dummy)
assert 'a' in inputs
assert inputs['a'] is None
assert 'b' in inputs
assert inputs['b'] is None
assert 'c' not in inputs
def test_wire_inputs_supply_default_param():
inputs = {'c': '^first'}
results = {'first': rand_str()}
inputs = wire_inputs(inputs, results, dummy)
assert 'a' in inputs
assert inputs['a'] is None
assert 'b' in inputs
assert inputs['b'] is None
assert 'c' in inputs
assert inputs['c'] == results['first']
def test_wire_inputs_input_pass_through():
for _ in range(100):
raw_inputs = {
rand_str(): rand_str()
for _ in range(random.randint(1, 5))
}
gold_inputs = deepcopy(raw_inputs)
chore = lambda **kwargs: None
inputs = wire_inputs(raw_inputs, {}, chore)
assert inputs == gold_inputs
def test_wire_inputs_missing_inputs():
inputs = {'a': '^first', 'b': '^second', 'c': '^third'}
og = deepcopy(inputs)
results = {'first': rand_str(), 'last': rand_str()}
inputs = wire_inputs(inputs, results, dummy)
for k, v in inputs.items():
if og[k].replace('^', '') in results:
assert v == results[og[k].replace('^', '')]
else:
assert v is None
assert inputs != results
def test_wire_inputs_all_inputs_present():
inputs = {'a': '^first', 'b': '^second', 'c': '^third'}
og = deepcopy(inputs)
results = {
'first': rand_str(),
'second': rand_str(),
'third': rand_str(),
'last': rand_str()
}
inputs = wire_inputs(inputs, results, dummy)
for k, v in inputs.items():
assert v == results[og[k].replace('^', '')]
assert inputs != results
def test_wire_inputs_list_of_inputs():
inputs = {'a': ['^first', '^second', '^third']}
og = deepcopy(inputs)
results = {
'first': rand_str(),
'second': rand_str(),
'third': rand_str(),
'last': rand_str()
}
inputs = wire_inputs(inputs, results, lambda a: None)
for k, vs in inputs.items():
for i in range(len(vs)):
assert vs[i] == results[og[k][i].replace('^', '')]
assert inputs != results
def run_chores(config: Dict, results: Dict) -> Dict:
"""Execute a chore graph
:param config: The list of chores and there deps
:param results: A set of upstream results that can be used
:return: The new results, updated with chore output
"""
graph = create_graph(config['chores'], results)
# Convert the graph from indices to names so it prints better.
named_graph = {
config['chores'][k]['name']: [config['chores'][v]['name'] for v in vs]
for k, vs in graph.items()
}
LOGGER.info(dot_graph(named_graph))
execution_order = topo_sort(graph)
# Track nodes that give up and all the nodes that are downstream of them.
to_skip = set()
children = find_children(graph)
LOGGER.info("Topological Sort %s", execution_order)
for step in execution_order:
params = config['chores'][step]
name = params.pop('name')
if step in to_skip:
# If an ancestor of yours gave up you should give up.
LOGGER.debug("Chore %s skipped because of parent", name)
continue
chore_type = params.pop('type')
params.pop(DEPENDENCY_KEY, None)
LOGGER.debug('Feeding Inputs: %s(%s)', name, _to_kwargs(params))
LOGGER.debug(results)
chore = create_chore(chore_type)
params = wire_inputs(params, results, chore)
LOGGER.info('Running: %s(%s)', name, _to_kwargs(params))
try:
res = chore(**params)
except ChoreSkippedException:
# I decided to give up. All my descendants should give up.
res = None
LOGGER.debug("Chore %s raised a ChoreSkippedException", name)
to_skip.add(step)
to_skip.update(children[step])
results[name] = format_output(res)
LOGGER.info("Chores %s skipped running.", to_skip)
return results