def test_run():
broker = dr.Broker()
broker["common"] = 3
graph = dr.get_dependency_graph(stage3)
graph.update(dr.get_dependency_graph(stage4))
broker = dr.run(graph, broker)
assert stage3 in broker.instances
assert broker[stage3] == 3
assert stage4 in broker.instances
assert broker[stage4] == 3
def test_run_incremental():
broker = dr.Broker()
broker["dep1"] = 1
broker["dep2"] = 2
broker["common"] = 3
graph = dr.get_dependency_graph(stage1)
graph.update(dr.get_dependency_graph(stage2))
graph.update(dr.get_dependency_graph(stage3))
graph.update(dr.get_dependency_graph(stage4))
brokers = list(dr.run_incremental(graph, broker))
assert len(brokers) == 3
def test_spec_factory():
hn = HostContext()
broker = dr.Broker()
broker[HostContext] = hn
broker = dr.run(dr.get_dependency_graph(dostuff), broker)
assert dostuff in broker, broker.tracebacks
assert broker[Stuff.smpl_file].content == file_content
def run_input_data(component, input_data):
broker = dr.Broker()
for k, v in input_data.data.items():
broker[k] = v
graph = dr.get_dependency_graph(component)
broker = dr.run(graph, broker=broker)
for v in broker.tracebacks.values():
print(v)
return broker
def add_filter(component, patterns):
"""
Add a filter or list of filters to a component. When the component is
a datasource, the filter will be directly added to that datasouce.
In cases when the component is a parser or combiner, the filter will be
added to underlying filterable datasources by traversing dependency graph.
A filter is a simple string, and it matches if it is contained anywhere
within a line.
Args:
component (component): The component to filter, can be datasource,
parser or combiner.
patterns (str, [str]): A string, list of strings, or set of strings to
add to the datasource's filters.
"""
def inner(component, patterns):
if component in _CACHE:
del _CACHE[component]
types = six.string_types + (list, set)
if not isinstance(patterns, types):
raise TypeError(
"Filter patterns must be of type string, list, or set.")
if isinstance(patterns, six.string_types):
patterns = set([patterns])
elif isinstance(patterns, list):
patterns = set(patterns)
for pat in patterns:
if not pat:
raise Exception("Filter patterns must not be empy.")
FILTERS[component] |= patterns
if not plugins.is_datasource(component):
for dep in dr.run_order(dr.get_dependency_graph(component)):
if plugins.is_datasource(dep):
d = dr.get_delegate(dep)
if d.filterable:
inner(dep, patterns)
else:
delegate = dr.get_delegate(component)
if delegate.raw:
raise Exception("Filters aren't applicable to raw datasources.")
if not delegate.filterable:
raise Exception("Filters aren't applicable to %s." %
dr.get_name(component))
inner(component, patterns)
