def test_run(): broker = dr.Broker() broker["common"] = 3 graph = dr.get_dependency_graph(stage3) graph.update(dr.get_dependency_graph(stage4)) broker = dr.run(graph, broker) assert stage3 in broker.instances assert broker[stage3] == 3 assert stage4 in broker.instances assert broker[stage4] == 3
def test_run_incremental(): broker = dr.Broker() broker["dep1"] = 1 broker["dep2"] = 2 broker["common"] = 3 graph = dr.get_dependency_graph(stage1) graph.update(dr.get_dependency_graph(stage2)) graph.update(dr.get_dependency_graph(stage3)) graph.update(dr.get_dependency_graph(stage4)) brokers = list(dr.run_incremental(graph, broker)) assert len(brokers) == 3
def test_spec_factory(): hn = HostContext() broker = dr.Broker() broker[HostContext] = hn broker = dr.run(dr.get_dependency_graph(dostuff), broker) assert dostuff in broker, broker.tracebacks assert broker[Stuff.smpl_file].content == file_content
def run_input_data(component, input_data): broker = dr.Broker() for k, v in input_data.data.items(): broker[k] = v graph = dr.get_dependency_graph(component) broker = dr.run(graph, broker=broker) for v in broker.tracebacks.values(): print(v) return broker
def add_filter(component, patterns): """ Add a filter or list of filters to a component. When the component is a datasource, the filter will be directly added to that datasouce. In cases when the component is a parser or combiner, the filter will be added to underlying filterable datasources by traversing dependency graph. A filter is a simple string, and it matches if it is contained anywhere within a line. Args: component (component): The component to filter, can be datasource, parser or combiner. patterns (str, [str]): A string, list of strings, or set of strings to add to the datasource's filters. """ def inner(component, patterns): if component in _CACHE: del _CACHE[component] types = six.string_types + (list, set) if not isinstance(patterns, types): raise TypeError( "Filter patterns must be of type string, list, or set.") if isinstance(patterns, six.string_types): patterns = set([patterns]) elif isinstance(patterns, list): patterns = set(patterns) for pat in patterns: if not pat: raise Exception("Filter patterns must not be empy.") FILTERS[component] |= patterns if not plugins.is_datasource(component): for dep in dr.run_order(dr.get_dependency_graph(component)): if plugins.is_datasource(dep): d = dr.get_delegate(dep) if d.filterable: inner(dep, patterns) else: delegate = dr.get_delegate(component) if delegate.raw: raise Exception("Filters aren't applicable to raw datasources.") if not delegate.filterable: raise Exception("Filters aren't applicable to %s." % dr.get_name(component)) inner(component, patterns)