def add(self, spec, content, path=None, do_filter=True):
if not path: # path must change to allow parsers to fire
path = self._make_path()
if not path.startswith("/"):
path = "/" + path
if dr.get_delegate(spec).raw:
content_iter = content
else:
if not isinstance(content, list):
content_iter = [
l.rstrip() for l in StringIO(content).readlines()
]
else:
content_iter = content
if do_filter:
content_iter = list(apply_filters(spec, content_iter))
content_provider = context_wrap(content_iter, path=path, split=False)
if dr.get_delegate(spec).multi_output:
if spec not in self.data:
self.data[spec] = []
self.data[spec].append(content_provider)
else:
self.data[spec] = content_provider
return self
def add(self, spec, content, path=None, do_filter=True):
if not path: # path must change to allow parsers to fire
path = str(next_gn()) + "BOGUS"
if not path.startswith("/"):
path = "/" + path
if dr.get_delegate(spec).raw:
content_iter = content
else:
if not isinstance(content, list):
content_iter = [
l.rstrip() for l in StringIO(content).readlines()
]
else:
content_iter = content
if do_filter:
content_iter = list(apply_filters(spec, content_iter))
content_provider = ContentProvider()
content_provider.path = path
content_provider._content = content_iter
if dr.get_delegate(spec).multi_output:
if spec not in self.data:
self.data[spec] = []
self.data[spec].append(content_provider)
else:
logger.warn("Replacing %s", spec.__name__)
logger.warn(traceback.format_stack())
self.data[spec] = content_provider
return self
def add_filter(component, patterns):
"""
Add a filter or list of filters to a component. When the component is
a datasource, the filter will be directly added to that datasouce.
In cases when the component is a parser or combiner, the filter will be
added to underlying filterable datasources by traversing dependency graph.
A filter is a simple string, and it matches if it is contained anywhere
within a line.
Args:
component (component): The component to filter, can be datasource,
parser or combiner.
patterns (str, [str]): A string, list of strings, or set of strings to
add to the datasource's filters.
"""
def inner(component, patterns):
if component in _CACHE:
del _CACHE[component]
types = six.string_types + (list, set)
if not isinstance(patterns, types):
raise TypeError(
"Filter patterns must be of type string, list, or set.")
if isinstance(patterns, six.string_types):
patterns = set([patterns])
elif isinstance(patterns, list):
patterns = set(patterns)
for pat in patterns:
if not pat:
raise Exception("Filter patterns must not be empy.")
FILTERS[component] |= patterns
if not plugins.is_datasource(component):
for dep in dr.run_order(dr.get_dependency_graph(component)):
if plugins.is_datasource(dep):
d = dr.get_delegate(dep)
if d.filterable:
inner(dep, patterns)
else:
delegate = dr.get_delegate(component)
if delegate.raw:
raise Exception("Filters aren't applicable to raw datasources.")
if not delegate.filterable:
raise Exception("Filters aren't applicable to %s." %
dr.get_name(component))
inner(component, patterns)
def add_filter(ds, patterns):
"""
Add a filter or list of filters to a datasource. A filter is a simple
string, and it matches if it is contained anywhere within a line.
Args:
ds (@datasource component): The datasource to filter
patterns (str, [str]): A string, list of strings, or set of strings to
add to the datasource's filters.
"""
if not plugins.is_datasource(ds):
raise Exception("Filters are applicable only to datasources.")
delegate = dr.get_delegate(ds)
if delegate.raw:
raise Exception("Filters aren't applicable to raw datasources.")
if not delegate.filterable:
raise Exception("Filters aren't applicable to %s." % dr.get_name(ds))
if ds in _CACHE:
del _CACHE[ds]
if isinstance(patterns, six.string_types):
FILTERS[ds].add(patterns)
elif isinstance(patterns, list):
FILTERS[ds] |= set(patterns)
elif isinstance(patterns, set):
FILTERS[ds] |= patterns
else:
raise TypeError("patterns must be string, list, or set.")
def _f(func):
metadata = kwargs.get("metadata", {})
c = StdTypes._datasource(*args, metadata=metadata, component_type=datasource)(func)
delegate = dr.get_delegate(c)
delegate.multi_output = kwargs.get("multi_output", False)
delegate.raw = kwargs.get("raw", False)
return c
def inner(comp, broker):
ct = dr.get_delegate(comp).__class__.__name__
res = StatResponse(name=dr.get_name(comp), component=ct)
if comp in broker:
stat_filler(broker[comp], res)
elif comp in broker.exceptions:
for e in broker.exceptions[comp]:
res.errors.append(broker.tracebacks[e])
self.results.append(attr.asdict(res))
def run_test(component, input_data, expected=None):
if filters.ENABLED:
mod = component.__module__
sup_mod = '.'.join(mod.split('.')[:-1])
rps = _get_registry_points(component)
filterable = set(d for d in rps if dr.get_delegate(d).filterable)
missing_filters = filterable - ADDED_FILTERS.get(mod, set()) - ADDED_FILTERS.get(sup_mod, set())
if missing_filters:
names = [dr.get_name(m) for m in missing_filters]
msg = "%s must add filters to %s"
raise Exception(msg % (mod, ", ".join(names)))
broker = run_input_data(component, input_data)
if expected:
deep_compare(broker.get(component), expected)
return broker.get(component)
def run_test(component, input_data, expected=None, return_make_none=False):
if filters.ENABLED:
mod = component.__module__
sup_mod = '.'.join(mod.split('.')[:-1])
rps = _get_registry_points(component)
filterable = set(d for d in rps if dr.get_delegate(d).filterable)
missing_filters = filterable - ADDED_FILTERS.get(
mod, set()) - ADDED_FILTERS.get(sup_mod, set())
if missing_filters:
names = [dr.get_name(m) for m in missing_filters]
msg = "%s must add filters to %s"
raise Exception(msg % (mod, ", ".join(names)))
broker = run_input_data(component, input_data)
result = broker.get(component)
if expected:
deep_compare(result, expected)
elif result == MAKE_NONE_RESULT and not return_make_none:
# Convert make_none() result to None as default unless
# make_none explicitly requested
return None
return result
def _resolve_registry_points(cls, base, dct):
module = cls.__module__
parents = [x for x in cls.__mro__ if x not in (cls, SpecSet, object)]
for k, v in dct.items():
if isinstance(v, RegistryPoint):
# add v under its name to this class's registry.
v.__name__ = k
cls.registry[k] = v
if is_datasource(v):
v.__qualname__ = ".".join([cls.__name__, k])
v.__name__ = k
v.__module__ = module
setattr(cls, k, SpecDescriptor(v))
if k in base.registry:
# if the datasource has the same name as a RegistryPoint in the
# base class, the datasource to the RegistryPoint.
point = base.registry[k]
# TODO: log when RegistryPoint and implementation properties
# TODO: aren't the same.
delegate = dr.get_delegate(v)
v.filterable = delegate.filterable = point.filterable
v.raw = delegate.raw = point.raw
v.multi_output = delegate.multi_output = point.multi_output
# the RegistryPoint gets the implementation datasource as a
# dependency
dr.add_dependency(point, v)
dr.mark_hidden(v)
# Datasources override previously defined datasources of the
# same name for contexts they all depend on. Here we tell
# datasources of the same name not to execute under contexts
# the new datasource will handle.
_register_context_handler(parents, v)
def __call__(self, broker):
for c in reversed(dr.get_delegate(self).deps):
if c in broker:
return broker[c]
raise dr.SkipComponent()