def get_component_set_from(config, components_lists):
"""
Use the Guesser API to fetch the components set from the given
components_lists
"""
all_set = set()
module_name = config.get(DEPMAKE_ACTION_NAME, GUESSER_MODULE_NAME)
_LOGGER.debug("Guesser module name: %s" , module_name)
module = __import__(to_str_from_unicode(module_name, should_be_uni=True))
# Go through sys.modules to find modules inside packages
module = sys.modules[module_name]
params = config.get(DEPMAKE_ACTION_NAME, GUESSER_PARAMS_NAME)
params = replace_if_none(params)
_LOGGER.debug("Guesser parameters: %s", params)
params = to_str_from_unicode(params)
guesser = module.get_guesser(params)
_LOGGER.info("Using guesser: %s", guesser)
for cl in components_lists:
cl = to_str_from_unicode(cl, 'utf-8', True)
(components_for, unknown) = guesser.guess_type(cl)
if len(unknown) != 0:
_LOGGER.warning("%s: [Unknown components]", unknown)
for table in components_for:
for type_ in components_for[table]:
components = NodeSet(components_for[table][type_])
for component in components:
comp2 = to_unicode(component)
all_set.add(Component(comp2, type_, table))
return all_set
def test_to_str_from_unicode(self):
uni = u"mmąöî"
nonascii = lib.to_str_from_unicode(uni, should_be_uni=True)
assert type(uni) == unicode
assert len(uni) == 5
assert type(nonascii) == str
assert len(nonascii) != 5
assert nonascii == "mmąöî"
def makedepgraph(config, rules, components_lists, options):
"""
Return the dependency graph for the given pair ('req_ruleset',
'components_lists').
"""
ruleset = RuleSet(rules.values())
all_set = get_component_set_from(config, components_lists)
force_opt = options.force
force_rule = force_opt.split(',') if force_opt is not None else []
docache = True if getattr(options, 'docache', 'yes') == 'yes' else False
_LOGGER.debug("Caching filter results (docache) is: %s", docache)
depgraph = ruleset.get_depgraph(all_set, force_rule, docache)
if _LOGGER.isEnabledFor(logging.DEBUG):
_LOGGER.debug("Components set: %s",
NodeSet.fromlist([to_str_from_unicode(x.id) for x in all_set]))
_LOGGER.debug("Remaining: %s",
NodeSet.fromlist([str(x) for x in depgraph.remaining_components]))
_LOGGER.debug("List: %s",
NodeSet.fromlist([str(x) for x in depgraph.components_map]))
return depgraph
def _update_hash(checksum, rule):
"""
Update the given checksum with all required fields of the given rule.
"""
checksum.update(to_str_from_unicode(rule.ruleset, should_be_uni=True))
checksum.update(to_str_from_unicode(rule.name, should_be_uni=True))
for type_ in rule.types:
checksum.update(to_str_from_unicode(type_, should_be_uni=True))
# Do not take filter into account
# ruleset_h.update(str(rule.filter))
checksum.update(to_str_from_unicode(replace_if_none_by_uni(rule.action), \
should_be_uni=True))
checksum.update(to_str_from_unicode(replace_if_none_by_uni(rule.depsfinder), \
should_be_uni=True))
checksum.update(to_str_from_unicode(replace_if_none_by_uni(rule.help), \
should_be_uni=True))
for dep in rule.dependson:
checksum.update(to_str_from_unicode(replace_if_none_by_uni(dep), \
should_be_uni=True))
def _filter_impl(self, component):
var_map = _get_var_map(component.id,
component.name,
component.type,
component.category,
self.rule.ruleset,
self.rule.name,
self.rule.help)
cmd_string = substitute(var_map, self.rule.filter)
cmd = shlex.split(to_str_from_unicode(cmd_string, should_be_uni=True))
_LOGGER.debug("%s: calling filter cmd: %s", self.rule.name, cmd)
try:
popen = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=-1) # Use system default
except OSError as ose:
_LOGGER.error("%s: can't call filter '%s': %s",
self.rule.name, cmd_string, ose)
return False
(msg_std, msg_err) = popen.communicate()
msg_std = msg_std.strip()
msg_err = msg_err.strip()
_LOGGER.debug("%s: output of filter command %s on component %s: %s",
self.rule.name, cmd_string, component, msg_std)
if len(msg_err) != 0:
_LOGGER.warning("%s: error when applying filter command " + \
"%s to component %s: %s",
self.rule.name, cmd_string,
component, msg_err)
_LOGGER.debug("%s: filter command: %s RC: %d",
self.rule.name,
cmd_string,
popen.returncode)
return popen.returncode == os.EX_OK
def _get_deps(self, component, rule):
"""
Find dependencies of a given component. This implies calling
the rule.depsfinder script. Substitution of variables is done.
Returns None if the given rule has already been applied on
the given component.
"""
result = dict()
depsfinder = rule.depsfinder
if rule.dependson is None or len(rule.dependson) == 0 or \
depsfinder is None or len(depsfinder) == 0:
_LOGGER.debug("No 'DepsFinder' or 'DependsOn' specified" + \
" in rule %s for component %s. Skipping.",
rule, component)
return result
var_map = _get_var_map(component.id,
component.name,
component.type,
component.category,
self.ruleset.name,
rule.name,
rule.help)
cmd = substitute(var_map, depsfinder)
_LOGGER.debug("Calling depsfinder for component %s: %s", component, cmd)
popen_args = shlex.split(to_str_from_unicode(cmd, should_be_uni=True))
try:
popen = subprocess.Popen(popen_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=-1) # Use system default
except OSError as ose:
_LOGGER.error("Can't call depsfinder '%s': %s", cmd, ose)
return result
(msg_std, msg_err) = popen.communicate()
msg_std = msg_std.strip()
msg_err = msg_err.strip()
if len(msg_err) != 0:
_LOGGER.warning("Depsfinder error when " + \
"applying rule %s to component %s: %s",
rule, component, msg_err)
deps = set()
with StringIO(to_unicode(msg_std)) as reader:
for dep in reader:
dep_id = dep.strip()
if len(dep_id) == 0:
continue
dependency = self.components_map.get(dep_id)
if dependency is None:
_LOGGER.debug("Creating dep for component %s with id: %r",
component, dep_id)
dependency = Component(dep_id)
self.components_map[dep_id] = dependency
deps.add(dependency)
_update_graph_with_node(self.dag, dep_id)
if _LOGGER.isEnabledFor(INFO):
_LOGGER.info("%s.depsfinder(%s): %s",
rule.name, component.id,
NodeSet.fromlist([str(x.id) for x in deps]))
# Find match only on rule.dependson
return _find_match([self.ruleset.rules_for[x] for x in rule.dependson],
deps)
