def knowntypes(db, config, args):
"""
This action fetches the rules from the DB sequencer table and calls
the DGM stage for the creation of the corresponding ruleset and to fetch
the root rules mapping. The result is then displayed on the screen.
"""
usage = "Usage: %prog [global_options] " + KNOWNTYPES_ACTION_NAME + \
" [action_options] ruleset"
doc = KNOWNTYPES_DOC + \
" For each displayed types, the starting rules that will" + \
" be applied on them for the" + \
" computation of the dependency graph is also given."
cmd = os.path.basename(sys.argv[0])
progname=to_unicode(cmd).encode('ascii', 'replace')
parser = optparse.OptionParser(usage, description=doc, prog=progname)
(options, action_args) = parser.parse_args(args)
if len(action_args) != 1:
parser.error(KNOWNTYPES_ACTION_NAME + ": ruleSet is missing.")
req_ruleset = action_args[0]
rules = db.get_rules_for(req_ruleset)
ruleset = RuleSet(rules.values())
mapping = ruleset.root_rules_for
tab_values = []
# Sort according to category
for type_ in sorted(mapping, key=lambda t: t[t.find('@') + 1]):
for rule in mapping[type_]:
line = [type_, rule.filter, rule.name, rule.action]
tab_values.append([u"NONE" if x is None else x for x in line])
_LOGGER.output(smart_display([u"Type",
u"Filter",
u"Rule Name",
u"Action"],
tab_values, vsep=u" | "))
def dbadd(db, config, args):
"""
Add a rule into the sequencer table.
"""
usage = "%prog [options] dbadd ruleset name types " + \
"filter action depsfinder dependson comments help"
doc = """Add a rule into the sequencer table. Multiple 'types'
can be specified using the comma separator ','. Multiple dependencies
can be specified using the comma separator ','. Both action and
depsfinder should be a valid shell command line. Quote must be used to
prevent mis-interpretation by the running shell. Special 'NONE' and
'NULL' strings are interpreted as the special NULL db value. Note that
errors might be raised if DB constraints are not fulfilled. The DB
specific error message should tell what the problem is. If unclear,
have a look to the DB constraints."""
cmd = os.path.basename(sys.argv[0])
progname=to_unicode(cmd).encode('ascii', 'replace')
parser = optparse.OptionParser(usage, description=doc, prog=progname)
(options, add_args) = parser.parse_args(args)
if len(add_args) != 9:
parser.error(DBADD_ACTION_NAME + \
": expected %d arguments, given %d" % \
(9, len(add_args)))
add_args = [None if x == "NONE" or x == "NULL" else x for x in add_args]
try:
db.add_rule(create_rule_from_strings_array(add_args))
except DuplicateRuleError as dre:
_LOGGER.error(DBADD_ACTION_NAME + \
": Rule %s.%s does already exists.",
dre.ruleset, dre.name)
def get_component_set_from(config, components_lists):
"""
Use the Guesser API to fetch the components set from the given
components_lists
"""
all_set = set()
module_name = config.get(DEPMAKE_ACTION_NAME, GUESSER_MODULE_NAME)
_LOGGER.debug("Guesser module name: %s" , module_name)
module = __import__(to_str_from_unicode(module_name, should_be_uni=True))
# Go through sys.modules to find modules inside packages
module = sys.modules[module_name]
params = config.get(DEPMAKE_ACTION_NAME, GUESSER_PARAMS_NAME)
params = replace_if_none(params)
_LOGGER.debug("Guesser parameters: %s", params)
params = to_str_from_unicode(params)
guesser = module.get_guesser(params)
_LOGGER.info("Using guesser: %s", guesser)
for cl in components_lists:
cl = to_str_from_unicode(cl, 'utf-8', True)
(components_for, unknown) = guesser.guess_type(cl)
if len(unknown) != 0:
_LOGGER.warning("%s: [Unknown components]", unknown)
for table in components_for:
for type_ in components_for[table]:
components = NodeSet(components_for[table][type_])
for component in components:
comp2 = to_unicode(component)
all_set.add(Component(comp2, type_, table))
return all_set
def dbremove(db, config, args):
"""
Remove a rule or a ruleset from the sequencer table.
"""
usage = "%prog [options] dbremove [--nodeps] " + \
"ruleset_name [rule_name...]"
doc = """Remove all (given) rules from the sequencer table."""
cmd = os.path.basename(sys.argv[0])
progname=to_unicode(cmd).encode('ascii', 'replace')
parser = optparse.OptionParser(usage, description=doc, prog=progname)
add_options_to(parser, ['--Enforce', '--nodeps'], config)
(options, remove_args) = parser.parse_args(args)
if len(remove_args) < 1:
parser.error(DBREMOVE_ACTION_NAME + \
": expected at least %d arguments, given %d" % \
(1, len(remove_args)))
ruleset = remove_args[0]
rules = remove_args[1:] if len(remove_args) > 1 else None
if not options.enforce:
prompt = "Confirm the removal of %s ruleset %s?" % \
(("rules %s from" % \
", ".join(rules)) if rules is not None else "whole", \
ruleset)
if not confirm(prompt, False):
_LOGGER.output("Canceled.")
sys.exit(os.EX_OK)
remaining = db.remove_rules(ruleset, rules, options.nodeps)
if remaining is not None and len(remaining) != 0:
_LOGGER.error(DBREMOVE_ACTION_NAME + \
": unable to remove following rules %s "
"from ruleset %s"
", ".join(remaining), ruleset)
def dbcopy(db, config, args):
"""
Copy a rule or ruleset of the sequencer table.
"""
usage = "%prog [options] dbcopy " + \
"ruleset_src[:rule_src] ruleset_dst"
doc = "Copy ruleset_src to ruleset_dst" + \
" or copy rule_src from ruleset_src to ruleset_dest."
cmd = os.path.basename(sys.argv[0])
progname=to_unicode(cmd).encode('ascii', 'replace')
parser = optparse.OptionParser(usage, description=doc, prog=progname)
(options, copy_args) = parser.parse_args(args)
if len(copy_args) < 2:
parser.error(DBCOPY_ACTION_NAME + \
": expected a minimum of %d arguments, given %d" % \
(2, len(copy_args)))
(ruleset_src, sep, rule_src) = copy_args[0].partition(":")
ruleset_dst = copy_args[1]
try:
src_set = db.get_rules_for(ruleset_src)
except UnknownRuleSet as urs:
_LOGGER.error(str(urs))
return 1
dst_set = dict()
try:
dst_set = db.get_rules_for(ruleset_dst)
except UnknownRuleSet:
pass
rule_src = replace_if_none(rule_src)
if rule_src is None and len(dst_set) != 0:
_LOGGER.error(DBCOPY_ACTION_NAME + \
": ruleset %s already exists!",
ruleset_dst)
return 1
if rule_src is not None and len(rule_src) != 0:
if rule_src in dst_set:
_LOGGER.error(DBCOPY_ACTION_NAME + \
": rule %s " + \
" already exists in " + \
"ruleset %s!",
rule_src, ruleset_dst)
return 1
if rule_src not in src_set:
_LOGGER.error(DBCOPY_ACTION_NAME + \
": unknown rule %s in ruleset %s",
rule_src, ruleset_src)
return 1
rule_dst = src_set[rule_src]
rule_dst.ruleset = ruleset_dst
db.add_rules([rule_dst])
return
for rule in src_set.values():
rule.ruleset = ruleset_dst
dst_set[rule.name] = rule
db.add_rules(dst_set.values())
def test_to_unicode(self):
nonascii = "mmąöî"
uni = lib.to_unicode(nonascii)
assert type(nonascii) == str
assert len(nonascii) != 5
assert type(uni) == unicode
assert len(uni) == 5
assert uni == u"mmąöî"
def test_get_basedir_base(self):
ret = lib.get_basedir(u"mmąöî")
cmdstr = lib.to_unicode(os.path.basename(sys.argv[0]))
cmdfile = lib.to_unicode(os.path.abspath(sys.argv[0]))
# Do not follow symbolic links
stat = os.lstat(cmdfile)
if stat.st_uid == 0:
# owner = root
expected = os.path.join(u"/etc", cmdstr, u"mmąöî")
else:
owner_data = pwd.getpwuid(stat.st_uid)
owner = lib.to_unicode(owner_data[0])
expected = os.path.join(owner_data[5], '.'+cmdstr, u"mmąöî")
assert type(ret) == unicode
assert ret == expected
def dbcreate(db, config, args):
"""
Create the sequencer table.
"""
usage = "%prog [options] dbcreate"
doc = DBCREATE_DOC
cmd = os.path.basename(sys.argv[0])
progname=to_unicode(cmd).encode('ascii', 'replace')
parser = optparse.OptionParser(usage, description=doc, prog=progname)
(options, dbcreate_args) = parser.parse_args(args)
if len(dbcreate_args) != 0:
parser.error(DBCREATE_ACTION_NAME + \
": expected %d arguments, given %d" % \
(0, len(dbcreate_args)))
db.create_table()
def dbchecksum(db, config, args):
"""
Display the checksums of rulesets and of each rule in that ruleset.
"""
usage = "%prog [options] dbchecksum [ruleset]"
doc = "Compute checksum for the specified ruleset " + \
" (all if not specified)"
cmd = os.path.basename(sys.argv[0])
progname = to_unicode(cmd).encode('ascii', 'replace')
parser = optparse.OptionParser(usage, description=doc, prog=progname)
(options, action_args) = parser.parse_args(args)
if len(action_args) > 1:
parser.error(DBCHECKSUM_ACTION_NAME + \
": too many arguments %d, maximum is %d" % \
(len(action_args), 1))
tab_values = []
if len(action_args) == 1:
ruleset_name = action_args[0]
try:
(ruleset_h, h_for) = db.checksum(ruleset_name)
for rulename in h_for:
tab_values.append([ruleset_name,
rulename,
h_for[rulename].hexdigest()])
tab_values.append([ruleset_name, FILL_EMPTY_ENTRY, \
ruleset_h.hexdigest()])
except UnknownRuleSet as urs:
_LOGGER.error(DBCHECKSUM_ACTION_NAME + str(urs))
return 1
else:
rules_map = db.get_rules_map()
for ruleset_name in rules_map:
(ruleset_h, h_for) = db.checksum(ruleset_name)
for rulename in h_for:
tab_values.append([ruleset_name,
rulename,
h_for[rulename].hexdigest()])
tab_values.append([ruleset_name, FILL_EMPTY_ENTRY,
ruleset_h.hexdigest()])
_LOGGER.output(smart_display(CHECKSUM_HEADER,
tab_values,
vsep=u' | '))
def _parse_depmake_cmdline(config, args):
"""
DGM Action Specific CLI Parser
"""
usage = "Usage: %prog [global_options] " + DEPMAKE_ACTION_NAME + \
" [action_options] ruleset components_list"
doc = DEPMAKE_DOC + \
" The output format is suitable for the 'seqmake' action."
cmd = os.path.basename(sys.argv[0])
progname=to_unicode(cmd).encode('ascii', 'replace')
parser = optparse.OptionParser(usage, description=doc, prog=progname)
parser.add_option('-F', '--Force',
metavar='RULE_LIST',
dest='force',
type='string',
help="Specify the force mode ('allowed'," + \
" 'always' or 'never')" + \
" that should be used for the execution of" + \
" each action related to the given comma" + \
" separated list of rule names. When prefixed" + \
" by " + NOT_FORCE_OP + ", " + \
" action execution will 'never' be" + \
" forced. Otherwise, it will 'always' be" + \
" forced. Action related to a rule that is " + \
" not specified in the list will see its" + \
" force mode set to 'allowed' meaning that" + \
" the decision is left to the Instruction " + \
" Sequence Executor (see '" + \
ise_cli.SEQEXEC_ACTION_NAME + "')")
add_options_to(parser, ['--out', '--depgraphto', '--docache'], config)
(options, action_args) = parser.parse_args(args)
if len(action_args) < 2:
parser.error(DEPMAKE_ACTION_NAME + \
": ruleSet and/or components lists missing.")
req_ruleset = action_args[0]
lists = action_args[1:]
components_lists = parse_components_lists(lists)
return (options, req_ruleset, components_lists)
def _parse(config, ism_args):
"""
ISM Action Specific CLI Parser
"""
usage = "Usage: %prog [global_options] " + SEQMAKE_ACTION_NAME + \
" [action_options]"
doc = SEQMAKE_DOC + \
" The input can be the output of the 'depmake' action." + \
" The output can be used as an input of the 'seqexec' action."
cmd = path.basename(sys.argv[0])
progname=to_unicode(cmd).encode('ascii', 'replace')
opt_parser = optparse.OptionParser(usage, description=doc, prog=progname)
add_options_to(opt_parser, ['--file', '--out', '--algo'], config)
(ism_options, action_args) = opt_parser.parse_args(ism_args)
if len(action_args) != 0:
opt_parser.error(SEQMAKE_ACTION_NAME + \
": wrong number of arguments.")
return (ism_options, action_args)
def _parse(basedir, config, args):
"""
ISE Action Specific CLI Parser
"""
usage = "Usage: %prog [global_options] " + SEQEXEC_ACTION_NAME + \
" [action_options]"
doc = SEQEXEC_DOC + \
" The input can be the output of the 'seqmake' action."
cmd = os.path.basename(sys.argv[0])
progname = to_unicode(cmd).encode('ascii', 'replace')
opt_parser = optparse.OptionParser(usage, description=doc, prog=progname)
opt_parser.add_option("-F", "--Force",
dest="force",
action='store_true',
default=False,
# If you want to read the default from a
# config file, you need a way to override
# it. Therefore, 'store_true' above is not a
# good action. You should provide either a
# --Force=no or a --no-Force option.
# default=config.getboolean(SEQEXEC_ACTION_NAME, \
# "Force"),
help="Do not stop the execution of an action when" + \
" one of its dependencies exits" + \
" with a WARNING error code.")
add_options_to(opt_parser, ['--file', '--actionsgraphto', '--progress',
'--doexec', '--report', '--dostats', '--fanout'],
config)
(ise_options, action_args) = opt_parser.parse_args(args)
if len(action_args) != 0:
opt_parser.error(SEQEXEC_ACTION_NAME + \
": wrong number of arguments.")
return (ise_options, action_args)
def dbdrop(db, config, args):
"""
Drop the sequencer table.
"""
usage = "%prog [options] dbdrop"
doc = DBDROP_DOC
cmd = os.path.basename(sys.argv[0])
progname=to_unicode(cmd).encode('ascii', 'replace')
parser = optparse.OptionParser(usage, description=doc, prog=progname)
add_options_to(parser, ['--Enforce'], config)
(options, dbdrop_args) = parser.parse_args(args)
if len(dbdrop_args) != 0:
parser.error(DBDROP_ACTION_NAME + \
": expected %d arguments, given %d" % \
(0, len(dbdrop_args)))
if not options.enforce:
if not confirm("Confirm the full deletion of the sequencer table?",
False):
_LOGGER.output("Canceled.")
sys.exit(os.EX_OK)
db.drop_table()
def graphrules(db, config, args):
"""
This action fetches the rules from the DB sequencer table and calls
the DGM stage for the computation of the related graph.
This graph is then given to the user in the DOT format.
"""
usage = "Usage: %prog [global_options] " + GRAPHRULES_ACTION_NAME + \
" [action_options] ruleset"
doc = GRAPHRULES_DOC + \
" The graph is given in DOT format."
cmd = os.path.basename(sys.argv[0])
progname=to_unicode(cmd).encode('ascii', 'replace')
parser = optparse.OptionParser(usage, description=doc, prog=progname)
add_options_to(parser, ['--out'], config)
(options, action_args) = parser.parse_args(args)
if len(action_args) != 1:
parser.error(GRAPHRULES_ACTION_NAME + ": ruleSet is missing.")
req_ruleset = action_args[0]
rules = db.get_rules_for(req_ruleset)
ruleset = RuleSet(rules.values())
write_graph_to(ruleset.get_rules_graph(), options.out)
def dbupdate(db, config, args):
"""
Update a rule of the sequencer table.
"""
usage = "%prog [options] dbupdate [--nodeps] " + \
"ruleset_name rule_name" + \
" <column1>=<value1> <column2>=<value2>..."
doc = "Update each columns <column1>, <column2>, ... " + \
"of the given rule ('ruleset', 'name') with values " + \
"<value1>, <value2>, ... respectively in the sequencer table."
cmd = os.path.basename(sys.argv[0])
progname=to_unicode(cmd).encode('ascii', 'replace')
parser = optparse.OptionParser(usage, description=doc, prog=progname)
add_options_to(parser, ['--nodeps'], config)
(options, update_args) = parser.parse_args(args)
if len(update_args) < 3:
parser.error(DBUPDATE_ACTION_NAME + \
": expected a minimum of %d arguments, given %d" % \
(3, len(update_args)))
ruleset = update_args[0]
rulename = update_args[1]
update_set = set()
for record in update_args[2:]:
(col, sep, val) = record.partition("=")
val = None if val == 'NONE' or val == 'NULL' else val
update_set.add((col, val))
try:
if not db.update_rule(ruleset, rulename,
update_set, options.nodeps):
_LOGGER.error(DBUPDATE_ACTION_NAME + \
": unable to update specified rule (%s, %s)",
ruleset, rulename)
except UnknownRuleSet as urs:
_LOGGER.error(DBUPDATE_ACTION_NAME + str(urs))
return 1
except NoSuchRuleError as nsre:
_LOGGER.error(DBUPDATE_ACTION_NAME + str(nsre))
return 1
def dbshow(db, config, args):
"""
Display the sequencer table.
"""
usage = "%prog [options] dbshow [--columns=<column:max>,...] [ruleset]"
doc = "Display the sequencer table (for the given " + \
"ruleset if specified)."
cmd = os.path.basename(sys.argv[0])
progname=to_unicode(cmd).encode('ascii', 'replace')
parser = optparse.OptionParser(usage, description=doc, prog=progname)
parser.add_option("", "--columns", dest="columns_list",
action='store',
help="Use the given list of 'column:max' for" + \
" the display where 'column' is the column" + \
" label, and 'max' is the maximum number of" + \
" character that should be used in that" + \
" column (0 means remove the column" + \
" completely). Note: 'max' should" + \
" be greater than %d" % TRUNCATION_MAX_SIZE + \
" if 'max' is not given at all, then only" + \
" specified columns are displayed")
(options, show_args) = parser.parse_args(args)
if len(show_args) > 1:
parser.error(DBSHOW_ACTION_NAME + \
": too many arguments %d, maximum is %d" % \
(len(show_args), 1))
columns_max = dict()
if options.columns_list is not None:
columns_list = options.columns_list.split(',')
for column_spec in columns_list:
(column, sep, maxchar_s) = column_spec.partition(':')
if column not in RULES_HEADER:
parser.error(DBSHOW_ACTION_NAME + \
": unknown column: %s" % column)
if len(maxchar_s) == 0:
maxchar = REMOVE_UNSPECIFIED_COLUMNS
else:
try:
maxchar = int(maxchar_s)
except ValueError:
parser.error(DBSHOW_ACTION_NAME + \
": invalid max value %s " % maxchar_s + \
" for column %s." % column + \
" Positive integer expected.")
if 0 < maxchar < TRUNCATION_MAX_SIZE or maxchar < 0:
parser.error(DBSHOW_ACTION_NAME + \
": given max: %d" % maxchar + \
" for column %s" % column + \
" should be greater" + \
" than %d" % TRUNCATION_MAX_SIZE)
columns_max[column] = int(maxchar)
_LOGGER.info("Reading from db: %s", db)
if len(show_args) == 1:
ruleset_name = show_args[0]
try:
ruleset = db.get_rules_for(ruleset_name)
_display(ruleset.values(), columns_max)
except UnknownRuleSet as urs:
_LOGGER.error(DBSHOW_ACTION_NAME + str(urs))
return 1
else:
rules_map = db.get_rules_map()
all_rules = []
for ruleset_name in rules_map:
all_rules.extend(rules_map[ruleset_name].values())
_display(all_rules, columns_max)
def _get_deps(self, component, rule):
"""
Find dependencies of a given component. This implies calling
the rule.depsfinder script. Substitution of variables is done.
Returns None if the given rule has already been applied on
the given component.
"""
result = dict()
depsfinder = rule.depsfinder
if rule.dependson is None or len(rule.dependson) == 0 or \
depsfinder is None or len(depsfinder) == 0:
_LOGGER.debug("No 'DepsFinder' or 'DependsOn' specified" + \
" in rule %s for component %s. Skipping.",
rule, component)
return result
var_map = _get_var_map(component.id,
component.name,
component.type,
component.category,
self.ruleset.name,
rule.name,
rule.help)
cmd = substitute(var_map, depsfinder)
_LOGGER.debug("Calling depsfinder for component %s: %s", component, cmd)
popen_args = shlex.split(to_str_from_unicode(cmd, should_be_uni=True))
try:
popen = subprocess.Popen(popen_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=-1) # Use system default
except OSError as ose:
_LOGGER.error("Can't call depsfinder '%s': %s", cmd, ose)
return result
(msg_std, msg_err) = popen.communicate()
msg_std = msg_std.strip()
msg_err = msg_err.strip()
if len(msg_err) != 0:
_LOGGER.warning("Depsfinder error when " + \
"applying rule %s to component %s: %s",
rule, component, msg_err)
deps = set()
with StringIO(to_unicode(msg_std)) as reader:
for dep in reader:
dep_id = dep.strip()
if len(dep_id) == 0:
continue
dependency = self.components_map.get(dep_id)
if dependency is None:
_LOGGER.debug("Creating dep for component %s with id: %r",
component, dep_id)
dependency = Component(dep_id)
self.components_map[dep_id] = dependency
deps.add(dependency)
_update_graph_with_node(self.dag, dep_id)
if _LOGGER.isEnabledFor(INFO):
_LOGGER.info("%s.depsfinder(%s): %s",
rule.name, component.id,
NodeSet.fromlist([str(x.id) for x in deps]))
# Find match only on rule.dependson
return _find_match([self.ruleset.rules_for[x] for x in rule.dependson],
deps)