def LoadRule(cls, rule):
"""Loads the rule.
Args:
rule: string: The rule that needs to be loaded.
Return:
boolean: True if rule is already present or successfully loaded and false
otherwise.
"""
# Check if the rule is loaded.
if cls.GetRule(rule): return True
(dirname, targetname) = os.path.split(rule)
rules_file = os.path.join(dirname, 'RULES')
if not dirname or not os.path.isfile(rules_file):
TermColor.Error('No rules file %s for target %s ' % (
rules_file, Utils.RuleDisplayName(rule)))
return False
try:
Rules.LoadRules(dirname)
return True
except Exception as e:
if type(e) == KeyboardInterrupt: raise e
TermColor.PrintException('Could not load %s. ' % Utils.RuleDisplayName(rule))
return False
def Run(cls):
"""Runs the command handler.
Return:
int: Exit status. 0 means no error.
"""
rules = cls._ComputeRules(Flags.ARGS.rule, Flags.ARGS.ignore_rules)
if not rules:
TermColor.Warning('Could not find any rules.')
return 101
(successful_rules, failed_rules) = cls.WorkHorse(rules)
if successful_rules:
TermColor.Info('')
TermColor.Success('No. of Rules: %d' % len(successful_rules))
TermColor.VInfo(
1, 'Successful Rules: %s' % json.dumps(
Utils.RulesDisplayNames(successful_rules), indent=2))
if failed_rules:
TermColor.Info('')
TermColor.Failure('No. of Rules: %d' % len(failed_rules))
TermColor.Failure(
'Rules: %s' %
json.dumps(Utils.RulesDisplayNames(failed_rules), indent=2))
return 102
return 0
def GetExpandedRules(cls, rules, allowed_rule_types=None):
"""Returns the expanded rules corresponding to input rules.
Args:
rules: list: List of rules for which the automake is to be generated.
allowed_rule_types: list: List of allowed rules to use from the RULES
file. e.g. ['cc_bin', 'cc_test'] will create make rules for all
'cc_bin' and 'cc_test' rules in the RULES file but not for 'cc_lib'
rules.
Return:
(list, list): Returns a tuple in the form (successful_rules, failed_rules)
specifying rules that were expanded successfully and ones that failed.
"""
if not allowed_rule_types:
allowed_rule_types = cls.PARSED_RULE_TYPES
successful_rules = []
failed_rules = []
for target in rules:
if not cls.LoadRule(target):
failed_rules += [target]
continue
expanded_targets = []
(dirname, targetname) = os.path.split(target)
if targetname == 'RULES':
expanded_targets = cls.GetRulesForDir(dirname, allowed_rule_types)
if not expanded_targets:
TermColor.Warning('No rules found in %s' % target)
continue
else:
expanded_targets = [targetname]
for item in expanded_targets:
item_rule = os.path.join(dirname, item)
rule_data = cls.GetRule(item_rule)
if not rule_data:
TermColor.Error('Unable to find a rule for %s' %
Utils.RuleDisplayName(item_rule))
failed_rules += [item_rule]
continue
rule_type = rule_data.get('_type' , 'invalid')
if not rule_type in allowed_rule_types:
TermColor.Error('Rule %s of type %s not allowed ' %
(Utils.RuleDisplayName(item_rule), rule_type))
failed_rules += [item_rule]
continue
# All good.
successful_rules += [item_rule]
return (successful_rules, failed_rules)
def Flatten(cls, new_dep, referrer, referrer_data):
"""Given a new dependency, flatten it into existing
Args:
new_dep: string: The new dependency which needs to be flattened.
referrer: string: The referrer for which the new dep is flattened.
referrer_data: dict: The rule data for the referrer.
Exceptions:
RulesParseError: Raises exception if parsing fails.
"""
TermColor.VInfo(5, '--- Resolving dependency %s' % new_dep)
(libdir, libname) = os.path.split(new_dep)
if not libdir:
err_str = ('Cannot resolve dependency [%s] (referred to by [%s])'
% (Utils.RuleDisplayName(new_dep),
Utils.RuleDisplayName(referrer)))
TermColor.Error(err_str)
raise RulesParseError(err_str)
# load the corresponding RULES file
cls.LoadRules(libdir)
new_dep_data = Rules.GetRule(new_dep)
if not new_dep_data:
err_str = 'Unable to find [%s] (referred to by [%s])' % (new_dep, referrer)
TermColor.Error(err_str)
raise RulesParseError(err_str)
referre_type_base = re.sub('_.*', '', referrer_data.get('_type', 'invalid'))
new_dep_type = new_dep_data.get('_type' , 'invalid')
if not new_dep_type in cls.FLATTENED_RULE_TYPES.get(referre_type_base, []):
err_str = ('Invalid rule [%s] of type [%s] (referred to by [%s])' %
(new_dep, new_dep_type, referrer))
TermColor.Error(err_str)
raise RulesParseError(err_str)
# Merge the data.
cls._MergeDepData(new_dep, new_dep_data, referrer, referrer_data)
# Flatten recursively.
for d in new_dep_data.get('dep', set()):
if d not in referrer_data.get('dep', set()):
with cls.LOAD_LOCK:
referrer_data['dep'] |= set([d])
Rules.Flatten(d, new_dep, referrer_data)
def Expand(cls, name):
"""Expand a file sname based on basedir.
Args:
name: string: The name of the rule.
Exceptions:
RulesParseError: Raises exception if parsing fails.
"""
sname = name.strip()
if not sname:
err_str = 'Empty names are not allowed in RULES specification'
TermColor.Error(err_str)
raise RulesParseError(err_str)
if not cls.basedir: # no basedir specified
return sname
elif sname[0] == '/': # absolute path
return Utils.RuleNormalizedName(sname)
else: # relative path
return Utils.RuleNormalizedName(os.path.join(cls.basedir, sname))
def _MakeSingeRule(cls, rule, makefile, deps_file):
"""Builds a Single Rule.
Args:
rule: string: The rule to build.
makefile: string: The *main* makefile name.
Return:
(int): Returns the result status.
The status is '1' for success, '0' for 'ignore', '-1' for fail.
"""
# Build the rule.
if Flags.ARGS.pool_size:
parallel_processes = Flags.ARGS.pool_size
else:
parallel_processes = max(multiprocessing.cpu_count(), 1)
(status, out) = ExecUtils.RunCmd('make -r -j%d -f %s %s' % (parallel_processes,
deps_file, rule))
if status:
TermColor.Failure('Failed Rule: %s' % Utils.RuleDisplayName(rule))
return -1
TermColor.VInfo(1, '%s Output: \n%s' % (Utils.RuleDisplayName(rule), out))
return 1
def _RunSingeRule(cls, rule, pipe_output):
"""Runs a Single Rule.
Args:
rule: string: The rule to run.
pipe_output: bool: Whether to pipe_output or dump it to STDOUT.
Return:
(int, string): Returns a tuple of the result status and the rule.
The status is '1' for success, '0' for 'ignore', '-1' for fail.
"""
TermColor.Info('Running %s' % Utils.RuleDisplayName(rule))
start = time.time()
bin_file = FileUtils.GetBinPathForFile(rule)
(status, out) = ExecUtils.RunCmd('%s %s' % (bin_file, Flags.ARGS.args),
Flags.ARGS.timeout, pipe_output)
if status:
TermColor.Failure('Failed Rule: %s' % Utils.RuleDisplayName(rule))
return (-1, rule)
TermColor.Info('Ran %s. Took %.2fs' %
(Utils.RuleDisplayName(rule), (time.time() - start)))
# Everything done. Mark the rule as successful.
return (1, rule)
def _ComputeRules(cls, targets, ignore_list=[]):
"""Computes the rules to be run given the input targets.
Args:
targets: list: List of input targets.
Return:
list: List of actual rules to be run.
"""
rules = []
for target in targets:
ignore = Utils.IgnoreRule(target, ignore_list)
if ignore:
TermColor.Warning(
'Ignored target %s as anything with [%s] is ignored.' %
(target, ignore))
continue
if os.path.isdir(target):
target = os.getcwd() if target == '.' else target
rule = os.path.join(target, 'RULES')
if os.path.isfile(rule):
rules += [Utils.RuleNormalizedName(rule)]
else:
TermColor.Warning('No RULES file in directory: %s' %
target)
elif os.path.isfile(target):
rules += [
Utils.RuleNormalizedName(os.path.splitext(target)[0])
]
elif os.path.basename(target) == '...':
dir = os.path.dirname(target)
if not dir: dir = os.getcwd()
dir = os.path.dirname(
Utils.RuleNormalizedName(os.path.join(dir, 'RULES')))
rules += Utils.GetRulesFilesFromSubdirs(dir, ignore_list)
else:
rules += [Utils.RuleNormalizedName(target)]
temp_list = []
seen = set()
for rule in rules:
if rule in seen: continue
temp_list += [rule]
seen |= set([rule])
rules = []
for rule in temp_list:
if ((os.path.basename(rule) != 'RULES') and
(os.path.join(os.path.dirname(rule), 'RULES') in seen)):
continue
rules += [rule]
return rules
def __init__(self, rule_name):
"""initializes the state
Args:
rule_name (string) - the rule name
Raises:
UnsupportedRuleError: raises exception if the rule type is not yet
supported. add to the RULE_TYPES lists
"""
# Create the user friendly link to bin dir if it doesn't already exist.
FileUtils.CreateLink(FileUtils.GetEDir(), FileUtils.GetBinDir())
rule_name = Utils.RuleNormalizedName(rule_name)
Rules.LoadRule(rule_name)
self._rule = Rules.GetRule(rule_name)
if not self._rule['_type'] in Packager.RULE_PACKAGER:
err = 'Rule type %s not supported' % self._rule._type
TermColor.Error(err)
raise UnsupportedRuleError(err)
self._packager = Packager.RULE_PACKAGER[self._rule['_type']]
def _MakeSingeRule(cls, rule, makefile, deps_file):
"""Builds a Single Rule.
Args:
rule: string: The rule to build.
makefile: string: The *main* makefile name.
Return:
(int): Returns the result status.
The status is '1' for success, '0' for 'ignore', '-1' for fail.
"""
# Get dependencies list for the rule. Run this with the original main file.
(status, out) = ExecUtils.RunCmd('make -f %s %s' %
(makefile, cls.GetDepsRuleName(rule)))
if status:
TermColor.Error('Could not make dependency for rule %s' %
Utils.RuleDisplayName(rule))
return -1
return super(CCRules, cls)._MakeSingeRule(rule, makefile, deps_file)
def _WorkHorse(cls, rule, makefile):
"""Workhorse for building a single rule.
Args:
rule: string: The rule to build.
makefile: string: The *main* makefile name.
Return:
(int, string): Returns a tuple of the result status and the rule.
The status is '1' for success, '0' for 'ignore', '-1' for fail.
"""
start = time.time()
ignore = Utils.IgnoreRule(rule, Flags.ARGS.ignore_rules)
if ignore:
TermColor.Warning('Ignored targets in %s as anything with [%s] is ignored' %
(Utils.RuleDisplayName(rule), ignore))
return (0, rule)
TermColor.Info('Building %s' % Utils.RuleDisplayName(rule))
deps_file = cls.GetDepsFileName(makefile, rule, '.main.')
try:
shutil.copy(makefile, deps_file)
cls._PrepareDepsFile(rule, deps_file)
except (OSError, IOError) as e:
TermColor.Error('Could not create makefile for rule %s' %
Utils.RuleDisplayName(rule))
return (-1, rule)
# Make the rule.
status = cls._MakeSingeRule(rule, makefile, deps_file)
if status != 1:
TermColor.Failure('Failed Rule: %s' % Utils.RuleDisplayName(rule))
return (status, rule)
TermColor.Info('Built %s. Took %.2fs' %
(Utils.RuleDisplayName(rule), (time.time() - start)))
# Everything done. Mark the rule as successful.
return (1, rule)
def GenAutoMakeFileFromRules(self, rules, allowed_rule_types=None):
"""Generates the automake file for the input set of rules.
Args:
rules: list: List of rules for which the automake is to be generated.
allowed_rule_types: list: List of allowed rules to use from the RULES
file. e.g. ['cc_bin', 'cc_test'] will create make rules for all
'cc_bin' and 'cc_test' rules in the RULES file but not for 'cc_lib'
rules.
Return:
(dict {string : list}, list): Returns a tuple in the form
({type: successful_rules}, failed_rules) specifying rules for which
the make rules were successfully generated and for which it failed.
"""
specs = {}
successful_rules = {}
(successful_expand,
failed_rules) = Rules.GetExpandedRules(rules, allowed_rule_types)
for target in successful_expand:
rule_data = Rules.GetRule(target)
# Expand dependency list.
seen_deps = set()
try:
# Copy the deps to a new set.
deps = set(rule_data.get('dep', set()))
for dep in deps:
if dep not in seen_deps:
seen_deps |= set([dep])
Rules.Flatten(dep, target, rule_data)
else:
TermColor.Warning('Rule %s has duplicate dep %s ' %
(Utils.RuleDisplayName(target),
Utils.RuleDisplayName(dep)))
except RulesParseError as e:
TermColor.Error('Could not flatten %s' % target)
failed_rules += [target]
continue
rule_type = rule_data.get('_type', '')
if rule_type == 'proto_lib':
# TODO(pramodg): Revisit when we want to add other code sources.
ProtoRules.UpdateProtoRuleWithFormattedData(
rule_data, 'cc_lib')
elif rule_type == 'swig_lib':
SwigRules.WriteMakefile(rule_data,
self.GetAutoMakeFileName('swig'))
SwigRules.UpdateSwigRuleWithFormattedData(rule_data)
# Get the rule type again as it may have been updated.
rule_type = rule_data.get('_type', '')
rule_type_base = re.sub('_.*', '', rule_type)
if not rule_type_base:
TermColor.Error('Invalid Rule type: [%s]' % rule_type)
failed_rules += [target]
continue
# Now we have a complete list of source files, compile flags and links
# for this target.
specs[rule_type_base] = specs.get(rule_type_base, []) + [rule_data]
successful_rules[rule_type_base] = (
successful_rules.get(rule_type_base, []) + [target])
# Generate the automake file for each rule type.
for (k, v) in list(specs.items()):
if k == 'cc':
CCRules.WriteMakefile(v, self.GetAutoMakeFileName('cc'))
elif k == 'js':
JSRules.WriteMakefile(v, self.GetAutoMakeFileName('js'))
elif k == 'ng':
NGRules.WriteMakefile(v, self.GetAutoMakeFileName('ng'))
elif k == 'nge2e':
NGe2eRules.WriteMakefile(v, self.GetAutoMakeFileName('nge2e'))
elif k == 'pkg':
PkgRules.WriteMakefile(v, self.GetAutoMakeFileName('pkg'))
elif k == 'pkg_bin':
PkgRules.WriteMakefile(v, self.GetAutoMakeFileName('pkg_bin'))
elif k == 'pkg_sys':
PkgRules.WriteMakefile(v, self.GetAutoMakeFileName('pkg_sys'))
elif k == 'py':
PyRules.WriteMakefile(v, self.GetAutoMakeFileName('py'))
else:
TermColor.Info('No make file to be generated for %s' % k)
return (successful_rules, failed_rules)
def _RunSingeRule(cls, rule):
"""Runs a Single Rule.
Args:
rule: string: The rule to run.
Return:
(int, string): Returns a tuple of the result status and the rule.
The status is '1' for success, '0' for 'ignore', '-1' for fail.
"""
TermColor.Info('Generating dependencies for %s' %
Utils.RuleDisplayName(rule))
start = time.time()
gr = digraph.digraph()
gr.add_node(rule)
nodes = [rule]
while len(nodes):
node = nodes.pop(0)
# The rule has already been processed. We assume if the node has outgoing
# edges, the we already processed it.
if gr.node_order(node) > 0: continue
# Add the dependencies of the rule to the graph.
if not Rules.LoadRule(node) or not Rules.GetRule(node):
TermColor.Warning(
'Could not load dependency %s for target %s ' %
(Utils.RuleDisplayName(node), Utils.RuleDisplayName(rule)))
return (-1, rule)
node_data = Rules.GetRule(node)
for dep in node_data.get('dep', set()):
nodes += [dep]
# Add the dep to the graph.
if not gr.has_node(dep): gr.add_node(dep)
if not gr.has_edge([node, dep]): gr.add_edge([node, dep])
# Now we have the graph, lets render it.
try:
dt = dot.write(gr)
dt = dt.replace('"%s";' % rule, ('"%s" [style=filled];' % rule), 1)
dt = dt.replace(FileUtils.GetSrcRoot(), '')
depgrah_file_name = cls.__GetDepGraphFileNameForRule(rule)
if Flags.ARGS.mode == 'gv':
gvv = gv.readstring(dt)
gv.layout(gvv, 'dot')
gv.render(gvv, 'pdf', depgrah_file_name)
if not Flags.ARGS.quiet:
subprocess.call('gv %s &' % depgrah_file_name, shell=True)
elif Flags.ARGS.mode == 'text':
FileUtils.CreateFileWithData(depgrah_file_name, dt)
TermColor.Info(
'Generated dependency graph (%d nodes) for %s at %s \tTook %.2fs'
% (len(gr.nodes()), Utils.RuleDisplayName(rule),
depgrah_file_name, (time.time() - start)))
return (1, rule)
except Exception as e:
TermColor.Error('Failed to render %s. Error: %s' %
(Utils.RuleDisplayName(rule), e))
if type(e) == KeyboardInterrupt: raise e
return (-1, rule)
def __GetDepGraphFileNameForRule(cls, rule):
"""Returns the file name for the dep graph of a given rule."""
display_rule = Utils.RuleDisplayName(rule)
return os.path.join(
'/tmp',
Utils.RuleDisplayName(rule).replace(os.sep, '_') + '.depgraph')