def LoadRule(cls, rule):
"""Loads the rule.
Args:
rule: string: The rule that needs to be loaded.
Return:
boolean: True if rule is already present or successfully loaded and false
otherwise.
"""
# Check if the rule is loaded.
if cls.GetRule(rule): return True
(dirname, targetname) = os.path.split(rule)
rules_file = os.path.join(dirname, 'RULES')
if not dirname or not os.path.isfile(rules_file):
TermColor.Error('No rules file %s for target %s ' % (
rules_file, Utils.RuleDisplayName(rule)))
return False
try:
Rules.LoadRules(dirname)
return True
except Exception as e:
if type(e) == KeyboardInterrupt: raise e
TermColor.PrintException('Could not load %s. ' % Utils.RuleDisplayName(rule))
return False
def GetExpandedRules(cls, rules, allowed_rule_types=None):
"""Returns the expanded rules corresponding to input rules.
Args:
rules: list: List of rules for which the automake is to be generated.
allowed_rule_types: list: List of allowed rules to use from the RULES
file. e.g. ['cc_bin', 'cc_test'] will create make rules for all
'cc_bin' and 'cc_test' rules in the RULES file but not for 'cc_lib'
rules.
Return:
(list, list): Returns a tuple in the form (successful_rules, failed_rules)
specifying rules that were expanded successfully and ones that failed.
"""
if not allowed_rule_types:
allowed_rule_types = cls.PARSED_RULE_TYPES
successful_rules = []
failed_rules = []
for target in rules:
if not cls.LoadRule(target):
failed_rules += [target]
continue
expanded_targets = []
(dirname, targetname) = os.path.split(target)
if targetname == 'RULES':
expanded_targets = cls.GetRulesForDir(dirname, allowed_rule_types)
if not expanded_targets:
TermColor.Warning('No rules found in %s' % target)
continue
else:
expanded_targets = [targetname]
for item in expanded_targets:
item_rule = os.path.join(dirname, item)
rule_data = cls.GetRule(item_rule)
if not rule_data:
TermColor.Error('Unable to find a rule for %s' %
Utils.RuleDisplayName(item_rule))
failed_rules += [item_rule]
continue
rule_type = rule_data.get('_type' , 'invalid')
if not rule_type in allowed_rule_types:
TermColor.Error('Rule %s of type %s not allowed ' %
(Utils.RuleDisplayName(item_rule), rule_type))
failed_rules += [item_rule]
continue
# All good.
successful_rules += [item_rule]
return (successful_rules, failed_rules)
def Flatten(cls, new_dep, referrer, referrer_data):
"""Given a new dependency, flatten it into existing
Args:
new_dep: string: The new dependency which needs to be flattened.
referrer: string: The referrer for which the new dep is flattened.
referrer_data: dict: The rule data for the referrer.
Exceptions:
RulesParseError: Raises exception if parsing fails.
"""
TermColor.VInfo(5, '--- Resolving dependency %s' % new_dep)
(libdir, libname) = os.path.split(new_dep)
if not libdir:
err_str = ('Cannot resolve dependency [%s] (referred to by [%s])'
% (Utils.RuleDisplayName(new_dep),
Utils.RuleDisplayName(referrer)))
TermColor.Error(err_str)
raise RulesParseError(err_str)
# load the corresponding RULES file
cls.LoadRules(libdir)
new_dep_data = Rules.GetRule(new_dep)
if not new_dep_data:
err_str = 'Unable to find [%s] (referred to by [%s])' % (new_dep, referrer)
TermColor.Error(err_str)
raise RulesParseError(err_str)
referre_type_base = re.sub('_.*', '', referrer_data.get('_type', 'invalid'))
new_dep_type = new_dep_data.get('_type' , 'invalid')
if not new_dep_type in cls.FLATTENED_RULE_TYPES.get(referre_type_base, []):
err_str = ('Invalid rule [%s] of type [%s] (referred to by [%s])' %
(new_dep, new_dep_type, referrer))
TermColor.Error(err_str)
raise RulesParseError(err_str)
# Merge the data.
cls._MergeDepData(new_dep, new_dep_data, referrer, referrer_data)
# Flatten recursively.
for d in new_dep_data.get('dep', set()):
if d not in referrer_data.get('dep', set()):
with cls.LOAD_LOCK:
referrer_data['dep'] |= set([d])
Rules.Flatten(d, new_dep, referrer_data)
def _WorkHorse(cls, rule, makefile):
"""Workhorse for building a single rule.
Args:
rule: string: The rule to build.
makefile: string: The *main* makefile name.
Return:
(int, string): Returns a tuple of the result status and the rule.
The status is '1' for success, '0' for 'ignore', '-1' for fail.
"""
start = time.time()
ignore = Utils.IgnoreRule(rule, Flags.ARGS.ignore_rules)
if ignore:
TermColor.Warning('Ignored targets in %s as anything with [%s] is ignored' %
(Utils.RuleDisplayName(rule), ignore))
return (0, rule)
TermColor.Info('Building %s' % Utils.RuleDisplayName(rule))
deps_file = cls.GetDepsFileName(makefile, rule, '.main.')
try:
shutil.copy(makefile, deps_file)
cls._PrepareDepsFile(rule, deps_file)
except (OSError, IOError) as e:
TermColor.Error('Could not create makefile for rule %s' %
Utils.RuleDisplayName(rule))
return (-1, rule)
# Make the rule.
status = cls._MakeSingeRule(rule, makefile, deps_file)
if status != 1:
TermColor.Failure('Failed Rule: %s' % Utils.RuleDisplayName(rule))
return (status, rule)
TermColor.Info('Built %s. Took %.2fs' %
(Utils.RuleDisplayName(rule), (time.time() - start)))
# Everything done. Mark the rule as successful.
return (1, rule)
def _MakeSingeRule(cls, rule, makefile, deps_file):
"""Builds a Single Rule.
Args:
rule: string: The rule to build.
makefile: string: The *main* makefile name.
Return:
(int): Returns the result status.
The status is '1' for success, '0' for 'ignore', '-1' for fail.
"""
# Build the rule.
if Flags.ARGS.pool_size:
parallel_processes = Flags.ARGS.pool_size
else:
parallel_processes = max(multiprocessing.cpu_count(), 1)
(status, out) = ExecUtils.RunCmd('make -r -j%d -f %s %s' % (parallel_processes,
deps_file, rule))
if status:
TermColor.Failure('Failed Rule: %s' % Utils.RuleDisplayName(rule))
return -1
TermColor.VInfo(1, '%s Output: \n%s' % (Utils.RuleDisplayName(rule), out))
return 1
def _RunSingeRule(cls, rule, pipe_output):
"""Runs a Single Rule.
Args:
rule: string: The rule to run.
pipe_output: bool: Whether to pipe_output or dump it to STDOUT.
Return:
(int, string): Returns a tuple of the result status and the rule.
The status is '1' for success, '0' for 'ignore', '-1' for fail.
"""
TermColor.Info('Running %s' % Utils.RuleDisplayName(rule))
start = time.time()
bin_file = FileUtils.GetBinPathForFile(rule)
(status, out) = ExecUtils.RunCmd('%s %s' % (bin_file, Flags.ARGS.args),
Flags.ARGS.timeout, pipe_output)
if status:
TermColor.Failure('Failed Rule: %s' % Utils.RuleDisplayName(rule))
return (-1, rule)
TermColor.Info('Ran %s. Took %.2fs' %
(Utils.RuleDisplayName(rule), (time.time() - start)))
# Everything done. Mark the rule as successful.
return (1, rule)
def _MakeSingeRule(cls, rule, makefile, deps_file):
"""Builds a Single Rule.
Args:
rule: string: The rule to build.
makefile: string: The *main* makefile name.
Return:
(int): Returns the result status.
The status is '1' for success, '0' for 'ignore', '-1' for fail.
"""
# Get dependencies list for the rule. Run this with the original main file.
(status, out) = ExecUtils.RunCmd('make -f %s %s' %
(makefile, cls.GetDepsRuleName(rule)))
if status:
TermColor.Error('Could not make dependency for rule %s' %
Utils.RuleDisplayName(rule))
return -1
return super(CCRules, cls)._MakeSingeRule(rule, makefile, deps_file)
def GenAutoMakeFileFromRules(self, rules, allowed_rule_types=None):
"""Generates the automake file for the input set of rules.
Args:
rules: list: List of rules for which the automake is to be generated.
allowed_rule_types: list: List of allowed rules to use from the RULES
file. e.g. ['cc_bin', 'cc_test'] will create make rules for all
'cc_bin' and 'cc_test' rules in the RULES file but not for 'cc_lib'
rules.
Return:
(dict {string : list}, list): Returns a tuple in the form
({type: successful_rules}, failed_rules) specifying rules for which
the make rules were successfully generated and for which it failed.
"""
specs = {}
successful_rules = {}
(successful_expand,
failed_rules) = Rules.GetExpandedRules(rules, allowed_rule_types)
for target in successful_expand:
rule_data = Rules.GetRule(target)
# Expand dependency list.
seen_deps = set()
try:
# Copy the deps to a new set.
deps = set(rule_data.get('dep', set()))
for dep in deps:
if dep not in seen_deps:
seen_deps |= set([dep])
Rules.Flatten(dep, target, rule_data)
else:
TermColor.Warning('Rule %s has duplicate dep %s ' %
(Utils.RuleDisplayName(target),
Utils.RuleDisplayName(dep)))
except RulesParseError as e:
TermColor.Error('Could not flatten %s' % target)
failed_rules += [target]
continue
rule_type = rule_data.get('_type', '')
if rule_type == 'proto_lib':
# TODO(pramodg): Revisit when we want to add other code sources.
ProtoRules.UpdateProtoRuleWithFormattedData(
rule_data, 'cc_lib')
elif rule_type == 'swig_lib':
SwigRules.WriteMakefile(rule_data,
self.GetAutoMakeFileName('swig'))
SwigRules.UpdateSwigRuleWithFormattedData(rule_data)
# Get the rule type again as it may have been updated.
rule_type = rule_data.get('_type', '')
rule_type_base = re.sub('_.*', '', rule_type)
if not rule_type_base:
TermColor.Error('Invalid Rule type: [%s]' % rule_type)
failed_rules += [target]
continue
# Now we have a complete list of source files, compile flags and links
# for this target.
specs[rule_type_base] = specs.get(rule_type_base, []) + [rule_data]
successful_rules[rule_type_base] = (
successful_rules.get(rule_type_base, []) + [target])
# Generate the automake file for each rule type.
for (k, v) in list(specs.items()):
if k == 'cc':
CCRules.WriteMakefile(v, self.GetAutoMakeFileName('cc'))
elif k == 'js':
JSRules.WriteMakefile(v, self.GetAutoMakeFileName('js'))
elif k == 'ng':
NGRules.WriteMakefile(v, self.GetAutoMakeFileName('ng'))
elif k == 'nge2e':
NGe2eRules.WriteMakefile(v, self.GetAutoMakeFileName('nge2e'))
elif k == 'pkg':
PkgRules.WriteMakefile(v, self.GetAutoMakeFileName('pkg'))
elif k == 'pkg_bin':
PkgRules.WriteMakefile(v, self.GetAutoMakeFileName('pkg_bin'))
elif k == 'pkg_sys':
PkgRules.WriteMakefile(v, self.GetAutoMakeFileName('pkg_sys'))
elif k == 'py':
PyRules.WriteMakefile(v, self.GetAutoMakeFileName('py'))
else:
TermColor.Info('No make file to be generated for %s' % k)
return (successful_rules, failed_rules)
def _RunSingeRule(cls, rule):
"""Runs a Single Rule.
Args:
rule: string: The rule to run.
Return:
(int, string): Returns a tuple of the result status and the rule.
The status is '1' for success, '0' for 'ignore', '-1' for fail.
"""
TermColor.Info('Generating dependencies for %s' %
Utils.RuleDisplayName(rule))
start = time.time()
gr = digraph.digraph()
gr.add_node(rule)
nodes = [rule]
while len(nodes):
node = nodes.pop(0)
# The rule has already been processed. We assume if the node has outgoing
# edges, the we already processed it.
if gr.node_order(node) > 0: continue
# Add the dependencies of the rule to the graph.
if not Rules.LoadRule(node) or not Rules.GetRule(node):
TermColor.Warning(
'Could not load dependency %s for target %s ' %
(Utils.RuleDisplayName(node), Utils.RuleDisplayName(rule)))
return (-1, rule)
node_data = Rules.GetRule(node)
for dep in node_data.get('dep', set()):
nodes += [dep]
# Add the dep to the graph.
if not gr.has_node(dep): gr.add_node(dep)
if not gr.has_edge([node, dep]): gr.add_edge([node, dep])
# Now we have the graph, lets render it.
try:
dt = dot.write(gr)
dt = dt.replace('"%s";' % rule, ('"%s" [style=filled];' % rule), 1)
dt = dt.replace(FileUtils.GetSrcRoot(), '')
depgrah_file_name = cls.__GetDepGraphFileNameForRule(rule)
if Flags.ARGS.mode == 'gv':
gvv = gv.readstring(dt)
gv.layout(gvv, 'dot')
gv.render(gvv, 'pdf', depgrah_file_name)
if not Flags.ARGS.quiet:
subprocess.call('gv %s &' % depgrah_file_name, shell=True)
elif Flags.ARGS.mode == 'text':
FileUtils.CreateFileWithData(depgrah_file_name, dt)
TermColor.Info(
'Generated dependency graph (%d nodes) for %s at %s \tTook %.2fs'
% (len(gr.nodes()), Utils.RuleDisplayName(rule),
depgrah_file_name, (time.time() - start)))
return (1, rule)
except Exception as e:
TermColor.Error('Failed to render %s. Error: %s' %
(Utils.RuleDisplayName(rule), e))
if type(e) == KeyboardInterrupt: raise e
return (-1, rule)
def __GetDepGraphFileNameForRule(cls, rule):
"""Returns the file name for the dep graph of a given rule."""
display_rule = Utils.RuleDisplayName(rule)
return os.path.join(
'/tmp',
Utils.RuleDisplayName(rule).replace(os.sep, '_') + '.depgraph')