def __init__(self, command_targets, load_targets, blade_path, working_dir,
build_path, blade_root_dir, blade_options, command):
"""init method.
"""
self.__command_targets = command_targets
self.__load_targets = load_targets
self.__blade_path = blade_path
self.__working_dir = working_dir
self.__build_path = build_path
self.__root_dir = blade_root_dir
self.__options = blade_options
self.__command = command
# Source dir of current loading BUILD file
self.__current_source_path = blade_root_dir
# The direct targets that are used for analyzing
self.__direct_targets = []
# All command targets, make sure that all targets specified with ...
# are all in the list now
self.__all_command_targets = []
# Given some targets specified in the command line, Blade will load
# BUILD files containing these command line targets; global target
# functions, i.e., cc_library, cc_binary and etc, in these BUILD
# files will register targets into target_database, which then becomes
# the input to dependency analyzer and SCons rules generator. It is
# notable that not all targets in target_database are dependencies of
# command line targets.
self.__target_database = {}
# targets to build after loading the build files.
self.__build_targets = {}
# The targets keys list after sorting by topological sorting method.
# Used to generate build rules in correct order.
self.__sorted_targets_keys = []
# The depended targets dict after topological sorting
self.__depended_targets = {}
# Indicate whether the deps list is expanded by expander or not
self.__targets_expanded = False
self.__build_time = time.time()
self.__build_platform = BuildPlatform()
self.build_environment = BuildEnvironment(self.__root_dir)
self.svn_root_dirs = []
self._verify_history_path = os.path.join(build_path,
'.blade_verify.json')
self._verify_history = {
'header_inclusion_dependencies':
{}, # path(.H) -> mtime(modification time)
}
def __init__(self,
command_targets,
blade_path,
working_dir,
build_path,
blade_root_dir,
blade_options,
command):
"""init method.
"""
self.__command_targets = command_targets
self.__blade_path = blade_path
self.__working_dir = working_dir
self.__build_path = build_path
self.__root_dir = blade_root_dir
self.__options = blade_options
self.__command = command
# Source dir of current loading BUILD file
self.__current_source_path = blade_root_dir
# The direct targets that are used for analyzing
self.__direct_targets = []
# All command targets, make sure that all targets specified with ...
# are all in the list now
self.__all_command_targets = []
# Given some targets specified in the command line, Blade will load
# BUILD files containing these command line targets; global target
# functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
# files will register targets into target_database, which then becomes
# the input to dependency analyzer and SCons rules generator. It is
# notable that not all targets in target_database are dependencies of
# command line targets.
self.__target_database = {}
# targets to build after loading the build files.
self.__build_targets = {}
# The targets keys list after sorting by topological sorting method.
# Used to generate build rules in correct order.
self.__sorted_targets_keys = []
# The depended targets dict after topological sorting
self.__depended_targets = {}
# Inidcating that whether the deps list is expanded by expander or not
self.__targets_expanded = False
self.__scons_platform = SconsPlatform()
self.build_environment = BuildEnvironment(self.__root_dir)
self.svn_root_dirs = []
def __init__(self,
command_targets,
blade_path,
working_dir,
build_path,
current_source_path,
blade_options,
**kwargs):
"""init method.
Mainly to hold the global data.
The directory which changes during the runtime of blade, and
contains BUILD file under current focus.
current_source_dir = "."
Given some targets specified in the command line, Blade will load
BUILD files containing these command line targets; global target
functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
files will register targets into target_database, which then becomes
the input to dependency analyzer and SCons rules generator. It is
notable that not all targets in target_database are dependencies of
command line targets.
target_database = {}
related targets after loading the build files and exec BUILD files
as python script
related_targets = {}
The map used by build rules to ensure that a source file occurres in
exactly one rule/target(only library target).
target_srcs_map = {}
The scons cache manager class string, which should be output to
scons script if ccache is not installed
scache_manager_class_str = ''
The targets keys list after sorting by topological sorting method.
sorted_targets_keys = []
Inidcating that whether the deps list is expanded by expander or not
False - not expanded
True - expanded
target_deps_expanded
All targets after expanding their dependency
all_targets = {}
The scons target objects registered into blade manager
scons_targets_map = {}
The vars which are depended by python binary
{key : 'python_files'}
self.python_binary_dep_source_cmd = {}
The files which are depended by python binary
{key : 'python_files'}
python_binary_dep_source_map = {}
The files which are depended by java jar file
{key : 'java_files'}
java_jar_dep_source_map = {}
The files which should be packed into java jar
{key : 'packing_files'}
java_jar_files_packing_map = {}
The jar files map
{key : 'jar_files_generated'}
java_jars_map = {}
The java compiling classpath parameter map
java_classpath_map = {}
{key : 'target_path'}
The java_jar dep var map, which should be added to dependency chain
java_jar_dep_vars = {}
The cc objects pool, a map to hold all the objects name.
cc_objects_pool = {}
The gen rule files map, which is used to generate the explict dependency
relationtion ship between gen_rule target and other targets
gen_rule_files_map = {}
The direct targets that are used for analyzing
direct_targets = []
All command targets, make sure that all targets specified with ...
are all in the list now
all_command_targets = []
The class to get platform info
SconsPlatform
The class to manage the cc flags
CcFlagsManager
The sources files that are needed to perform explict dependency
sources_explict_dependency_map = {}
The prebuilt cc_library file map which is needed to establish
symbolic links while testing
prebuilt_cc_library_file_map = {}
"""
self.command_targets = command_targets
self.direct_targets = []
self.all_command_targets = []
self.blade_path = blade_path
self.working_dir = working_dir
self.build_path = build_path
self.current_source_path = current_source_path
self.target_database = {}
self.related_targets = {}
self.target_srcs_map = {}
self.scache_manager_class_str = ''
self.options = blade_options
self.sorted_targets_keys = []
self.target_deps_expanded = False
self.all_targets_expanded = {}
self.scons_targets_map = {}
self.java_jar_dep_source_map = {}
self.java_jar_files_packing_map = {}
self.java_jars_map = {}
self.java_classpath_map = {}
self.java_jar_dep_vars = {}
self.python_binary_dep_source_cmd = {}
self.python_binary_dep_source_map = {}
self.cc_objects_pool = {}
self.deps_expander = None
self.build_rules_generator = None
self.gen_rule_files_map = {}
self.scons_platform = SconsPlatform()
self.ccflags_manager = CcFlagsManager(self.options)
self.sources_explict_dependency_map = {}
self.prebuilt_cc_library_file_map = {}
self.distcc_enabled = configparse.blade_config.get_config(
'distcc_config')['enabled']
self.build_environment = BuildEnvironment(self.current_source_path)
self.svn_root_dirs = []
self.kwargs = kwargs
class Blade(object):
"""Blade. A blade manager class. """
def __init__(self,
command_targets,
blade_path,
working_dir,
build_path,
current_source_path,
blade_options,
**kwargs):
"""init method.
Mainly to hold the global data.
The directory which changes during the runtime of blade, and
contains BUILD file under current focus.
current_source_dir = "."
Given some targets specified in the command line, Blade will load
BUILD files containing these command line targets; global target
functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
files will register targets into target_database, which then becomes
the input to dependency analyzer and SCons rules generator. It is
notable that not all targets in target_database are dependencies of
command line targets.
target_database = {}
related targets after loading the build files and exec BUILD files
as python script
related_targets = {}
The map used by build rules to ensure that a source file occurres in
exactly one rule/target(only library target).
target_srcs_map = {}
The scons cache manager class string, which should be output to
scons script if ccache is not installed
scache_manager_class_str = ''
The targets keys list after sorting by topological sorting method.
sorted_targets_keys = []
Inidcating that whether the deps list is expanded by expander or not
False - not expanded
True - expanded
target_deps_expanded
All targets after expanding their dependency
all_targets = {}
The scons target objects registered into blade manager
scons_targets_map = {}
The vars which are depended by python binary
{key : 'python_files'}
self.python_binary_dep_source_cmd = {}
The files which are depended by python binary
{key : 'python_files'}
python_binary_dep_source_map = {}
The files which are depended by java jar file
{key : 'java_files'}
java_jar_dep_source_map = {}
The files which should be packed into java jar
{key : 'packing_files'}
java_jar_files_packing_map = {}
The jar files map
{key : 'jar_files_generated'}
java_jars_map = {}
The java compiling classpath parameter map
java_classpath_map = {}
{key : 'target_path'}
The java_jar dep var map, which should be added to dependency chain
java_jar_dep_vars = {}
The cc objects pool, a map to hold all the objects name.
cc_objects_pool = {}
The gen rule files map, which is used to generate the explict dependency
relationtion ship between gen_rule target and other targets
gen_rule_files_map = {}
The direct targets that are used for analyzing
direct_targets = []
All command targets, make sure that all targets specified with ...
are all in the list now
all_command_targets = []
The class to get platform info
SconsPlatform
The class to manage the cc flags
CcFlagsManager
The sources files that are needed to perform explict dependency
sources_explict_dependency_map = {}
The prebuilt cc_library file map which is needed to establish
symbolic links while testing
prebuilt_cc_library_file_map = {}
"""
self.command_targets = command_targets
self.direct_targets = []
self.all_command_targets = []
self.blade_path = blade_path
self.working_dir = working_dir
self.build_path = build_path
self.current_source_path = current_source_path
self.target_database = {}
self.related_targets = {}
self.target_srcs_map = {}
self.scache_manager_class_str = ''
self.options = blade_options
self.sorted_targets_keys = []
self.target_deps_expanded = False
self.all_targets_expanded = {}
self.scons_targets_map = {}
self.java_jar_dep_source_map = {}
self.java_jar_files_packing_map = {}
self.java_jars_map = {}
self.java_classpath_map = {}
self.java_jar_dep_vars = {}
self.python_binary_dep_source_cmd = {}
self.python_binary_dep_source_map = {}
self.cc_objects_pool = {}
self.deps_expander = None
self.build_rules_generator = None
self.gen_rule_files_map = {}
self.scons_platform = SconsPlatform()
self.ccflags_manager = CcFlagsManager(self.options)
self.sources_explict_dependency_map = {}
self.prebuilt_cc_library_file_map = {}
self.distcc_enabled = configparse.blade_config.get_config(
'distcc_config')['enabled']
self.build_environment = BuildEnvironment(self.current_source_path)
self.svn_root_dirs = []
self.kwargs = kwargs
def _get_normpath_target(self, command_target):
"""returns a tuple (path, name).
path is a full path from BLADE_ROOT
"""
target_path = relative_path(self.working_dir, self.current_source_path)
path, name = command_target.split(':')
if target_path != '.':
if path:
path = target_path + '/' + path
else:
path = target_path
path = os.path.normpath(path)
return path, name
def load_targets(self):
"""Load the targets. """
console.info("loading BUILDs...")
if self.kwargs.get('blade_command', '') == 'query':
working_dir = self.current_source_path
if '...' not in self.command_targets:
new_target_list = []
for target in self.command_targets:
new_target_list.append("%s:%s" %
self._get_normpath_target(target))
self.command_targets = new_target_list
else:
working_dir = self.working_dir
(self.direct_targets,
self.all_command_targets) = load_targets(self.command_targets,
working_dir,
self.current_source_path,
self)
console.info("loading done.")
return self.direct_targets, self.all_command_targets
def analyze_targets(self):
"""Expand the targets. """
console.info("analyzing dependency graph...")
self.deps_analyzer = DependenciesAnalyzer(self)
self.deps_analyzer.analyze_deps()
console.info("analyzing done.")
return self.all_targets_expanded
def generate_build_rules(self):
"""Generate the constructing rules. """
console.info("generating build rules...")
self.build_rules_generator = SconsRulesGenerator('SConstruct',
self.blade_path, self)
rules_buf = self.build_rules_generator.generate_scons_script()
console.info("generating done.")
return rules_buf
def generate(self):
"""Build the targets. """
self.load_targets()
self.analyze_targets()
self.generate_build_rules()
def run(self, target):
"""Run the target. """
key = self._get_normpath_target(target)
runner = BinaryRunner(self.all_targets_expanded,
self.options,
self.prebuilt_cc_library_file_map,
self.target_database)
return runner.run_target(key)
def test(self):
"""Run tests. """
test_runner = TestRunner(self.all_targets_expanded,
self.options,
self.prebuilt_cc_library_file_map,
self.target_database,
self.direct_targets)
return test_runner.run()
def query(self, targets):
"""Query the targets. """
print_deps = hasattr(self.options, 'deps') and (
self.options.deps)
print_depended = hasattr(self.options, 'depended') and (
self.options.depended)
dot_file = ""
if hasattr(self.options, 'output-to-dot'):
dot_file = self.options.output_to_dot
result_map = self.query_helper(targets)
if dot_file:
print_mode = 0
if print_deps:
print_mode = 0
if print_depended:
print_mode = 1
if not dot_file.startswith("/"):
dot_file = self.working_dir + "/" + dot_file
self.output_dot(result_map, print_mode, dot_file)
else:
if print_deps:
for key in result_map.keys():
print "\n"
deps = result_map[key][0]
console.info("//%s:%s depends on the following targets:" % (
key[0], key[1]))
for d in deps:
print "%s:%s" % (d[0], d[1])
if print_depended:
for key in result_map.keys():
print "\n"
depended_by = result_map[key][1]
console.info("//%s:%s is depended by the following targets:" % (
key[0], key[1]))
depended_by.sort(key=lambda x: x, reverse=False)
for d in depended_by:
print "%s:%s" % (d[0], d[1])
return 0
def print_dot_node(self, output_file, node):
print >>output_file, '"%s:%s" [label = "%s:%s"]' % (node[0],
node[1],
node[0],
node[1])
def print_dot_deps(self, output_file, node, target_set):
targets = self.related_targets
deps = targets.get(node, {}).get('direct_deps', [])
for i in deps:
if not i in target_set:
continue
print >>output_file, '"%s:%s" -> "%s:%s"' % (node[0],
node[1],
i[0],
i[1])
def output_dot(self, result_map, print_mode, dot_file):
f = open(dot_file, 'w')
targets = result_map.keys()
nodes = set(targets)
for key in targets:
nodes |= set(result_map[key][print_mode])
print >>f, "digraph blade {"
for i in nodes:
self.print_dot_node(f, i)
for i in nodes:
self.print_dot_deps(f, i, nodes)
print >>f, "}"
f.close()
def query_helper(self, targets):
"""Query the targets helper method. """
all_targets = self.all_targets_expanded
query_list = []
target_path = relative_path(self.working_dir, self.current_source_path)
t_path = ''
for t in targets:
key = t.split(':')
if target_path == '.':
t_path = key[0]
else:
t_path = target_path + '/' + key[0]
t_path = os.path.normpath(t_path)
query_list.append((t_path, key[1]))
result_map = {}
for key in query_list:
result_map[key] = ([], [])
deps = all_targets.get(key, {}).get('deps', [])
deps.sort(key=lambda x: x, reverse=False)
depended_by = []
for tkey in all_targets.keys():
if key in all_targets[tkey]['deps']:
depended_by.append(tkey)
depended_by.sort(key=lambda x: x, reverse=False)
result_map[key] = (list(deps), list(depended_by))
return result_map
def get_blade_path(self):
"""Return the blade archive path. """
return self.blade_path
def get_build_path(self):
"""The current building path. """
return self.build_path
def set_current_source_path(self, current_source_path):
"""Set the current source path. """
self.current_source_path = current_source_path
def get_current_source_path(self):
"""Get the current source path. """
return self.current_source_path
def get_target_database(self):
"""Get the whole target database that haven't been expanded. """
return self.target_database
def set_related_targets(self, related_targets):
"""Set the related targets. """
self.related_targets = dict(related_targets)
def get_related_targets(self):
"""Get the related targets. """
return self.related_targets
def get_direct_targets(self):
"""Return the direct targets. """
return self.direct_targets
def get_all_command_targets(self):
"""Return all command targets. """
return self.all_command_targets
def set_sorted_targets_keys(self, sorted_keys_list):
"""Set the keys list from expaned targets. """
self.sorted_targets_keys = list(sorted_keys_list)
def get_sorted_targets_keys(self):
"""Get the keys list from expaned targets. """
return self.sorted_targets_keys
def set_all_targets_expanded(self, all_targets):
"""Set the targets that have been expanded by expander. """
self.all_targets_expanded = dict(all_targets)
self.target_deps_expanded = True
def get_all_targets_expanded(self):
"""Get all the targets that expaned. """
return self.all_targets_expanded
def get_target_srcs_map(self):
"""Get the targets source files map.
It is used in generating cc object rules.
"""
return self.target_srcs_map
def get_options(self):
"""Get the global command options. """
return self.options
def get_expanded(self):
"""Whether the targets are expanded. """
return self.target_deps_expanded
def register_scons_target(self, target_key, scons_target):
"""Register scons targets into the scons targets map.
It is used to do quick looking.
"""
# check that whether there is already a key in database
if target_key in self.scons_targets_map.keys():
console.error_exit(
"target name %s is duplicate in //%s/BUILD" % (
target_key[1], target_key[0]))
self.scons_targets_map[target_key] = scons_target
def get_scons_target(self, target_key):
"""Get scons target according to the key. """
return self.scons_targets_map.get(target_key, None)
def get_java_jar_dep_source_map(self):
"""The map mainly to hold the java files from swig or proto rules.
These files maybe depended by java_jar target.
"""
return self.java_jar_dep_source_map
def get_java_jar_files_packing_map(self):
"""The map to hold the files that should be packed into java jar. """
return self.java_jar_files_packing_map
def get_java_jars_map(self):
"""The map to hold the java jar files generated by blade. """
return self.java_jars_map
def get_java_classpath_map(self):
"""The classpath list which is needed by java compling. """
return self.java_classpath_map
def get_java_jar_dep_vars(self):
"""The vars map which is prerequiste of the java jar target. """
return self.java_jar_dep_vars
def get_cc_objects_pool(self):
"""The cc objects pool which is used when generating the cc object rules. """
return self.cc_objects_pool
def _is_scons_object_type(self, target_type):
"""The types that shouldn't be registered into blade manager.
Sholdn't invoke scons_rule method when it is not a scons target which
could not be registered into blade manager, like system library.
1. system_library
"""
if target_type == 'system_library':
return False
else:
return True
def get_targets_rules(self):
"""Get the build rules and return to the object who queries this. """
rules_buf = []
skip_test_targets = False
if hasattr(self.options, 'no_test') and self.options.no_test:
skip_test_targets = True
for k in self.sorted_targets_keys:
target = self.all_targets_expanded[k]
if not self._is_scons_object_type(target['type']):
continue
scons_object = self.scons_targets_map.get(k, None)
if not scons_object:
console.warning('not registered scons object, key %s' % str(k))
continue
if skip_test_targets and (target['type'] == 'cc_test' or
target['type'] == 'dynamic_cc_test'):
continue
scons_object.scons_rules()
rules_buf += scons_object.get_rules()
return rules_buf
def set_gen_rule_files_map(self, files_map):
"""Set the gen_rule files map. """
self.gen_rule_files_map = dict(files_map)
def get_gen_rule_files_map(self):
"""Get the gen_rule files map. """
return self.gen_rule_files_map
def get_scons_platform(self):
"""Return handle of the platform class. """
return self.scons_platform
def get_ccflags_manager(self):
"""Return handle of the ccflags manager class. """
return self.ccflags_manager
def get_sources_keyword_list(self):
"""This keywords list is used to check the source files path.
Ex, when users specifies warning=no, it could be used to check that
the source files is under thirdparty or not. If not, it will warn
users that this flag is used incorrectly.
"""
keywords = ['thirdparty']
return keywords
def get_sources_explict_dependency_map(self):
"""Returns the handle of sources_explict_dependency_map. """
return self.sources_explict_dependency_map
def get_prebuilt_cc_library_file_map(self):
"""Returns the prebuilt_cc_library_file_map. """
return self.prebuilt_cc_library_file_map
def tune_parallel_jobs_num(self):
"""Tune the jobs num. """
user_jobs_num = self.options.jobs
jobs_num = 0
cpu_core_num = cpu_count()
if self.distcc_enabled and self.build_environment.distcc_env_prepared:
jobs_num = int(1.5 * len(self.build_environment.get_distcc_hosts_list())) + 1
if jobs_num > 20:
jobs_num = 20
if jobs_num and self.options.jobs != jobs_num:
self.options.jobs = jobs_num
elif self.options.jobs < 1:
if cpu_core_num <= 4:
self.options.jobs = 2 * cpu_core_num
else:
self.options.jobs = cpu_core_num
if self.options.jobs > 8:
self.options.jobs = 8
if self.options.jobs != user_jobs_num:
console.info("tunes the parallel jobs number(-j N) to be %d" % (
self.options.jobs))
return self.options.jobs
class Blade(object):
"""Blade. A blade manager class. """
def __init__(self,
command_targets,
blade_path,
working_dir,
build_path,
blade_root_dir,
blade_options,
command):
"""init method.
"""
self.__command_targets = command_targets
self.__blade_path = blade_path
self.__working_dir = working_dir
self.__build_path = build_path
self.__root_dir = blade_root_dir
self.__options = blade_options
self.__command = command
# Source dir of current loading BUILD file
self.__current_source_path = blade_root_dir
# The direct targets that are used for analyzing
self.__direct_targets = []
# All command targets, make sure that all targets specified with ...
# are all in the list now
self.__all_command_targets = []
# Given some targets specified in the command line, Blade will load
# BUILD files containing these command line targets; global target
# functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
# files will register targets into target_database, which then becomes
# the input to dependency analyzer and SCons rules generator. It is
# notable that not all targets in target_database are dependencies of
# command line targets.
self.__target_database = {}
# targets to build after loading the build files.
self.__build_targets = {}
# The targets keys list after sorting by topological sorting method.
# Used to generate build rules in correct order.
self.__sorted_targets_keys = []
# The depended targets dict after topological sorting
self.__depended_targets = {}
# Inidcating that whether the deps list is expanded by expander or not
self.__targets_expanded = False
self.__scons_platform = SconsPlatform()
self.build_environment = BuildEnvironment(self.__root_dir)
self.svn_root_dirs = []
def _get_normpath_target(self, command_target):
"""returns a tuple (path, name).
path is a full path from BLADE_ROOT
"""
target_path = relative_path(self.__working_dir, self.__root_dir)
path, name = command_target.split(':')
if target_path != '.':
if path:
path = target_path + '/' + path
else:
path = target_path
path = os.path.normpath(path)
return path, name
def load_targets(self):
"""Load the targets. """
console.info('loading BUILDs...')
if self.__command == 'query' and getattr(self.__options,
'depended', False):
# For query depended command, always start from root with target ...
# that is scanning the whole tree
working_dir = self.__root_dir
else:
working_dir = self.__working_dir
(self.__direct_targets,
self.__all_command_targets,
self.__build_targets) = load_targets(self.__command_targets,
working_dir,
self.__root_dir,
self)
console.info('loading done.')
return self.__direct_targets, self.__all_command_targets # For test
def analyze_targets(self):
"""Expand the targets. """
console.info('analyzing dependency graph...')
(self.__sorted_targets_keys,
self.__depended_targets) = analyze_deps(self.__build_targets)
self.__targets_expanded = True
console.info('analyzing done.')
return self.__build_targets # For test
def generate_build_rules(self):
"""Generate the constructing rules. """
console.info('generating build rules...')
build_rules_generator = SconsRulesGenerator('SConstruct',
self.__blade_path, self)
rules_buf = build_rules_generator.generate_scons_script()
console.info('generating done.')
return rules_buf
def generate(self):
"""Generate the build script. """
self.load_targets()
self.analyze_targets()
if self.__command != 'query':
self.generate_build_rules()
def run(self, target):
"""Run the target. """
key = self._get_normpath_target(target)
runner = BinaryRunner(self.__build_targets,
self.__options,
self.__target_database)
return runner.run_target(key)
def test(self):
"""Run tests. """
test_runner = TestRunner(self.__build_targets,
self.__options,
self.__target_database,
self.__direct_targets)
return test_runner.run()
def query(self, targets):
"""Query the targets. """
print_deps = getattr(self.__options, 'deps', False)
print_depended = getattr(self.__options, 'depended', False)
dot_file = getattr(self.__options, 'output_to_dot', '')
print_dep_tree = getattr(self.__options, 'output_tree', False)
result_map = self.query_helper(targets)
if dot_file:
print_mode = 0
if print_deps:
print_mode = 0
if print_depended:
print_mode = 1
dot_file = os.path.join(self.__working_dir, dot_file)
self.output_dot(result_map, print_mode, dot_file)
else:
if print_deps:
if print_dep_tree:
self.query_dependency_tree(targets)
else:
for key in result_map:
print '\n'
deps = result_map[key][0]
console.info('//%s:%s depends on the following targets:' % (
key[0], key[1]))
for d in deps:
print '%s:%s' % (d[0], d[1])
if print_depended:
for key in result_map:
print '\n'
depended_by = result_map[key][1]
console.info('//%s:%s is depended by the following targets:' % (
key[0], key[1]))
for d in depended_by:
print '%s:%s' % (d[0], d[1])
return 0
def print_dot_node(self, output_file, node):
print >>output_file, '"%s:%s" [label = "%s:%s"]' % (node[0],
node[1],
node[0],
node[1])
def print_dot_deps(self, output_file, node, target_set):
targets = self.__build_targets
deps = targets[node].deps
for i in deps:
if not i in target_set:
continue
print >>output_file, '"%s:%s" -> "%s:%s"' % (node[0],
node[1],
i[0],
i[1])
def output_dot(self, result_map, print_mode, dot_file):
f = open(dot_file, 'w')
targets = result_map.keys()
nodes = set(targets)
for key in targets:
nodes |= set(result_map[key][print_mode])
print >>f, 'digraph blade {'
for i in nodes:
self.print_dot_node(f, i)
for i in nodes:
self.print_dot_deps(f, i, nodes)
print >>f, '}'
f.close()
def query_helper(self, targets):
"""Query the targets helper method. """
all_targets = self.__build_targets
query_list = []
target_path = relative_path(self.__working_dir, self.__root_dir)
t_path = ''
for t in targets:
if t.find(':') != -1:
key = t.split(':')
if target_path == '.':
t_path = key[0]
else:
t_path = target_path + '/' + key[0]
t_path = os.path.normpath(t_path)
query_list.append((t_path, key[1]))
elif t.endswith('...'):
t_path = os.path.normpath(target_path + '/' + t[:-3])
for tkey in all_targets:
if tkey[0].startswith(t_path):
query_list.append((tkey[0], tkey[1]))
else:
t_path = os.path.normpath(target_path + '/' + t)
for tkey in all_targets:
if tkey[0] == t_path:
query_list.append((t_path, tkey[1]))
result_map = {}
for key in query_list:
deps = all_targets[key].expanded_deps
# depended_by = [k for k in all_targets if key in all_targets[k].expanded_deps]
depended_by = self.__depended_targets[key]
result_map[key] = (sorted(deps), sorted(depended_by))
return result_map
def query_dependency_tree(self, targets):
"""Query the dependency tree of the specified targets. """
query_targets = []
for target in targets:
if ':' not in target:
console.error_exit(
'Target %s is not supported by dependency tree query. '
'The target should be in the format directory:name.' % target)
path, name = target.split(':')
relpath = os.path.relpath(self.__working_dir, self.__root_dir)
path = os.path.normpath(os.path.join(relpath, path))
query_targets.append((path, name))
for key in query_targets:
console.info('')
self._query_dependency_tree(key, 0, self.__build_targets)
console.info('')
def _query_dependency_tree(self, key, level, build_targets):
"""Query the dependency tree of the specified target recursively. """
path, name = key
if level == 0:
output = '%s:%s' % (path, name)
elif level == 1:
output = '%s %s:%s' % ('+-', path, name)
else:
output = '%s%s %s:%s' % ('| ' * (level - 1), '+-', path, name)
console.info(console.colors('end') + console.colors('gray') + output)
for dkey in build_targets[key].deps:
self._query_dependency_tree(dkey, level + 1, build_targets)
def get_build_path(self):
"""The current building path. """
return self.__build_path
def get_root_dir(self):
"""Return the blade root path. """
return self.__root_dir
def set_current_source_path(self, current_source_path):
"""Set the current source path. """
self.__current_source_path = current_source_path
def get_current_source_path(self):
"""Get the current source path. """
return self.__current_source_path
def get_target_database(self):
"""Get the whole target database that haven't been expanded. """
return self.__target_database
def get_direct_targets(self):
"""Return the direct targets. """
return self.__direct_targets
def get_build_targets(self):
"""Get all the targets to be build. """
return self.__build_targets
def get_options(self):
"""Get the global command options. """
return self.__options
def is_expanded(self):
"""Whether the targets are expanded. """
return self.__targets_expanded
def register_target(self, target):
"""Register scons targets into the scons targets map.
It is used to do quick looking.
"""
key = target.key
# Check whether there is already a key in database
if key in self.__target_database:
console.error_exit('Target %s is duplicate in //%s/BUILD' % (
target.name, target.path))
self.__target_database[key] = target
def _is_scons_object_type(self, target_type):
"""The types that shouldn't be registered into blade manager.
Sholdn't invoke scons_rule method when it is not a scons target which
could not be registered into blade manager, like system library.
1. system_library
"""
return target_type != 'system_library'
def gen_targets_rules(self):
"""Get the build rules and return to the object who queries this. """
rules_buf = []
skip_test = getattr(self.__options, 'no_test', False)
skip_package = not getattr(self.__options, 'generate_package', False)
for k in self.__sorted_targets_keys:
target = self.__build_targets[k]
if not self._is_scons_object_type(target.type):
continue
scons_object = self.__target_database.get(k, None)
if not scons_object:
console.warning('not registered scons object, key %s' % str(k))
continue
if (skip_test and target.type.endswith('_test')
and k not in self.__direct_targets):
continue
if (skip_package and target.type == 'package'
and k not in self.__direct_targets):
continue
scons_object.scons_rules()
rules = scons_object.get_rules()
if rules:
rules_buf.append('\n')
rules_buf += rules
return rules_buf
def get_scons_platform(self):
"""Return handle of the platform class. """
return self.__scons_platform
def get_sources_keyword_list(self):
"""This keywords list is used to check the source files path.
Ex, when users specifies warning=no, it could be used to check that
the source files is under thirdparty or not. If not, it will warn
users that this flag is used incorrectly.
"""
keywords = ['thirdparty']
return keywords
def parallel_jobs_num(self):
"""Tune the jobs num. """
# User has the highest priority
user_jobs_num = self.__options.jobs
if user_jobs_num > 0:
return user_jobs_num
# Calculate job numbers smartly
jobs_num = 0
distcc_enabled = configparse.blade_config.get_config('distcc_config')['enabled']
if distcc_enabled and self.build_environment.distcc_env_prepared:
# Distcc cost doesn;t much local cpu, jobs can be quite large.
distcc_num = len(self.build_environment.get_distcc_hosts_list())
jobs_num = min(max(int(1.5 * distcc_num), 1), 20)
else:
cpu_core_num = cpu_count()
# machines with cpu_core_num > 4 is usually shared by multiple users,
# set an upper bound to avoid interfering other users
jobs_num = min(2 * cpu_core_num, 8)
if jobs_num != user_jobs_num:
console.info('tunes the parallel jobs number(-j N) to be %d' % (
jobs_num))
return jobs_num
class Blade(object):
"""Blade. A blade manager class. """
# pylint: disable=too-many-public-methods
def __init__(self, command_targets, load_targets, blade_path, working_dir,
build_path, blade_root_dir, blade_options, command):
"""init method.
"""
self.__command_targets = command_targets
self.__load_targets = load_targets
self.__blade_path = blade_path
self.__working_dir = working_dir
self.__build_path = build_path
self.__root_dir = blade_root_dir
self.__options = blade_options
self.__command = command
# Source dir of current loading BUILD file
self.__current_source_path = blade_root_dir
# The direct targets that are used for analyzing
self.__direct_targets = []
# All command targets, make sure that all targets specified with ...
# are all in the list now
self.__all_command_targets = []
# Given some targets specified in the command line, Blade will load
# BUILD files containing these command line targets; global target
# functions, i.e., cc_library, cc_binary and etc, in these BUILD
# files will register targets into target_database, which then becomes
# the input to dependency analyzer and SCons rules generator. It is
# notable that not all targets in target_database are dependencies of
# command line targets.
self.__target_database = {}
# targets to build after loading the build files.
self.__build_targets = {}
# The targets keys list after sorting by topological sorting method.
# Used to generate build rules in correct order.
self.__sorted_targets_keys = []
# The depended targets dict after topological sorting
self.__depended_targets = {}
# Indicate whether the deps list is expanded by expander or not
self.__targets_expanded = False
self.__build_time = time.time()
self.__build_platform = BuildPlatform()
self.build_environment = BuildEnvironment(self.__root_dir)
self.svn_root_dirs = []
self._verify_history_path = os.path.join(build_path,
'.blade_verify.json')
self._verify_history = {
'header_inclusion_dependencies':
{}, # path(.H) -> mtime(modification time)
}
def load_targets(self):
"""Load the targets. """
console.info('loading BUILDs...')
(self.__direct_targets, self.__all_command_targets,
self.__build_targets) = load_targets(self.__load_targets,
self.__root_dir, self)
if self.__command_targets != self.__load_targets:
# In query dependents mode, we must use command targets to execute query
self.__all_command_targets = self._expand_command_targets()
console.info('loading done.')
return self.__direct_targets, self.__all_command_targets # For test
def _expand_command_targets(self):
"""Expand command line targets to targets list"""
all_targets = self.__build_targets
all_command_targets = []
for t in self.__command_targets:
t_path, name = t.split(':')
if name == '...':
for tkey in all_targets:
if tkey[0].startswith(t_path):
all_command_targets.append(tkey)
elif name == '*':
for tkey in all_targets:
if tkey[0] == t_path:
all_command_targets.append(tkey)
else:
all_command_targets.append((t_path, name))
return all_command_targets
def analyze_targets(self):
"""Expand the targets. """
console.info('analyzing dependency graph...')
(self.__sorted_targets_keys,
self.__depended_targets) = analyze_deps(self.__build_targets)
self.__targets_expanded = True
console.info('analyzing done.')
return self.__build_targets # For test
def new_build_rules_generator(self):
if config.get_item('global_config', 'native_builder') == 'ninja':
return NinjaRulesGenerator('build.ninja', self.__blade_path, self)
else:
return SconsRulesGenerator('SConstruct', self.__blade_path, self)
def generate_build_rules(self):
"""Generate the constructing rules. """
console.info('generating build rules...')
generator = self.new_build_rules_generator()
rules = generator.generate_build_script()
self.__all_rule_names = generator.get_all_rule_names()
console.info('generating done.')
return rules
def generate(self):
"""Generate the build script. """
if self.__command != 'query':
self.generate_build_rules()
def verify(self):
"""Verify specific targets after build is complete. """
verify_history = self._load_verify_history()
error = 0
header_inclusion_dependencies = config.get_item(
'cc_config', 'header_inclusion_dependencies')
header_inclusion_history = verify_history[
'header_inclusion_dependencies']
for k in self.__sorted_targets_keys:
target = self.__build_targets[k]
if (header_inclusion_dependencies and target.type == 'cc_library'
and target.srcs):
if not target.verify_header_inclusion_dependencies(
header_inclusion_history):
error += 1
self._dump_verify_history()
return error == 0
def run(self, target):
"""Run the target. """
runner = BinaryRunner(self.__build_targets, self.__options,
self.__target_database)
return runner.run_target(target)
def test(self):
"""Run tests. """
test_runner = TestRunner(self.__build_targets, self.__options,
self.__target_database, self.__direct_targets)
return test_runner.run()
def query(self):
"""Query the targets. """
output_file_name = self.__options.output_file
if output_file_name:
output_file_name = os.path.join(self.__working_dir,
output_file_name)
output_file = open(output_file_name, 'w')
console.info('query result will be written to file "%s"' %
self.__options.output_file)
else:
output_file = sys.stdout
console.info('query result:')
output_format = self.__options.output_format
if output_format == 'dot':
self.query_dependency_dot(output_file)
elif output_format == 'tree':
self.query_dependency_tree(output_file)
else:
self.query_dependency_plain(output_file)
if output_file_name:
output_file.close()
return 0
def query_dependency_plain(self, output_file):
result_map = self.query_helper()
if self.__options.deps:
for key in result_map:
print(file=output_file)
deps = result_map[key][0]
print('//%s:%s depends on the following targets:' %
(key[0], key[1]),
file=output_file)
for d in deps:
print('%s:%s' % (d[0], d[1]), file=output_file)
if self.__options.dependents:
for key in result_map:
print(file=output_file)
depended_by = result_map[key][1]
print('//%s:%s is depended by the following targets:' %
(key[0], key[1]),
file=output_file)
for d in depended_by:
print('%s:%s' % (d[0], d[1]), file=output_file)
def print_dot_node(self, output_file, node):
print('"%s:%s" [label = "%s:%s"]' %
(node[0], node[1], node[0], node[1]),
file=output_file)
def print_dot_deps(self, output_file, node, target_set):
targets = self.__build_targets
deps = targets[node].deps
for i in deps:
if not i in target_set:
continue
print('"%s:%s" -> "%s:%s"' % (node[0], node[1], i[0], i[1]),
file=output_file)
def __print_dot_graph(self, result_map, name, print_mode, output_file):
# print_mode = 0: deps, 1: dependents
targets = result_map.keys()
nodes = set(targets)
for key in targets:
nodes |= set(result_map[key][print_mode])
print('digraph %s {' % name, file=output_file)
for i in nodes:
self.print_dot_node(output_file, i)
for i in nodes:
self.print_dot_deps(output_file, i, nodes)
print('}', file=output_file)
pass
def query_dependency_dot(self, output_file):
result_map = self.query_helper()
if self.__options.deps:
self.__print_dot_graph(result_map, 'deps', 0, output_file)
if self.__options.dependents:
self.__print_dot_graph(result_map, 'dependents', 1, output_file)
def query_helper(self):
"""Query the targets helper method. """
all_targets = self.__build_targets
query_list = self.__all_command_targets
result_map = {}
for key in query_list:
deps = all_targets[key].expanded_deps
# depended_by = [k for k in all_targets if key in all_targets[k].expanded_deps]
depended_by = self.__depended_targets[key]
result_map[key] = (sorted(deps), sorted(depended_by))
return result_map
def query_dependency_tree(self, output_file):
"""Query the dependency tree of the specified targets. """
if self.__options.dependents:
console.error_exit(
'only query --deps can be output as tree format')
print(file=output_file)
for key in self.__all_command_targets:
self._query_dependency_tree(key, 0, self.__build_targets,
output_file)
print(file=output_file)
def _query_dependency_tree(self, key, level, build_targets, output_file):
"""Query the dependency tree of the specified target recursively. """
path, name = key
if level == 0:
output = '%s:%s' % (path, name)
elif level == 1:
output = '%s %s:%s' % ('+-', path, name)
else:
output = '%s%s %s:%s' % ('| ' * (level - 1), '+-', path, name)
print(output, file=output_file)
for dkey in build_targets[key].deps:
self._query_dependency_tree(dkey, level + 1, build_targets,
output_file)
def get_build_time(self):
return self.__build_time
def get_build_path(self):
"""The current building path. """
return self.__build_path
def get_root_dir(self):
"""Return the blade root path. """
return self.__root_dir
def get_command(self):
"""Get the blade command. """
return self.__command
def set_current_source_path(self, current_source_path):
"""Set the current source path. """
self.__current_source_path = current_source_path
def get_current_source_path(self):
"""Get the current source path. """
return self.__current_source_path
def get_target_database(self):
"""Get the whole target database that haven't been expanded. """
return self.__target_database
def get_direct_targets(self):
"""Return the direct targets. """
return self.__direct_targets
def get_build_targets(self):
"""Get all the targets to be build. """
return self.__build_targets
def get_depended_target_database(self):
"""Get depended target database that query dependent targets directly. """
return self.__depended_targets
def get_options(self):
"""Get the global command options. """
return self.__options
def is_expanded(self):
"""Whether the targets are expanded. """
return self.__targets_expanded
def register_target(self, target):
"""Register a target into blade target database.
It is used to do quick looking.
"""
key = target.key
# Check whether there is already a key in database
if key in self.__target_database:
console.error_exit('Target %s is duplicate in //%s/BUILD' %
(target.name, target.path))
self.__target_database[key] = target
def _is_scons_object_type(self, target_type):
"""The types that shouldn't be registered into blade manager.
Sholdn't invoke scons_rule method when it is not a scons target which
could not be registered into blade manager, like system library.
1. system_library
"""
return target_type != 'system_library'
def gen_targets_rules(self):
"""Get the build rules and return to the object who queries this. """
rules_buf = []
skip_test = getattr(self.__options, 'no_test', False)
skip_package = not getattr(self.__options, 'generate_package', False)
native_builder = config.get_item('global_config', 'native_builder')
for k in self.__sorted_targets_keys:
target = self.__build_targets[k]
if not self._is_scons_object_type(target.type):
continue
blade_object = self.__target_database.get(k, None)
if not blade_object:
console.warning('not registered blade object, key %s' % str(k))
continue
if (skip_test and target.type.endswith('_test')
and k not in self.__direct_targets):
continue
if (skip_package and target.type == 'package'
and k not in self.__direct_targets):
continue
if native_builder == 'ninja':
blade_object.ninja_rules()
else:
blade_object.scons_rules()
rules = blade_object.get_rules()
if rules:
rules_buf.append('\n')
rules_buf += rules
return rules_buf
def get_scons_platform(self):
"""Return handle of the platform class. """
return self.__build_platform
def get_sources_keyword_list(self):
"""This keywords list is used to check the source files path.
Ex, when users specifies warning=no, it could be used to check that
the source files is under thirdparty or not. If not, it will warn
users that this flag is used incorrectly.
"""
keywords = ['thirdparty']
return keywords
def _load_verify_history(self):
if os.path.exists(self._verify_history_path):
with open(self._verify_history_path) as f:
self._verify_history = json.load(f)
return self._verify_history
def _dump_verify_history(self):
with open(self._verify_history_path, 'w') as f:
json.dump(self._verify_history, f)
def parallel_jobs_num(self):
"""Tune the jobs num. """
# User has the highest priority
user_jobs_num = self.__options.jobs
if user_jobs_num > 0:
return user_jobs_num
# Calculate job numbers smartly
jobs_num = 0
distcc_enabled = config.get_item('distcc_config', 'enabled')
if distcc_enabled and self.build_environment.distcc_env_prepared:
# Distcc cost doesn;t much local cpu, jobs can be quite large.
distcc_num = len(self.build_environment.get_distcc_hosts_list())
jobs_num = min(max(int(1.5 * distcc_num), 1), 20)
else:
cpu_core_num = cpu_count()
# machines with cpu_core_num > 4 is usually shared by multiple users,
# set an upper bound to avoid interfering other users
jobs_num = min(2 * cpu_core_num, 8)
if jobs_num != user_jobs_num:
console.info('tunes the parallel jobs number(-j N) to be %d' %
(jobs_num))
return jobs_num
def get_all_rule_names(self):
return self.__all_rule_names
class Blade(object):
"""Blade. A blade manager class. """
def __init__(self, command_targets, blade_path, working_dir, build_path,
blade_root_dir, blade_options, command):
"""init method.
"""
self.__command_targets = command_targets
self.__blade_path = blade_path
self.__working_dir = working_dir
self.__build_path = build_path
self.__root_dir = blade_root_dir
self.__options = blade_options
self.__command = command
# Source dir of current loading BUILD file
self.__current_source_path = blade_root_dir
# The direct targets that are used for analyzing
self.__direct_targets = []
# All command targets, make sure that all targets specified with ...
# are all in the list now
self.__all_command_targets = []
# Given some targets specified in the command line, Blade will load
# BUILD files containing these command line targets; global target
# functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
# files will register targets into target_database, which then becomes
# the input to dependency analyzer and SCons rules generator. It is
# notable that not all targets in target_database are dependencies of
# command line targets.
self.__target_database = {}
# targets to build after loading the build files.
self.__build_targets = {}
# The targets keys list after sorting by topological sorting method.
# Used to generate build rules in correct order.
self.__sorted_targets_keys = []
# Inidcating that whether the deps list is expanded by expander or not
self.__targets_expanded = False
self.__scons_platform = SconsPlatform()
self.build_environment = BuildEnvironment(self.__root_dir)
self.svn_root_dirs = []
def _get_normpath_target(self, command_target):
"""returns a tuple (path, name).
path is a full path from BLADE_ROOT
"""
target_path = relative_path(self.__working_dir, self.__root_dir)
path, name = command_target.split(':')
if target_path != '.':
if path:
path = target_path + '/' + path
else:
path = target_path
path = os.path.normpath(path)
return path, name
def load_targets(self):
"""Load the targets. """
console.info('loading BUILDs...')
if self.__command == 'query':
working_dir = self.__root_dir
if '...' not in self.__command_targets:
new_target_list = []
for target in self.__command_targets:
new_target_list.append('%s:%s' %
self._get_normpath_target(target))
self.__command_targets = new_target_list
else:
working_dir = self.__working_dir
(self.__direct_targets, self.__all_command_targets,
self.__build_targets) = load_targets(self.__command_targets,
working_dir, self.__root_dir,
self)
console.info('loading done.')
return self.__direct_targets, self.__all_command_targets # For test
def analyze_targets(self):
"""Expand the targets. """
console.info('analyzing dependency graph...')
self.__sorted_targets_keys = analyze_deps(self.__build_targets)
self.__targets_expanded = True
console.info('analyzing done.')
return self.__build_targets # For test
def generate_build_rules(self):
"""Generate the constructing rules. """
console.info('generating build rules...')
build_rules_generator = SconsRulesGenerator('SConstruct',
self.__blade_path, self)
rules_buf = build_rules_generator.generate_scons_script()
console.info('generating done.')
return rules_buf
def generate(self):
"""Generate the build script. """
self.load_targets()
self.analyze_targets()
self.generate_build_rules()
def run(self, target):
"""Run the target. """
key = self._get_normpath_target(target)
runner = BinaryRunner(self.__build_targets, self.__options,
self.__target_database)
return runner.run_target(key)
def test(self):
"""Run tests. """
test_runner = TestRunner(self.__build_targets, self.__options,
self.__target_database, self.__direct_targets)
return test_runner.run()
def query(self, targets):
"""Query the targets. """
print_deps = getattr(self.__options, 'deps', False)
print_depended = getattr(self.__options, 'depended', False)
dot_file = getattr(self.__options, 'output_to_dot', '')
result_map = self.query_helper(targets)
if dot_file:
print_mode = 0
if print_deps:
print_mode = 0
if print_depended:
print_mode = 1
dot_file = os.path.join(self.__working_dir, dot_file)
self.output_dot(result_map, print_mode, dot_file)
else:
if print_deps:
for key in result_map:
print '\n'
deps = result_map[key][0]
console.info('//%s:%s depends on the following targets:' %
(key[0], key[1]))
for d in deps:
print '%s:%s' % (d[0], d[1])
if print_depended:
for key in result_map:
print '\n'
depended_by = result_map[key][1]
console.info(
'//%s:%s is depended by the following targets:' %
(key[0], key[1]))
depended_by.sort(key=lambda x: x, reverse=False)
for d in depended_by:
print '%s:%s' % (d[0], d[1])
return 0
def print_dot_node(self, output_file, node):
print >> output_file, '"%s:%s" [label = "%s:%s"]' % (node[0], node[1],
node[0], node[1])
def print_dot_deps(self, output_file, node, target_set):
targets = self.__build_targets
deps = targets[node].deps
for i in deps:
if not i in target_set:
continue
print >> output_file, '"%s:%s" -> "%s:%s"' % (node[0], node[1],
i[0], i[1])
def output_dot(self, result_map, print_mode, dot_file):
f = open(dot_file, 'w')
targets = result_map.keys()
nodes = set(targets)
for key in targets:
nodes |= set(result_map[key][print_mode])
print >> f, 'digraph blade {'
for i in nodes:
self.print_dot_node(f, i)
for i in nodes:
self.print_dot_deps(f, i, nodes)
print >> f, '}'
f.close()
def query_helper(self, targets):
"""Query the targets helper method. """
all_targets = self.__build_targets
query_list = []
target_path = relative_path(self.__working_dir, self.__root_dir)
t_path = ''
for t in targets:
key = t.split(':')
if target_path == '.':
t_path = key[0]
else:
t_path = target_path + '/' + key[0]
t_path = os.path.normpath(t_path)
query_list.append((t_path, key[1]))
result_map = {}
for key in query_list:
result_map[key] = ([], [])
deps = all_targets[key].expanded_deps
deps.sort(key=lambda x: x, reverse=False)
depended_by = []
for tkey in all_targets:
if key in all_targets[tkey].expanded_deps:
depended_by.append(tkey)
depended_by.sort(key=lambda x: x, reverse=False)
result_map[key] = (list(deps), list(depended_by))
return result_map
def get_build_path(self):
"""The current building path. """
return self.__build_path
def get_root_dir(self):
"""Return the blade root path. """
return self.__root_dir
def set_current_source_path(self, current_source_path):
"""Set the current source path. """
self.__current_source_path = current_source_path
def get_current_source_path(self):
"""Get the current source path. """
return self.__current_source_path
def get_target_database(self):
"""Get the whole target database that haven't been expanded. """
return self.__target_database
def get_direct_targets(self):
"""Return the direct targets. """
return self.__direct_targets
def get_build_targets(self):
"""Get all the targets to be build. """
return self.__build_targets
def get_options(self):
"""Get the global command options. """
return self.__options
def is_expanded(self):
"""Whether the targets are expanded. """
return self.__targets_expanded
def register_target(self, target):
"""Register scons targets into the scons targets map.
It is used to do quick looking.
"""
target_key = target.key
# check that whether there is already a key in database
if target_key in self.__target_database:
print self.__target_database
console.error_exit('target name %s is duplicate in //%s/BUILD' %
(target.name, target.path))
self.__target_database[target_key] = target
def _is_scons_object_type(self, target_type):
"""The types that shouldn't be registered into blade manager.
Sholdn't invoke scons_rule method when it is not a scons target which
could not be registered into blade manager, like system library.
1. system_library
"""
return target_type != 'system_library'
def gen_targets_rules(self):
"""Get the build rules and return to the object who queries this. """
rules_buf = []
skip_test_targets = False
if getattr(self.__options, 'no_test', False):
skip_test_targets = True
for k in self.__sorted_targets_keys:
target = self.__build_targets[k]
if not self._is_scons_object_type(target.type):
continue
scons_object = self.__target_database.get(k, None)
if not scons_object:
console.warning('not registered scons object, key %s' % str(k))
continue
if skip_test_targets and target.type == 'cc_test':
continue
scons_object.scons_rules()
rules_buf.append('\n')
rules_buf += scons_object.get_rules()
return rules_buf
def get_scons_platform(self):
"""Return handle of the platform class. """
return self.__scons_platform
def get_sources_keyword_list(self):
"""This keywords list is used to check the source files path.
Ex, when users specifies warning=no, it could be used to check that
the source files is under thirdparty or not. If not, it will warn
users that this flag is used incorrectly.
"""
keywords = ['thirdparty']
return keywords
def tune_parallel_jobs_num(self):
"""Tune the jobs num. """
user_jobs_num = self.__options.jobs
jobs_num = 0
cpu_core_num = cpu_count()
distcc_enabled = configparse.blade_config.get_config(
'distcc_config')['enabled']
if distcc_enabled and self.build_environment.distcc_env_prepared:
jobs_num = int(
1.5 * len(self.build_environment.get_distcc_hosts_list())) + 1
if jobs_num > 20:
jobs_num = 20
if jobs_num and self.__options.jobs != jobs_num:
self.__options.jobs = jobs_num
elif self.__options.jobs < 1:
if cpu_core_num <= 4:
self.__options.jobs = 2 * cpu_core_num
else:
self.__options.jobs = cpu_core_num
if self.__options.jobs > 8:
self.__options.jobs = 8
if self.__options.jobs != user_jobs_num:
console.info('tunes the parallel jobs number(-j N) to be %d' %
(self.__options.jobs))
return self.__options.jobs
def __init__(self, command_targets, blade_path, working_dir, build_path,
current_source_path, blade_options, **kwargs):
"""init method.
Mainly to hold the global data.
The directory which changes during the runtime of blade, and
contains BUILD file under current focus.
current_source_dir = "."
Given some targets specified in the command line, Blade will load
BUILD files containing these command line targets; global target
functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
files will register targets into target_database, which then becomes
the input to dependency analyzer and SCons rules generator. It is
notable that not all targets in target_database are dependencies of
command line targets.
target_database = {}
related targets after loading the build files and exec BUILD files
as python script
related_targets = {}
The map used by build rules to ensure that a source file occurres in
exactly one rule/target(only library target).
target_srcs_map = {}
The scons cache manager class string, which should be output to
scons script if ccache is not installed
scache_manager_class_str = ''
The targets keys list after sorting by topological sorting method.
sorted_targets_keys = []
Inidcating that whether the deps list is expanded by expander or not
False - not expanded
True - expanded
target_deps_expanded
All targets after expanding their dependency
all_targets = {}
The scons target objects registered into blade manager
scons_targets_map = {}
The vars which are depended by python binary
{key : 'python_files'}
self.python_binary_dep_source_cmd = {}
The files which are depended by python binary
{key : 'python_files'}
python_binary_dep_source_map = {}
The files which are depended by java jar file
{key : 'java_files'}
java_jar_dep_source_map = {}
The files which should be packed into java jar
{key : 'packing_files'}
java_jar_files_packing_map = {}
The jar files map
{key : 'jar_files_generated'}
java_jars_map = {}
The java compiling classpath parameter map
java_classpath_map = {}
{key : 'target_path'}
The java_jar dep var map, which should be added to dependency chain
java_jar_dep_vars = {}
The cc objects pool, a map to hold all the objects name.
cc_objects_pool = {}
The gen rule files map, which is used to generate the explict dependency
relationtion ship between gen_rule target and other targets
gen_rule_files_map = {}
The direct targets that are used for analyzing
direct_targets = []
All command targets, make sure that all targets specified with ...
are all in the list now
all_command_targets = []
The class to get platform info
SconsPlatform
The class to manage the cc flags
CcFlagsManager
The sources files that are needed to perform explict dependency
sources_explict_dependency_map = {}
The prebuilt cc_library file map which is needed to establish
symbolic links while testing
prebuilt_cc_library_file_map = {}
"""
self.command_targets = command_targets
self.direct_targets = []
self.all_command_targets = []
self.blade_path = blade_path
self.working_dir = working_dir
self.build_path = build_path
self.current_source_path = current_source_path
self.target_database = {}
self.related_targets = {}
self.target_srcs_map = {}
self.scache_manager_class_str = ''
self.options = blade_options
self.sorted_targets_keys = []
self.target_deps_expanded = False
self.all_targets_expanded = {}
self.scons_targets_map = {}
self.java_jar_dep_source_map = {}
self.java_jar_files_packing_map = {}
self.java_jars_map = {}
self.java_classpath_map = {}
self.java_jar_dep_vars = {}
self.python_binary_dep_source_cmd = {}
self.python_binary_dep_source_map = {}
self.cc_objects_pool = {}
self.deps_expander = None
self.build_rules_generator = None
self.gen_rule_files_map = {}
self.scons_platform = SconsPlatform()
self.ccflags_manager = CcFlagsManager(self.options)
self.sources_explict_dependency_map = {}
self.prebuilt_cc_library_file_map = {}
self.distcc_enabled = configparse.blade_config.get_config(
'distcc_config')['enabled']
self.build_environment = BuildEnvironment(self.current_source_path)
self.svn_root_dirs = []
self.kwargs = kwargs
class Blade(object):
"""Blade. A blade manager class. """
def __init__(self, command_targets, blade_path, working_dir, build_path,
current_source_path, blade_options, **kwargs):
"""init method.
Mainly to hold the global data.
The directory which changes during the runtime of blade, and
contains BUILD file under current focus.
current_source_dir = "."
Given some targets specified in the command line, Blade will load
BUILD files containing these command line targets; global target
functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
files will register targets into target_database, which then becomes
the input to dependency analyzer and SCons rules generator. It is
notable that not all targets in target_database are dependencies of
command line targets.
target_database = {}
related targets after loading the build files and exec BUILD files
as python script
related_targets = {}
The map used by build rules to ensure that a source file occurres in
exactly one rule/target(only library target).
target_srcs_map = {}
The scons cache manager class string, which should be output to
scons script if ccache is not installed
scache_manager_class_str = ''
The targets keys list after sorting by topological sorting method.
sorted_targets_keys = []
Inidcating that whether the deps list is expanded by expander or not
False - not expanded
True - expanded
target_deps_expanded
All targets after expanding their dependency
all_targets = {}
The scons target objects registered into blade manager
scons_targets_map = {}
The vars which are depended by python binary
{key : 'python_files'}
self.python_binary_dep_source_cmd = {}
The files which are depended by python binary
{key : 'python_files'}
python_binary_dep_source_map = {}
The files which are depended by java jar file
{key : 'java_files'}
java_jar_dep_source_map = {}
The files which should be packed into java jar
{key : 'packing_files'}
java_jar_files_packing_map = {}
The jar files map
{key : 'jar_files_generated'}
java_jars_map = {}
The java compiling classpath parameter map
java_classpath_map = {}
{key : 'target_path'}
The java_jar dep var map, which should be added to dependency chain
java_jar_dep_vars = {}
The cc objects pool, a map to hold all the objects name.
cc_objects_pool = {}
The gen rule files map, which is used to generate the explict dependency
relationtion ship between gen_rule target and other targets
gen_rule_files_map = {}
The direct targets that are used for analyzing
direct_targets = []
All command targets, make sure that all targets specified with ...
are all in the list now
all_command_targets = []
The class to get platform info
SconsPlatform
The class to manage the cc flags
CcFlagsManager
The sources files that are needed to perform explict dependency
sources_explict_dependency_map = {}
The prebuilt cc_library file map which is needed to establish
symbolic links while testing
prebuilt_cc_library_file_map = {}
"""
self.command_targets = command_targets
self.direct_targets = []
self.all_command_targets = []
self.blade_path = blade_path
self.working_dir = working_dir
self.build_path = build_path
self.current_source_path = current_source_path
self.target_database = {}
self.related_targets = {}
self.target_srcs_map = {}
self.scache_manager_class_str = ''
self.options = blade_options
self.sorted_targets_keys = []
self.target_deps_expanded = False
self.all_targets_expanded = {}
self.scons_targets_map = {}
self.java_jar_dep_source_map = {}
self.java_jar_files_packing_map = {}
self.java_jars_map = {}
self.java_classpath_map = {}
self.java_jar_dep_vars = {}
self.python_binary_dep_source_cmd = {}
self.python_binary_dep_source_map = {}
self.cc_objects_pool = {}
self.deps_expander = None
self.build_rules_generator = None
self.gen_rule_files_map = {}
self.scons_platform = SconsPlatform()
self.ccflags_manager = CcFlagsManager(self.options)
self.sources_explict_dependency_map = {}
self.prebuilt_cc_library_file_map = {}
self.distcc_enabled = configparse.blade_config.get_config(
'distcc_config')['enabled']
self.build_environment = BuildEnvironment(self.current_source_path)
self.svn_root_dirs = []
self.kwargs = kwargs
def _get_normpath_target(self, command_target):
"""returns a tuple (path, name).
path is a full path from BLADE_ROOT
"""
target_path = relative_path(self.working_dir, self.current_source_path)
path, name = command_target.split(':')
if target_path != '.':
if path:
path = target_path + '/' + path
else:
path = target_path
path = os.path.normpath(path)
return path, name
def load_targets(self):
"""Load the targets. """
console.info("loading BUILDs...")
if self.kwargs.get('blade_command', '') == 'query':
working_dir = self.current_source_path
if '...' not in self.command_targets:
new_target_list = []
for target in self.command_targets:
new_target_list.append("%s:%s" %
self._get_normpath_target(target))
self.command_targets = new_target_list
else:
working_dir = self.working_dir
(self.direct_targets, self.all_command_targets) = load_targets(
self.command_targets, working_dir, self.current_source_path, self)
console.info("loading done.")
return self.direct_targets, self.all_command_targets
def analyze_targets(self):
"""Expand the targets. """
console.info("analyzing dependency graph...")
self.deps_analyzer = DependenciesAnalyzer(self)
self.deps_analyzer.analyze_deps()
console.info("analyzing done.")
return self.all_targets_expanded
def generate_build_rules(self):
"""Generate the constructing rules. """
console.info("generating build rules...")
self.build_rules_generator = SconsRulesGenerator(
'SConstruct', self.blade_path, self)
rules_buf = self.build_rules_generator.generate_scons_script()
console.info("generating done.")
return rules_buf
def generate(self):
"""Build the targets. """
self.load_targets()
self.analyze_targets()
self.generate_build_rules()
def run(self, target):
"""Run the target. """
key = self._get_normpath_target(target)
runner = BinaryRunner(self.all_targets_expanded, self.options,
self.prebuilt_cc_library_file_map,
self.target_database)
return runner.run_target(key)
def test(self):
"""Run tests. """
test_runner = TestRunner(self.all_targets_expanded, self.options,
self.prebuilt_cc_library_file_map,
self.target_database)
return test_runner.run()
def query(self, targets):
"""Query the targets. """
print_deps = hasattr(self.options, 'deps') and (self.options.deps)
print_depended = hasattr(self.options,
'depended') and (self.options.depended)
dot_file = ""
if hasattr(self.options, 'output-to-dot'):
dot_file = self.options.output_to_dot
result_map = self.query_helper(targets)
if dot_file:
print_mode = 0
if print_deps:
print_mode = 0
if print_depended:
print_mode = 1
if not dot_file.startswith("/"):
dot_file = self.working_dir + "/" + dot_file
self.output_dot(result_map, print_mode, dot_file)
else:
if print_deps:
for key in result_map.keys():
print "\n"
deps = result_map[key][0]
console.info("//%s:%s depends on the following targets:" %
(key[0], key[1]))
for d in deps:
print "%s:%s" % (d[0], d[1])
if print_depended:
for key in result_map.keys():
print "\n"
depended_by = result_map[key][1]
console.info(
"//%s:%s is depended by the following targets:" %
(key[0], key[1]))
depended_by.sort(key=lambda x: x, reverse=False)
for d in depended_by:
print "%s:%s" % (d[0], d[1])
return 0
def print_dot_node(self, output_file, node):
print >> output_file, '"%s:%s" [label = "%s:%s"]' % (node[0], node[1],
node[0], node[1])
def print_dot_deps(self, output_file, node, target_set):
targets = self.related_targets
deps = targets.get(node, {}).get('direct_deps', [])
for i in deps:
if not i in target_set:
continue
print >> output_file, '"%s:%s" -> "%s:%s"' % (node[0], node[1],
i[0], i[1])
def output_dot(self, result_map, print_mode, dot_file):
f = open(dot_file, 'w')
targets = result_map.keys()
nodes = set(targets)
for key in targets:
nodes |= set(result_map[key][print_mode])
print >> f, "digraph blade {"
for i in nodes:
self.print_dot_node(f, i)
for i in nodes:
self.print_dot_deps(f, i, nodes)
print >> f, "}"
f.close()
def query_helper(self, targets):
"""Query the targets helper method. """
all_targets = self.all_targets_expanded
query_list = []
target_path = relative_path(self.working_dir, self.current_source_path)
t_path = ''
for t in targets:
key = t.split(':')
if target_path == '.':
t_path = key[0]
else:
t_path = target_path + '/' + key[0]
t_path = os.path.normpath(t_path)
query_list.append((t_path, key[1]))
result_map = {}
for key in query_list:
result_map[key] = ([], [])
deps = all_targets.get(key, {}).get('deps', [])
deps.sort(key=lambda x: x, reverse=False)
depended_by = []
for tkey in all_targets.keys():
if key in all_targets[tkey]['deps']:
depended_by.append(tkey)
depended_by.sort(key=lambda x: x, reverse=False)
result_map[key] = (list(deps), list(depended_by))
return result_map
def get_blade_path(self):
"""Return the blade archive path. """
return self.blade_path
def get_build_path(self):
"""The current building path. """
return self.build_path
def set_current_source_path(self, current_source_path):
"""Set the current source path. """
self.current_source_path = current_source_path
def get_current_source_path(self):
"""Get the current source path. """
return self.current_source_path
def get_target_database(self):
"""Get the whole target database that haven't been expanded. """
return self.target_database
def set_related_targets(self, related_targets):
"""Set the related targets. """
self.related_targets = dict(related_targets)
def get_related_targets(self):
"""Get the related targets. """
return self.related_targets
def get_direct_targets(self):
"""Return the direct targets. """
return self.direct_targets
def get_all_command_targets(self):
"""Return all command targets. """
return self.all_command_targets
def set_sorted_targets_keys(self, sorted_keys_list):
"""Set the keys list from expaned targets. """
self.sorted_targets_keys = list(sorted_keys_list)
def get_sorted_targets_keys(self):
"""Get the keys list from expaned targets. """
return self.sorted_targets_keys
def set_all_targets_expanded(self, all_targets):
"""Set the targets that have been expanded by expander. """
self.all_targets_expanded = dict(all_targets)
self.target_deps_expanded = True
def get_all_targets_expanded(self):
"""Get all the targets that expaned. """
return self.all_targets_expanded
def get_target_srcs_map(self):
"""Get the targets source files map.
It is used in generating cc object rules.
"""
return self.target_srcs_map
def get_options(self):
"""Get the global command options. """
return self.options
def get_expanded(self):
"""Whether the targets are expanded. """
return self.target_deps_expanded
def register_scons_target(self, target_key, scons_target):
"""Register scons targets into the scons targets map.
It is used to do quick looking.
"""
# check that whether there is already a key in database
if target_key in self.scons_targets_map.keys():
console.error_exit("target name %s is duplicate in //%s/BUILD" %
(target_key[1], target_key[0]))
self.scons_targets_map[target_key] = scons_target
def get_scons_target(self, target_key):
"""Get scons target according to the key. """
return self.scons_targets_map.get(target_key, None)
def get_java_jar_dep_source_map(self):
"""The map mainly to hold the java files from swig or proto rules.
These files maybe depended by java_jar target.
"""
return self.java_jar_dep_source_map
def get_java_jar_files_packing_map(self):
"""The map to hold the files that should be packed into java jar. """
return self.java_jar_files_packing_map
def get_java_jars_map(self):
"""The map to hold the java jar files generated by blade. """
return self.java_jars_map
def get_java_classpath_map(self):
"""The classpath list which is needed by java compling. """
return self.java_classpath_map
def get_java_jar_dep_vars(self):
"""The vars map which is prerequiste of the java jar target. """
return self.java_jar_dep_vars
def get_cc_objects_pool(self):
"""The cc objects pool which is used when generating the cc object rules. """
return self.cc_objects_pool
def _is_scons_object_type(self, target_type):
"""The types that shouldn't be registered into blade manager.
Sholdn't invoke scons_rule method when it is not a scons target which
could not be registered into blade manager, like system library.
1. system_library
"""
if target_type == 'system_library':
return False
else:
return True
def get_targets_rules(self):
"""Get the build rules and return to the object who queries this. """
rules_buf = []
skip_test_targets = False
if hasattr(self.options, 'no_test') and self.options.no_test:
skip_test_targets = True
for k in self.sorted_targets_keys:
target = self.all_targets_expanded[k]
if not self._is_scons_object_type(target['type']):
continue
scons_object = self.scons_targets_map.get(k, None)
if not scons_object:
console.warning('not registered scons object, key %s' % str(k))
continue
if skip_test_targets and (target['type'] == 'cc_test'
or target['type'] == 'dynamic_cc_test'):
continue
scons_object.scons_rules()
rules_buf += scons_object.get_rules()
return rules_buf
def set_gen_rule_files_map(self, files_map):
"""Set the gen_rule files map. """
self.gen_rule_files_map = dict(files_map)
def get_gen_rule_files_map(self):
"""Get the gen_rule files map. """
return self.gen_rule_files_map
def get_scons_platform(self):
"""Return handle of the platform class. """
return self.scons_platform
def get_ccflags_manager(self):
"""Return handle of the ccflags manager class. """
return self.ccflags_manager
def get_sources_keyword_list(self):
"""This keywords list is used to check the source files path.
Ex, when users specifies warning=no, it could be used to check that
the source files is under thirdparty or not. If not, it will warn
users that this flag is used incorrectly.
"""
keywords = ['thirdparty']
return keywords
def get_sources_explict_dependency_map(self):
"""Returns the handle of sources_explict_dependency_map. """
return self.sources_explict_dependency_map
def get_prebuilt_cc_library_file_map(self):
"""Returns the prebuilt_cc_library_file_map. """
return self.prebuilt_cc_library_file_map
def tune_parallel_jobs_num(self):
"""Tune the jobs num. """
user_jobs_num = self.options.jobs
jobs_num = 0
cpu_core_num = multiprocessing.cpu_count()
if self.distcc_enabled and self.build_environment.distcc_env_prepared:
jobs_num = int(
1.5 * len(self.build_environment.get_distcc_hosts_list())) + 1
if jobs_num > 20:
jobs_num = 20
if jobs_num and self.options.jobs != jobs_num:
self.options.jobs = jobs_num
elif self.options.jobs < 1:
if cpu_core_num <= 4:
self.options.jobs = 2 * cpu_core_num
else:
self.options.jobs = cpu_core_num
if self.options.jobs > 8:
self.options.jobs = 8
if self.options.jobs != user_jobs_num:
console.info("tunes the parallel jobs number(-j N) to be %d" %
(self.options.jobs))
return self.options.jobs
class Blade(object):
"""Blade. A blade manager class. """
def __init__(self,
command_targets,
blade_path,
working_dir,
build_path,
blade_root_dir,
blade_options,
command):
"""init method.
"""
self.__command_targets = command_targets
self.__blade_path = blade_path
self.__working_dir = working_dir
self.__build_path = build_path
self.__root_dir = blade_root_dir
self.__options = blade_options
self.__command = command
# Source dir of current loading BUILD file
self.__current_source_path = blade_root_dir
# The direct targets that are used for analyzing
self.__direct_targets = []
# All command targets, make sure that all targets specified with ...
# are all in the list now
self.__all_command_targets = []
# Given some targets specified in the command line, Blade will load
# BUILD files containing these command line targets; global target
# functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
# files will register targets into target_database, which then becomes
# the input to dependency analyzer and SCons rules generator. It is
# notable that not all targets in target_database are dependencies of
# command line targets.
self.__target_database = {}
# targets to build after loading the build files.
self.__build_targets = {}
# The targets keys list after sorting by topological sorting method.
# Used to generate build rules in correct order.
self.__sorted_targets_keys = []
# Inidcating that whether the deps list is expanded by expander or not
self.__targets_expanded = False
self.__scons_platform = SconsPlatform()
self.build_environment = BuildEnvironment(self.__root_dir)
self.svn_root_dirs = []
def _get_normpath_target(self, command_target):
"""returns a tuple (path, name).
path is a full path from BLADE_ROOT
"""
target_path = relative_path(self.__working_dir, self.__root_dir)
path, name = command_target.split(':')
if target_path != '.':
if path:
path = target_path + '/' + path
else:
path = target_path
path = os.path.normpath(path)
return path, name
def load_targets(self):
"""Load the targets. """
console.info('loading BUILDs...')
if self.__command == 'query' and getattr(self.__options,
'depended', False):
# For query depended command, always start from root with target ...
# that is scanning the whole tree
working_dir = self.__root_dir
else:
working_dir = self.__working_dir
(self.__direct_targets,
self.__all_command_targets,
self.__build_targets) = load_targets(self.__command_targets,
working_dir,
self.__root_dir,
self)
console.info('loading done.')
return self.__direct_targets, self.__all_command_targets # For test
def analyze_targets(self):
"""Expand the targets. """
console.info('analyzing dependency graph...')
self.__sorted_targets_keys = analyze_deps(self.__build_targets)
self.__targets_expanded = True
console.info('analyzing done.')
return self.__build_targets # For test
def generate_build_rules(self):
"""Generate the constructing rules. """
console.info('generating build rules...')
build_rules_generator = SconsRulesGenerator('SConstruct',
self.__blade_path, self)
rules_buf = build_rules_generator.generate_scons_script()
console.info('generating done.')
return rules_buf
def generate(self):
"""Generate the build script. """
self.load_targets()
self.analyze_targets()
self.generate_build_rules()
def run(self, target):
"""Run the target. """
key = self._get_normpath_target(target)
runner = BinaryRunner(self.__build_targets,
self.__options,
self.__target_database)
return runner.run_target(key)
def test(self):
"""Run tests. """
test_runner = TestRunner(self.__build_targets,
self.__options,
self.__target_database,
self.__direct_targets)
return test_runner.run()
def query(self, targets):
"""Query the targets. """
print_deps = getattr(self.__options, 'deps', False)
print_depended = getattr(self.__options, 'depended', False)
dot_file = getattr(self.__options, 'output_to_dot', '')
result_map = self.query_helper(targets)
if dot_file:
print_mode = 0
if print_deps:
print_mode = 0
if print_depended:
print_mode = 1
dot_file = os.path.join(self.__working_dir, dot_file)
self.output_dot(result_map, print_mode, dot_file)
else:
if print_deps:
for key in result_map:
print '\n'
deps = result_map[key][0]
console.info('//%s:%s depends on the following targets:' % (
key[0], key[1]))
for d in deps:
print '%s:%s' % (d[0], d[1])
if print_depended:
for key in result_map:
print '\n'
depended_by = result_map[key][1]
console.info('//%s:%s is depended by the following targets:' % (
key[0], key[1]))
depended_by.sort(key=lambda x: x, reverse=False)
for d in depended_by:
print '%s:%s' % (d[0], d[1])
return 0
def print_dot_node(self, output_file, node):
print >>output_file, '"%s:%s" [label = "%s:%s"]' % (node[0],
node[1],
node[0],
node[1])
def print_dot_deps(self, output_file, node, target_set):
targets = self.__build_targets
deps = targets[node].deps
for i in deps:
if not i in target_set:
continue
print >>output_file, '"%s:%s" -> "%s:%s"' % (node[0],
node[1],
i[0],
i[1])
def output_dot(self, result_map, print_mode, dot_file):
f = open(dot_file, 'w')
targets = result_map.keys()
nodes = set(targets)
for key in targets:
nodes |= set(result_map[key][print_mode])
print >>f, 'digraph blade {'
for i in nodes:
self.print_dot_node(f, i)
for i in nodes:
self.print_dot_deps(f, i, nodes)
print >>f, '}'
f.close()
def query_helper(self, targets):
"""Query the targets helper method. """
all_targets = self.__build_targets
query_list = []
target_path = relative_path(self.__working_dir, self.__root_dir)
t_path = ''
for t in targets:
if t.find(':') != -1:
key = t.split(':')
if target_path == '.':
t_path = key[0]
else:
t_path = target_path + '/' + key[0]
t_path = os.path.normpath(t_path)
query_list.append((t_path, key[1]))
elif t.endswith('...'):
t_path = os.path.normpath(target_path + '/' + t[:-3])
for tkey in all_targets:
if tkey[0].startswith(t_path):
query_list.append((tkey[0], tkey[1]))
else:
t_path = os.path.normpath(target_path + '/' + t)
for tkey in all_targets:
if tkey[0] == t_path:
query_list.append((t_path, tkey[1]))
result_map = {}
for key in query_list:
result_map[key] = ([], [])
deps = all_targets[key].expanded_deps
deps.sort(key=lambda x: x, reverse=False)
depended_by = []
for tkey in all_targets:
if key in all_targets[tkey].expanded_deps:
depended_by.append(tkey)
depended_by.sort(key=lambda x: x, reverse=False)
result_map[key] = (list(deps), list(depended_by))
return result_map
def get_build_path(self):
"""The current building path. """
return self.__build_path
def get_root_dir(self):
"""Return the blade root path. """
return self.__root_dir
def set_current_source_path(self, current_source_path):
"""Set the current source path. """
self.__current_source_path = current_source_path
def get_current_source_path(self):
"""Get the current source path. """
return self.__current_source_path
def get_target_database(self):
"""Get the whole target database that haven't been expanded. """
return self.__target_database
def get_direct_targets(self):
"""Return the direct targets. """
return self.__direct_targets
def get_build_targets(self):
"""Get all the targets to be build. """
return self.__build_targets
def get_options(self):
"""Get the global command options. """
return self.__options
def is_expanded(self):
"""Whether the targets are expanded. """
return self.__targets_expanded
def register_target(self, target):
"""Register scons targets into the scons targets map.
It is used to do quick looking.
"""
target_key = target.key
# check that whether there is already a key in database
if target_key in self.__target_database:
print self.__target_database
console.error_exit(
'target name %s is duplicate in //%s/BUILD' % (
target.name, target.path))
self.__target_database[target_key] = target
def _is_scons_object_type(self, target_type):
"""The types that shouldn't be registered into blade manager.
Sholdn't invoke scons_rule method when it is not a scons target which
could not be registered into blade manager, like system library.
1. system_library
"""
return target_type != 'system_library'
def gen_targets_rules(self):
"""Get the build rules and return to the object who queries this. """
rules_buf = []
skip_test_targets = False
if getattr(self.__options, 'no_test', False):
skip_test_targets = True
for k in self.__sorted_targets_keys:
target = self.__build_targets[k]
if not self._is_scons_object_type(target.type):
continue
scons_object = self.__target_database.get(k, None)
if not scons_object:
console.warning('not registered scons object, key %s' % str(k))
continue
if skip_test_targets and target.type == 'cc_test':
continue
scons_object.scons_rules()
rules_buf.append('\n')
rules_buf += scons_object.get_rules()
return rules_buf
def get_scons_platform(self):
"""Return handle of the platform class. """
return self.__scons_platform
def get_sources_keyword_list(self):
"""This keywords list is used to check the source files path.
Ex, when users specifies warning=no, it could be used to check that
the source files is under thirdparty or not. If not, it will warn
users that this flag is used incorrectly.
"""
keywords = ['thirdparty']
return keywords
def tune_parallel_jobs_num(self):
"""Tune the jobs num. """
user_jobs_num = self.__options.jobs
jobs_num = 0
cpu_core_num = cpu_count()
distcc_enabled = configparse.blade_config.get_config('distcc_config')['enabled']
if distcc_enabled and self.build_environment.distcc_env_prepared:
jobs_num = int(1.5 * len(self.build_environment.get_distcc_hosts_list())) + 1
if jobs_num > 20:
jobs_num = 20
if jobs_num and self.__options.jobs != jobs_num:
self.__options.jobs = jobs_num
elif self.__options.jobs < 1:
if cpu_core_num <= 4:
self.__options.jobs = 2 * cpu_core_num
else:
self.__options.jobs = cpu_core_num
if self.__options.jobs > 8:
self.__options.jobs = 8
if self.__options.jobs != user_jobs_num:
console.info('tunes the parallel jobs number(-j N) to be %d' % (
self.__options.jobs))
return self.__options.jobs
class Blade(object):
"""Blade. A blade manager class. """
def __init__(self,
command_targets,
blade_path,
working_dir,
build_path,
blade_root_dir,
blade_options,
command):
"""init method.
"""
self.__command_targets = command_targets
self.__blade_path = blade_path
self.__working_dir = working_dir
self.__build_path = build_path
self.__root_dir = blade_root_dir
self.__options = blade_options
self.__command = command
# Source dir of current loading BUILD file
self.__current_source_path = blade_root_dir
# The direct targets that are used for analyzing
self.__direct_targets = []
# All command targets, make sure that all targets specified with ...
# are all in the list now
self.__all_command_targets = []
# Given some targets specified in the command line, Blade will load
# BUILD files containing these command line targets; global target
# functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
# files will register targets into target_database, which then becomes
# the input to dependency analyzer and SCons rules generator. It is
# notable that not all targets in target_database are dependencies of
# command line targets.
self.__target_database = {}
# targets to build after loading the build files.
self.__build_targets = {}
# The targets keys list after sorting by topological sorting method.
# Used to generate build rules in correct order.
self.__sorted_targets_keys = []
# The depended targets dict after topological sorting
self.__depended_targets = {}
# Inidcating that whether the deps list is expanded by expander or not
self.__targets_expanded = False
self.__scons_platform = SconsPlatform()
self.build_environment = BuildEnvironment(self.__root_dir)
self.svn_root_dirs = []
def _get_normpath_target(self, command_target):
"""returns a tuple (path, name).
path is a full path from BLADE_ROOT
"""
target_path = relative_path(self.__working_dir, self.__root_dir)
path, name = command_target.split(':')
if target_path != '.':
if path:
path = target_path + '/' + path
else:
path = target_path
path = os.path.normpath(path)
return path, name
def load_targets(self):
"""Load the targets. """
console.info('loading BUILDs...')
if self.__command == 'query' and getattr(self.__options,
'depended', False):
# For query depended command, always start from root with target ...
# that is scanning the whole tree
working_dir = self.__root_dir
else:
working_dir = self.__working_dir
(self.__direct_targets,
self.__all_command_targets,
self.__build_targets) = load_targets(self.__command_targets,
working_dir,
self.__root_dir,
self)
console.info('loading done.')
return self.__direct_targets, self.__all_command_targets # For test
def analyze_targets(self):
"""Expand the targets. """
console.info('analyzing dependency graph...')
(self.__sorted_targets_keys,
self.__depended_targets) = analyze_deps(self.__build_targets)
self.__targets_expanded = True
console.info('analyzing done.')
return self.__build_targets # For test
def generate_build_rules(self):
"""Generate the constructing rules. """
console.info('generating build rules...')
build_rules_generator = SconsRulesGenerator('SConstruct',
self.__blade_path, self)
rules_buf = build_rules_generator.generate_scons_script()
console.info('generating done.')
return rules_buf
def generate(self):
"""Generate the build script. """
self.load_targets()
self.analyze_targets()
if self.__command != 'query':
self.generate_build_rules()
def run(self, target):
"""Run the target. """
key = self._get_normpath_target(target)
runner = BinaryRunner(self.__build_targets,
self.__options,
self.__target_database)
return runner.run_target(key)
def test(self):
"""Run tests. """
test_runner = TestRunner(self.__build_targets,
self.__options,
self.__target_database,
self.__direct_targets)
return test_runner.run()
def query(self, targets):
"""Query the targets. """
print_deps = getattr(self.__options, 'deps', False)
print_depended = getattr(self.__options, 'depended', False)
dot_file = getattr(self.__options, 'output_to_dot', '')
print_dep_tree = getattr(self.__options, 'output_tree', False)
result_map = self.query_helper(targets)
if dot_file:
print_mode = 0
if print_deps:
print_mode = 0
if print_depended:
print_mode = 1
dot_file = os.path.join(self.__working_dir, dot_file)
self.output_dot(result_map, print_mode, dot_file)
else:
if print_deps:
if print_dep_tree:
self.query_dependency_tree(targets)
else:
for key in result_map:
print '\n'
deps = result_map[key][0]
console.info('//%s:%s depends on the following targets:' % (
key[0], key[1]))
for d in deps:
print '%s:%s' % (d[0], d[1])
if print_depended:
for key in result_map:
print '\n'
depended_by = result_map[key][1]
console.info('//%s:%s is depended by the following targets:' % (
key[0], key[1]))
for d in depended_by:
print '%s:%s' % (d[0], d[1])
return 0
def print_dot_node(self, output_file, node):
print >>output_file, '"%s:%s" [label = "%s:%s"]' % (node[0],
node[1],
node[0],
node[1])
def print_dot_deps(self, output_file, node, target_set):
targets = self.__build_targets
deps = targets[node].deps
for i in deps:
if not i in target_set:
continue
print >>output_file, '"%s:%s" -> "%s:%s"' % (node[0],
node[1],
i[0],
i[1])
def output_dot(self, result_map, print_mode, dot_file):
f = open(dot_file, 'w')
targets = result_map.keys()
nodes = set(targets)
for key in targets:
nodes |= set(result_map[key][print_mode])
print >>f, 'digraph blade {'
for i in nodes:
self.print_dot_node(f, i)
for i in nodes:
self.print_dot_deps(f, i, nodes)
print >>f, '}'
f.close()
def query_helper(self, targets):
"""Query the targets helper method. """
all_targets = self.__build_targets
query_list = []
target_path = relative_path(self.__working_dir, self.__root_dir)
t_path = ''
for t in targets:
if t.find(':') != -1:
key = t.split(':')
if target_path == '.':
t_path = key[0]
else:
t_path = target_path + '/' + key[0]
t_path = os.path.normpath(t_path)
query_list.append((t_path, key[1]))
elif t.endswith('...'):
t_path = os.path.normpath(target_path + '/' + t[:-3])
for tkey in all_targets:
if tkey[0].startswith(t_path):
query_list.append((tkey[0], tkey[1]))
else:
t_path = os.path.normpath(target_path + '/' + t)
for tkey in all_targets:
if tkey[0] == t_path:
query_list.append((t_path, tkey[1]))
result_map = {}
for key in query_list:
deps = all_targets[key].expanded_deps
# depended_by = [k for k in all_targets if key in all_targets[k].expanded_deps]
depended_by = self.__depended_targets[key]
result_map[key] = (sorted(deps), sorted(depended_by))
return result_map
def query_dependency_tree(self, targets):
"""Query the dependency tree of the specified targets. """
query_targets = []
for target in targets:
if ':' not in target:
console.error_exit(
'Target %s is not supported by dependency tree query. '
'The target should be in the format directory:name.' % target)
path, name = target.split(':')
relpath = os.path.relpath(self.__working_dir, self.__root_dir)
path = os.path.normpath(os.path.join(relpath, path))
query_targets.append((path, name))
for key in query_targets:
console.info('')
self._query_dependency_tree(key, 0, self.__build_targets)
console.info('')
def _query_dependency_tree(self, key, level, build_targets):
"""Query the dependency tree of the specified target recursively. """
path, name = key
if level == 0:
output = '%s:%s' % (path, name)
elif level == 1:
output = '%s %s:%s' % ('+-', path, name)
else:
output = '%s%s %s:%s' % ('| ' * (level - 1), '+-', path, name)
console.info(console.colors('end') + console.colors('gray') + output)
for dkey in build_targets[key].deps:
self._query_dependency_tree(dkey, level + 1, build_targets)
def get_build_path(self):
"""The current building path. """
return self.__build_path
def get_root_dir(self):
"""Return the blade root path. """
return self.__root_dir
def set_current_source_path(self, current_source_path):
"""Set the current source path. """
self.__current_source_path = current_source_path
def get_current_source_path(self):
"""Get the current source path. """
return self.__current_source_path
def get_target_database(self):
"""Get the whole target database that haven't been expanded. """
return self.__target_database
def get_direct_targets(self):
"""Return the direct targets. """
return self.__direct_targets
def get_build_targets(self):
"""Get all the targets to be build. """
return self.__build_targets
def get_options(self):
"""Get the global command options. """
return self.__options
def is_expanded(self):
"""Whether the targets are expanded. """
return self.__targets_expanded
def register_target(self, target):
"""Register scons targets into the scons targets map.
It is used to do quick looking.
"""
key = target.key
# Check whether there is already a key in database
if key in self.__target_database:
console.error_exit('Target %s is duplicate in //%s/BUILD' % (
target.name, target.path))
self.__target_database[key] = target
def _is_scons_object_type(self, target_type):
"""The types that shouldn't be registered into blade manager.
Sholdn't invoke scons_rule method when it is not a scons target which
could not be registered into blade manager, like system library.
1. system_library
"""
return target_type != 'system_library'
def gen_targets_rules(self):
"""Get the build rules and return to the object who queries this. """
rules_buf = []
skip_test = getattr(self.__options, 'no_test', False)
skip_package = not getattr(self.__options, 'generate_package', False)
for k in self.__sorted_targets_keys:
target = self.__build_targets[k]
if not self._is_scons_object_type(target.type):
continue
scons_object = self.__target_database.get(k, None)
if not scons_object:
console.warning('not registered scons object, key %s' % str(k))
continue
if (skip_test and target.type.endswith('_test')
and k not in self.__direct_targets):
continue
if (skip_package and target.type == 'package'
and k not in self.__direct_targets):
continue
scons_object.scons_rules()
rules_buf.append('\n')
rules_buf += scons_object.get_rules()
return rules_buf
def get_scons_platform(self):
"""Return handle of the platform class. """
return self.__scons_platform
def get_sources_keyword_list(self):
"""This keywords list is used to check the source files path.
Ex, when users specifies warning=no, it could be used to check that
the source files is under thirdparty or not. If not, it will warn
users that this flag is used incorrectly.
"""
keywords = ['thirdparty']
return keywords
def parallel_jobs_num(self):
"""Tune the jobs num. """
# User has the highest priority
user_jobs_num = self.__options.jobs
if user_jobs_num > 0:
return user_jobs_num
# Calculate job numbers smartly
jobs_num = 0
distcc_enabled = configparse.blade_config.get_config('distcc_config')['enabled']
if distcc_enabled and self.build_environment.distcc_env_prepared:
# Distcc cost doesn;t much local cpu, jobs can be quite large.
distcc_num = len(self.build_environment.get_distcc_hosts_list())
jobs_num = min(max(int(1.5 * distcc_num), 1), 20)
else:
cpu_core_num = cpu_count()
# machines with cpu_core_num > 4 is usually shared by multiple users,
# set an upper bound to avoid interfering other users
jobs_num = min(2 * cpu_core_num, 8)
if jobs_num != user_jobs_num:
console.info('tunes the parallel jobs number(-j N) to be %d' % (
jobs_num))
return jobs_num