Esempio n. 1
0
    def __init__(self, command_targets, load_targets, blade_path, working_dir,
                 build_path, blade_root_dir, blade_options, command):
        """init method.

        """
        self.__command_targets = command_targets
        self.__load_targets = load_targets
        self.__blade_path = blade_path
        self.__working_dir = working_dir
        self.__build_path = build_path
        self.__root_dir = blade_root_dir
        self.__options = blade_options
        self.__command = command

        # Source dir of current loading BUILD file
        self.__current_source_path = blade_root_dir

        # The direct targets that are used for analyzing
        self.__direct_targets = []

        # All command targets, make sure that all targets specified with ...
        # are all in the list now
        self.__all_command_targets = []

        # Given some targets specified in the command line, Blade will load
        # BUILD files containing these command line targets; global target
        # functions, i.e., cc_library, cc_binary and etc, in these BUILD
        # files will register targets into target_database, which then becomes
        # the input to dependency analyzer and SCons rules generator.  It is
        # notable that not all targets in target_database are dependencies of
        # command line targets.
        self.__target_database = {}

        # targets to build after loading the build files.
        self.__build_targets = {}

        # The targets keys list after sorting by topological sorting method.
        # Used to generate build rules in correct order.
        self.__sorted_targets_keys = []

        # The depended targets dict after topological sorting
        self.__depended_targets = {}

        # Indicate whether the deps list is expanded by expander or not
        self.__targets_expanded = False

        self.__build_time = time.time()

        self.__build_platform = BuildPlatform()
        self.build_environment = BuildEnvironment(self.__root_dir)

        self.svn_root_dirs = []

        self._verify_history_path = os.path.join(build_path,
                                                 '.blade_verify.json')
        self._verify_history = {
            'header_inclusion_dependencies':
            {},  # path(.H) -> mtime(modification time)
        }
Esempio n. 2
0
    def __init__(self,
                 command_targets,
                 blade_path,
                 working_dir,
                 build_path,
                 blade_root_dir,
                 blade_options,
                 command):
        """init method.

        """
        self.__command_targets = command_targets
        self.__blade_path = blade_path
        self.__working_dir = working_dir
        self.__build_path = build_path
        self.__root_dir = blade_root_dir
        self.__options = blade_options
        self.__command = command

        # Source dir of current loading BUILD file
        self.__current_source_path = blade_root_dir

        # The direct targets that are used for analyzing
        self.__direct_targets = []

        # All command targets, make sure that all targets specified with ...
        # are all in the list now
        self.__all_command_targets = []

        # Given some targets specified in the command line, Blade will load
        # BUILD files containing these command line targets; global target
        # functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
        # files will register targets into target_database, which then becomes
        # the input to dependency analyzer and SCons rules generator.  It is
        # notable that not all targets in target_database are dependencies of
        # command line targets.
        self.__target_database = {}

        # targets to build after loading the build files.
        self.__build_targets = {}

        # The targets keys list after sorting by topological sorting method.
        # Used to generate build rules in correct order.
        self.__sorted_targets_keys = []

        # The depended targets dict after topological sorting
        self.__depended_targets = {}

        # Inidcating that whether the deps list is expanded by expander or not
        self.__targets_expanded = False

        self.__scons_platform = SconsPlatform()
        self.build_environment = BuildEnvironment(self.__root_dir)

        self.svn_root_dirs = []
Esempio n. 3
0
    def __init__(self,
                 command_targets,
                 blade_path,
                 working_dir,
                 build_path,
                 current_source_path,
                 blade_options,
                 **kwargs):
        """init method.

        Mainly to hold the global data.
        The directory which changes during the runtime of blade, and
        contains BUILD file under current focus.
        current_source_dir = "."

        Given some targets specified in the command line, Blade will load
        BUILD files containing these command line targets; global target
        functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
        files will register targets into target_database, which then becomes
        the input to dependency analyzer and SCons rules generator.  It is
        notable that not all targets in target_database are dependencies of
        command line targets.
        target_database = {}

        related targets after loading the build files and exec BUILD files
        as python script
        related_targets = {}

        The map used by build rules to ensure that a source file occurres in
        exactly one rule/target(only library target).
        target_srcs_map = {}

        The scons cache manager class string, which should be output to
        scons script if ccache is not installed
        scache_manager_class_str = ''

        The targets keys list after sorting by topological sorting method.
        sorted_targets_keys = []

        Inidcating that whether the deps list is expanded by expander or not
        False - not expanded
        True - expanded
        target_deps_expanded

        All targets after expanding their dependency
        all_targets = {}

        The scons target objects registered into blade manager
        scons_targets_map = {}

        The vars which are depended by python binary
        {key : 'python_files'}
        self.python_binary_dep_source_cmd = {}

        The files which are depended by python binary
        {key : 'python_files'}
        python_binary_dep_source_map = {}

        The files which are depended by java jar file
        {key : 'java_files'}
        java_jar_dep_source_map = {}

        The files which should be packed into java jar
        {key : 'packing_files'}
        java_jar_files_packing_map = {}

        The jar files map
        {key : 'jar_files_generated'}
        java_jars_map = {}

        The java compiling classpath parameter map
        java_classpath_map = {}
        {key : 'target_path'}

        The java_jar dep var map, which should be added to dependency chain
        java_jar_dep_vars = {}

        The cc objects pool, a map to hold all the objects name.
        cc_objects_pool = {}

        The gen rule files map, which is used to generate the explict dependency
        relationtion ship between gen_rule target and other targets
        gen_rule_files_map = {}

        The direct targets that are used for analyzing
        direct_targets = []

        All command targets, make sure that all targets specified with ...
        are all in the list now
        all_command_targets = []

        The class to get platform info
        SconsPlatform

        The class to manage the cc flags
        CcFlagsManager

        The sources files that are needed to perform explict dependency
        sources_explict_dependency_map = {}

        The prebuilt cc_library file map which is needed to establish
        symbolic links while testing
        prebuilt_cc_library_file_map = {}

        """
        self.command_targets = command_targets
        self.direct_targets = []
        self.all_command_targets = []
        self.blade_path = blade_path
        self.working_dir = working_dir
        self.build_path = build_path
        self.current_source_path = current_source_path
        self.target_database = {}
        self.related_targets = {}
        self.target_srcs_map = {}
        self.scache_manager_class_str = ''
        self.options = blade_options
        self.sorted_targets_keys = []
        self.target_deps_expanded = False
        self.all_targets_expanded = {}
        self.scons_targets_map = {}
        self.java_jar_dep_source_map = {}
        self.java_jar_files_packing_map = {}
        self.java_jars_map = {}
        self.java_classpath_map = {}
        self.java_jar_dep_vars = {}
        self.python_binary_dep_source_cmd = {}
        self.python_binary_dep_source_map = {}
        self.cc_objects_pool = {}

        self.deps_expander = None
        self.build_rules_generator = None

        self.gen_rule_files_map = {}

        self.scons_platform = SconsPlatform()
        self.ccflags_manager = CcFlagsManager(self.options)
        self.sources_explict_dependency_map = {}
        self.prebuilt_cc_library_file_map = {}

        self.distcc_enabled = configparse.blade_config.get_config(
                'distcc_config')['enabled']

        self.build_environment = BuildEnvironment(self.current_source_path)

        self.svn_root_dirs = []

        self.kwargs = kwargs
Esempio n. 4
0
class Blade(object):
    """Blade. A blade manager class. """
    def __init__(self,
                 command_targets,
                 blade_path,
                 working_dir,
                 build_path,
                 current_source_path,
                 blade_options,
                 **kwargs):
        """init method.

        Mainly to hold the global data.
        The directory which changes during the runtime of blade, and
        contains BUILD file under current focus.
        current_source_dir = "."

        Given some targets specified in the command line, Blade will load
        BUILD files containing these command line targets; global target
        functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
        files will register targets into target_database, which then becomes
        the input to dependency analyzer and SCons rules generator.  It is
        notable that not all targets in target_database are dependencies of
        command line targets.
        target_database = {}

        related targets after loading the build files and exec BUILD files
        as python script
        related_targets = {}

        The map used by build rules to ensure that a source file occurres in
        exactly one rule/target(only library target).
        target_srcs_map = {}

        The scons cache manager class string, which should be output to
        scons script if ccache is not installed
        scache_manager_class_str = ''

        The targets keys list after sorting by topological sorting method.
        sorted_targets_keys = []

        Inidcating that whether the deps list is expanded by expander or not
        False - not expanded
        True - expanded
        target_deps_expanded

        All targets after expanding their dependency
        all_targets = {}

        The scons target objects registered into blade manager
        scons_targets_map = {}

        The vars which are depended by python binary
        {key : 'python_files'}
        self.python_binary_dep_source_cmd = {}

        The files which are depended by python binary
        {key : 'python_files'}
        python_binary_dep_source_map = {}

        The files which are depended by java jar file
        {key : 'java_files'}
        java_jar_dep_source_map = {}

        The files which should be packed into java jar
        {key : 'packing_files'}
        java_jar_files_packing_map = {}

        The jar files map
        {key : 'jar_files_generated'}
        java_jars_map = {}

        The java compiling classpath parameter map
        java_classpath_map = {}
        {key : 'target_path'}

        The java_jar dep var map, which should be added to dependency chain
        java_jar_dep_vars = {}

        The cc objects pool, a map to hold all the objects name.
        cc_objects_pool = {}

        The gen rule files map, which is used to generate the explict dependency
        relationtion ship between gen_rule target and other targets
        gen_rule_files_map = {}

        The direct targets that are used for analyzing
        direct_targets = []

        All command targets, make sure that all targets specified with ...
        are all in the list now
        all_command_targets = []

        The class to get platform info
        SconsPlatform

        The class to manage the cc flags
        CcFlagsManager

        The sources files that are needed to perform explict dependency
        sources_explict_dependency_map = {}

        The prebuilt cc_library file map which is needed to establish
        symbolic links while testing
        prebuilt_cc_library_file_map = {}

        """
        self.command_targets = command_targets
        self.direct_targets = []
        self.all_command_targets = []
        self.blade_path = blade_path
        self.working_dir = working_dir
        self.build_path = build_path
        self.current_source_path = current_source_path
        self.target_database = {}
        self.related_targets = {}
        self.target_srcs_map = {}
        self.scache_manager_class_str = ''
        self.options = blade_options
        self.sorted_targets_keys = []
        self.target_deps_expanded = False
        self.all_targets_expanded = {}
        self.scons_targets_map = {}
        self.java_jar_dep_source_map = {}
        self.java_jar_files_packing_map = {}
        self.java_jars_map = {}
        self.java_classpath_map = {}
        self.java_jar_dep_vars = {}
        self.python_binary_dep_source_cmd = {}
        self.python_binary_dep_source_map = {}
        self.cc_objects_pool = {}

        self.deps_expander = None
        self.build_rules_generator = None

        self.gen_rule_files_map = {}

        self.scons_platform = SconsPlatform()
        self.ccflags_manager = CcFlagsManager(self.options)
        self.sources_explict_dependency_map = {}
        self.prebuilt_cc_library_file_map = {}

        self.distcc_enabled = configparse.blade_config.get_config(
                'distcc_config')['enabled']

        self.build_environment = BuildEnvironment(self.current_source_path)

        self.svn_root_dirs = []

        self.kwargs = kwargs

    def _get_normpath_target(self, command_target):
        """returns a tuple (path, name).

        path is a full path from BLADE_ROOT

        """
        target_path = relative_path(self.working_dir, self.current_source_path)
        path, name = command_target.split(':')
        if target_path != '.':
            if path:
                path = target_path + '/' + path
            else:
                path = target_path
        path = os.path.normpath(path)
        return path, name

    def load_targets(self):
        """Load the targets. """
        console.info("loading BUILDs...")
        if self.kwargs.get('blade_command', '') == 'query':
            working_dir = self.current_source_path

            if '...' not in self.command_targets:
                new_target_list = []
                for target in self.command_targets:
                    new_target_list.append("%s:%s" %
                            self._get_normpath_target(target))
                self.command_targets = new_target_list
        else:
            working_dir = self.working_dir
        (self.direct_targets,
         self.all_command_targets) = load_targets(self.command_targets,
                                                  working_dir,
                                                  self.current_source_path,
                                                  self)
        console.info("loading done.")
        return self.direct_targets, self.all_command_targets

    def analyze_targets(self):
        """Expand the targets. """
        console.info("analyzing dependency graph...")
        self.deps_analyzer = DependenciesAnalyzer(self)
        self.deps_analyzer.analyze_deps()
        console.info("analyzing done.")
        return self.all_targets_expanded

    def generate_build_rules(self):
        """Generate the constructing rules. """
        console.info("generating build rules...")
        self.build_rules_generator = SconsRulesGenerator('SConstruct',
                                                         self.blade_path, self)
        rules_buf = self.build_rules_generator.generate_scons_script()
        console.info("generating done.")
        return rules_buf

    def generate(self):
        """Build the targets. """
        self.load_targets()
        self.analyze_targets()
        self.generate_build_rules()

    def run(self, target):
        """Run the target. """
        key = self._get_normpath_target(target)
        runner = BinaryRunner(self.all_targets_expanded,
                              self.options,
                              self.prebuilt_cc_library_file_map,
                              self.target_database)
        return runner.run_target(key)

    def test(self):
        """Run tests. """
        test_runner = TestRunner(self.all_targets_expanded,
                                 self.options,
                                 self.prebuilt_cc_library_file_map,
                                 self.target_database,
                                 self.direct_targets)
        return test_runner.run()

    def query(self, targets):
        """Query the targets. """
        print_deps = hasattr(self.options, 'deps') and (
                        self.options.deps)
        print_depended = hasattr(self.options, 'depended') and (
                        self.options.depended)
        dot_file = ""
        if hasattr(self.options, 'output-to-dot'):
            dot_file = self.options.output_to_dot
        result_map = self.query_helper(targets)
        if dot_file:
            print_mode = 0
            if print_deps:
                print_mode = 0
            if print_depended:
                print_mode = 1
            if not dot_file.startswith("/"):
                dot_file = self.working_dir + "/" + dot_file
            self.output_dot(result_map, print_mode, dot_file)
        else:
            if print_deps:
                for key in result_map.keys():
                    print "\n"
                    deps = result_map[key][0]
                    console.info("//%s:%s depends on the following targets:" % (
                            key[0], key[1]))
                    for d in deps:
                        print "%s:%s" % (d[0], d[1])
            if print_depended:
                for key in result_map.keys():
                    print "\n"
                    depended_by = result_map[key][1]
                    console.info("//%s:%s is depended by the following targets:" % (
                            key[0], key[1]))
                    depended_by.sort(key=lambda x: x, reverse=False)
                    for d in depended_by:
                        print "%s:%s" % (d[0], d[1])
        return 0

    def print_dot_node(self, output_file, node):
        print >>output_file, '"%s:%s" [label = "%s:%s"]' % (node[0],
                                                            node[1],
                                                            node[0],
                                                            node[1])

    def print_dot_deps(self, output_file, node, target_set):
        targets = self.related_targets
        deps = targets.get(node, {}).get('direct_deps', [])
        for i in deps:
            if not i in target_set:
                continue
            print >>output_file, '"%s:%s" -> "%s:%s"' % (node[0],
                                                         node[1],
                                                         i[0],
                                                         i[1])

    def output_dot(self, result_map, print_mode, dot_file):
        f = open(dot_file, 'w')
        targets = result_map.keys()
        nodes = set(targets)
        for key in targets:
            nodes |= set(result_map[key][print_mode])
        print >>f, "digraph blade {"
        for i in nodes:
            self.print_dot_node(f, i)
        for i in nodes:
            self.print_dot_deps(f, i, nodes)
        print >>f, "}"
        f.close()

    def query_helper(self, targets):
        """Query the targets helper method. """
        all_targets = self.all_targets_expanded
        query_list = []
        target_path = relative_path(self.working_dir, self.current_source_path)
        t_path = ''
        for t in targets:
            key = t.split(':')
            if target_path == '.':
                t_path = key[0]
            else:
                t_path = target_path + '/' + key[0]
            t_path = os.path.normpath(t_path)
            query_list.append((t_path, key[1]))
        result_map = {}
        for key in query_list:
            result_map[key] = ([], [])
            deps = all_targets.get(key, {}).get('deps', [])
            deps.sort(key=lambda x: x, reverse=False)
            depended_by = []
            for tkey in all_targets.keys():
                if key in all_targets[tkey]['deps']:
                    depended_by.append(tkey)
            depended_by.sort(key=lambda x: x, reverse=False)
            result_map[key] = (list(deps), list(depended_by))
        return result_map

    def get_blade_path(self):
        """Return the blade archive path. """
        return self.blade_path

    def get_build_path(self):
        """The current building path. """
        return self.build_path

    def set_current_source_path(self, current_source_path):
        """Set the current source path. """
        self.current_source_path = current_source_path

    def get_current_source_path(self):
        """Get the current source path. """
        return self.current_source_path

    def get_target_database(self):
        """Get the whole target database that haven't been expanded. """
        return self.target_database

    def set_related_targets(self, related_targets):
        """Set the related targets. """
        self.related_targets = dict(related_targets)

    def get_related_targets(self):
        """Get the related targets. """
        return self.related_targets

    def get_direct_targets(self):
        """Return the direct targets. """
        return self.direct_targets

    def get_all_command_targets(self):
        """Return all command targets. """
        return self.all_command_targets

    def set_sorted_targets_keys(self, sorted_keys_list):
        """Set the keys list from expaned targets. """
        self.sorted_targets_keys = list(sorted_keys_list)

    def get_sorted_targets_keys(self):
        """Get the keys list from expaned targets. """
        return self.sorted_targets_keys

    def set_all_targets_expanded(self, all_targets):
        """Set the targets that have been expanded by expander. """
        self.all_targets_expanded = dict(all_targets)
        self.target_deps_expanded = True

    def get_all_targets_expanded(self):
        """Get all the targets that expaned. """
        return self.all_targets_expanded

    def get_target_srcs_map(self):
        """Get the targets source files map.

        It is used in generating cc object rules.

        """
        return self.target_srcs_map

    def get_options(self):
        """Get the global command options. """
        return self.options

    def get_expanded(self):
        """Whether the targets are expanded. """
        return self.target_deps_expanded

    def register_scons_target(self, target_key, scons_target):
        """Register scons targets into the scons targets map.

        It is used to do quick looking.

        """
        # check that whether there is already a key in database
        if target_key in self.scons_targets_map.keys():
            console.error_exit(
                    "target name %s is duplicate in //%s/BUILD" % (
                        target_key[1], target_key[0]))
        self.scons_targets_map[target_key] = scons_target

    def get_scons_target(self, target_key):
        """Get scons target according to the key. """
        return self.scons_targets_map.get(target_key, None)

    def get_java_jar_dep_source_map(self):
        """The map mainly to hold the java files from swig or proto rules.

        These files maybe depended by java_jar target.

        """
        return self.java_jar_dep_source_map

    def get_java_jar_files_packing_map(self):
        """The map to hold the files that should be packed into java jar. """
        return  self.java_jar_files_packing_map

    def get_java_jars_map(self):
        """The map to hold the java jar files generated by blade. """
        return self.java_jars_map

    def get_java_classpath_map(self):
        """The classpath list which is needed by java compling. """
        return self.java_classpath_map

    def get_java_jar_dep_vars(self):
        """The vars map which is prerequiste of the java jar target. """
        return self.java_jar_dep_vars

    def get_cc_objects_pool(self):
        """The cc objects pool which is used when generating the cc object rules. """
        return self.cc_objects_pool

    def _is_scons_object_type(self, target_type):
        """The types that shouldn't be registered into blade manager.

        Sholdn't invoke scons_rule method when it is not a scons target which
        could not be registered into blade manager, like system library.

        1. system_library

        """
        if target_type == 'system_library':
            return False
        else:
            return True

    def get_targets_rules(self):
        """Get the build rules and return to the object who queries this. """
        rules_buf = []
        skip_test_targets = False
        if hasattr(self.options, 'no_test') and self.options.no_test:
            skip_test_targets = True
        for k in self.sorted_targets_keys:
            target = self.all_targets_expanded[k]
            if not self._is_scons_object_type(target['type']):
                continue
            scons_object = self.scons_targets_map.get(k, None)
            if not scons_object:
                console.warning('not registered scons object, key %s' % str(k))
                continue
            if skip_test_targets and (target['type'] == 'cc_test' or
                                      target['type'] == 'dynamic_cc_test'):
                continue
            scons_object.scons_rules()
            rules_buf += scons_object.get_rules()
        return rules_buf

    def set_gen_rule_files_map(self, files_map):
        """Set the gen_rule files map. """
        self.gen_rule_files_map = dict(files_map)

    def get_gen_rule_files_map(self):
        """Get the gen_rule files map. """
        return self.gen_rule_files_map

    def get_scons_platform(self):
        """Return handle of the platform class. """
        return self.scons_platform

    def get_ccflags_manager(self):
        """Return handle of the ccflags manager class. """
        return self.ccflags_manager

    def get_sources_keyword_list(self):
        """This keywords list is used to check the source files path.

        Ex, when users specifies warning=no, it could be used to check that
        the source files is under thirdparty or not. If not, it will warn
        users that this flag is used incorrectly.

        """
        keywords = ['thirdparty']
        return keywords

    def get_sources_explict_dependency_map(self):
        """Returns the handle of sources_explict_dependency_map. """
        return self.sources_explict_dependency_map

    def get_prebuilt_cc_library_file_map(self):
        """Returns the prebuilt_cc_library_file_map. """
        return self.prebuilt_cc_library_file_map

    def tune_parallel_jobs_num(self):
        """Tune the jobs num. """
        user_jobs_num = self.options.jobs
        jobs_num = 0
        cpu_core_num = cpu_count()
        if self.distcc_enabled and self.build_environment.distcc_env_prepared:
            jobs_num = int(1.5 * len(self.build_environment.get_distcc_hosts_list())) + 1
            if jobs_num > 20:
                jobs_num = 20
            if jobs_num and self.options.jobs != jobs_num:
                self.options.jobs = jobs_num
        elif self.options.jobs < 1:
            if cpu_core_num <= 4:
                self.options.jobs = 2 * cpu_core_num
            else:
                self.options.jobs = cpu_core_num
                if self.options.jobs > 8:
                    self.options.jobs = 8
        if self.options.jobs != user_jobs_num:
            console.info("tunes the parallel jobs number(-j N) to be %d" % (
                self.options.jobs))
        return self.options.jobs
Esempio n. 5
0
class Blade(object):
    """Blade. A blade manager class. """
    def __init__(self,
                 command_targets,
                 blade_path,
                 working_dir,
                 build_path,
                 blade_root_dir,
                 blade_options,
                 command):
        """init method.

        """
        self.__command_targets = command_targets
        self.__blade_path = blade_path
        self.__working_dir = working_dir
        self.__build_path = build_path
        self.__root_dir = blade_root_dir
        self.__options = blade_options
        self.__command = command

        # Source dir of current loading BUILD file
        self.__current_source_path = blade_root_dir

        # The direct targets that are used for analyzing
        self.__direct_targets = []

        # All command targets, make sure that all targets specified with ...
        # are all in the list now
        self.__all_command_targets = []

        # Given some targets specified in the command line, Blade will load
        # BUILD files containing these command line targets; global target
        # functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
        # files will register targets into target_database, which then becomes
        # the input to dependency analyzer and SCons rules generator.  It is
        # notable that not all targets in target_database are dependencies of
        # command line targets.
        self.__target_database = {}

        # targets to build after loading the build files.
        self.__build_targets = {}

        # The targets keys list after sorting by topological sorting method.
        # Used to generate build rules in correct order.
        self.__sorted_targets_keys = []

        # The depended targets dict after topological sorting
        self.__depended_targets = {}

        # Inidcating that whether the deps list is expanded by expander or not
        self.__targets_expanded = False

        self.__scons_platform = SconsPlatform()
        self.build_environment = BuildEnvironment(self.__root_dir)

        self.svn_root_dirs = []

    def _get_normpath_target(self, command_target):
        """returns a tuple (path, name).

        path is a full path from BLADE_ROOT

        """
        target_path = relative_path(self.__working_dir, self.__root_dir)
        path, name = command_target.split(':')
        if target_path != '.':
            if path:
                path = target_path + '/' + path
            else:
                path = target_path
        path = os.path.normpath(path)
        return path, name

    def load_targets(self):
        """Load the targets. """
        console.info('loading BUILDs...')
        if self.__command == 'query' and getattr(self.__options,
            'depended', False):
            # For query depended command, always start from root with target ...
            # that is scanning the whole tree
            working_dir = self.__root_dir
        else:
            working_dir = self.__working_dir
        (self.__direct_targets,
         self.__all_command_targets,
         self.__build_targets) = load_targets(self.__command_targets,
                                                  working_dir,
                                                  self.__root_dir,
                                                  self)
        console.info('loading done.')
        return self.__direct_targets, self.__all_command_targets  # For test

    def analyze_targets(self):
        """Expand the targets. """
        console.info('analyzing dependency graph...')
        (self.__sorted_targets_keys,
         self.__depended_targets) = analyze_deps(self.__build_targets)
        self.__targets_expanded = True

        console.info('analyzing done.')
        return self.__build_targets  # For test

    def generate_build_rules(self):
        """Generate the constructing rules. """
        console.info('generating build rules...')
        build_rules_generator = SconsRulesGenerator('SConstruct',
                                                    self.__blade_path, self)
        rules_buf = build_rules_generator.generate_scons_script()
        console.info('generating done.')
        return rules_buf

    def generate(self):
        """Generate the build script. """
        self.load_targets()
        self.analyze_targets()
        if self.__command != 'query':
            self.generate_build_rules()

    def run(self, target):
        """Run the target. """
        key = self._get_normpath_target(target)
        runner = BinaryRunner(self.__build_targets,
                              self.__options,
                              self.__target_database)
        return runner.run_target(key)

    def test(self):
        """Run tests. """
        test_runner = TestRunner(self.__build_targets,
                                 self.__options,
                                 self.__target_database,
                                 self.__direct_targets)
        return test_runner.run()

    def query(self, targets):
        """Query the targets. """
        print_deps = getattr(self.__options, 'deps', False)
        print_depended = getattr(self.__options, 'depended', False)
        dot_file = getattr(self.__options, 'output_to_dot', '')
        print_dep_tree = getattr(self.__options, 'output_tree', False)
        result_map = self.query_helper(targets)
        if dot_file:
            print_mode = 0
            if print_deps:
                print_mode = 0
            if print_depended:
                print_mode = 1
            dot_file = os.path.join(self.__working_dir, dot_file)
            self.output_dot(result_map, print_mode, dot_file)
        else:
            if print_deps:
                if print_dep_tree:
                    self.query_dependency_tree(targets)
                else:
                    for key in result_map:
                        print '\n'
                        deps = result_map[key][0]
                        console.info('//%s:%s depends on the following targets:' % (
                                key[0], key[1]))
                        for d in deps:
                            print '%s:%s' % (d[0], d[1])
            if print_depended:
                for key in result_map:
                    print '\n'
                    depended_by = result_map[key][1]
                    console.info('//%s:%s is depended by the following targets:' % (
                            key[0], key[1]))
                    for d in depended_by:
                        print '%s:%s' % (d[0], d[1])
        return 0

    def print_dot_node(self, output_file, node):
        print >>output_file, '"%s:%s" [label = "%s:%s"]' % (node[0],
                                                            node[1],
                                                            node[0],
                                                            node[1])

    def print_dot_deps(self, output_file, node, target_set):
        targets = self.__build_targets
        deps = targets[node].deps
        for i in deps:
            if not i in target_set:
                continue
            print >>output_file, '"%s:%s" -> "%s:%s"' % (node[0],
                                                         node[1],
                                                         i[0],
                                                         i[1])

    def output_dot(self, result_map, print_mode, dot_file):
        f = open(dot_file, 'w')
        targets = result_map.keys()
        nodes = set(targets)
        for key in targets:
            nodes |= set(result_map[key][print_mode])
        print >>f, 'digraph blade {'
        for i in nodes:
            self.print_dot_node(f, i)
        for i in nodes:
            self.print_dot_deps(f, i, nodes)
        print >>f, '}'
        f.close()

    def query_helper(self, targets):
        """Query the targets helper method. """
        all_targets = self.__build_targets
        query_list = []
        target_path = relative_path(self.__working_dir, self.__root_dir)
        t_path = ''
        for t in targets:
            if t.find(':') != -1:
                key = t.split(':')
                if target_path == '.':
                    t_path = key[0]
                else:
                    t_path = target_path + '/' + key[0]
                t_path = os.path.normpath(t_path)
                query_list.append((t_path, key[1]))
            elif t.endswith('...'):
                t_path = os.path.normpath(target_path + '/' + t[:-3])
                for tkey in all_targets:
                    if tkey[0].startswith(t_path):
                        query_list.append((tkey[0], tkey[1]))
            else:
                t_path = os.path.normpath(target_path + '/' + t)
                for tkey in all_targets:
                    if tkey[0] == t_path:
                        query_list.append((t_path, tkey[1]))
        result_map = {}
        for key in query_list:
            deps = all_targets[key].expanded_deps
            # depended_by = [k for k in all_targets if key in all_targets[k].expanded_deps]
            depended_by = self.__depended_targets[key]
            result_map[key] = (sorted(deps), sorted(depended_by))
        return result_map

    def query_dependency_tree(self, targets):
        """Query the dependency tree of the specified targets. """
        query_targets = []
        for target in targets:
            if ':' not in target:
                console.error_exit(
                    'Target %s is not supported by dependency tree query. '
                    'The target should be in the format directory:name.' % target)
            path, name = target.split(':')
            relpath = os.path.relpath(self.__working_dir, self.__root_dir)
            path = os.path.normpath(os.path.join(relpath, path))
            query_targets.append((path, name))

        for key in query_targets:
            console.info('')
            self._query_dependency_tree(key, 0, self.__build_targets)
            console.info('')

    def _query_dependency_tree(self, key, level, build_targets):
        """Query the dependency tree of the specified target recursively. """
        path, name = key
        if level == 0:
            output = '%s:%s' % (path, name)
        elif level == 1:
            output = '%s %s:%s' % ('+-', path, name)
        else:
            output = '%s%s %s:%s' % ('|  ' * (level - 1), '+-', path, name)
        console.info(console.colors('end') + console.colors('gray') + output)
        for dkey in build_targets[key].deps:
            self._query_dependency_tree(dkey, level + 1, build_targets)

    def get_build_path(self):
        """The current building path. """
        return self.__build_path

    def get_root_dir(self):
        """Return the blade root path. """
        return self.__root_dir

    def set_current_source_path(self, current_source_path):
        """Set the current source path. """
        self.__current_source_path = current_source_path

    def get_current_source_path(self):
        """Get the current source path. """
        return self.__current_source_path

    def get_target_database(self):
        """Get the whole target database that haven't been expanded. """
        return self.__target_database

    def get_direct_targets(self):
        """Return the direct targets. """
        return self.__direct_targets

    def get_build_targets(self):
        """Get all the targets to be build. """
        return self.__build_targets

    def get_options(self):
        """Get the global command options. """
        return self.__options

    def is_expanded(self):
        """Whether the targets are expanded. """
        return self.__targets_expanded

    def register_target(self, target):
        """Register scons targets into the scons targets map.

        It is used to do quick looking.

        """
        key = target.key
        # Check whether there is already a key in database
        if key in self.__target_database:
            console.error_exit('Target %s is duplicate in //%s/BUILD' % (
                               target.name, target.path))
        self.__target_database[key] = target

    def _is_scons_object_type(self, target_type):
        """The types that shouldn't be registered into blade manager.

        Sholdn't invoke scons_rule method when it is not a scons target which
        could not be registered into blade manager, like system library.

        1. system_library

        """
        return target_type != 'system_library'

    def gen_targets_rules(self):
        """Get the build rules and return to the object who queries this. """
        rules_buf = []
        skip_test = getattr(self.__options, 'no_test', False)
        skip_package = not getattr(self.__options, 'generate_package', False)
        for k in self.__sorted_targets_keys:
            target = self.__build_targets[k]
            if not self._is_scons_object_type(target.type):
                continue
            scons_object = self.__target_database.get(k, None)
            if not scons_object:
                console.warning('not registered scons object, key %s' % str(k))
                continue
            if (skip_test and target.type.endswith('_test')
                and k not in self.__direct_targets):
                continue
            if (skip_package and target.type == 'package'
                and k not in self.__direct_targets):
                continue
            scons_object.scons_rules()
            rules = scons_object.get_rules()
            if rules:
                rules_buf.append('\n')
                rules_buf += rules
        return rules_buf

    def get_scons_platform(self):
        """Return handle of the platform class. """
        return self.__scons_platform

    def get_sources_keyword_list(self):
        """This keywords list is used to check the source files path.

        Ex, when users specifies warning=no, it could be used to check that
        the source files is under thirdparty or not. If not, it will warn
        users that this flag is used incorrectly.

        """
        keywords = ['thirdparty']
        return keywords

    def parallel_jobs_num(self):
        """Tune the jobs num. """
        # User has the highest priority
        user_jobs_num = self.__options.jobs
        if user_jobs_num > 0:
            return user_jobs_num

        # Calculate job numbers smartly
        jobs_num = 0
        distcc_enabled = configparse.blade_config.get_config('distcc_config')['enabled']

        if distcc_enabled and self.build_environment.distcc_env_prepared:
            # Distcc cost doesn;t much local cpu, jobs can be quite large.
            distcc_num = len(self.build_environment.get_distcc_hosts_list())
            jobs_num = min(max(int(1.5 * distcc_num), 1), 20)
        else:
            cpu_core_num = cpu_count()
            # machines with cpu_core_num > 4 is usually shared by multiple users,
            # set an upper bound to avoid interfering other users
            jobs_num = min(2 * cpu_core_num, 8)

        if jobs_num != user_jobs_num:
            console.info('tunes the parallel jobs number(-j N) to be %d' % (
                jobs_num))
        return jobs_num
Esempio n. 6
0
class Blade(object):
    """Blade. A blade manager class. """

    # pylint: disable=too-many-public-methods
    def __init__(self, command_targets, load_targets, blade_path, working_dir,
                 build_path, blade_root_dir, blade_options, command):
        """init method.

        """
        self.__command_targets = command_targets
        self.__load_targets = load_targets
        self.__blade_path = blade_path
        self.__working_dir = working_dir
        self.__build_path = build_path
        self.__root_dir = blade_root_dir
        self.__options = blade_options
        self.__command = command

        # Source dir of current loading BUILD file
        self.__current_source_path = blade_root_dir

        # The direct targets that are used for analyzing
        self.__direct_targets = []

        # All command targets, make sure that all targets specified with ...
        # are all in the list now
        self.__all_command_targets = []

        # Given some targets specified in the command line, Blade will load
        # BUILD files containing these command line targets; global target
        # functions, i.e., cc_library, cc_binary and etc, in these BUILD
        # files will register targets into target_database, which then becomes
        # the input to dependency analyzer and SCons rules generator.  It is
        # notable that not all targets in target_database are dependencies of
        # command line targets.
        self.__target_database = {}

        # targets to build after loading the build files.
        self.__build_targets = {}

        # The targets keys list after sorting by topological sorting method.
        # Used to generate build rules in correct order.
        self.__sorted_targets_keys = []

        # The depended targets dict after topological sorting
        self.__depended_targets = {}

        # Indicate whether the deps list is expanded by expander or not
        self.__targets_expanded = False

        self.__build_time = time.time()

        self.__build_platform = BuildPlatform()
        self.build_environment = BuildEnvironment(self.__root_dir)

        self.svn_root_dirs = []

        self._verify_history_path = os.path.join(build_path,
                                                 '.blade_verify.json')
        self._verify_history = {
            'header_inclusion_dependencies':
            {},  # path(.H) -> mtime(modification time)
        }

    def load_targets(self):
        """Load the targets. """
        console.info('loading BUILDs...')
        (self.__direct_targets, self.__all_command_targets,
         self.__build_targets) = load_targets(self.__load_targets,
                                              self.__root_dir, self)
        if self.__command_targets != self.__load_targets:
            # In query dependents mode, we must use command targets to execute query
            self.__all_command_targets = self._expand_command_targets()
        console.info('loading done.')
        return self.__direct_targets, self.__all_command_targets  # For test

    def _expand_command_targets(self):
        """Expand command line targets to targets list"""
        all_targets = self.__build_targets
        all_command_targets = []
        for t in self.__command_targets:
            t_path, name = t.split(':')
            if name == '...':
                for tkey in all_targets:
                    if tkey[0].startswith(t_path):
                        all_command_targets.append(tkey)
            elif name == '*':
                for tkey in all_targets:
                    if tkey[0] == t_path:
                        all_command_targets.append(tkey)
            else:
                all_command_targets.append((t_path, name))
        return all_command_targets

    def analyze_targets(self):
        """Expand the targets. """
        console.info('analyzing dependency graph...')
        (self.__sorted_targets_keys,
         self.__depended_targets) = analyze_deps(self.__build_targets)
        self.__targets_expanded = True

        console.info('analyzing done.')
        return self.__build_targets  # For test

    def new_build_rules_generator(self):
        if config.get_item('global_config', 'native_builder') == 'ninja':
            return NinjaRulesGenerator('build.ninja', self.__blade_path, self)
        else:
            return SconsRulesGenerator('SConstruct', self.__blade_path, self)

    def generate_build_rules(self):
        """Generate the constructing rules. """
        console.info('generating build rules...')
        generator = self.new_build_rules_generator()
        rules = generator.generate_build_script()
        self.__all_rule_names = generator.get_all_rule_names()
        console.info('generating done.')
        return rules

    def generate(self):
        """Generate the build script. """
        if self.__command != 'query':
            self.generate_build_rules()

    def verify(self):
        """Verify specific targets after build is complete. """
        verify_history = self._load_verify_history()
        error = 0
        header_inclusion_dependencies = config.get_item(
            'cc_config', 'header_inclusion_dependencies')
        header_inclusion_history = verify_history[
            'header_inclusion_dependencies']
        for k in self.__sorted_targets_keys:
            target = self.__build_targets[k]
            if (header_inclusion_dependencies and target.type == 'cc_library'
                    and target.srcs):
                if not target.verify_header_inclusion_dependencies(
                        header_inclusion_history):
                    error += 1
        self._dump_verify_history()
        return error == 0

    def run(self, target):
        """Run the target. """
        runner = BinaryRunner(self.__build_targets, self.__options,
                              self.__target_database)
        return runner.run_target(target)

    def test(self):
        """Run tests. """
        test_runner = TestRunner(self.__build_targets, self.__options,
                                 self.__target_database, self.__direct_targets)
        return test_runner.run()

    def query(self):
        """Query the targets. """
        output_file_name = self.__options.output_file
        if output_file_name:
            output_file_name = os.path.join(self.__working_dir,
                                            output_file_name)
            output_file = open(output_file_name, 'w')
            console.info('query result will be written to file "%s"' %
                         self.__options.output_file)
        else:
            output_file = sys.stdout
            console.info('query result:')

        output_format = self.__options.output_format
        if output_format == 'dot':
            self.query_dependency_dot(output_file)
        elif output_format == 'tree':
            self.query_dependency_tree(output_file)
        else:
            self.query_dependency_plain(output_file)
        if output_file_name:
            output_file.close()
        return 0

    def query_dependency_plain(self, output_file):
        result_map = self.query_helper()
        if self.__options.deps:
            for key in result_map:
                print(file=output_file)
                deps = result_map[key][0]
                print('//%s:%s depends on the following targets:' %
                      (key[0], key[1]),
                      file=output_file)
                for d in deps:
                    print('%s:%s' % (d[0], d[1]), file=output_file)
        if self.__options.dependents:
            for key in result_map:
                print(file=output_file)
                depended_by = result_map[key][1]
                print('//%s:%s is depended by the following targets:' %
                      (key[0], key[1]),
                      file=output_file)
                for d in depended_by:
                    print('%s:%s' % (d[0], d[1]), file=output_file)

    def print_dot_node(self, output_file, node):
        print('"%s:%s" [label = "%s:%s"]' %
              (node[0], node[1], node[0], node[1]),
              file=output_file)

    def print_dot_deps(self, output_file, node, target_set):
        targets = self.__build_targets
        deps = targets[node].deps
        for i in deps:
            if not i in target_set:
                continue
            print('"%s:%s" -> "%s:%s"' % (node[0], node[1], i[0], i[1]),
                  file=output_file)

    def __print_dot_graph(self, result_map, name, print_mode, output_file):
        # print_mode = 0: deps, 1: dependents
        targets = result_map.keys()
        nodes = set(targets)
        for key in targets:
            nodes |= set(result_map[key][print_mode])
        print('digraph %s {' % name, file=output_file)
        for i in nodes:
            self.print_dot_node(output_file, i)
        for i in nodes:
            self.print_dot_deps(output_file, i, nodes)
        print('}', file=output_file)
        pass

    def query_dependency_dot(self, output_file):
        result_map = self.query_helper()
        if self.__options.deps:
            self.__print_dot_graph(result_map, 'deps', 0, output_file)
        if self.__options.dependents:
            self.__print_dot_graph(result_map, 'dependents', 1, output_file)

    def query_helper(self):
        """Query the targets helper method. """
        all_targets = self.__build_targets
        query_list = self.__all_command_targets

        result_map = {}
        for key in query_list:
            deps = all_targets[key].expanded_deps
            # depended_by = [k for k in all_targets if key in all_targets[k].expanded_deps]
            depended_by = self.__depended_targets[key]
            result_map[key] = (sorted(deps), sorted(depended_by))
        return result_map

    def query_dependency_tree(self, output_file):
        """Query the dependency tree of the specified targets. """
        if self.__options.dependents:
            console.error_exit(
                'only query --deps can be output as tree format')
        print(file=output_file)
        for key in self.__all_command_targets:
            self._query_dependency_tree(key, 0, self.__build_targets,
                                        output_file)
            print(file=output_file)

    def _query_dependency_tree(self, key, level, build_targets, output_file):
        """Query the dependency tree of the specified target recursively. """
        path, name = key
        if level == 0:
            output = '%s:%s' % (path, name)
        elif level == 1:
            output = '%s %s:%s' % ('+-', path, name)
        else:
            output = '%s%s %s:%s' % ('|  ' * (level - 1), '+-', path, name)
        print(output, file=output_file)
        for dkey in build_targets[key].deps:
            self._query_dependency_tree(dkey, level + 1, build_targets,
                                        output_file)

    def get_build_time(self):
        return self.__build_time

    def get_build_path(self):
        """The current building path. """
        return self.__build_path

    def get_root_dir(self):
        """Return the blade root path. """
        return self.__root_dir

    def get_command(self):
        """Get the blade command. """
        return self.__command

    def set_current_source_path(self, current_source_path):
        """Set the current source path. """
        self.__current_source_path = current_source_path

    def get_current_source_path(self):
        """Get the current source path. """
        return self.__current_source_path

    def get_target_database(self):
        """Get the whole target database that haven't been expanded. """
        return self.__target_database

    def get_direct_targets(self):
        """Return the direct targets. """
        return self.__direct_targets

    def get_build_targets(self):
        """Get all the targets to be build. """
        return self.__build_targets

    def get_depended_target_database(self):
        """Get depended target database that query dependent targets directly. """
        return self.__depended_targets

    def get_options(self):
        """Get the global command options. """
        return self.__options

    def is_expanded(self):
        """Whether the targets are expanded. """
        return self.__targets_expanded

    def register_target(self, target):
        """Register a target into blade target database.
        It is used to do quick looking.
        """
        key = target.key
        # Check whether there is already a key in database
        if key in self.__target_database:
            console.error_exit('Target %s is duplicate in //%s/BUILD' %
                               (target.name, target.path))
        self.__target_database[key] = target

    def _is_scons_object_type(self, target_type):
        """The types that shouldn't be registered into blade manager.

        Sholdn't invoke scons_rule method when it is not a scons target which
        could not be registered into blade manager, like system library.

        1. system_library

        """
        return target_type != 'system_library'

    def gen_targets_rules(self):
        """Get the build rules and return to the object who queries this. """
        rules_buf = []
        skip_test = getattr(self.__options, 'no_test', False)
        skip_package = not getattr(self.__options, 'generate_package', False)
        native_builder = config.get_item('global_config', 'native_builder')
        for k in self.__sorted_targets_keys:
            target = self.__build_targets[k]
            if not self._is_scons_object_type(target.type):
                continue
            blade_object = self.__target_database.get(k, None)
            if not blade_object:
                console.warning('not registered blade object, key %s' % str(k))
                continue
            if (skip_test and target.type.endswith('_test')
                    and k not in self.__direct_targets):
                continue
            if (skip_package and target.type == 'package'
                    and k not in self.__direct_targets):
                continue

            if native_builder == 'ninja':
                blade_object.ninja_rules()
            else:
                blade_object.scons_rules()
            rules = blade_object.get_rules()
            if rules:
                rules_buf.append('\n')
                rules_buf += rules
        return rules_buf

    def get_scons_platform(self):
        """Return handle of the platform class. """
        return self.__build_platform

    def get_sources_keyword_list(self):
        """This keywords list is used to check the source files path.

        Ex, when users specifies warning=no, it could be used to check that
        the source files is under thirdparty or not. If not, it will warn
        users that this flag is used incorrectly.

        """
        keywords = ['thirdparty']
        return keywords

    def _load_verify_history(self):
        if os.path.exists(self._verify_history_path):
            with open(self._verify_history_path) as f:
                self._verify_history = json.load(f)
        return self._verify_history

    def _dump_verify_history(self):
        with open(self._verify_history_path, 'w') as f:
            json.dump(self._verify_history, f)

    def parallel_jobs_num(self):
        """Tune the jobs num. """
        # User has the highest priority
        user_jobs_num = self.__options.jobs
        if user_jobs_num > 0:
            return user_jobs_num

        # Calculate job numbers smartly
        jobs_num = 0
        distcc_enabled = config.get_item('distcc_config', 'enabled')

        if distcc_enabled and self.build_environment.distcc_env_prepared:
            # Distcc cost doesn;t much local cpu, jobs can be quite large.
            distcc_num = len(self.build_environment.get_distcc_hosts_list())
            jobs_num = min(max(int(1.5 * distcc_num), 1), 20)
        else:
            cpu_core_num = cpu_count()
            # machines with cpu_core_num > 4 is usually shared by multiple users,
            # set an upper bound to avoid interfering other users
            jobs_num = min(2 * cpu_core_num, 8)

        if jobs_num != user_jobs_num:
            console.info('tunes the parallel jobs number(-j N) to be %d' %
                         (jobs_num))
        return jobs_num

    def get_all_rule_names(self):
        return self.__all_rule_names
Esempio n. 7
0
class Blade(object):
    """Blade. A blade manager class. """
    def __init__(self, command_targets, blade_path, working_dir, build_path,
                 blade_root_dir, blade_options, command):
        """init method.

        """
        self.__command_targets = command_targets
        self.__blade_path = blade_path
        self.__working_dir = working_dir
        self.__build_path = build_path
        self.__root_dir = blade_root_dir
        self.__options = blade_options
        self.__command = command

        # Source dir of current loading BUILD file
        self.__current_source_path = blade_root_dir

        # The direct targets that are used for analyzing
        self.__direct_targets = []

        # All command targets, make sure that all targets specified with ...
        # are all in the list now
        self.__all_command_targets = []

        # Given some targets specified in the command line, Blade will load
        # BUILD files containing these command line targets; global target
        # functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
        # files will register targets into target_database, which then becomes
        # the input to dependency analyzer and SCons rules generator.  It is
        # notable that not all targets in target_database are dependencies of
        # command line targets.
        self.__target_database = {}

        # targets to build after loading the build files.
        self.__build_targets = {}

        # The targets keys list after sorting by topological sorting method.
        # Used to generate build rules in correct order.
        self.__sorted_targets_keys = []

        # Inidcating that whether the deps list is expanded by expander or not
        self.__targets_expanded = False

        self.__scons_platform = SconsPlatform()
        self.build_environment = BuildEnvironment(self.__root_dir)

        self.svn_root_dirs = []

    def _get_normpath_target(self, command_target):
        """returns a tuple (path, name).

        path is a full path from BLADE_ROOT

        """
        target_path = relative_path(self.__working_dir, self.__root_dir)
        path, name = command_target.split(':')
        if target_path != '.':
            if path:
                path = target_path + '/' + path
            else:
                path = target_path
        path = os.path.normpath(path)
        return path, name

    def load_targets(self):
        """Load the targets. """
        console.info('loading BUILDs...')
        if self.__command == 'query':
            working_dir = self.__root_dir

            if '...' not in self.__command_targets:
                new_target_list = []
                for target in self.__command_targets:
                    new_target_list.append('%s:%s' %
                                           self._get_normpath_target(target))
                self.__command_targets = new_target_list
        else:
            working_dir = self.__working_dir
        (self.__direct_targets, self.__all_command_targets,
         self.__build_targets) = load_targets(self.__command_targets,
                                              working_dir, self.__root_dir,
                                              self)
        console.info('loading done.')
        return self.__direct_targets, self.__all_command_targets  # For test

    def analyze_targets(self):
        """Expand the targets. """
        console.info('analyzing dependency graph...')
        self.__sorted_targets_keys = analyze_deps(self.__build_targets)
        self.__targets_expanded = True

        console.info('analyzing done.')
        return self.__build_targets  # For test

    def generate_build_rules(self):
        """Generate the constructing rules. """
        console.info('generating build rules...')
        build_rules_generator = SconsRulesGenerator('SConstruct',
                                                    self.__blade_path, self)
        rules_buf = build_rules_generator.generate_scons_script()
        console.info('generating done.')
        return rules_buf

    def generate(self):
        """Generate the build script. """
        self.load_targets()
        self.analyze_targets()
        self.generate_build_rules()

    def run(self, target):
        """Run the target. """
        key = self._get_normpath_target(target)
        runner = BinaryRunner(self.__build_targets, self.__options,
                              self.__target_database)
        return runner.run_target(key)

    def test(self):
        """Run tests. """
        test_runner = TestRunner(self.__build_targets, self.__options,
                                 self.__target_database, self.__direct_targets)
        return test_runner.run()

    def query(self, targets):
        """Query the targets. """
        print_deps = getattr(self.__options, 'deps', False)
        print_depended = getattr(self.__options, 'depended', False)
        dot_file = getattr(self.__options, 'output_to_dot', '')
        result_map = self.query_helper(targets)
        if dot_file:
            print_mode = 0
            if print_deps:
                print_mode = 0
            if print_depended:
                print_mode = 1
            dot_file = os.path.join(self.__working_dir, dot_file)
            self.output_dot(result_map, print_mode, dot_file)
        else:
            if print_deps:
                for key in result_map:
                    print '\n'
                    deps = result_map[key][0]
                    console.info('//%s:%s depends on the following targets:' %
                                 (key[0], key[1]))
                    for d in deps:
                        print '%s:%s' % (d[0], d[1])
            if print_depended:
                for key in result_map:
                    print '\n'
                    depended_by = result_map[key][1]
                    console.info(
                        '//%s:%s is depended by the following targets:' %
                        (key[0], key[1]))
                    depended_by.sort(key=lambda x: x, reverse=False)
                    for d in depended_by:
                        print '%s:%s' % (d[0], d[1])
        return 0

    def print_dot_node(self, output_file, node):
        print >> output_file, '"%s:%s" [label = "%s:%s"]' % (node[0], node[1],
                                                             node[0], node[1])

    def print_dot_deps(self, output_file, node, target_set):
        targets = self.__build_targets
        deps = targets[node].deps
        for i in deps:
            if not i in target_set:
                continue
            print >> output_file, '"%s:%s" -> "%s:%s"' % (node[0], node[1],
                                                          i[0], i[1])

    def output_dot(self, result_map, print_mode, dot_file):
        f = open(dot_file, 'w')
        targets = result_map.keys()
        nodes = set(targets)
        for key in targets:
            nodes |= set(result_map[key][print_mode])
        print >> f, 'digraph blade {'
        for i in nodes:
            self.print_dot_node(f, i)
        for i in nodes:
            self.print_dot_deps(f, i, nodes)
        print >> f, '}'
        f.close()

    def query_helper(self, targets):
        """Query the targets helper method. """
        all_targets = self.__build_targets
        query_list = []
        target_path = relative_path(self.__working_dir, self.__root_dir)
        t_path = ''
        for t in targets:
            key = t.split(':')
            if target_path == '.':
                t_path = key[0]
            else:
                t_path = target_path + '/' + key[0]
            t_path = os.path.normpath(t_path)
            query_list.append((t_path, key[1]))
        result_map = {}
        for key in query_list:
            result_map[key] = ([], [])
            deps = all_targets[key].expanded_deps
            deps.sort(key=lambda x: x, reverse=False)
            depended_by = []
            for tkey in all_targets:
                if key in all_targets[tkey].expanded_deps:
                    depended_by.append(tkey)
            depended_by.sort(key=lambda x: x, reverse=False)
            result_map[key] = (list(deps), list(depended_by))
        return result_map

    def get_build_path(self):
        """The current building path. """
        return self.__build_path

    def get_root_dir(self):
        """Return the blade root path. """
        return self.__root_dir

    def set_current_source_path(self, current_source_path):
        """Set the current source path. """
        self.__current_source_path = current_source_path

    def get_current_source_path(self):
        """Get the current source path. """
        return self.__current_source_path

    def get_target_database(self):
        """Get the whole target database that haven't been expanded. """
        return self.__target_database

    def get_direct_targets(self):
        """Return the direct targets. """
        return self.__direct_targets

    def get_build_targets(self):
        """Get all the targets to be build. """
        return self.__build_targets

    def get_options(self):
        """Get the global command options. """
        return self.__options

    def is_expanded(self):
        """Whether the targets are expanded. """
        return self.__targets_expanded

    def register_target(self, target):
        """Register scons targets into the scons targets map.

        It is used to do quick looking.

        """
        target_key = target.key
        # check that whether there is already a key in database
        if target_key in self.__target_database:
            print self.__target_database
            console.error_exit('target name %s is duplicate in //%s/BUILD' %
                               (target.name, target.path))
        self.__target_database[target_key] = target

    def _is_scons_object_type(self, target_type):
        """The types that shouldn't be registered into blade manager.

        Sholdn't invoke scons_rule method when it is not a scons target which
        could not be registered into blade manager, like system library.

        1. system_library

        """
        return target_type != 'system_library'

    def gen_targets_rules(self):
        """Get the build rules and return to the object who queries this. """
        rules_buf = []
        skip_test_targets = False
        if getattr(self.__options, 'no_test', False):
            skip_test_targets = True
        for k in self.__sorted_targets_keys:
            target = self.__build_targets[k]
            if not self._is_scons_object_type(target.type):
                continue
            scons_object = self.__target_database.get(k, None)
            if not scons_object:
                console.warning('not registered scons object, key %s' % str(k))
                continue
            if skip_test_targets and target.type == 'cc_test':
                continue
            scons_object.scons_rules()
            rules_buf.append('\n')
            rules_buf += scons_object.get_rules()
        return rules_buf

    def get_scons_platform(self):
        """Return handle of the platform class. """
        return self.__scons_platform

    def get_sources_keyword_list(self):
        """This keywords list is used to check the source files path.

        Ex, when users specifies warning=no, it could be used to check that
        the source files is under thirdparty or not. If not, it will warn
        users that this flag is used incorrectly.

        """
        keywords = ['thirdparty']
        return keywords

    def tune_parallel_jobs_num(self):
        """Tune the jobs num. """
        user_jobs_num = self.__options.jobs
        jobs_num = 0
        cpu_core_num = cpu_count()
        distcc_enabled = configparse.blade_config.get_config(
            'distcc_config')['enabled']

        if distcc_enabled and self.build_environment.distcc_env_prepared:
            jobs_num = int(
                1.5 * len(self.build_environment.get_distcc_hosts_list())) + 1
            if jobs_num > 20:
                jobs_num = 20
            if jobs_num and self.__options.jobs != jobs_num:
                self.__options.jobs = jobs_num
        elif self.__options.jobs < 1:
            if cpu_core_num <= 4:
                self.__options.jobs = 2 * cpu_core_num
            else:
                self.__options.jobs = cpu_core_num
                if self.__options.jobs > 8:
                    self.__options.jobs = 8
        if self.__options.jobs != user_jobs_num:
            console.info('tunes the parallel jobs number(-j N) to be %d' %
                         (self.__options.jobs))
        return self.__options.jobs
Esempio n. 8
0
    def __init__(self, command_targets, blade_path, working_dir, build_path,
                 current_source_path, blade_options, **kwargs):
        """init method.

        Mainly to hold the global data.
        The directory which changes during the runtime of blade, and
        contains BUILD file under current focus.
        current_source_dir = "."

        Given some targets specified in the command line, Blade will load
        BUILD files containing these command line targets; global target
        functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
        files will register targets into target_database, which then becomes
        the input to dependency analyzer and SCons rules generator.  It is
        notable that not all targets in target_database are dependencies of
        command line targets.
        target_database = {}

        related targets after loading the build files and exec BUILD files
        as python script
        related_targets = {}

        The map used by build rules to ensure that a source file occurres in
        exactly one rule/target(only library target).
        target_srcs_map = {}

        The scons cache manager class string, which should be output to
        scons script if ccache is not installed
        scache_manager_class_str = ''

        The targets keys list after sorting by topological sorting method.
        sorted_targets_keys = []

        Inidcating that whether the deps list is expanded by expander or not
        False - not expanded
        True - expanded
        target_deps_expanded

        All targets after expanding their dependency
        all_targets = {}

        The scons target objects registered into blade manager
        scons_targets_map = {}

        The vars which are depended by python binary
        {key : 'python_files'}
        self.python_binary_dep_source_cmd = {}

        The files which are depended by python binary
        {key : 'python_files'}
        python_binary_dep_source_map = {}

        The files which are depended by java jar file
        {key : 'java_files'}
        java_jar_dep_source_map = {}

        The files which should be packed into java jar
        {key : 'packing_files'}
        java_jar_files_packing_map = {}

        The jar files map
        {key : 'jar_files_generated'}
        java_jars_map = {}

        The java compiling classpath parameter map
        java_classpath_map = {}
        {key : 'target_path'}

        The java_jar dep var map, which should be added to dependency chain
        java_jar_dep_vars = {}

        The cc objects pool, a map to hold all the objects name.
        cc_objects_pool = {}

        The gen rule files map, which is used to generate the explict dependency
        relationtion ship between gen_rule target and other targets
        gen_rule_files_map = {}

        The direct targets that are used for analyzing
        direct_targets = []

        All command targets, make sure that all targets specified with ...
        are all in the list now
        all_command_targets = []

        The class to get platform info
        SconsPlatform

        The class to manage the cc flags
        CcFlagsManager

        The sources files that are needed to perform explict dependency
        sources_explict_dependency_map = {}

        The prebuilt cc_library file map which is needed to establish
        symbolic links while testing
        prebuilt_cc_library_file_map = {}

        """
        self.command_targets = command_targets
        self.direct_targets = []
        self.all_command_targets = []
        self.blade_path = blade_path
        self.working_dir = working_dir
        self.build_path = build_path
        self.current_source_path = current_source_path
        self.target_database = {}
        self.related_targets = {}
        self.target_srcs_map = {}
        self.scache_manager_class_str = ''
        self.options = blade_options
        self.sorted_targets_keys = []
        self.target_deps_expanded = False
        self.all_targets_expanded = {}
        self.scons_targets_map = {}
        self.java_jar_dep_source_map = {}
        self.java_jar_files_packing_map = {}
        self.java_jars_map = {}
        self.java_classpath_map = {}
        self.java_jar_dep_vars = {}
        self.python_binary_dep_source_cmd = {}
        self.python_binary_dep_source_map = {}
        self.cc_objects_pool = {}

        self.deps_expander = None
        self.build_rules_generator = None

        self.gen_rule_files_map = {}

        self.scons_platform = SconsPlatform()
        self.ccflags_manager = CcFlagsManager(self.options)
        self.sources_explict_dependency_map = {}
        self.prebuilt_cc_library_file_map = {}

        self.distcc_enabled = configparse.blade_config.get_config(
            'distcc_config')['enabled']

        self.build_environment = BuildEnvironment(self.current_source_path)

        self.svn_root_dirs = []

        self.kwargs = kwargs
Esempio n. 9
0
class Blade(object):
    """Blade. A blade manager class. """
    def __init__(self, command_targets, blade_path, working_dir, build_path,
                 current_source_path, blade_options, **kwargs):
        """init method.

        Mainly to hold the global data.
        The directory which changes during the runtime of blade, and
        contains BUILD file under current focus.
        current_source_dir = "."

        Given some targets specified in the command line, Blade will load
        BUILD files containing these command line targets; global target
        functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
        files will register targets into target_database, which then becomes
        the input to dependency analyzer and SCons rules generator.  It is
        notable that not all targets in target_database are dependencies of
        command line targets.
        target_database = {}

        related targets after loading the build files and exec BUILD files
        as python script
        related_targets = {}

        The map used by build rules to ensure that a source file occurres in
        exactly one rule/target(only library target).
        target_srcs_map = {}

        The scons cache manager class string, which should be output to
        scons script if ccache is not installed
        scache_manager_class_str = ''

        The targets keys list after sorting by topological sorting method.
        sorted_targets_keys = []

        Inidcating that whether the deps list is expanded by expander or not
        False - not expanded
        True - expanded
        target_deps_expanded

        All targets after expanding their dependency
        all_targets = {}

        The scons target objects registered into blade manager
        scons_targets_map = {}

        The vars which are depended by python binary
        {key : 'python_files'}
        self.python_binary_dep_source_cmd = {}

        The files which are depended by python binary
        {key : 'python_files'}
        python_binary_dep_source_map = {}

        The files which are depended by java jar file
        {key : 'java_files'}
        java_jar_dep_source_map = {}

        The files which should be packed into java jar
        {key : 'packing_files'}
        java_jar_files_packing_map = {}

        The jar files map
        {key : 'jar_files_generated'}
        java_jars_map = {}

        The java compiling classpath parameter map
        java_classpath_map = {}
        {key : 'target_path'}

        The java_jar dep var map, which should be added to dependency chain
        java_jar_dep_vars = {}

        The cc objects pool, a map to hold all the objects name.
        cc_objects_pool = {}

        The gen rule files map, which is used to generate the explict dependency
        relationtion ship between gen_rule target and other targets
        gen_rule_files_map = {}

        The direct targets that are used for analyzing
        direct_targets = []

        All command targets, make sure that all targets specified with ...
        are all in the list now
        all_command_targets = []

        The class to get platform info
        SconsPlatform

        The class to manage the cc flags
        CcFlagsManager

        The sources files that are needed to perform explict dependency
        sources_explict_dependency_map = {}

        The prebuilt cc_library file map which is needed to establish
        symbolic links while testing
        prebuilt_cc_library_file_map = {}

        """
        self.command_targets = command_targets
        self.direct_targets = []
        self.all_command_targets = []
        self.blade_path = blade_path
        self.working_dir = working_dir
        self.build_path = build_path
        self.current_source_path = current_source_path
        self.target_database = {}
        self.related_targets = {}
        self.target_srcs_map = {}
        self.scache_manager_class_str = ''
        self.options = blade_options
        self.sorted_targets_keys = []
        self.target_deps_expanded = False
        self.all_targets_expanded = {}
        self.scons_targets_map = {}
        self.java_jar_dep_source_map = {}
        self.java_jar_files_packing_map = {}
        self.java_jars_map = {}
        self.java_classpath_map = {}
        self.java_jar_dep_vars = {}
        self.python_binary_dep_source_cmd = {}
        self.python_binary_dep_source_map = {}
        self.cc_objects_pool = {}

        self.deps_expander = None
        self.build_rules_generator = None

        self.gen_rule_files_map = {}

        self.scons_platform = SconsPlatform()
        self.ccflags_manager = CcFlagsManager(self.options)
        self.sources_explict_dependency_map = {}
        self.prebuilt_cc_library_file_map = {}

        self.distcc_enabled = configparse.blade_config.get_config(
            'distcc_config')['enabled']

        self.build_environment = BuildEnvironment(self.current_source_path)

        self.svn_root_dirs = []

        self.kwargs = kwargs

    def _get_normpath_target(self, command_target):
        """returns a tuple (path, name).

        path is a full path from BLADE_ROOT

        """
        target_path = relative_path(self.working_dir, self.current_source_path)
        path, name = command_target.split(':')
        if target_path != '.':
            if path:
                path = target_path + '/' + path
            else:
                path = target_path
        path = os.path.normpath(path)
        return path, name

    def load_targets(self):
        """Load the targets. """
        console.info("loading BUILDs...")
        if self.kwargs.get('blade_command', '') == 'query':
            working_dir = self.current_source_path

            if '...' not in self.command_targets:
                new_target_list = []
                for target in self.command_targets:
                    new_target_list.append("%s:%s" %
                                           self._get_normpath_target(target))
                self.command_targets = new_target_list
        else:
            working_dir = self.working_dir
        (self.direct_targets, self.all_command_targets) = load_targets(
            self.command_targets, working_dir, self.current_source_path, self)
        console.info("loading done.")
        return self.direct_targets, self.all_command_targets

    def analyze_targets(self):
        """Expand the targets. """
        console.info("analyzing dependency graph...")
        self.deps_analyzer = DependenciesAnalyzer(self)
        self.deps_analyzer.analyze_deps()
        console.info("analyzing done.")
        return self.all_targets_expanded

    def generate_build_rules(self):
        """Generate the constructing rules. """
        console.info("generating build rules...")
        self.build_rules_generator = SconsRulesGenerator(
            'SConstruct', self.blade_path, self)
        rules_buf = self.build_rules_generator.generate_scons_script()
        console.info("generating done.")
        return rules_buf

    def generate(self):
        """Build the targets. """
        self.load_targets()
        self.analyze_targets()
        self.generate_build_rules()

    def run(self, target):
        """Run the target. """
        key = self._get_normpath_target(target)
        runner = BinaryRunner(self.all_targets_expanded, self.options,
                              self.prebuilt_cc_library_file_map,
                              self.target_database)
        return runner.run_target(key)

    def test(self):
        """Run tests. """
        test_runner = TestRunner(self.all_targets_expanded, self.options,
                                 self.prebuilt_cc_library_file_map,
                                 self.target_database)
        return test_runner.run()

    def query(self, targets):
        """Query the targets. """
        print_deps = hasattr(self.options, 'deps') and (self.options.deps)
        print_depended = hasattr(self.options,
                                 'depended') and (self.options.depended)
        dot_file = ""
        if hasattr(self.options, 'output-to-dot'):
            dot_file = self.options.output_to_dot
        result_map = self.query_helper(targets)
        if dot_file:
            print_mode = 0
            if print_deps:
                print_mode = 0
            if print_depended:
                print_mode = 1
            if not dot_file.startswith("/"):
                dot_file = self.working_dir + "/" + dot_file
            self.output_dot(result_map, print_mode, dot_file)
        else:
            if print_deps:
                for key in result_map.keys():
                    print "\n"
                    deps = result_map[key][0]
                    console.info("//%s:%s depends on the following targets:" %
                                 (key[0], key[1]))
                    for d in deps:
                        print "%s:%s" % (d[0], d[1])
            if print_depended:
                for key in result_map.keys():
                    print "\n"
                    depended_by = result_map[key][1]
                    console.info(
                        "//%s:%s is depended by the following targets:" %
                        (key[0], key[1]))
                    depended_by.sort(key=lambda x: x, reverse=False)
                    for d in depended_by:
                        print "%s:%s" % (d[0], d[1])
        return 0

    def print_dot_node(self, output_file, node):
        print >> output_file, '"%s:%s" [label = "%s:%s"]' % (node[0], node[1],
                                                             node[0], node[1])

    def print_dot_deps(self, output_file, node, target_set):
        targets = self.related_targets
        deps = targets.get(node, {}).get('direct_deps', [])
        for i in deps:
            if not i in target_set:
                continue
            print >> output_file, '"%s:%s" -> "%s:%s"' % (node[0], node[1],
                                                          i[0], i[1])

    def output_dot(self, result_map, print_mode, dot_file):
        f = open(dot_file, 'w')
        targets = result_map.keys()
        nodes = set(targets)
        for key in targets:
            nodes |= set(result_map[key][print_mode])
        print >> f, "digraph blade {"
        for i in nodes:
            self.print_dot_node(f, i)
        for i in nodes:
            self.print_dot_deps(f, i, nodes)
        print >> f, "}"
        f.close()

    def query_helper(self, targets):
        """Query the targets helper method. """
        all_targets = self.all_targets_expanded
        query_list = []
        target_path = relative_path(self.working_dir, self.current_source_path)
        t_path = ''
        for t in targets:
            key = t.split(':')
            if target_path == '.':
                t_path = key[0]
            else:
                t_path = target_path + '/' + key[0]
            t_path = os.path.normpath(t_path)
            query_list.append((t_path, key[1]))
        result_map = {}
        for key in query_list:
            result_map[key] = ([], [])
            deps = all_targets.get(key, {}).get('deps', [])
            deps.sort(key=lambda x: x, reverse=False)
            depended_by = []
            for tkey in all_targets.keys():
                if key in all_targets[tkey]['deps']:
                    depended_by.append(tkey)
            depended_by.sort(key=lambda x: x, reverse=False)
            result_map[key] = (list(deps), list(depended_by))
        return result_map

    def get_blade_path(self):
        """Return the blade archive path. """
        return self.blade_path

    def get_build_path(self):
        """The current building path. """
        return self.build_path

    def set_current_source_path(self, current_source_path):
        """Set the current source path. """
        self.current_source_path = current_source_path

    def get_current_source_path(self):
        """Get the current source path. """
        return self.current_source_path

    def get_target_database(self):
        """Get the whole target database that haven't been expanded. """
        return self.target_database

    def set_related_targets(self, related_targets):
        """Set the related targets. """
        self.related_targets = dict(related_targets)

    def get_related_targets(self):
        """Get the related targets. """
        return self.related_targets

    def get_direct_targets(self):
        """Return the direct targets. """
        return self.direct_targets

    def get_all_command_targets(self):
        """Return all command targets. """
        return self.all_command_targets

    def set_sorted_targets_keys(self, sorted_keys_list):
        """Set the keys list from expaned targets. """
        self.sorted_targets_keys = list(sorted_keys_list)

    def get_sorted_targets_keys(self):
        """Get the keys list from expaned targets. """
        return self.sorted_targets_keys

    def set_all_targets_expanded(self, all_targets):
        """Set the targets that have been expanded by expander. """
        self.all_targets_expanded = dict(all_targets)
        self.target_deps_expanded = True

    def get_all_targets_expanded(self):
        """Get all the targets that expaned. """
        return self.all_targets_expanded

    def get_target_srcs_map(self):
        """Get the targets source files map.

        It is used in generating cc object rules.

        """
        return self.target_srcs_map

    def get_options(self):
        """Get the global command options. """
        return self.options

    def get_expanded(self):
        """Whether the targets are expanded. """
        return self.target_deps_expanded

    def register_scons_target(self, target_key, scons_target):
        """Register scons targets into the scons targets map.

        It is used to do quick looking.

        """
        # check that whether there is already a key in database
        if target_key in self.scons_targets_map.keys():
            console.error_exit("target name %s is duplicate in //%s/BUILD" %
                               (target_key[1], target_key[0]))
        self.scons_targets_map[target_key] = scons_target

    def get_scons_target(self, target_key):
        """Get scons target according to the key. """
        return self.scons_targets_map.get(target_key, None)

    def get_java_jar_dep_source_map(self):
        """The map mainly to hold the java files from swig or proto rules.

        These files maybe depended by java_jar target.

        """
        return self.java_jar_dep_source_map

    def get_java_jar_files_packing_map(self):
        """The map to hold the files that should be packed into java jar. """
        return self.java_jar_files_packing_map

    def get_java_jars_map(self):
        """The map to hold the java jar files generated by blade. """
        return self.java_jars_map

    def get_java_classpath_map(self):
        """The classpath list which is needed by java compling. """
        return self.java_classpath_map

    def get_java_jar_dep_vars(self):
        """The vars map which is prerequiste of the java jar target. """
        return self.java_jar_dep_vars

    def get_cc_objects_pool(self):
        """The cc objects pool which is used when generating the cc object rules. """
        return self.cc_objects_pool

    def _is_scons_object_type(self, target_type):
        """The types that shouldn't be registered into blade manager.

        Sholdn't invoke scons_rule method when it is not a scons target which
        could not be registered into blade manager, like system library.

        1. system_library

        """
        if target_type == 'system_library':
            return False
        else:
            return True

    def get_targets_rules(self):
        """Get the build rules and return to the object who queries this. """
        rules_buf = []
        skip_test_targets = False
        if hasattr(self.options, 'no_test') and self.options.no_test:
            skip_test_targets = True
        for k in self.sorted_targets_keys:
            target = self.all_targets_expanded[k]
            if not self._is_scons_object_type(target['type']):
                continue
            scons_object = self.scons_targets_map.get(k, None)
            if not scons_object:
                console.warning('not registered scons object, key %s' % str(k))
                continue
            if skip_test_targets and (target['type'] == 'cc_test'
                                      or target['type'] == 'dynamic_cc_test'):
                continue
            scons_object.scons_rules()
            rules_buf += scons_object.get_rules()
        return rules_buf

    def set_gen_rule_files_map(self, files_map):
        """Set the gen_rule files map. """
        self.gen_rule_files_map = dict(files_map)

    def get_gen_rule_files_map(self):
        """Get the gen_rule files map. """
        return self.gen_rule_files_map

    def get_scons_platform(self):
        """Return handle of the platform class. """
        return self.scons_platform

    def get_ccflags_manager(self):
        """Return handle of the ccflags manager class. """
        return self.ccflags_manager

    def get_sources_keyword_list(self):
        """This keywords list is used to check the source files path.

        Ex, when users specifies warning=no, it could be used to check that
        the source files is under thirdparty or not. If not, it will warn
        users that this flag is used incorrectly.

        """
        keywords = ['thirdparty']
        return keywords

    def get_sources_explict_dependency_map(self):
        """Returns the handle of sources_explict_dependency_map. """
        return self.sources_explict_dependency_map

    def get_prebuilt_cc_library_file_map(self):
        """Returns the prebuilt_cc_library_file_map. """
        return self.prebuilt_cc_library_file_map

    def tune_parallel_jobs_num(self):
        """Tune the jobs num. """
        user_jobs_num = self.options.jobs
        jobs_num = 0
        cpu_core_num = multiprocessing.cpu_count()
        if self.distcc_enabled and self.build_environment.distcc_env_prepared:
            jobs_num = int(
                1.5 * len(self.build_environment.get_distcc_hosts_list())) + 1
            if jobs_num > 20:
                jobs_num = 20
            if jobs_num and self.options.jobs != jobs_num:
                self.options.jobs = jobs_num
        elif self.options.jobs < 1:
            if cpu_core_num <= 4:
                self.options.jobs = 2 * cpu_core_num
            else:
                self.options.jobs = cpu_core_num
                if self.options.jobs > 8:
                    self.options.jobs = 8
        if self.options.jobs != user_jobs_num:
            console.info("tunes the parallel jobs number(-j N) to be %d" %
                         (self.options.jobs))
        return self.options.jobs
Esempio n. 10
0
class Blade(object):
    """Blade. A blade manager class. """
    def __init__(self,
                 command_targets,
                 blade_path,
                 working_dir,
                 build_path,
                 blade_root_dir,
                 blade_options,
                 command):
        """init method.

        """
        self.__command_targets = command_targets
        self.__blade_path = blade_path
        self.__working_dir = working_dir
        self.__build_path = build_path
        self.__root_dir = blade_root_dir
        self.__options = blade_options
        self.__command = command

        # Source dir of current loading BUILD file
        self.__current_source_path = blade_root_dir

        # The direct targets that are used for analyzing
        self.__direct_targets = []

        # All command targets, make sure that all targets specified with ...
        # are all in the list now
        self.__all_command_targets = []

        # Given some targets specified in the command line, Blade will load
        # BUILD files containing these command line targets; global target
        # functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
        # files will register targets into target_database, which then becomes
        # the input to dependency analyzer and SCons rules generator.  It is
        # notable that not all targets in target_database are dependencies of
        # command line targets.
        self.__target_database = {}

        # targets to build after loading the build files.
        self.__build_targets = {}

        # The targets keys list after sorting by topological sorting method.
        # Used to generate build rules in correct order.
        self.__sorted_targets_keys = []

        # Inidcating that whether the deps list is expanded by expander or not
        self.__targets_expanded = False

        self.__scons_platform = SconsPlatform()
        self.build_environment = BuildEnvironment(self.__root_dir)

        self.svn_root_dirs = []

    def _get_normpath_target(self, command_target):
        """returns a tuple (path, name).

        path is a full path from BLADE_ROOT

        """
        target_path = relative_path(self.__working_dir, self.__root_dir)
        path, name = command_target.split(':')
        if target_path != '.':
            if path:
                path = target_path + '/' + path
            else:
                path = target_path
        path = os.path.normpath(path)
        return path, name

    def load_targets(self):
        """Load the targets. """
        console.info('loading BUILDs...')
        if self.__command == 'query' and getattr(self.__options,
            'depended', False):
            # For query depended command, always start from root with target ...
            # that is scanning the whole tree
            working_dir = self.__root_dir
        else:
            working_dir = self.__working_dir
        (self.__direct_targets,
         self.__all_command_targets,
         self.__build_targets) = load_targets(self.__command_targets,
                                                  working_dir,
                                                  self.__root_dir,
                                                  self)
        console.info('loading done.')
        return self.__direct_targets, self.__all_command_targets  # For test

    def analyze_targets(self):
        """Expand the targets. """
        console.info('analyzing dependency graph...')
        self.__sorted_targets_keys = analyze_deps(self.__build_targets)
        self.__targets_expanded = True

        console.info('analyzing done.')
        return self.__build_targets  # For test

    def generate_build_rules(self):
        """Generate the constructing rules. """
        console.info('generating build rules...')
        build_rules_generator = SconsRulesGenerator('SConstruct',
                                                    self.__blade_path, self)
        rules_buf = build_rules_generator.generate_scons_script()
        console.info('generating done.')
        return rules_buf

    def generate(self):
        """Generate the build script. """
        self.load_targets()
        self.analyze_targets()
        self.generate_build_rules()

    def run(self, target):
        """Run the target. """
        key = self._get_normpath_target(target)
        runner = BinaryRunner(self.__build_targets,
                              self.__options,
                              self.__target_database)
        return runner.run_target(key)

    def test(self):
        """Run tests. """
        test_runner = TestRunner(self.__build_targets,
                                 self.__options,
                                 self.__target_database,
                                 self.__direct_targets)
        return test_runner.run()

    def query(self, targets):
        """Query the targets. """
        print_deps = getattr(self.__options, 'deps', False)
        print_depended = getattr(self.__options, 'depended', False)
        dot_file = getattr(self.__options, 'output_to_dot', '')
        result_map = self.query_helper(targets)
        if dot_file:
            print_mode = 0
            if print_deps:
                print_mode = 0
            if print_depended:
                print_mode = 1
            dot_file = os.path.join(self.__working_dir, dot_file)
            self.output_dot(result_map, print_mode, dot_file)
        else:
            if print_deps:
                for key in result_map:
                    print '\n'
                    deps = result_map[key][0]
                    console.info('//%s:%s depends on the following targets:' % (
                            key[0], key[1]))
                    for d in deps:
                        print '%s:%s' % (d[0], d[1])
            if print_depended:
                for key in result_map:
                    print '\n'
                    depended_by = result_map[key][1]
                    console.info('//%s:%s is depended by the following targets:' % (
                            key[0], key[1]))
                    depended_by.sort(key=lambda x: x, reverse=False)
                    for d in depended_by:
                        print '%s:%s' % (d[0], d[1])
        return 0

    def print_dot_node(self, output_file, node):
        print >>output_file, '"%s:%s" [label = "%s:%s"]' % (node[0],
                                                            node[1],
                                                            node[0],
                                                            node[1])

    def print_dot_deps(self, output_file, node, target_set):
        targets = self.__build_targets
        deps = targets[node].deps
        for i in deps:
            if not i in target_set:
                continue
            print >>output_file, '"%s:%s" -> "%s:%s"' % (node[0],
                                                         node[1],
                                                         i[0],
                                                         i[1])

    def output_dot(self, result_map, print_mode, dot_file):
        f = open(dot_file, 'w')
        targets = result_map.keys()
        nodes = set(targets)
        for key in targets:
            nodes |= set(result_map[key][print_mode])
        print >>f, 'digraph blade {'
        for i in nodes:
            self.print_dot_node(f, i)
        for i in nodes:
            self.print_dot_deps(f, i, nodes)
        print >>f, '}'
        f.close()

    def query_helper(self, targets):
        """Query the targets helper method. """
        all_targets = self.__build_targets
        query_list = []
        target_path = relative_path(self.__working_dir, self.__root_dir)
        t_path = ''
        for t in targets:
            if t.find(':') != -1:
                key = t.split(':')
                if target_path == '.':
                    t_path = key[0]
                else:
                    t_path = target_path + '/' + key[0]
                t_path = os.path.normpath(t_path)
                query_list.append((t_path, key[1]))
            elif t.endswith('...'):
                t_path = os.path.normpath(target_path + '/' + t[:-3])
                for tkey in all_targets:
                    if tkey[0].startswith(t_path):
                        query_list.append((tkey[0], tkey[1]))
            else:
                t_path = os.path.normpath(target_path + '/' + t)
                for tkey in all_targets:
                    if tkey[0] == t_path:
                        query_list.append((t_path, tkey[1]))
        result_map = {}
        for key in query_list:
            result_map[key] = ([], [])
            deps = all_targets[key].expanded_deps
            deps.sort(key=lambda x: x, reverse=False)
            depended_by = []
            for tkey in all_targets:
                if key in all_targets[tkey].expanded_deps:
                    depended_by.append(tkey)
            depended_by.sort(key=lambda x: x, reverse=False)
            result_map[key] = (list(deps), list(depended_by))
        return result_map

    def get_build_path(self):
        """The current building path. """
        return self.__build_path

    def get_root_dir(self):
        """Return the blade root path. """
        return self.__root_dir

    def set_current_source_path(self, current_source_path):
        """Set the current source path. """
        self.__current_source_path = current_source_path

    def get_current_source_path(self):
        """Get the current source path. """
        return self.__current_source_path

    def get_target_database(self):
        """Get the whole target database that haven't been expanded. """
        return self.__target_database

    def get_direct_targets(self):
        """Return the direct targets. """
        return self.__direct_targets

    def get_build_targets(self):
        """Get all the targets to be build. """
        return self.__build_targets

    def get_options(self):
        """Get the global command options. """
        return self.__options

    def is_expanded(self):
        """Whether the targets are expanded. """
        return self.__targets_expanded

    def register_target(self, target):
        """Register scons targets into the scons targets map.

        It is used to do quick looking.

        """
        target_key = target.key
        # check that whether there is already a key in database
        if target_key in self.__target_database:
            print self.__target_database
            console.error_exit(
                    'target name %s is duplicate in //%s/BUILD' % (
                        target.name, target.path))
        self.__target_database[target_key] = target

    def _is_scons_object_type(self, target_type):
        """The types that shouldn't be registered into blade manager.

        Sholdn't invoke scons_rule method when it is not a scons target which
        could not be registered into blade manager, like system library.

        1. system_library

        """
        return target_type != 'system_library'

    def gen_targets_rules(self):
        """Get the build rules and return to the object who queries this. """
        rules_buf = []
        skip_test_targets = False
        if getattr(self.__options, 'no_test', False):
            skip_test_targets = True
        for k in self.__sorted_targets_keys:
            target = self.__build_targets[k]
            if not self._is_scons_object_type(target.type):
                continue
            scons_object = self.__target_database.get(k, None)
            if not scons_object:
                console.warning('not registered scons object, key %s' % str(k))
                continue
            if skip_test_targets and target.type == 'cc_test':
                continue
            scons_object.scons_rules()
            rules_buf.append('\n')
            rules_buf += scons_object.get_rules()
        return rules_buf

    def get_scons_platform(self):
        """Return handle of the platform class. """
        return self.__scons_platform

    def get_sources_keyword_list(self):
        """This keywords list is used to check the source files path.

        Ex, when users specifies warning=no, it could be used to check that
        the source files is under thirdparty or not. If not, it will warn
        users that this flag is used incorrectly.

        """
        keywords = ['thirdparty']
        return keywords

    def tune_parallel_jobs_num(self):
        """Tune the jobs num. """
        user_jobs_num = self.__options.jobs
        jobs_num = 0
        cpu_core_num = cpu_count()
        distcc_enabled = configparse.blade_config.get_config('distcc_config')['enabled']

        if distcc_enabled and self.build_environment.distcc_env_prepared:
            jobs_num = int(1.5 * len(self.build_environment.get_distcc_hosts_list())) + 1
            if jobs_num > 20:
                jobs_num = 20
            if jobs_num and self.__options.jobs != jobs_num:
                self.__options.jobs = jobs_num
        elif self.__options.jobs < 1:
            if cpu_core_num <= 4:
                self.__options.jobs = 2 * cpu_core_num
            else:
                self.__options.jobs = cpu_core_num
                if self.__options.jobs > 8:
                    self.__options.jobs = 8
        if self.__options.jobs != user_jobs_num:
            console.info('tunes the parallel jobs number(-j N) to be %d' % (
                self.__options.jobs))
        return self.__options.jobs
Esempio n. 11
0
class Blade(object):
    """Blade. A blade manager class. """
    def __init__(self,
                 command_targets,
                 blade_path,
                 working_dir,
                 build_path,
                 blade_root_dir,
                 blade_options,
                 command):
        """init method.

        """
        self.__command_targets = command_targets
        self.__blade_path = blade_path
        self.__working_dir = working_dir
        self.__build_path = build_path
        self.__root_dir = blade_root_dir
        self.__options = blade_options
        self.__command = command

        # Source dir of current loading BUILD file
        self.__current_source_path = blade_root_dir

        # The direct targets that are used for analyzing
        self.__direct_targets = []

        # All command targets, make sure that all targets specified with ...
        # are all in the list now
        self.__all_command_targets = []

        # Given some targets specified in the command line, Blade will load
        # BUILD files containing these command line targets; global target
        # functions, i.e., cc_libarary, cc_binary and etc, in these BUILD
        # files will register targets into target_database, which then becomes
        # the input to dependency analyzer and SCons rules generator.  It is
        # notable that not all targets in target_database are dependencies of
        # command line targets.
        self.__target_database = {}

        # targets to build after loading the build files.
        self.__build_targets = {}

        # The targets keys list after sorting by topological sorting method.
        # Used to generate build rules in correct order.
        self.__sorted_targets_keys = []

        # The depended targets dict after topological sorting
        self.__depended_targets = {}

        # Inidcating that whether the deps list is expanded by expander or not
        self.__targets_expanded = False

        self.__scons_platform = SconsPlatform()
        self.build_environment = BuildEnvironment(self.__root_dir)

        self.svn_root_dirs = []

    def _get_normpath_target(self, command_target):
        """returns a tuple (path, name).

        path is a full path from BLADE_ROOT

        """
        target_path = relative_path(self.__working_dir, self.__root_dir)
        path, name = command_target.split(':')
        if target_path != '.':
            if path:
                path = target_path + '/' + path
            else:
                path = target_path
        path = os.path.normpath(path)
        return path, name

    def load_targets(self):
        """Load the targets. """
        console.info('loading BUILDs...')
        if self.__command == 'query' and getattr(self.__options,
            'depended', False):
            # For query depended command, always start from root with target ...
            # that is scanning the whole tree
            working_dir = self.__root_dir
        else:
            working_dir = self.__working_dir
        (self.__direct_targets,
         self.__all_command_targets,
         self.__build_targets) = load_targets(self.__command_targets,
                                                  working_dir,
                                                  self.__root_dir,
                                                  self)
        console.info('loading done.')
        return self.__direct_targets, self.__all_command_targets  # For test

    def analyze_targets(self):
        """Expand the targets. """
        console.info('analyzing dependency graph...')
        (self.__sorted_targets_keys,
         self.__depended_targets) = analyze_deps(self.__build_targets)
        self.__targets_expanded = True

        console.info('analyzing done.')
        return self.__build_targets  # For test

    def generate_build_rules(self):
        """Generate the constructing rules. """
        console.info('generating build rules...')
        build_rules_generator = SconsRulesGenerator('SConstruct',
                                                    self.__blade_path, self)
        rules_buf = build_rules_generator.generate_scons_script()
        console.info('generating done.')
        return rules_buf

    def generate(self):
        """Generate the build script. """
        self.load_targets()
        self.analyze_targets()
        if self.__command != 'query':
            self.generate_build_rules()

    def run(self, target):
        """Run the target. """
        key = self._get_normpath_target(target)
        runner = BinaryRunner(self.__build_targets,
                              self.__options,
                              self.__target_database)
        return runner.run_target(key)

    def test(self):
        """Run tests. """
        test_runner = TestRunner(self.__build_targets,
                                 self.__options,
                                 self.__target_database,
                                 self.__direct_targets)
        return test_runner.run()

    def query(self, targets):
        """Query the targets. """
        print_deps = getattr(self.__options, 'deps', False)
        print_depended = getattr(self.__options, 'depended', False)
        dot_file = getattr(self.__options, 'output_to_dot', '')
        print_dep_tree = getattr(self.__options, 'output_tree', False)
        result_map = self.query_helper(targets)
        if dot_file:
            print_mode = 0
            if print_deps:
                print_mode = 0
            if print_depended:
                print_mode = 1
            dot_file = os.path.join(self.__working_dir, dot_file)
            self.output_dot(result_map, print_mode, dot_file)
        else:
            if print_deps:
                if print_dep_tree:
                    self.query_dependency_tree(targets)
                else:
                    for key in result_map:
                        print '\n'
                        deps = result_map[key][0]
                        console.info('//%s:%s depends on the following targets:' % (
                                key[0], key[1]))
                        for d in deps:
                            print '%s:%s' % (d[0], d[1])
            if print_depended:
                for key in result_map:
                    print '\n'
                    depended_by = result_map[key][1]
                    console.info('//%s:%s is depended by the following targets:' % (
                            key[0], key[1]))
                    for d in depended_by:
                        print '%s:%s' % (d[0], d[1])
        return 0

    def print_dot_node(self, output_file, node):
        print >>output_file, '"%s:%s" [label = "%s:%s"]' % (node[0],
                                                            node[1],
                                                            node[0],
                                                            node[1])

    def print_dot_deps(self, output_file, node, target_set):
        targets = self.__build_targets
        deps = targets[node].deps
        for i in deps:
            if not i in target_set:
                continue
            print >>output_file, '"%s:%s" -> "%s:%s"' % (node[0],
                                                         node[1],
                                                         i[0],
                                                         i[1])

    def output_dot(self, result_map, print_mode, dot_file):
        f = open(dot_file, 'w')
        targets = result_map.keys()
        nodes = set(targets)
        for key in targets:
            nodes |= set(result_map[key][print_mode])
        print >>f, 'digraph blade {'
        for i in nodes:
            self.print_dot_node(f, i)
        for i in nodes:
            self.print_dot_deps(f, i, nodes)
        print >>f, '}'
        f.close()

    def query_helper(self, targets):
        """Query the targets helper method. """
        all_targets = self.__build_targets
        query_list = []
        target_path = relative_path(self.__working_dir, self.__root_dir)
        t_path = ''
        for t in targets:
            if t.find(':') != -1:
                key = t.split(':')
                if target_path == '.':
                    t_path = key[0]
                else:
                    t_path = target_path + '/' + key[0]
                t_path = os.path.normpath(t_path)
                query_list.append((t_path, key[1]))
            elif t.endswith('...'):
                t_path = os.path.normpath(target_path + '/' + t[:-3])
                for tkey in all_targets:
                    if tkey[0].startswith(t_path):
                        query_list.append((tkey[0], tkey[1]))
            else:
                t_path = os.path.normpath(target_path + '/' + t)
                for tkey in all_targets:
                    if tkey[0] == t_path:
                        query_list.append((t_path, tkey[1]))
        result_map = {}
        for key in query_list:
            deps = all_targets[key].expanded_deps
            # depended_by = [k for k in all_targets if key in all_targets[k].expanded_deps]
            depended_by = self.__depended_targets[key]
            result_map[key] = (sorted(deps), sorted(depended_by))
        return result_map

    def query_dependency_tree(self, targets):
        """Query the dependency tree of the specified targets. """
        query_targets = []
        for target in targets:
            if ':' not in target:
                console.error_exit(
                    'Target %s is not supported by dependency tree query. '
                    'The target should be in the format directory:name.' % target)
            path, name = target.split(':')
            relpath = os.path.relpath(self.__working_dir, self.__root_dir)
            path = os.path.normpath(os.path.join(relpath, path))
            query_targets.append((path, name))

        for key in query_targets:
            console.info('')
            self._query_dependency_tree(key, 0, self.__build_targets)
            console.info('')

    def _query_dependency_tree(self, key, level, build_targets):
        """Query the dependency tree of the specified target recursively. """
        path, name = key
        if level == 0:
            output = '%s:%s' % (path, name)
        elif level == 1:
            output = '%s %s:%s' % ('+-', path, name)
        else:
            output = '%s%s %s:%s' % ('|  ' * (level - 1), '+-', path, name)
        console.info(console.colors('end') + console.colors('gray') + output)
        for dkey in build_targets[key].deps:
            self._query_dependency_tree(dkey, level + 1, build_targets)

    def get_build_path(self):
        """The current building path. """
        return self.__build_path

    def get_root_dir(self):
        """Return the blade root path. """
        return self.__root_dir

    def set_current_source_path(self, current_source_path):
        """Set the current source path. """
        self.__current_source_path = current_source_path

    def get_current_source_path(self):
        """Get the current source path. """
        return self.__current_source_path

    def get_target_database(self):
        """Get the whole target database that haven't been expanded. """
        return self.__target_database

    def get_direct_targets(self):
        """Return the direct targets. """
        return self.__direct_targets

    def get_build_targets(self):
        """Get all the targets to be build. """
        return self.__build_targets

    def get_options(self):
        """Get the global command options. """
        return self.__options

    def is_expanded(self):
        """Whether the targets are expanded. """
        return self.__targets_expanded

    def register_target(self, target):
        """Register scons targets into the scons targets map.

        It is used to do quick looking.

        """
        key = target.key
        # Check whether there is already a key in database
        if key in self.__target_database:
            console.error_exit('Target %s is duplicate in //%s/BUILD' % (
                               target.name, target.path))
        self.__target_database[key] = target

    def _is_scons_object_type(self, target_type):
        """The types that shouldn't be registered into blade manager.

        Sholdn't invoke scons_rule method when it is not a scons target which
        could not be registered into blade manager, like system library.

        1. system_library

        """
        return target_type != 'system_library'

    def gen_targets_rules(self):
        """Get the build rules and return to the object who queries this. """
        rules_buf = []
        skip_test = getattr(self.__options, 'no_test', False)
        skip_package = not getattr(self.__options, 'generate_package', False)
        for k in self.__sorted_targets_keys:
            target = self.__build_targets[k]
            if not self._is_scons_object_type(target.type):
                continue
            scons_object = self.__target_database.get(k, None)
            if not scons_object:
                console.warning('not registered scons object, key %s' % str(k))
                continue
            if (skip_test and target.type.endswith('_test')
                and k not in self.__direct_targets):
                continue
            if (skip_package and target.type == 'package'
                and k not in self.__direct_targets):
                continue
            scons_object.scons_rules()
            rules_buf.append('\n')
            rules_buf += scons_object.get_rules()
        return rules_buf

    def get_scons_platform(self):
        """Return handle of the platform class. """
        return self.__scons_platform

    def get_sources_keyword_list(self):
        """This keywords list is used to check the source files path.

        Ex, when users specifies warning=no, it could be used to check that
        the source files is under thirdparty or not. If not, it will warn
        users that this flag is used incorrectly.

        """
        keywords = ['thirdparty']
        return keywords

    def parallel_jobs_num(self):
        """Tune the jobs num. """
        # User has the highest priority
        user_jobs_num = self.__options.jobs
        if user_jobs_num > 0:
            return user_jobs_num

        # Calculate job numbers smartly
        jobs_num = 0
        distcc_enabled = configparse.blade_config.get_config('distcc_config')['enabled']

        if distcc_enabled and self.build_environment.distcc_env_prepared:
            # Distcc cost doesn;t much local cpu, jobs can be quite large.
            distcc_num = len(self.build_environment.get_distcc_hosts_list())
            jobs_num = min(max(int(1.5 * distcc_num), 1), 20)
        else:
            cpu_core_num = cpu_count()
            # machines with cpu_core_num > 4 is usually shared by multiple users,
            # set an upper bound to avoid interfering other users
            jobs_num = min(2 * cpu_core_num, 8)

        if jobs_num != user_jobs_num:
            console.info('tunes the parallel jobs number(-j N) to be %d' % (
                jobs_num))
        return jobs_num