Пример #1
0
    def generateTestDirList(self, builddir, dirname, source_files, component, immediate_dependencies, toplevel=False, module_is_empty=False):
        logger.debug('generate CMakeLists.txt for directory: %s' % os.path.join(component.path, dirname))

        link_dependencies = [x for x in immediate_dependencies]
        fname = os.path.join(builddir, dirname, 'CMakeLists.txt')

        # group the list of source files by subdirectory: generate one test for
        # each subdirectory, and one test for each file at the top level
        subdirs = defaultdict(list)
        toplevel_srcs = []
        for f in source_files:
            if f.lang in ('c', 'cpp', 'objc', 's'):
                subrelpath = os.path.relpath(f.relpath, dirname)
                subdir = fsutils.fullySplitPath(subrelpath)[0]
                if subdir and subdir != subrelpath:
                    subdirs[subdir].append(f)
                else:
                    toplevel_srcs.append(f)

        tests = []
        for f in toplevel_srcs:
            object_name = '%s-test-%s' % (
                component.getName(), os.path.basename(os.path.splitext(str(f))[0]).lower()
            )
            tests.append([[str(f)], object_name, [f.lang]])
        for subdirname, sources in sorted(subdirs.items(), key=lambda x: x[0]):
            object_name = '%s-test-%s' % (
                component.getName(), fsutils.fullySplitPath(subdirname)[0].lower()
            )
            tests.append([[str(f) for f in sources], object_name, [f.lang for f in sources]])

        # link tests against the main executable
        if not module_is_empty:
            link_dependencies.append(component.getName())

        # Find cmake files
        cmake_files = []
        for root, dires, files in os.walk(os.path.join(component.path, dirname)):
            for f in files:
                name, ext = os.path.splitext(f)
                if ext.lower() == '.cmake' and not component.ignores(os.path.relpath(os.path.join(root, f), component.path)):
                    cmake_files.append(os.path.join(root, f))

        test_template = jinja_environment.get_template('test_CMakeLists.txt')

        file_contents = test_template.render({ #pylint: disable=no-member
             'source_directory':os.path.join(component.path, dirname),
                        'tests':tests,
            'link_dependencies':link_dependencies,
                  'cmake_files': cmake_files,
             'exclude_from_all': (not toplevel),
            'test_dependencies': [x[1] for x in immediate_dependencies.items() if x[1].isTestDependency()]
        })

        self._writeFile(fname, file_contents)
Пример #2
0
    def generateTestDirList(self, builddir, dirname, source_files, component, immediate_dependencies, toplevel=False, module_is_empty=False):
        logger.debug('generate CMakeLists.txt for directory: %s' % os.path.join(component.path, dirname))

        link_dependencies = [x for x in immediate_dependencies]
        fname = os.path.join(builddir, dirname, 'CMakeLists.txt')

        # group the list of source files by subdirectory: generate one test for
        # each subdirectory, and one test for each file at the top level
        subdirs = defaultdict(list)
        toplevel_srcs = []
        for f in source_files:
            if f.lang in ('c', 'cpp', 'objc', 's'):
                subrelpath = os.path.relpath(f.relpath, dirname)
                subdir = fsutils.fullySplitPath(subrelpath)[0]
                if subdir and subdir != subrelpath:
                    subdirs[subdir].append(f)
                else:
                    toplevel_srcs.append(f)

        tests = []
        for f in toplevel_srcs:
            object_name = '%s-test-%s' % (
                component.getName(), os.path.basename(os.path.splitext(str(f))[0]).lower()
            )
            tests.append([[str(f)], object_name, [f.lang]])
        for subdirname, sources in sorted(subdirs.items(), key=lambda x: x[0]):
            object_name = '%s-test-%s' % (
                component.getName(), fsutils.fullySplitPath(subdirname)[0].lower()
            )
            tests.append([[str(f) for f in sources], object_name, [f.lang for f in sources]])

        # link tests against the main executable
        if not module_is_empty:
            link_dependencies.append(component.getName())

        # Find cmake files
        cmake_files = []
        for root, dires, files in os.walk(os.path.join(component.path, dirname)):
            for f in files:
                name, ext = os.path.splitext(f)
                if ext.lower() == '.cmake' and not component.ignores(os.path.relpath(os.path.join(root, f), component.path)):
                    cmake_files.append(os.path.join(root, f))

        test_template = jinja_environment.get_template('test_CMakeLists.txt')

        file_contents = test_template.render({ #pylint: disable=no-member
             'source_directory':os.path.join(component.path, dirname),
                        'tests':tests,
            'link_dependencies':link_dependencies,
                  'cmake_files': cmake_files,
             'exclude_from_all': (not toplevel),
            'test_dependencies': [x[1] for x in immediate_dependencies.items() if x[1].isTestDependency()]
        })

        self._writeFile(fname, file_contents)
Пример #3
0
def unpackFrom(tar_file_path, to_directory):
    # first unpack into a sibling directory of the specified directory, and
    # then move it into place.

    # we expect our tarballs to contain a single top-level directory. We strip
    # off this name as we extract to minimise the path length

    into_parent_dir = os.path.dirname(to_directory)
    fsutils.mkDirP(into_parent_dir)
    temp_directory = tempfile.mkdtemp(dir=into_parent_dir)
    try:
        with tarfile.open(tar_file_path) as tf:
            strip_dirname = ''
            # get the extraction directory name from the first part of the
            # extraction paths: it should be the same for all members of
            # the archive
            for m in tf.getmembers():
                split_path = fsutils.fullySplitPath(m.name)
                logger.debug('process member: %s %s', m.name, split_path)
                if os.path.isabs(m.name) or '..' in split_path:
                    raise ValueError('archive uses invalid paths')
                if not strip_dirname:
                    if len(split_path) != 1 or not len(split_path[0]):
                        raise ValueError(
                            'archive does not appear to contain a single module'
                        )
                    strip_dirname = split_path[0]
                    continue
                else:
                    if split_path[0] != strip_dirname:
                        raise ValueError(
                            'archive does not appear to contain a single module'
                        )
                m.name = os.path.join(*split_path[1:])
                tf.extract(m, path=temp_directory)
        # make sure the destination directory doesn't exist:
        fsutils.rmRf(to_directory)
        shutil.move(temp_directory, to_directory)
        temp_directory = None
        logger.debug('extraction complete %s', to_directory)
    except IOError as e:
        if e.errno != errno.ENOENT:
            logger.error('failed to extract tarfile %s', e)
            fsutils.rmF(tar_file_path)
        raise
    finally:
        if temp_directory is not None:
            # if anything has failed, cleanup
            fsutils.rmRf(temp_directory)
Пример #4
0
def unpackFrom(tar_file_path, to_directory):
    # first unpack into a sibling directory of the specified directory, and
    # then move it into place.

    # we expect our tarballs to contain a single top-level directory. We strip
    # off this name as we extract to minimise the path length

    into_parent_dir = os.path.dirname(to_directory)
    fsutils.mkDirP(into_parent_dir)
    temp_directory = tempfile.mkdtemp(dir=into_parent_dir)
    try:
        with tarfile.open(tar_file_path) as tf:
            strip_dirname = ''
            # get the extraction directory name from the first part of the
            # extraction paths: it should be the same for all members of
            # the archive
            for m in tf.getmembers():
                split_path = fsutils.fullySplitPath(m.name)
                logger.debug('process member: %s %s', m.name, split_path)
                if os.path.isabs(m.name) or '..' in split_path:
                    raise ValueError('archive uses invalid paths')
                if not strip_dirname:
                    if len(split_path) != 1 or not len(split_path[0]):
                        raise ValueError('archive does not appear to contain a single module')
                    strip_dirname = split_path[0]
                    continue
                else:
                    if split_path[0] != strip_dirname:
                        raise ValueError('archive does not appear to contain a single module')
                m.name = os.path.join(*split_path[1:])
                tf.extract(m, path=temp_directory)
        # make sure the destination directory doesn't exist:
        fsutils.rmRf(to_directory)
        shutil.move(temp_directory, to_directory)
        temp_directory = None
        logger.debug('extraction complete %s', to_directory)
    except IOError as e:
        if e.errno != errno.ENOENT:
            logger.error('failed to extract tarfile %s', e)
            fsutils.rmF(tar_file_path)
        raise
    finally:
        if temp_directory is not None:
            # if anything has failed, cleanup
            fsutils.rmRf(temp_directory)
Пример #5
0
def moduleFromDirname(build_subdir, all_modules, toplevel_module):
    modtop = True
    submod = False
    module = toplevel_module
    # <topdir> /ym/<submod>/ym/<submod2>/somedir/somedir --> submod2
    for part in fsutils.fullySplitPath(build_subdir):
        if submod:
            if part in all_modules:
                module = all_modules[part]
            modtop = True
            submod = False
        else:
            if part == 'ym' and modtop:
                submod = True
            else:
                submod = False
                modtop = False
    return module
Пример #6
0
def moduleFromDirname(build_subdir, all_modules, toplevel_module):
    modtop = True
    submod = False
    module = toplevel_module
    # <topdir> /ym/<submod>/ym/<submod2>/somedir/somedir --> submod2
    for part in fsutils.fullySplitPath(build_subdir):
        if submod:
            if part in all_modules:
                module = all_modules[part]
            modtop = True
            submod = False
        else:
            if part == 'ym' and modtop:
                submod = True
            else:
                submod = False
                modtop = False
    return module
Пример #7
0
    def _listSubDirectories(self, component, toplevel):
        ''' return: {
                manual: [list of subdirectories with manual CMakeLists],
                  auto: [list of pairs: (subdirectories name to autogenerate, a list of source files in that dir)],
                   bin: {dictionary of subdirectory name to binary name},
                   lib: {dictionary of subdirectory name to binary name},
                  test: [list of directories that build tests],
              resource: [list of directories that contain resources]
            }
        '''
        manual_subdirs = []
        auto_subdirs = []
        header_subdirs = []
        lib_subdirs = component.getLibs()
        bin_subdirs = component.getBinaries()
        test_subdirs = []
        resource_subdirs = []
        # if the application or library is set to get the sources from top level ("."),
        # they'll be acumulated into a single array (top_sources below).
        top_sources = []
        start_on_top = "." in [os.path.normpath(x) for x in list(lib_subdirs.keys()) + list(bin_subdirs.keys())]
        for f in sorted(os.listdir(component.path)):
            if f in Ignore_Subdirs or f.startswith('.') or f.startswith('_'):
                continue
            check_cmakefile_path = os.path.join(f, 'CMakeLists.txt')
            if os.path.isfile(os.path.join(component.path, check_cmakefile_path)) and not \
                    component.ignores(check_cmakefile_path):
                self.checkStandardSourceDir(f, component)
                # if the subdirectory has a CMakeLists.txt in it (and it isn't
                # ignored), then delegate to that:
                manual_subdirs.append(f)
                # tests only supported in the `test` directory for now
                if f in ('test',):
                    test_subdirs.append(f)
            else:
                if os.path.isfile(os.path.join(component.path, f)):
                    # top level source: check if it should be included
                    if not component.ignores(f) and start_on_top:
                        sf = self.createSourceFile(f, os.path.join(component.path, f), ".")
                        if sf is not None:
                            top_sources.append(sf)
                else:
                    # otherwise, if the directory has source files, and is listed
                    # as a source/test directory, generate a CMakeLists in the
                    # corresponding temporary directory, and add that.
                    sources = self.containsSourceFiles(os.path.join(component.path, f), component)
                    if sources:
                        if f in ('test',):
                            auto_subdirs.append((f, sources))
                            test_subdirs.append(f)
                        elif start_on_top:
                            # include the sources in this directory only if it's not
                            # a potential test directory
                            from yotta.lib import validate
                            if not validate.isPotentialTestDir(f):
                                top_sources.extend(sources)
                                if f == component.getName():
                                    header_subdirs.append((f, sources))
                        elif os.path.normpath(f) in [fsutils.fullySplitPath(x)[0] for x in lib_subdirs] or \
                             os.path.normpath(f) in [fsutils.fullySplitPath(x)[0] for x in bin_subdirs]:
                            for full_subpath in list(lib_subdirs.keys()) + list(bin_subdirs.keys()):
                                if fsutils.fullySplitPath(full_subpath)[0] == os.path.normpath(f):
                                    # this might be a sub-sub directory, in which
                                    # case we need to re-calculate the sources just
                                    # for the part we care about:
                                    sources = self.containsSourceFiles(os.path.join(component.path, full_subpath), component)
                                    auto_subdirs.append((full_subpath, sources))
                        elif f == component.getName():
                            header_subdirs.append((f, sources))
                    elif toplevel and \
                         ((f in ('test',)) or \
                          (os.path.normpath(f) in lib_subdirs or start_on_top) or \
                          (os.path.normpath(f) in bin_subdirs or start_on_top) and not \
                          component.ignores(f)):
                        # (if there aren't any source files then do nothing)
                        # !!! FIXME: ensure this warning is covered in tests
                        logger.warning("subdirectory \"%s\" of %s was ignored because it doesn't appear to contain any source files", f, component)

            # 'resource' directory also has special meaning, but there's no
            # pattern for the files which might be in here:
            if f in ('resource',):
                resource_subdirs.append(os.path.join(component.path, f))

            # issue a warning if a differently cased or common misspelling of a
            # standard directory name was encountered:
            check_directory_name_cases = list(lib_subdirs.keys()) + list(bin_subdirs.keys()) + ['test', 'resource']
            if f.lower() in check_directory_name_cases + ['src'] and not \
               f in check_directory_name_cases and not \
               component.ignores(f):
                self.checkStandardSourceDir(f, component)
        if top_sources:
            # all the top level sources are grouped into a single cmake-generated directory
            # which is given the same name as the component
            auto_subdirs.append((component.getName(), top_sources))

        return {
            "manual": manual_subdirs,
              "auto": auto_subdirs,
           "headers": header_subdirs,
               "bin": {component.getName(): component.getName()} if (start_on_top and component.isApplication()) else bin_subdirs,
               "lib": {component.getName(): component.getName()} if (start_on_top and not component.isApplication()) else lib_subdirs,
              "test": test_subdirs,
          "resource": resource_subdirs
        }