def generateTestDirList(self, builddir, dirname, source_files, component, immediate_dependencies, toplevel=False, module_is_empty=False):
logger.debug('generate CMakeLists.txt for directory: %s' % os.path.join(component.path, dirname))
link_dependencies = [x for x in immediate_dependencies]
fname = os.path.join(builddir, dirname, 'CMakeLists.txt')
# group the list of source files by subdirectory: generate one test for
# each subdirectory, and one test for each file at the top level
subdirs = defaultdict(list)
toplevel_srcs = []
for f in source_files:
if f.lang in ('c', 'cpp', 'objc', 's'):
subrelpath = os.path.relpath(f.relpath, dirname)
subdir = fsutils.fullySplitPath(subrelpath)[0]
if subdir and subdir != subrelpath:
subdirs[subdir].append(f)
else:
toplevel_srcs.append(f)
tests = []
for f in toplevel_srcs:
object_name = '%s-test-%s' % (
component.getName(), os.path.basename(os.path.splitext(str(f))[0]).lower()
)
tests.append([[str(f)], object_name, [f.lang]])
for subdirname, sources in sorted(subdirs.items(), key=lambda x: x[0]):
object_name = '%s-test-%s' % (
component.getName(), fsutils.fullySplitPath(subdirname)[0].lower()
)
tests.append([[str(f) for f in sources], object_name, [f.lang for f in sources]])
# link tests against the main executable
if not module_is_empty:
link_dependencies.append(component.getName())
# Find cmake files
cmake_files = []
for root, dires, files in os.walk(os.path.join(component.path, dirname)):
for f in files:
name, ext = os.path.splitext(f)
if ext.lower() == '.cmake' and not component.ignores(os.path.relpath(os.path.join(root, f), component.path)):
cmake_files.append(os.path.join(root, f))
test_template = jinja_environment.get_template('test_CMakeLists.txt')
file_contents = test_template.render({ #pylint: disable=no-member
'source_directory':os.path.join(component.path, dirname),
'tests':tests,
'link_dependencies':link_dependencies,
'cmake_files': cmake_files,
'exclude_from_all': (not toplevel),
'test_dependencies': [x[1] for x in immediate_dependencies.items() if x[1].isTestDependency()]
})
self._writeFile(fname, file_contents)
def generateTestDirList(self, builddir, dirname, source_files, component, immediate_dependencies, toplevel=False, module_is_empty=False):
logger.debug('generate CMakeLists.txt for directory: %s' % os.path.join(component.path, dirname))
link_dependencies = [x for x in immediate_dependencies]
fname = os.path.join(builddir, dirname, 'CMakeLists.txt')
# group the list of source files by subdirectory: generate one test for
# each subdirectory, and one test for each file at the top level
subdirs = defaultdict(list)
toplevel_srcs = []
for f in source_files:
if f.lang in ('c', 'cpp', 'objc', 's'):
subrelpath = os.path.relpath(f.relpath, dirname)
subdir = fsutils.fullySplitPath(subrelpath)[0]
if subdir and subdir != subrelpath:
subdirs[subdir].append(f)
else:
toplevel_srcs.append(f)
tests = []
for f in toplevel_srcs:
object_name = '%s-test-%s' % (
component.getName(), os.path.basename(os.path.splitext(str(f))[0]).lower()
)
tests.append([[str(f)], object_name, [f.lang]])
for subdirname, sources in sorted(subdirs.items(), key=lambda x: x[0]):
object_name = '%s-test-%s' % (
component.getName(), fsutils.fullySplitPath(subdirname)[0].lower()
)
tests.append([[str(f) for f in sources], object_name, [f.lang for f in sources]])
# link tests against the main executable
if not module_is_empty:
link_dependencies.append(component.getName())
# Find cmake files
cmake_files = []
for root, dires, files in os.walk(os.path.join(component.path, dirname)):
for f in files:
name, ext = os.path.splitext(f)
if ext.lower() == '.cmake' and not component.ignores(os.path.relpath(os.path.join(root, f), component.path)):
cmake_files.append(os.path.join(root, f))
test_template = jinja_environment.get_template('test_CMakeLists.txt')
file_contents = test_template.render({ #pylint: disable=no-member
'source_directory':os.path.join(component.path, dirname),
'tests':tests,
'link_dependencies':link_dependencies,
'cmake_files': cmake_files,
'exclude_from_all': (not toplevel),
'test_dependencies': [x[1] for x in immediate_dependencies.items() if x[1].isTestDependency()]
})
self._writeFile(fname, file_contents)
def unpackFrom(tar_file_path, to_directory):
# first unpack into a sibling directory of the specified directory, and
# then move it into place.
# we expect our tarballs to contain a single top-level directory. We strip
# off this name as we extract to minimise the path length
into_parent_dir = os.path.dirname(to_directory)
fsutils.mkDirP(into_parent_dir)
temp_directory = tempfile.mkdtemp(dir=into_parent_dir)
try:
with tarfile.open(tar_file_path) as tf:
strip_dirname = ''
# get the extraction directory name from the first part of the
# extraction paths: it should be the same for all members of
# the archive
for m in tf.getmembers():
split_path = fsutils.fullySplitPath(m.name)
logger.debug('process member: %s %s', m.name, split_path)
if os.path.isabs(m.name) or '..' in split_path:
raise ValueError('archive uses invalid paths')
if not strip_dirname:
if len(split_path) != 1 or not len(split_path[0]):
raise ValueError(
'archive does not appear to contain a single module'
)
strip_dirname = split_path[0]
continue
else:
if split_path[0] != strip_dirname:
raise ValueError(
'archive does not appear to contain a single module'
)
m.name = os.path.join(*split_path[1:])
tf.extract(m, path=temp_directory)
# make sure the destination directory doesn't exist:
fsutils.rmRf(to_directory)
shutil.move(temp_directory, to_directory)
temp_directory = None
logger.debug('extraction complete %s', to_directory)
except IOError as e:
if e.errno != errno.ENOENT:
logger.error('failed to extract tarfile %s', e)
fsutils.rmF(tar_file_path)
raise
finally:
if temp_directory is not None:
# if anything has failed, cleanup
fsutils.rmRf(temp_directory)
def unpackFrom(tar_file_path, to_directory):
# first unpack into a sibling directory of the specified directory, and
# then move it into place.
# we expect our tarballs to contain a single top-level directory. We strip
# off this name as we extract to minimise the path length
into_parent_dir = os.path.dirname(to_directory)
fsutils.mkDirP(into_parent_dir)
temp_directory = tempfile.mkdtemp(dir=into_parent_dir)
try:
with tarfile.open(tar_file_path) as tf:
strip_dirname = ''
# get the extraction directory name from the first part of the
# extraction paths: it should be the same for all members of
# the archive
for m in tf.getmembers():
split_path = fsutils.fullySplitPath(m.name)
logger.debug('process member: %s %s', m.name, split_path)
if os.path.isabs(m.name) or '..' in split_path:
raise ValueError('archive uses invalid paths')
if not strip_dirname:
if len(split_path) != 1 or not len(split_path[0]):
raise ValueError('archive does not appear to contain a single module')
strip_dirname = split_path[0]
continue
else:
if split_path[0] != strip_dirname:
raise ValueError('archive does not appear to contain a single module')
m.name = os.path.join(*split_path[1:])
tf.extract(m, path=temp_directory)
# make sure the destination directory doesn't exist:
fsutils.rmRf(to_directory)
shutil.move(temp_directory, to_directory)
temp_directory = None
logger.debug('extraction complete %s', to_directory)
except IOError as e:
if e.errno != errno.ENOENT:
logger.error('failed to extract tarfile %s', e)
fsutils.rmF(tar_file_path)
raise
finally:
if temp_directory is not None:
# if anything has failed, cleanup
fsutils.rmRf(temp_directory)
def moduleFromDirname(build_subdir, all_modules, toplevel_module):
modtop = True
submod = False
module = toplevel_module
# <topdir> /ym/<submod>/ym/<submod2>/somedir/somedir --> submod2
for part in fsutils.fullySplitPath(build_subdir):
if submod:
if part in all_modules:
module = all_modules[part]
modtop = True
submod = False
else:
if part == 'ym' and modtop:
submod = True
else:
submod = False
modtop = False
return module
def moduleFromDirname(build_subdir, all_modules, toplevel_module):
modtop = True
submod = False
module = toplevel_module
# <topdir> /ym/<submod>/ym/<submod2>/somedir/somedir --> submod2
for part in fsutils.fullySplitPath(build_subdir):
if submod:
if part in all_modules:
module = all_modules[part]
modtop = True
submod = False
else:
if part == 'ym' and modtop:
submod = True
else:
submod = False
modtop = False
return module
def _listSubDirectories(self, component, toplevel):
''' return: {
manual: [list of subdirectories with manual CMakeLists],
auto: [list of pairs: (subdirectories name to autogenerate, a list of source files in that dir)],
bin: {dictionary of subdirectory name to binary name},
lib: {dictionary of subdirectory name to binary name},
test: [list of directories that build tests],
resource: [list of directories that contain resources]
}
'''
manual_subdirs = []
auto_subdirs = []
header_subdirs = []
lib_subdirs = component.getLibs()
bin_subdirs = component.getBinaries()
test_subdirs = []
resource_subdirs = []
# if the application or library is set to get the sources from top level ("."),
# they'll be acumulated into a single array (top_sources below).
top_sources = []
start_on_top = "." in [os.path.normpath(x) for x in list(lib_subdirs.keys()) + list(bin_subdirs.keys())]
for f in sorted(os.listdir(component.path)):
if f in Ignore_Subdirs or f.startswith('.') or f.startswith('_'):
continue
check_cmakefile_path = os.path.join(f, 'CMakeLists.txt')
if os.path.isfile(os.path.join(component.path, check_cmakefile_path)) and not \
component.ignores(check_cmakefile_path):
self.checkStandardSourceDir(f, component)
# if the subdirectory has a CMakeLists.txt in it (and it isn't
# ignored), then delegate to that:
manual_subdirs.append(f)
# tests only supported in the `test` directory for now
if f in ('test',):
test_subdirs.append(f)
else:
if os.path.isfile(os.path.join(component.path, f)):
# top level source: check if it should be included
if not component.ignores(f) and start_on_top:
sf = self.createSourceFile(f, os.path.join(component.path, f), ".")
if sf is not None:
top_sources.append(sf)
else:
# otherwise, if the directory has source files, and is listed
# as a source/test directory, generate a CMakeLists in the
# corresponding temporary directory, and add that.
sources = self.containsSourceFiles(os.path.join(component.path, f), component)
if sources:
if f in ('test',):
auto_subdirs.append((f, sources))
test_subdirs.append(f)
elif start_on_top:
# include the sources in this directory only if it's not
# a potential test directory
from yotta.lib import validate
if not validate.isPotentialTestDir(f):
top_sources.extend(sources)
if f == component.getName():
header_subdirs.append((f, sources))
elif os.path.normpath(f) in [fsutils.fullySplitPath(x)[0] for x in lib_subdirs] or \
os.path.normpath(f) in [fsutils.fullySplitPath(x)[0] for x in bin_subdirs]:
for full_subpath in list(lib_subdirs.keys()) + list(bin_subdirs.keys()):
if fsutils.fullySplitPath(full_subpath)[0] == os.path.normpath(f):
# this might be a sub-sub directory, in which
# case we need to re-calculate the sources just
# for the part we care about:
sources = self.containsSourceFiles(os.path.join(component.path, full_subpath), component)
auto_subdirs.append((full_subpath, sources))
elif f == component.getName():
header_subdirs.append((f, sources))
elif toplevel and \
((f in ('test',)) or \
(os.path.normpath(f) in lib_subdirs or start_on_top) or \
(os.path.normpath(f) in bin_subdirs or start_on_top) and not \
component.ignores(f)):
# (if there aren't any source files then do nothing)
# !!! FIXME: ensure this warning is covered in tests
logger.warning("subdirectory \"%s\" of %s was ignored because it doesn't appear to contain any source files", f, component)
# 'resource' directory also has special meaning, but there's no
# pattern for the files which might be in here:
if f in ('resource',):
resource_subdirs.append(os.path.join(component.path, f))
# issue a warning if a differently cased or common misspelling of a
# standard directory name was encountered:
check_directory_name_cases = list(lib_subdirs.keys()) + list(bin_subdirs.keys()) + ['test', 'resource']
if f.lower() in check_directory_name_cases + ['src'] and not \
f in check_directory_name_cases and not \
component.ignores(f):
self.checkStandardSourceDir(f, component)
if top_sources:
# all the top level sources are grouped into a single cmake-generated directory
# which is given the same name as the component
auto_subdirs.append((component.getName(), top_sources))
return {
"manual": manual_subdirs,
"auto": auto_subdirs,
"headers": header_subdirs,
"bin": {component.getName(): component.getName()} if (start_on_top and component.isApplication()) else bin_subdirs,
"lib": {component.getName(): component.getName()} if (start_on_top and not component.isApplication()) else lib_subdirs,
"test": test_subdirs,
"resource": resource_subdirs
}