def build(self, options, args):
self.process_configuration_lines()
# It is crucial that we do not run 'make' with spurious
# libraries in LD_LIBRARY_PATH, because 'make' has no safeguard
# against it.
check_ld_library_path_error(fatal=False)
timeout = self.configuration.general_section.subprocess_timeout
timeout = getattr(options, 'subprocess_timeout', timeout)
if options.clean or self.clean_build.upper() == 'ON':
if self.clean_commands:
clean_opts = ['-b']
else:
clean_opts = []
my_path = os.path.dirname(__file__)
bv_clean = os.path.join(my_path, 'bv_clean_build_tree')
print('cleaning build tree', self.directory)
# don't remove empty dirs here since configure may have created
# directories which will be used during build
subprocess.call(
[sys.executable, bv_clean] + clean_opts + [self.directory],
env=self.get_environ())
print('Building directory:', self.directory)
system(cwd=self.directory, *(['make'] + self.make_options),
env=self.get_environ(),
timeout=timeout)
# make / update run scripts/symlinks from a container
if os.environ.get('CASA_SYSTEM'):
try:
casa_distro = 'casa_distro'
casa_distro = distutils.spawn.find_executable(casa_distro)
if not casa_distro:
casa_distro = distutils.spawn.find_executable(
'casa_container')
if casa_distro:
casa_distro = os.path.dirname(os.path.dirname(
os.path.realpath(casa_distro)))
script = os.path.join(
casa_distro, 'share', 'scripts',
'casa_build_host_links')
if os.path.exists(script):
print('updating run scripts for casa-distro')
env = dict(os.environ)
if 'PYTHONPATH' in env:
pypath = ':'.join(
[os.path.join(casa_distro, 'python'),
env['PYTHONPATH']])
env['PYTHONPATH'] = pypath
else:
env['PYTHONPATH'] = os.path.join(casa_distro,
'python')
subprocess.call([sys.executable, script], env=env)
except Exception as e:
print(e)
pass
def doc(self):
self.process_configuration_lines()
print('Building docs in directory:', self.directory)
timeout = self.configuration.general_section.subprocess_timeout
system(cwd=self.directory, *
(['make'] + self.make_options + ['doc']),
env=self.get_environ(),
timeout=timeout)
def svncommand(self, *svnargs, **subprocess_kwargs):
cmd = ['svn', svnargs[0]]
if self.configuration.username:
cmd += ['--username', self.configuration.username]
if self.configuration.username == 'brainvisa':
cmd += ['--password', 'Soma2009']
cmd.extend(svnargs[1:])
system(*cmd, **subprocess_kwargs)
def run_and_log_tests(cwd=None, env=None, options=None, projects=None, timeout=None):
# get test labels to assign them to projects
test_labels = system_output_on_error(
['ctest', '--print-labels'] + options, echo=False, cwd=cwd)
lines = test_labels.strip().split('\n')
if 'All Labels:' in lines:
labels_index = lines.index('All Labels:')
labels = [line.strip() for line in lines[labels_index+1:]]
elif 'No Labels Exist' in lines:
labels = [] # no tests
else:
raise RuntimeError(
'ctest --print-labels produced an unexpected output:\n'
+ '\n'.join(lines))
logs = {}
if timeout is None:
timeout = brainvisa.maker.configuration.default_subprocess_timeout
for label in labels:
if projects is not None and label not in projects:
# skip this test, it's not part of the packaging/build config
continue
logfile = tempfile.mkstemp(prefix='bv_test_%s' % label, suffix='.log')
os.close(logfile[0])
start_time = time.localtime()
try:
system(cwd=cwd,
env=env,
timeout = timeout,
*(['ctest', '-L', '^%s$' % label, '--output-on-failure',
'-O', logfile[1]] + options))
except Exception as e:
logitem = {}
logitem['log_file'] = logfile[1]
logitem['exception'] = e
logitem['start_time'] = start_time
logitem['stop_time'] = time.localtime()
logs[label] = logitem
else:
logitem = {}
logitem['log_file'] = logfile[1]
logitem['exception'] = None
logitem['start_time'] = start_time
logitem['stop_time'] = time.localtime()
logs[label] = logitem
#os.unlink(logfile[1])
# FIXME DEBUG
with open(logfile[1], 'a') as f:
print('-------------------------------------', file=f)
print('projects to test: %s' % repr(projects), file=f)
print('labels to test: %s' % repr(labels), file=f)
print('current label: %s' % label, file=f)
return logs
def virtualenv_command(self, env_path):
timeout = self.configuration.general_section.subprocess_timeout
if not self.which("virtualenv"):
raise ValueError("Cannot find virtual. Please install virtualenv.")
active_path = os.path.join(env_path, "bin", "activate")
if not os.path.isfile(active_path):
cmd = ["virtualenv", "--system-site-packages"]
cmd.append(env_path)
system(*cmd,
env=self.get_environ(),
timeout=timeout)
else:
print("No need to virtualenv init '%s' since it is already initialized." \
% env_path)
pass
def configure(self, options, args):
self.process_configuration_lines()
timeout = self.configuration.general_section.subprocess_timeout
timeout = getattr(options, 'subprocess_timeout', timeout)
# Order of projects and components is important for dependencies
sortedProjects = [p for p in brainvisa_projects.ordered_projects
if p in self.projects]
sortedComponents = []
components = set(self.components)
for project in sortedProjects:
for component \
in brainvisa_projects.components_per_project[project]:
if component in components:
sortedComponents.append(component)
components.remove(component)
sortedComponents.extend(components)
if not os.path.exists(self.directory):
os.makedirs(self.directory)
if options.clean or self.clean_config.upper == 'ON':
my_path = os.path.dirname(__file__)
bv_clean = os.path.join(my_path, 'bv_clean_build_tree')
print('cleaning build tree', self.directory)
# clean and remove empty dirs. Don't use -b option here
# because configuration has to be done first.
subprocess.call([sys.executable, bv_clean, '-d', self.directory],
env=self.get_environ())
# Create a sitecustomize Python package that imports all modules it
# contains during Python startup. This is mainly used to modify
# sys.path to include pure Python components source (see module
# brainvisa.maker.build_models.pure_python). This package is used only
# in build directory, it is not installed in packages (to date there is
# one exception to this in axon component, see Axon's CMakeLists.txt).
sitecustomize_dir = os.path.join(
self.directory, 'python', 'sitecustomize')
if not os.path.exists(sitecustomize_dir):
os.makedirs(sitecustomize_dir)
with open(os.path.join(sitecustomize_dir, '__init__.py'), 'w') as f:
f.write(self.sitecustomize_content)
# Remove existing sitecustomize.py (was generated by older Axon)
for i in glob.glob(sitecustomize_dir + '.py*'):
os.remove(i)
if not os.path.exists(self.directory):
os.makedirs(self.directory)
cross_compiling_directories = {}
for k, s in six.iteritems(self.configuration.sourcesDirectories):
if s.cross_compiling_dirs is not None:
if len(self.cross_compiling_prefix) > 0:
cross_compiling_dir = \
s.cross_compiling_dirs.get(
self.cross_compiling_prefix)
if cross_compiling_dir is not None:
cross_compiling_directories[s.directory] = cross_compiling_dir
#print('==== Toolchain:', self.cross_compiling_prefix,
# 'directories:', cross_compiling_directories)
self.buildModelPerComponent = {}
for component in sortedComponents:
# find build model
build_model = self.components[component][3]
if build_model is None:
build_model = brainvisa_projects.info_per_component.get(
component, {}).get('build_model')
if build_model is not None:
build_model_class = getattr(__import__(
'brainvisa.maker.build_models',
fromlist=[six.ensure_str('pure_python')], level=0),
build_model)
build_model = build_model_class(
component, self.components[component][0], self,
cross_compiling_directories, options=options, args=args)
self.buildModelPerComponent[component] = build_model
cmakeFile = os.path.join(self.directory, 'bv_maker.cmake')
with open(cmakeFile, 'w') as out:
print('set( BRAINVISA_PROJECTS', ' '.join(
sortedProjects), 'CACHE STRING "BrainVISA Projects list" FORCE )',
file=out)
print('set( _BRAINVISA_PROJECTS', ' '.join(
sortedProjects), 'CACHE STRING "BrainVISA Projects list" FORCE )',
file=out)
print('set( BRAINVISA_COMPONENTS',
' '.join(sortedComponents),
'CACHE STRING "BrainVISA components list" FORCE )',
file=out)
print('set( _BRAINVISA_COMPONENTS',
' '.join(sortedComponents),
'CACHE STRING "BrainVISA components list" FORCE )',
file=out)
print(file=out)
for component, directory_version_model in six.iteritems(self.components):
directory, selected_version, version, build_model = directory_version_model
if component in self.buildModelPerComponent:
print('set( BRAINVISA_SOURCES_' + component + ' "' \
+ cmake_path(self.directory ) + '/build_files/' \
+ component + '_src' \
+ '" CACHE STRING "Sources directory for component ' \
+ component + '" FORCE )',
file=out)
else:
print('set( BRAINVISA_SOURCES_' + component + ' "' \
+ cmake_path(directory) \
+ '" CACHE STRING "Sources directory for component ' \
+ component + '" FORCE )',
file=out)
print('set( ' + component + '_DIR "' \
+ cmake_path(self.directory ) + '/share/' + component + \
'-' + version + \
'/cmake" CACHE STRING "Directory used for find_package( ' + \
component + \
' )" FORCE )',
file=out)
print('set( ' + component + '_VERSION "' + version + '" )',
file=out)
cmakeLists = os.path.join(self.directory, 'CMakeLists.txt')
with open(cmakeLists, 'w') as out:
print('''
cmake_minimum_required( VERSION 2.6 )
set( CMAKE_PREFIX_PATH "${CMAKE_BINARY_DIR}" ${CMAKE_PREFIX_PATH} )
find_package( brainvisa-cmake NO_POLICY_SCOPE )
include( "${brainvisa-cmake_DIR}/brainvisa-compilation.cmake" )
''', file=out)
exe_suffix = ''
if sys.platform == 'win32':
command_base = ['cmake', '-G', 'MSYS Makefiles']
exe_suffix = '.exe'
else:
command_base = ['cmake']
command_options = list(self.cmake_options)
command_options += ['-DCMAKE_BUILD_TYPE:STRING=' + self.build_type]
if self.packaging_thirdparty.upper() == 'ON':
print('ERROR: the packaging_thirdparty option is no longer '
'supported, it will be ignored.')
config_dir = cmake_path(self.directory)
for component, build_model \
in six.iteritems(self.buildModelPerComponent):
build_model.configure()
# set bv_maker path, so that cmake finds its modules
os.environ['PATH'] = os.path.dirname(this_script) + os.pathsep \
+ os.getenv('PATH')
# cross compilation options
cross_compiling_prefix = self.cross_compiling_prefix.strip()
if len(cross_compiling_prefix) > 0:
cross_compiling_prefix_path = os.path.join( cmake_root,
'toolchains',
cross_compiling_prefix )
cross_compiling_options = ['-DBRAINVISA_CMAKE_OPTIONS:STRING=' \
'CMAKE_CROSSCOMPILING;COMPILER_PREFIX;' \
'CMAKE_TOOLCHAIN_FILE', \
'-DCOMPILER_PREFIX:STRING=%s' % \
self.cross_compiling_prefix.strip(), \
'-DCMAKE_CROSSCOMPILING:BOOL=ON']
cross_compiling_toolchain_path = os.path.join(
cross_compiling_prefix_path,
'toolchain.cmake' )
cross_compiling_init_cache_path = os.path.join(
cross_compiling_prefix_path,
'init-cache.cmake' )
#print("=== cross_compiling_prefix:", cross_compiling_prefix, "===")
#print("=== cross_compiling_prefix_path:", cross_compiling_prefix_path, "===")
#print("=== cross_compiling_toolchain_path:", cross_compiling_toolchain_path, "===")
#print("=== cross_compiling_init_cache_path:", cross_compiling_init_cache_path, "===")
if os.path.exists( cross_compiling_toolchain_path ):
cross_compiling_options += ['-DCMAKE_TOOLCHAIN_FILE:PATH=%s' % \
cmake_path(
cross_compiling_toolchain_path),]
if os.path.exists( cross_compiling_init_cache_path ):
cross_compiling_options += ['-C',
cmake_path(
cross_compiling_init_cache_path),]
#print('cross compiling using toolchain:', cross_compiling_prefix)
#print(' with options:', *cross_compiling_options)
else:
cross_compiling_options = []
# special case: if bv-cmake is part of the build directory, run cmake
# in 2 passes: once to reinstall bv-cmake from sources, and a second
# time to actually configure all projects using the newly installed
# bv-cmake.
if 'brainvisa-cmake' in self.components:
print('=== bootstraping brainvisa-cmake project ===')
bvcmake_dir = os.path.join(self.directory, 'brainvisa-cmake')
if not os.path.exists(bvcmake_dir):
os.makedirs(bvcmake_dir)
# pass it Qt version if we have any info
bvcmake_options = []
qt_opt = [x for x in self.cmake_options
if x.startswith('DESIRED_QT_VERSION')]
if qt_opt:
qt_opt = qt_opt[0].split('=')[1].strip()
bvcmake_options.append('-DDESIRED_QT_VERSION=%s' %qt_opt)
elif os.path.exists(os.path.join(self.directory, 'CMakeCache.txt')):
with open(os.path.join(self.directory, 'CMakeCache.txt')) as f:
for l in f.readlines():
if l.startswith('DESIRED_QT_VERSION:'):
qt_opt = l.split('=')[1].strip()
bvcmake_options.append(
'-DDESIRED_QT_VERSION=%s' %qt_opt)
system(cwd=bvcmake_dir,
*(command_base
+ [self.components['brainvisa-cmake'][0],
'-DBRAINVISA_CMAKE_BUILD_TYPE=brainvisa-cmake-only',
'-DCMAKE_INSTALL_PREFIX=%s' % self.directory]
+ bvcmake_options),
env=self.get_environ(),
timeout=timeout)
system(cwd=bvcmake_dir, *['make', 'install'],
env=self.get_environ(),
timeout=timeout)
print('=== now configuring all other projects ===')
# run with this local bv-cmake environment
system(cwd=self.directory,
*( [os.path.join(self.directory, 'bin',
'bv_env_host%s' % exe_suffix)]
+ command_base
+ command_options
+ cross_compiling_options
+ ["-DBRAINVISA_CMAKE_BUILD_TYPE=no-brainvisa-cmake"]
+ [config_dir]),
env=self.get_environ(),
timeout=timeout)
else:
# run cmake in a regular way
system(cwd=self.directory, *( command_base
+ command_options
+ cross_compiling_options
+ [config_dir]),
env=self.get_environ(),
timeout=timeout)
# After a first configuration, the global version file of the build
# directory has been generated, and package directories variables,
# must be updated
for p in list(self.configuration.packageDirectories.values()) \
+ list(self.configuration.publicationDirectories.values()):
if p.get_build_dir() is self:
version = self.get_version()
if version:
p.update_python_vars({'version': version})
def process(self, options, args):
self._git_only_failed = False
if options.ignore_git_failure:
self.ignore_git_failure = True
if not os.path.exists(self.directory):
os.makedirs(self.directory)
with open(os.path.join(self.directory, 'bv_maker.cfg'),
'w') as clientFile:
print('\n'.join(self.configurationLines), file=clientFile)
repositoryDirectory = os.path.join(self.directory, '.repository')
checkout = False
use_rcs = self.revision_control.upper() in ('', 'ON')
if use_rcs and options.svn and not os.path.exists(repositoryDirectory):
os.makedirs(repositoryDirectory)
# Because of a bug in svnadmin on MacOS, I cannot use an absolute name for the directory to create.
# When I try "svnadmin create /neurospin/brainvisa/cmake_mac/", I have the following error:
# svnadmin: '/neurospin/brainvisa/cmake_mac' is a subdirectory of an existing repository rooted at '/neurospin'
# But it works if I do "cd /neurospin/brainvisa && vnadmin create
# cmake_mac"
cwd, dir = os.path.split(repositoryDirectory)
system('svnadmin', 'create', dir, cwd=cwd)
if len(os.path.splitdrive(repositoryDirectory)[0]) > 0:
# This is for windows absolute pathes
repositoryDirectory = '/' + \
repositoryDirectory.replace(os.path.sep, "/")
self.svncommand(
'checkout', 'file://' + repositoryDirectory, self.directory)
checkout = True
source_directories = []
# Update SVN repositories
# Go to the sources directory
externalsFileName = os.path.join(self.directory, 'bv_maker.externals')
with open(externalsFileName, 'w') as externalsFile:
for component_version, url, dest_directory, bv_version in set(self.svnComponents):
print(dest_directory, url, file=externalsFile)
source_directories.append((dest_directory, bv_version))
if use_rcs and options.svn:
if options.cleanup:
self.svncommand('cleanup', self.directory, cwd=self.directory)
self.svncommand('propset', 'svn:externals',
'--file', externalsFileName, self.directory,
cwd=self.directory)
self.svncommand('commit', '-m', '', self.directory,
cwd=self.directory)
self.svncommand('update', self.directory, cwd=self.directory)
# update Git Repositories
git_status_list = []
git_update_failure = False
for component_version, url, git_tag, dest_directory, bv_version \
in self.gitComponents:
if dest_directory is None:
dest_directory = url.rsplit('/', 1)[-1]
if dest_directory.endswith('.git'):
dest_directory = dest_directory[:-4]
if use_rcs and options.git:
gr = GitRepository(self.directory, dest_directory,
remote_url=url, remote_ref=git_tag)
try:
gr.update_or_clone(source_dir=self)
except GitUpdateError as exc:
update_message = u'✗ ' + six.ensure_text(str(exc))
git_update_failure = True
else:
update_message = u'✓'
status_dict = gr.get_status_dict()
status_dict['update_message'] = update_message
git_status_list.append(status_dict)
source_directories.append((dest_directory, bv_version))
print_git_status_summary(self.directory, git_status_list)
components_sources = {}
for dest_path, bv_version in source_directories:
pinfo = brainvisa_projects.read_project_info(
os.path.join(self.directory, dest_path),
version_format=version_format_short
)
if pinfo:
project, component, version, build_model = pinfo
version = str(version)
components_sources.setdefault(component, {})[
bv_version or version] = (dest_path, build_model)
else:
print('WARNING: directory %s will be ignored because project_info.cmake, python/*/info.py or */info.py cannot be found or used' % os.path.join(self.directory, dest_path))
with open(os.path.join(self.directory, 'components_sources.json'),
'w') as f:
json.dump(components_sources, f, indent=2)
if git_update_failure:
self._git_only_failed = True
raise RuntimeError('Error updating one or more Git repositories')