def build(self, options, args): self.process_configuration_lines() # It is crucial that we do not run 'make' with spurious # libraries in LD_LIBRARY_PATH, because 'make' has no safeguard # against it. check_ld_library_path_error(fatal=False) timeout = self.configuration.general_section.subprocess_timeout timeout = getattr(options, 'subprocess_timeout', timeout) if options.clean or self.clean_build.upper() == 'ON': if self.clean_commands: clean_opts = ['-b'] else: clean_opts = [] my_path = os.path.dirname(__file__) bv_clean = os.path.join(my_path, 'bv_clean_build_tree') print('cleaning build tree', self.directory) # don't remove empty dirs here since configure may have created # directories which will be used during build subprocess.call( [sys.executable, bv_clean] + clean_opts + [self.directory], env=self.get_environ()) print('Building directory:', self.directory) system(cwd=self.directory, *(['make'] + self.make_options), env=self.get_environ(), timeout=timeout) # make / update run scripts/symlinks from a container if os.environ.get('CASA_SYSTEM'): try: casa_distro = 'casa_distro' casa_distro = distutils.spawn.find_executable(casa_distro) if not casa_distro: casa_distro = distutils.spawn.find_executable( 'casa_container') if casa_distro: casa_distro = os.path.dirname(os.path.dirname( os.path.realpath(casa_distro))) script = os.path.join( casa_distro, 'share', 'scripts', 'casa_build_host_links') if os.path.exists(script): print('updating run scripts for casa-distro') env = dict(os.environ) if 'PYTHONPATH' in env: pypath = ':'.join( [os.path.join(casa_distro, 'python'), env['PYTHONPATH']]) env['PYTHONPATH'] = pypath else: env['PYTHONPATH'] = os.path.join(casa_distro, 'python') subprocess.call([sys.executable, script], env=env) except Exception as e: print(e) pass
def doc(self): self.process_configuration_lines() print('Building docs in directory:', self.directory) timeout = self.configuration.general_section.subprocess_timeout system(cwd=self.directory, * (['make'] + self.make_options + ['doc']), env=self.get_environ(), timeout=timeout)
def svncommand(self, *svnargs, **subprocess_kwargs): cmd = ['svn', svnargs[0]] if self.configuration.username: cmd += ['--username', self.configuration.username] if self.configuration.username == 'brainvisa': cmd += ['--password', 'Soma2009'] cmd.extend(svnargs[1:]) system(*cmd, **subprocess_kwargs)
def run_and_log_tests(cwd=None, env=None, options=None, projects=None, timeout=None): # get test labels to assign them to projects test_labels = system_output_on_error( ['ctest', '--print-labels'] + options, echo=False, cwd=cwd) lines = test_labels.strip().split('\n') if 'All Labels:' in lines: labels_index = lines.index('All Labels:') labels = [line.strip() for line in lines[labels_index+1:]] elif 'No Labels Exist' in lines: labels = [] # no tests else: raise RuntimeError( 'ctest --print-labels produced an unexpected output:\n' + '\n'.join(lines)) logs = {} if timeout is None: timeout = brainvisa.maker.configuration.default_subprocess_timeout for label in labels: if projects is not None and label not in projects: # skip this test, it's not part of the packaging/build config continue logfile = tempfile.mkstemp(prefix='bv_test_%s' % label, suffix='.log') os.close(logfile[0]) start_time = time.localtime() try: system(cwd=cwd, env=env, timeout = timeout, *(['ctest', '-L', '^%s$' % label, '--output-on-failure', '-O', logfile[1]] + options)) except Exception as e: logitem = {} logitem['log_file'] = logfile[1] logitem['exception'] = e logitem['start_time'] = start_time logitem['stop_time'] = time.localtime() logs[label] = logitem else: logitem = {} logitem['log_file'] = logfile[1] logitem['exception'] = None logitem['start_time'] = start_time logitem['stop_time'] = time.localtime() logs[label] = logitem #os.unlink(logfile[1]) # FIXME DEBUG with open(logfile[1], 'a') as f: print('-------------------------------------', file=f) print('projects to test: %s' % repr(projects), file=f) print('labels to test: %s' % repr(labels), file=f) print('current label: %s' % label, file=f) return logs
def virtualenv_command(self, env_path): timeout = self.configuration.general_section.subprocess_timeout if not self.which("virtualenv"): raise ValueError("Cannot find virtual. Please install virtualenv.") active_path = os.path.join(env_path, "bin", "activate") if not os.path.isfile(active_path): cmd = ["virtualenv", "--system-site-packages"] cmd.append(env_path) system(*cmd, env=self.get_environ(), timeout=timeout) else: print("No need to virtualenv init '%s' since it is already initialized." \ % env_path) pass
def configure(self, options, args): self.process_configuration_lines() timeout = self.configuration.general_section.subprocess_timeout timeout = getattr(options, 'subprocess_timeout', timeout) # Order of projects and components is important for dependencies sortedProjects = [p for p in brainvisa_projects.ordered_projects if p in self.projects] sortedComponents = [] components = set(self.components) for project in sortedProjects: for component \ in brainvisa_projects.components_per_project[project]: if component in components: sortedComponents.append(component) components.remove(component) sortedComponents.extend(components) if not os.path.exists(self.directory): os.makedirs(self.directory) if options.clean or self.clean_config.upper == 'ON': my_path = os.path.dirname(__file__) bv_clean = os.path.join(my_path, 'bv_clean_build_tree') print('cleaning build tree', self.directory) # clean and remove empty dirs. Don't use -b option here # because configuration has to be done first. subprocess.call([sys.executable, bv_clean, '-d', self.directory], env=self.get_environ()) # Create a sitecustomize Python package that imports all modules it # contains during Python startup. This is mainly used to modify # sys.path to include pure Python components source (see module # brainvisa.maker.build_models.pure_python). This package is used only # in build directory, it is not installed in packages (to date there is # one exception to this in axon component, see Axon's CMakeLists.txt). sitecustomize_dir = os.path.join( self.directory, 'python', 'sitecustomize') if not os.path.exists(sitecustomize_dir): os.makedirs(sitecustomize_dir) with open(os.path.join(sitecustomize_dir, '__init__.py'), 'w') as f: f.write(self.sitecustomize_content) # Remove existing sitecustomize.py (was generated by older Axon) for i in glob.glob(sitecustomize_dir + '.py*'): os.remove(i) if not os.path.exists(self.directory): os.makedirs(self.directory) cross_compiling_directories = {} for k, s in six.iteritems(self.configuration.sourcesDirectories): if s.cross_compiling_dirs is not None: if len(self.cross_compiling_prefix) > 0: cross_compiling_dir = \ s.cross_compiling_dirs.get( self.cross_compiling_prefix) if cross_compiling_dir is not None: cross_compiling_directories[s.directory] = cross_compiling_dir #print('==== Toolchain:', self.cross_compiling_prefix, # 'directories:', cross_compiling_directories) self.buildModelPerComponent = {} for component in sortedComponents: # find build model build_model = self.components[component][3] if build_model is None: build_model = brainvisa_projects.info_per_component.get( component, {}).get('build_model') if build_model is not None: build_model_class = getattr(__import__( 'brainvisa.maker.build_models', fromlist=[six.ensure_str('pure_python')], level=0), build_model) build_model = build_model_class( component, self.components[component][0], self, cross_compiling_directories, options=options, args=args) self.buildModelPerComponent[component] = build_model cmakeFile = os.path.join(self.directory, 'bv_maker.cmake') with open(cmakeFile, 'w') as out: print('set( BRAINVISA_PROJECTS', ' '.join( sortedProjects), 'CACHE STRING "BrainVISA Projects list" FORCE )', file=out) print('set( _BRAINVISA_PROJECTS', ' '.join( sortedProjects), 'CACHE STRING "BrainVISA Projects list" FORCE )', file=out) print('set( BRAINVISA_COMPONENTS', ' '.join(sortedComponents), 'CACHE STRING "BrainVISA components list" FORCE )', file=out) print('set( _BRAINVISA_COMPONENTS', ' '.join(sortedComponents), 'CACHE STRING "BrainVISA components list" FORCE )', file=out) print(file=out) for component, directory_version_model in six.iteritems(self.components): directory, selected_version, version, build_model = directory_version_model if component in self.buildModelPerComponent: print('set( BRAINVISA_SOURCES_' + component + ' "' \ + cmake_path(self.directory ) + '/build_files/' \ + component + '_src' \ + '" CACHE STRING "Sources directory for component ' \ + component + '" FORCE )', file=out) else: print('set( BRAINVISA_SOURCES_' + component + ' "' \ + cmake_path(directory) \ + '" CACHE STRING "Sources directory for component ' \ + component + '" FORCE )', file=out) print('set( ' + component + '_DIR "' \ + cmake_path(self.directory ) + '/share/' + component + \ '-' + version + \ '/cmake" CACHE STRING "Directory used for find_package( ' + \ component + \ ' )" FORCE )', file=out) print('set( ' + component + '_VERSION "' + version + '" )', file=out) cmakeLists = os.path.join(self.directory, 'CMakeLists.txt') with open(cmakeLists, 'w') as out: print(''' cmake_minimum_required( VERSION 2.6 ) set( CMAKE_PREFIX_PATH "${CMAKE_BINARY_DIR}" ${CMAKE_PREFIX_PATH} ) find_package( brainvisa-cmake NO_POLICY_SCOPE ) include( "${brainvisa-cmake_DIR}/brainvisa-compilation.cmake" ) ''', file=out) exe_suffix = '' if sys.platform == 'win32': command_base = ['cmake', '-G', 'MSYS Makefiles'] exe_suffix = '.exe' else: command_base = ['cmake'] command_options = list(self.cmake_options) command_options += ['-DCMAKE_BUILD_TYPE:STRING=' + self.build_type] if self.packaging_thirdparty.upper() == 'ON': print('ERROR: the packaging_thirdparty option is no longer ' 'supported, it will be ignored.') config_dir = cmake_path(self.directory) for component, build_model \ in six.iteritems(self.buildModelPerComponent): build_model.configure() # set bv_maker path, so that cmake finds its modules os.environ['PATH'] = os.path.dirname(this_script) + os.pathsep \ + os.getenv('PATH') # cross compilation options cross_compiling_prefix = self.cross_compiling_prefix.strip() if len(cross_compiling_prefix) > 0: cross_compiling_prefix_path = os.path.join( cmake_root, 'toolchains', cross_compiling_prefix ) cross_compiling_options = ['-DBRAINVISA_CMAKE_OPTIONS:STRING=' \ 'CMAKE_CROSSCOMPILING;COMPILER_PREFIX;' \ 'CMAKE_TOOLCHAIN_FILE', \ '-DCOMPILER_PREFIX:STRING=%s' % \ self.cross_compiling_prefix.strip(), \ '-DCMAKE_CROSSCOMPILING:BOOL=ON'] cross_compiling_toolchain_path = os.path.join( cross_compiling_prefix_path, 'toolchain.cmake' ) cross_compiling_init_cache_path = os.path.join( cross_compiling_prefix_path, 'init-cache.cmake' ) #print("=== cross_compiling_prefix:", cross_compiling_prefix, "===") #print("=== cross_compiling_prefix_path:", cross_compiling_prefix_path, "===") #print("=== cross_compiling_toolchain_path:", cross_compiling_toolchain_path, "===") #print("=== cross_compiling_init_cache_path:", cross_compiling_init_cache_path, "===") if os.path.exists( cross_compiling_toolchain_path ): cross_compiling_options += ['-DCMAKE_TOOLCHAIN_FILE:PATH=%s' % \ cmake_path( cross_compiling_toolchain_path),] if os.path.exists( cross_compiling_init_cache_path ): cross_compiling_options += ['-C', cmake_path( cross_compiling_init_cache_path),] #print('cross compiling using toolchain:', cross_compiling_prefix) #print(' with options:', *cross_compiling_options) else: cross_compiling_options = [] # special case: if bv-cmake is part of the build directory, run cmake # in 2 passes: once to reinstall bv-cmake from sources, and a second # time to actually configure all projects using the newly installed # bv-cmake. if 'brainvisa-cmake' in self.components: print('=== bootstraping brainvisa-cmake project ===') bvcmake_dir = os.path.join(self.directory, 'brainvisa-cmake') if not os.path.exists(bvcmake_dir): os.makedirs(bvcmake_dir) # pass it Qt version if we have any info bvcmake_options = [] qt_opt = [x for x in self.cmake_options if x.startswith('DESIRED_QT_VERSION')] if qt_opt: qt_opt = qt_opt[0].split('=')[1].strip() bvcmake_options.append('-DDESIRED_QT_VERSION=%s' %qt_opt) elif os.path.exists(os.path.join(self.directory, 'CMakeCache.txt')): with open(os.path.join(self.directory, 'CMakeCache.txt')) as f: for l in f.readlines(): if l.startswith('DESIRED_QT_VERSION:'): qt_opt = l.split('=')[1].strip() bvcmake_options.append( '-DDESIRED_QT_VERSION=%s' %qt_opt) system(cwd=bvcmake_dir, *(command_base + [self.components['brainvisa-cmake'][0], '-DBRAINVISA_CMAKE_BUILD_TYPE=brainvisa-cmake-only', '-DCMAKE_INSTALL_PREFIX=%s' % self.directory] + bvcmake_options), env=self.get_environ(), timeout=timeout) system(cwd=bvcmake_dir, *['make', 'install'], env=self.get_environ(), timeout=timeout) print('=== now configuring all other projects ===') # run with this local bv-cmake environment system(cwd=self.directory, *( [os.path.join(self.directory, 'bin', 'bv_env_host%s' % exe_suffix)] + command_base + command_options + cross_compiling_options + ["-DBRAINVISA_CMAKE_BUILD_TYPE=no-brainvisa-cmake"] + [config_dir]), env=self.get_environ(), timeout=timeout) else: # run cmake in a regular way system(cwd=self.directory, *( command_base + command_options + cross_compiling_options + [config_dir]), env=self.get_environ(), timeout=timeout) # After a first configuration, the global version file of the build # directory has been generated, and package directories variables, # must be updated for p in list(self.configuration.packageDirectories.values()) \ + list(self.configuration.publicationDirectories.values()): if p.get_build_dir() is self: version = self.get_version() if version: p.update_python_vars({'version': version})
def process(self, options, args): self._git_only_failed = False if options.ignore_git_failure: self.ignore_git_failure = True if not os.path.exists(self.directory): os.makedirs(self.directory) with open(os.path.join(self.directory, 'bv_maker.cfg'), 'w') as clientFile: print('\n'.join(self.configurationLines), file=clientFile) repositoryDirectory = os.path.join(self.directory, '.repository') checkout = False use_rcs = self.revision_control.upper() in ('', 'ON') if use_rcs and options.svn and not os.path.exists(repositoryDirectory): os.makedirs(repositoryDirectory) # Because of a bug in svnadmin on MacOS, I cannot use an absolute name for the directory to create. # When I try "svnadmin create /neurospin/brainvisa/cmake_mac/", I have the following error: # svnadmin: '/neurospin/brainvisa/cmake_mac' is a subdirectory of an existing repository rooted at '/neurospin' # But it works if I do "cd /neurospin/brainvisa && vnadmin create # cmake_mac" cwd, dir = os.path.split(repositoryDirectory) system('svnadmin', 'create', dir, cwd=cwd) if len(os.path.splitdrive(repositoryDirectory)[0]) > 0: # This is for windows absolute pathes repositoryDirectory = '/' + \ repositoryDirectory.replace(os.path.sep, "/") self.svncommand( 'checkout', 'file://' + repositoryDirectory, self.directory) checkout = True source_directories = [] # Update SVN repositories # Go to the sources directory externalsFileName = os.path.join(self.directory, 'bv_maker.externals') with open(externalsFileName, 'w') as externalsFile: for component_version, url, dest_directory, bv_version in set(self.svnComponents): print(dest_directory, url, file=externalsFile) source_directories.append((dest_directory, bv_version)) if use_rcs and options.svn: if options.cleanup: self.svncommand('cleanup', self.directory, cwd=self.directory) self.svncommand('propset', 'svn:externals', '--file', externalsFileName, self.directory, cwd=self.directory) self.svncommand('commit', '-m', '', self.directory, cwd=self.directory) self.svncommand('update', self.directory, cwd=self.directory) # update Git Repositories git_status_list = [] git_update_failure = False for component_version, url, git_tag, dest_directory, bv_version \ in self.gitComponents: if dest_directory is None: dest_directory = url.rsplit('/', 1)[-1] if dest_directory.endswith('.git'): dest_directory = dest_directory[:-4] if use_rcs and options.git: gr = GitRepository(self.directory, dest_directory, remote_url=url, remote_ref=git_tag) try: gr.update_or_clone(source_dir=self) except GitUpdateError as exc: update_message = u'✗ ' + six.ensure_text(str(exc)) git_update_failure = True else: update_message = u'✓' status_dict = gr.get_status_dict() status_dict['update_message'] = update_message git_status_list.append(status_dict) source_directories.append((dest_directory, bv_version)) print_git_status_summary(self.directory, git_status_list) components_sources = {} for dest_path, bv_version in source_directories: pinfo = brainvisa_projects.read_project_info( os.path.join(self.directory, dest_path), version_format=version_format_short ) if pinfo: project, component, version, build_model = pinfo version = str(version) components_sources.setdefault(component, {})[ bv_version or version] = (dest_path, build_model) else: print('WARNING: directory %s will be ignored because project_info.cmake, python/*/info.py or */info.py cannot be found or used' % os.path.join(self.directory, dest_path)) with open(os.path.join(self.directory, 'components_sources.json'), 'w') as f: json.dump(components_sources, f, indent=2) if git_update_failure: self._git_only_failed = True raise RuntimeError('Error updating one or more Git repositories')