def _lazy_update_library_list(env, emitting, libraries, prebuilt_libraries, add_dependents, linktype, boost, stage_dir): def build_with_library_name(library): return library == 'log_setup' and 'log' or library if add_dependents: if not emitting: libraries = set( build_with_library_name(l) for l in add_dependent_libraries(boost, linktype, libraries)) else: libraries = add_dependent_libraries(boost, linktype, libraries) if not stage_dir in prebuilt_libraries: logger.trace("Lazy update libraries list for [{}] to [{}]".format( as_info(stage_dir), colour_items(str(l) for l in libraries))) prebuilt_libraries[stage_dir] = set(libraries) else: logger.trace( "Lazy read libraries list for [{}]: libraries are [{}]".format( as_info(stage_dir), colour_items(str(l) for l in libraries))) libraries = [ l for l in libraries if l not in prebuilt_libraries[stage_dir] ] prebuilt_libraries[stage_dir].update(libraries) return libraries
def get_option( cls, option, default=None ): if option in cls._cached_options: return cls._cached_options[ option ] value = SCons.Script.GetOption( option ) source = None if value == None or value == '': if cls._options['default_options'] and option in cls._options['default_options']: value = cls._options['default_options'][ option ] source = "in the sconstruct file" elif default: value = default source = "using default" else: source = "on command-line" if option in cls._options['configured_options']: source = "using configure" if value: logger.debug( "option [{}] set {} as [{}]".format( as_info( option ), source, as_info( str(value) ) ) ) cls._cached_options[option] = value return value
def _determine_latest_boost_verion(offline): current_release = "1.71.0" if not offline: try: boost_version_url = 'https://www.boost.org/users/download/' logger.info("Checking current boost version from {}...".format( as_info(boost_version_url))) html = lxml.html.parse(urlopen(boost_version_url)) current_release = html.xpath( "/html/body/div[2]/div/div[1]/div/div/div[2]/h3[1]/span" )[0].text current_release = str( re.search(r'(\d[.]\d+([.]\d+)?)', current_release).group(1)) logger.info("Latest boost release detected as [{}]".format( as_info(current_release))) except Exception as e: logger.warn( "Cannot determine latest version of boost - [{}]. Assuming [{}]." .format(str(e), current_release)) else: logger.info( "In offline mode. No version of boost specified so assuming [{}]". format(as_info(current_release))) return current_release
def build_library_from_source( self, env, sources=None, library_name=None, linktype=None ): from SCons.Script import Flatten if not self._source_path and not sources: logger.warn( "Attempting to build library when source path is None" ) return None if not library_name: library_name = self._name if not linktype: linktype = self._linktype variant_key = env['tool_variant_dir'] prebuilt_objects = self.lazy_create_node( variant_key, self._prebuilt_objects ) prebuilt_libraries = self.lazy_create_node( variant_key, self._prebuilt_libraries ) local_dir = self._location.local() local_folder = self._location.local_folder() build_dir = os.path.abspath( os.path.join( env['abs_build_root'], local_folder, env['tool_variant_working_dir'] ) ) final_dir = os.path.abspath( os.path.normpath( os.path.join( build_dir, env['final_dir'] ) ) ) logger.debug( "build_dir for [{}] = [{}]".format( as_info(self._name), build_dir ) ) logger.debug( "final_dir for [{}] = [{}]".format( as_info(self._name), final_dir ) ) obj_suffix = env['OBJSUFFIX'] obj_builder = env.StaticObject lib_builder = env.BuildStaticLib if linktype == "shared": obj_suffix = env['SHOBJSUFFIX'] obj_builder = env.SharedObject lib_builder = env.BuildSharedLib if not sources: sources = env.RecursiveGlob( "*.cpp", start=self._source_path, exclude_dirs=[ env['build_dir'] ] ) sources.extend( env.RecursiveGlob( "*.cc", start=self._source_path, exclude_dirs=[ env['build_dir'] ] ) ) sources.extend( env.RecursiveGlob( "*.c", start=self._source_path, exclude_dirs=[ env['build_dir'] ] ) ) objects = [] for source in Flatten( [sources] ): rel_path = os.path.relpath( str(source), local_dir ) rel_obj_path = os.path.splitext( rel_path )[0] + obj_suffix obj_path = os.path.join( build_dir, rel_obj_path ) if not rel_obj_path in prebuilt_objects: prebuilt_objects[rel_obj_path] = obj_builder( obj_path, source ) objects.append( prebuilt_objects[rel_obj_path] ) if not linktype in prebuilt_libraries: library = lib_builder( library_name, objects, final_dir = final_dir ) if linktype == "shared": library = env.Install( env['abs_final_dir'], library ) prebuilt_libraries[linktype] = library else: logger.trace( "using existing library = [{}]".format( str(prebuilt_libraries[linktype]) ) ) return prebuilt_libraries[linktype]
def get_cached_archive( self, cache_root, path ): logger.debug( "Checking for cached archive [{}]...".format( as_info( path ) ) ) for archive in os.listdir(cache_root): if fnmatch.fnmatch( archive, path ): logger.debug( "Found cached archive [{}] skipping download".format( as_info( archive ) ) ) return os.path.join( cache_root, archive ) return None
def get_branch(cls, path): branch = None try: result = cls.execute_command( "{git} symbolic-ref HEAD".format(git=cls.binary()), path) branch = result.replace("refs/heads/", "").strip() logger.trace("Branch (using symbolic-ref) for [{}] is [{}]".format( as_notice(path), as_info(branch))) return branch except cls.Error: pass # In case we have a detached head we can fallback to this result = cls.execute_command( "{git} show -s --pretty=\%d HEAD".format(git=cls.binary()), path) match = re.search(r'[(]HEAD[^,]*[,] (?P<branches>[^)]+)[)]', result) if match: branches = [b.strip() for b in match.group("branches").split(',')] logger.trace("Branches (using show) for [{}] are [{}]".format( as_notice(path), colour_items(branches))) if len(branches) == 1: # If this returns a tag: tag_name replace the ": " with "/" and then extract the tag_name # otherwise this will simply extract the branch_name as expected branch = branches[0].replace(': ', '/').split('/')[1] else: branch = branches[-2].split('/')[1] logger.trace("Branch (using show) for [{}] is [{}]".format( as_notice(path), as_info(branch))) else: logger.warn("No branch found from [{}]".format(result)) return branch
def __init__( self, cuppa_env, location, branch=None, extra_sub_path=None, name_hint=None ): self._location = location self._full_url = urlparse.urlparse( location ) self._sub_dir = "" self._name_hint = name_hint if extra_sub_path: if os.path.isabs( extra_sub_path ): raise LocationException( "Error extra sub path [{}] is not relative".format(extra_sub_path) ) else: self._sub_dir = os.path.normpath( extra_sub_path ) ## Get the location for the source dependency. If the location is a URL or an Archive we'll need to ## retrieve the URL and extract the archive. get_local_directory() returns the location of the source ## once this is done local_directory, use_sub_dir = self.get_local_directory( cuppa_env, location, self._sub_dir, branch, self._full_url ) self._base_local_directory = local_directory self._local_directory = use_sub_dir and os.path.join( local_directory, self._sub_dir ) or local_directory ## Now that we have a locally accessible version of the dependency we can try to collate some information ## about it to allow us to specify what we are building with. self._url, self._repository, self._branch, self._revision = self.get_info( self._location, self._local_directory, self._full_url ) self._version, self._revision = self.ver_rev_summary( self._branch, self._revision, self._full_url.path ) logger.debug( "Using [{}]{} at [{}] stored in [{}]".format( as_info( location ), ( branch and ":[{}]".format( as_info( str(branch) ) ) or "" ), as_info( self._version ), as_notice( self._local_directory ) ) )
def __call__( self, target, source, env ): logger.trace( "target = [{}]".format( colour_items( [ str(node) for node in target ] ) ) ) logger.trace( "source = [{}]".format( colour_items( [ str(node) for node in source ] ) ) ) for html_report_src_tgt, json_report_src_tgt in zip(*[iter(itertools.izip( source, target ))]*2): html_report = html_report_src_tgt[0] json_report = json_report_src_tgt[0] html_target = html_report_src_tgt[1] json_target = json_report_src_tgt[1] logger.trace( "html_report = [{}]".format( as_notice( str(html_report) ) ) ) logger.trace( "json_report = [{}]".format( as_info( str(json_report) ) ) ) logger.trace( "html_target = [{}]".format( as_notice( str(html_target) ) ) ) logger.trace( "json_target = [{}]".format( as_info( str(json_target) ) ) ) destination = env['abs_final_dir'] if self._destination: destination = self._destination + destination_subdir( env ) logger.trace( "report_summary = {}".format( str( self._read( str(json_report) ) ) ) ) env.Execute( Copy( html_target, html_report ) ) env.Execute( Copy( json_target, json_report ) ) return None
def get_cached_archive( self, cache_root, path ): logger.debug( "Checking for cached archive [{}]...".format( as_info( path ) ) ) for archive in os.listdir(cache_root): if fnmatch.fnmatch( archive, path ): logger.debug( "Found cached archive [{}] skipping download".format( as_info( archive ) ) ) return os.path.join( cache_root, archive ) return None
def GenerateHtmlTestReport( self, target, source, env ): self._base_uri = "" if self._auto_link_tests: self._base_uri = initialise_test_linking( env, link_style=self._link_style ) # Each source will result in one or more targets so we need to slice the targets to pick up # the gcov target (the first one) before we perform the zip iteration for s, t in itertools.izip( source, itertools.islice( target, 0, None, len(target)/len(source) ) ): #for s, t in itertools.izip( source, target ): test_suites = {} logger.trace( "source = [{}]".format( as_info(str(s)) ) ) logger.trace( "target = [{}]".format( as_info(str(t)) ) ) test_cases = self._read( s.abspath ) for test_case in test_cases: if not 'assertions_count' in test_case: if 'assertions' in test_case: test_case['assertions_count'] = test_case['assertions'] test_case['assertions_passed'] = test_case['passed'] test_case['assertions_failed'] = test_case['failed'] test_case['assertions_aborted'] = test_case['aborted'] else: test_case['assertions_count'] = 0 test_case['assertions_passed'] = 0 test_case['assertions_failed'] = 0 test_case['assertions_aborted'] = 0 self._add_to_test_suites( test_suites, test_case ) self._write( str(t), env, test_suites, sort_test_cases=self._sort_test_cases ) return None
def get_local_directory_for_non_url( self, location, sub_dir, branch_path, base ): if pip_is_archive_file( location ): self._local_folder = self.folder_name_from_path( location ) local_directory = os.path.join( base, self._local_folder ) local_dir_with_sub_dir = os.path.join( local_directory, sub_dir and sub_dir or "" ) if os.path.exists( local_dir_with_sub_dir ): try: os.rmdir( local_dir_with_sub_dir ) except: return local_directory self.extract( location, local_dir_with_sub_dir ) logger.debug( "(local archive) Location = [{}]".format( as_info( location ) ) ) logger.debug( "(local archive) Local folder = [{}]".format( as_info( self._local_folder ) ) ) else: local_directory = branch_path and os.path.join( location, branch_path ) or location self._local_folder = self.folder_name_from_path( location ) logger.debug( "(local file) Location = [{}]".format( as_info( location ) ) ) logger.debug( "(local file) Local folder = [{}]".format( as_info( self._local_folder ) ) ) return local_directory
def load( self ): self._show = self._env.get_option( 'show_conf' ) self._save = self._env.get_option( 'save_conf' ) self._remove = self._env.get_option( 'remove_settings' ) self._update = self._env.get_option( 'update_conf' ) self._clear = self._env.get_option( 'clear_conf' ) self._configure = self._save or self._remove or self._update self._clean = self._env.get_option( 'clean' ) self._unconfigure = ( self._save and self._clean ) or self._clear if self._unconfigure: self._configure = False logger.info( "{}".format( as_notice( "Clear configuration requested..." ) ) ) if os.path.exists( self._conf_path ): logger.info( "Removing configure file [{}]".format( as_info( self._conf_path ) ) ) os.remove( self._conf_path ) else: logger.info( "Configure file [{}] does not exist. Unconfigure not needed".format( as_info( self._conf_path ) ) ) return elif self._configure: print logger.info( "{}".format( as_notice( "Update configuration requested..." ) ) ) if not self._save: self._loaded_options = self._load_conf() else: self._loaded_options = {} self._env['configured_options'] = self._loaded_options self._env['default_options'].update( self._loaded_options )
def obtain_from_repository( self, location, full_url, local_dir_with_sub_dir, vc_type, vcs_backend ): rev_options = self.get_rev_options( vc_type, vcs_backend ) action = "Cloning" if vc_type == "svn": action = "Checking out" max_attempts = 2 attempt = 1 while attempt <= max_attempts: logger.info( "{} [{}] into [{}]{}".format( action, as_info( location ), as_info( local_dir_with_sub_dir ), attempt > 1 and "(attempt {})".format( str(attempt) ) or "" ) ) try: obtain( vcs_backend, local_dir_with_sub_dir, vcs_backend.url ) logger.debug( "Successfully retrieved [{}]".format( as_info( location ) ) ) break except pip_exceptions.PipError as error: attempt = attempt + 1 log_as = logger.warn if attempt > max_attempts: log_as = logger.error log_as( "Could not retrieve [{}] into [{}]{} due to error [{}]".format( as_info( location ), as_notice( local_dir_with_sub_dir ), ( rev_options and " to {}".format( as_notice( str(rev_options) ) ) or ""), as_error( str(error) ) ) ) if attempt > max_attempts: raise LocationException( str(error) )
def apply_patch_if_needed(home, version_string): patch_applied_path = os.path.join(home, "cuppa_test_patch_applied.txt") expected_diff_file = os.path.join( os.path.split(__file__)[0], "boost_test_patch_{}.diff".format(version_string)) available_diff_files = sorted(glob.glob( os.path.join(os.path.split(__file__)[0], "boost_test_patch_*.diff")), reverse=True) for diff_file in available_diff_files: if diff_file <= expected_diff_file: break logger.debug("Using diff file [{}]".format(as_info(diff_file))) if os.path.exists(patch_applied_path): logger.debug("[{}] already applied".format(as_info(diff_file))) return command = "patch --batch -p1 --input={}".format(diff_file) logger.info("Applying [{}] using [{}] in [{}]".format( as_info(diff_file), as_info(command), as_info(home))) if subprocess.call(shlex.split(command), cwd=home) != 0: logger.error("Could not apply [{}]".format(diff_file)) else: with open(patch_applied_path, "w") as patch_applied_file: pass
def _clear_config( self, conf_path ): if os.path.exists( conf_path ): logger.info( "Removing configure file [{}]".format( as_info( conf_path ) ) ) os.remove( conf_path ) else: logger.info( "Configure file [{}] does not exist. Unconfigure not needed".format( as_info( conf_path ) ) )
def _get_boost_location( cls, env, location, version, base, patched ): logger.debug( "Identify boost using location = [{}], version = [{}], base = [{}], patched = [{}]".format( as_info( str(location) ), as_info( str(version) ), as_info( str(base) ), as_info( str(patched) ) ) ) boost_home = None boost_location = None extra_sub_path = 'clean' if patched: extra_sub_path = 'patched' if location: location = cls.location_from_boost_version( location ) if not location: # use version as a fallback in case both at specified location = cls.location_from_boost_version( version ) boost_location = cuppa.location.Location( env, location, extra_sub_path=extra_sub_path, name_hint="boost" ) elif base: # Find boost locally if not os.path.isabs( base ): base = os.path.abspath( base ) if not version: boost_home = base elif version: search_list = [ os.path.join( base, 'boost', version, 'source' ), os.path.join( base, 'boost', 'boost_' + version ), os.path.join( base, 'boost', version ), os.path.join( base, 'boost_' + version ), ] def exists_in( locations ): for location in locations: home = cls._home_from_path( location ) if home: return home return None boost_home = exists_in( search_list ) if not boost_home: raise BoostException("Cannot construct Boost Object. Home for Version [{}] cannot be found. Seached in [{}]".format(version, str([l for l in search_list]))) else: raise BoostException("Cannot construct Boost Object. No Home or Version specified") logger.debug( "Using boost found at [{}]".format( as_info( boost_home ) ) ) boost_location = cuppa.location.Location( env, boost_home, extra_sub_path=extra_sub_path ) else: location = cls.location_from_boost_version( version ) boost_location = cuppa.location.Location( env, location, extra_sub_path=extra_sub_path ) if patched: cls.apply_patch_if_needed( boost_location.local() ) return boost_location
def add_to_env( cls, env, add_toolchain, add_to_supported ): for version in cls.supported_versions(): add_to_supported( version ) for version, gcc in cls.available_versions().iteritems(): logger.debug( "Adding toolchain [{}] reported as [{}] with cxx_version [g++{}] at [{}]" .format( as_info(version), as_info(gcc['version']), as_info(gcc['cxx_version']), as_notice(gcc['path']) ) ) add_toolchain( version, cls( version, gcc['cxx_version'], gcc['version'], gcc['path'] ) )
def add_to_env( cls, env, add_toolchain, add_to_supported ): for version in cls.supported_versions(): add_to_supported( version ) for version, gcc in cls.available_versions().iteritems(): logger.debug( "Adding toolchain [{}] reported as [{}] with cxx_version [g++{}] at [{}]" .format( as_info(version), as_info(gcc['version']), as_info(gcc['cxx_version']), as_notice(gcc['path']) ) ) add_toolchain( version, cls( version, gcc['cxx_version'], gcc['version'], gcc['path'] ) )
def _command_line_from_settings(self, settings): commands = [] for key, value in six.iteritems(settings): command = as_emphasised("--" + key) if value != True and value != False: if not isinstance(value, list): command += "=" + as_info(str(value)) else: command += "=" + as_info(",".join(value)) commands.append(command) commands.sort() return " ".join(commands)
def _command_line_from_settings( self, settings ): commands = [] for key, value in settings.iteritems(): command = as_emphasised( "--" + key ) if value != True and value != False: if not isinstance( value, list ): command += "=" + as_info( str(value) ) else: command += "=" + as_info( ",".join( value ) ) commands.append( command ) commands.sort() return " ".join( commands )
def get_local_directory_for_download_url(self, location, sub_dir, local_directory): logger.debug("[{}] is an archive download".format(as_info(location))) local_dir_with_sub_dir = os.path.join(local_directory, sub_dir and sub_dir or "") # First we check to see if we already downloaded and extracted this archive before if os.path.exists(local_dir_with_sub_dir): try: # If not empty this will fail os.rmdir(local_dir_with_sub_dir) except: # Not empty so we'll return this as the local_directory logger.debug("(already present) Location = [{}]".format( as_info(location))) logger.debug("(already present) Local folder = [{}]".format( as_info(str(self._local_folder)))) return local_directory if self._cuppa_env['dump'] or self._cuppa_env['clean']: return local_directory # If not we then check to see if we cached the download cached_archive = self.get_cached_archive(self._cuppa_env['cache_root'], self._local_folder) if cached_archive: logger.debug("Cached archive [{}] found for [{}]".format( as_info(cached_archive), as_info(location))) self.extract(cached_archive, local_dir_with_sub_dir) else: logger.info("Downloading [{}]...".format(as_info(location))) try: report_hook = None if logger.isEnabledFor(logging.INFO): report_hook = ReportDownloadProgress() filename, headers = urlretrieve(location, reporthook=report_hook) name, extension = os.path.splitext(filename) logger.info("[{}] successfully downloaded to [{}]".format( as_info(location), as_info(filename))) self.extract(filename, local_dir_with_sub_dir) if self._cuppa_env['cache_root']: cached_archive = os.path.join( self._cuppa_env['cache_root'], self._local_folder) logger.debug("Caching downloaded file as [{}]".format( as_info(cached_archive))) shutil.copyfile(filename, cached_archive) except ContentTooShortError as error: logger.error("Download of [{}] failed with error [{}]".format( as_error(location), as_error(str(error)))) raise LocationException(error) return local_directory
def log_exception( error, suppress=None ): from cuppa.log import logger from cuppa.colourise import as_info if not suppress: logger.fatal( "Cuppa terminated by exception [{}: {}]".format( as_info( error.__class__.__name__ ), as_info( str(error) ) ) ) if not logger.isEnabledFor( logging.EXCEPTION ): logger.warn( "Use {} (or above) to see the stack".format( as_info( "--verbosity=exception" ) ) ) logger.exception( traceback.format_exc() )
def __call__( self, blocks_transferred, block_size, total_size ): percent = 100.0 * float(blocks_transferred) * float(block_size) / float(total_size) if percent >= self._expected: if percent >= 100.0: sys.stdout.write( "={} Complete\n".format( as_info("|") ) ) sys.stdout.flush() else: sys.stdout.write( "=" ) if percent >= float(self._report_percent): sys.stdout.write( as_info( str(self._report_percent) + "%" ) ) self._report_percent += self._percent_step sys.stdout.flush() self._expected += self._step
def log_exception( error, suppress=None ): from cuppa.log import logger from cuppa.colourise import as_info if not suppress: logger.fatal( "Cuppa terminated by exception [{}: {}]".format( as_info( error.__class__.__name__ ), as_info( str(error) ) ) ) if not logger.isEnabledFor( logging.EXCEPTION ): logger.warn( "Use {} (or above) to see the stack".format( as_info( "--verbosity=exception" ) ) ) logger.exception( traceback.format_exc() )
def __call__( self, blocks_transferred, block_size, total_size ): percent = 100.0 * float(blocks_transferred) * float(block_size) / float(total_size) if percent >= self._expected: if percent >= 100.0: sys.stdout.write( "={} Complete\n".format( as_info("|") ) ) sys.stdout.flush() else: sys.stdout.write( "=" ) if percent >= float(self._report_percent): sys.stdout.write( as_info( str(self._report_percent) + "%" ) ) self._report_percent += self._percent_step sys.stdout.flush() self._expected += self._step
def __call_classmethod_for_classes_in_module(package, name, path, method, *args, **kwargs): try: filehandle, pathname, description = imp.find_module( name, path and [path] or None) try: try: qualified_name = package and package + "." + name or name module = sys.modules[qualified_name] except KeyError as error: module = imp.load_module(name, filehandle, pathname, description) for member_name in dir(module): member = getattr(module, member_name) if inspect.ismodule(member): if package: parent_package = package + "." + name else: parent_package = name __call_classmethod_for_classes_in_module( parent_package, member_name, pathname, method, *args, **kwargs) elif inspect.isclass(member): try: function = getattr(member, method) if callable(function): try: function(*args, **kwargs) except Exception as error: if logger.isEnabledFor(logging.EXCEPTION): logger.error( "[{}] in [{}] failed with error [{}]". format(as_info(str(method)), as_notice(str(member)), as_info(str(error)))) traceback.print_exc() raise error except AttributeError as ignore: pass finally: if filehandle: filehandle.close() except ImportError as error: pass
def remote_default_branch(cls, repository): command = "{git} ls-remote --symref {repository} HEAD".format( git=cls.binary(), repository=repository) result = cls.execute_command(command) if result: branch_pattern = r'ref[:]\s+refs/heads/(?P<default_branch>[^\s]+)\s+HEAD' match = re.search(branch_pattern, result) logger.trace( "When searching for default branch name for repoistory [{}] using regex [{}] the following match [{}] was returned" .format(as_info(repository), as_notice(branch_pattern), as_info(str(match)))) if match: return match.group('default_branch') return None
def _lazy_update_library_list( env, emitting, libraries, prebuilt_libraries, add_dependents, linktype, boost, stage_dir ): def build_with_library_name( library ): if library == 'log_setup': return 'log' elif library in { 'prg_exec_monitor', 'test_exec_monitor', 'unit_test_framework' }: return 'test' else: return library if add_dependents: if not emitting: libraries = set( build_with_library_name(l) for l in add_dependent_libraries( boost, linktype, libraries ) ) else: libraries = add_dependent_libraries( boost, linktype, libraries ) if not stage_dir in prebuilt_libraries: logger.trace( "Lazy update libraries list for [{}] to [{}]".format( as_info(stage_dir), colour_items(str(l) for l in libraries) ) ) prebuilt_libraries[ stage_dir ] = set( libraries ) else: logger.trace( "Lazy read libraries list for [{}]: libraries are [{}]".format( as_info(stage_dir), colour_items(str(l) for l in libraries) ) ) libraries = [ l for l in libraries if l not in prebuilt_libraries[ stage_dir ] ] prebuilt_libraries[ stage_dir ].update( libraries ) return libraries
def get_local_directory( self, location, sub_dir, branch_path, full_url ): logger.debug( "Determine local directory for [{location}] when {offline}".format( location=as_info(location), offline= self._offline and as_info_label("OFFLINE") or "online" ) ) local_directory = None base = self._cuppa_env['download_root'] if not os.path.isabs( base ): base = os.path.join( self._cuppa_env['working_dir'], base ) if location.startswith( 'file:' ): location = pip_download.url_to_path( location ) if not pip_is_url( location ): return self.get_local_directory_for_non_url( location, sub_dir, branch_path, base ) else: self._local_folder = self.folder_name_from_path( full_url ) local_directory = os.path.join( base, self._local_folder ) if full_url.scheme.startswith( 'http' ) and self.url_is_download_archive_url( full_url.path ): return self.get_local_directory_for_download_url( location, sub_dir, local_directory ) elif '+' in full_url.scheme: return self.get_local_directory_for_repository( location, sub_dir, full_url, local_directory ) return local_directory
def __init__( self ): self._step = 2.5 self._percent_step = 10 self._report_percent = self._percent_step self._expected = self._step sys.stdout.write( "cuppa: location: [info] Download progress {}".format( as_info("|") ) ) sys.stdout.flush()
def get_branch(cls, path): branch = None remote = None # In case we have a detached head we use this result = as_str( cls.execute_command( "{git} show -s --pretty=\%d HEAD".format(git=cls.binary()), path)) match = re.search(r'[(]HEAD[^,]*[,] (?P<branches>[^)]+)[)]', result) if match: branches = [b.strip() for b in match.group("branches").split(',')] logger.trace("Branches (using show) for [{}] are [{}]".format( as_notice(path), colour_items(branches))) if len(branches) == 1: # If this returns a tag: tag_name replace the ": " with "/" and then extract the tag_name # otherwise this will simply extract the branch_name as expected if not branches[0].startswith('tag:'): remote = branches[0] branch = branches[0].replace(': ', '/').split('/')[1] else: remote = branches[-2] branch = remote.split('/')[1] logger.trace("Branch (using show) for [{}] is [{}]".format( as_notice(path), as_info(branch))) else: logger.warn("No branch found from [{}]".format(result)) return branch, remote
def __init__( self ): self._step = 2.5 self._percent_step = 10 self._report_percent = self._percent_step self._expected = self._step sys.stdout.write( "cuppa: location: [info] Download progress {}".format( as_info("|") ) ) sys.stdout.flush()
def remove_common_top_directory_under( self, path ): dirs = os.listdir( path ) if not dirs: raise LocationException( "Uncompressed archive [{}] is empty".format( path ) ) top_dir = os.path.join( path, dirs[0] ) if len(dirs) == 1 and os.path.isdir( top_dir ): logger.debug( "Removing redundant top directory [{}] from [{}]".format( as_info( dirs[0] ), as_info( path ) ) ) # we have a single top-level directory move_dirs = os.listdir( top_dir ) for d in move_dirs: shutil.move( os.path.join( top_dir, d ), os.path.join( path, d ) ) shutil.rmtree( top_dir ) return True return False
def summary_name(cls, filename, toolchain_variant_dir, offset_dir, sconscript_name): name = os.path.splitext(filename)[0] if name.startswith(coverage_index_marker): name = name.replace(coverage_index_marker, "") logger.trace( "filename = [{}], toolchain_variant_dir = [{}], offset_dir = [{}], sconscript_name = [{}]" .format( as_info(filename), as_notice(toolchain_variant_dir), as_info(offset_dir), as_info(sconscript_name), )) return "./{}/{}".format(offset_dir, sconscript_name and sconscript_name or "*")
def remove_common_top_directory_under(cls, path): dirs = os.listdir(path) if not dirs: raise LocationException( "Uncompressed archive [{}] is empty".format(path)) top_dir = os.path.join(path, dirs[0]) if len(dirs) == 1 and os.path.isdir(top_dir): logger.debug( "Removing redundant top directory [{}] from [{}]".format( as_info(dirs[0]), as_info(path))) # we have a single top-level directory move_dirs = os.listdir(top_dir) for d in move_dirs: shutil.move(os.path.join(top_dir, d), os.path.join(path, d)) shutil.rmtree(top_dir) return True return False
def get_rev_options(self, vc_type, vcs_backend, local_remote=None): url, rev = get_url_rev(vcs_backend) logger.debug( "make_rev_options for [{}] at url [{}] with rev [{}]/[{}]".format( as_info(vc_type), as_notice(str(url)), as_notice(str(rev)), as_notice(str(local_remote)))) return make_rev_options(vc_type, vcs_backend, url, rev, local_remote)
def run(*args, **kwargs): from inspect import getframeinfo, stack caller = getframeinfo(stack()[1][0]) sconsctruct_path = caller.filename import traceback from cuppa.log import logger, initialise_logging from cuppa.colourise import as_info import logging initialise_logging() try: import cuppa.core cuppa.core.run(sconsctruct_path, *args, **kwargs) except Exception as error: logger.error("Cuppa terminated by exception [{}: {}]".format( as_info(error.__class__.__name__), as_info(str(error)))) if not logger.isEnabledFor(logging.EXCEPTION): logger.error("Use {} (or above) to see the stack".format( as_info("--verbosity=exception"))) logger.exception(traceback.format_exc())
def extract( cls, filename, target_dir ): os.makedirs( target_dir ) if tarfile.is_tarfile( filename ): logger.debug( "Extracting [{}] into [{}]".format( as_info( filename ), as_info( target_dir ) ) ) try: with tarfile.TarFile( filename ) as tf: tf.extractall( target_dir ) except tarfile.ReadError: command = "tar -xf {filename}".format( filename=filename ) if subprocess.call( shlex.split( command ), cwd=target_dir ) != 0: raise LocationException( "Could not untar downloaded file from [{}]".format( filename ) ) if zipfile.is_zipfile( filename ): logger.debug( "Extracting [{}] into [{}]".format( as_info( filename ), as_info( target_dir ) ) ) with zipfile.ZipFile( filename ) as zf: zf.extractall( target_dir ) while cls.remove_common_top_directory_under( target_dir ): pass
def extract( self, filename, target_dir ): os.makedirs( target_dir ) if tarfile.is_tarfile( filename ): logger.debug( "Extracting [{}] into [{}]".format( as_info( filename ), as_info( target_dir ) ) ) try: with tarfile.TarFile( filename ) as tf: tf.extractall( target_dir ) except tarfile.ReadError: command = "tar -xf {filename}".format( filename=filename ) if subprocess.call( shlex.split( command ), cwd=target_dir ) != 0: raise LocationException( "Could not untar downloaded file from [{}]".format( filename ) ) if zipfile.is_zipfile( filename ): logger.debug( "Extracting [{}] into [{}]".format( as_info( filename ), as_info( target_dir ) ) ) with zipfile.ZipFile( filename ) as zf: zf.extractall( target_dir ) while self.remove_common_top_directory_under( target_dir ): pass
def _read(cls, json_report_path, default={}): with open(json_report_path, "r") as report: try: report = json.load(report) return report except ValueError as error: logger.error( "Test Report [{}] does not contain valid JSON. Error [{}] encountered while parsing" .format(as_info(json_report_path), as_error(str(error)))) return default
def _determine_latest_boost_verion( offline ): current_release = "1.69.0" if not offline: try: boost_version_url = 'https://www.boost.org/users/download/' logger.info( "Checking current boost version from {}...".format( as_info( boost_version_url ) ) ) html = lxml.html.parse( urllib2.urlopen( boost_version_url ) ) current_release = html.xpath("/html/body/div[2]/div/div[1]/div/div/div[2]/h3[1]/span")[0].text current_release = str( re.search( r'(\d[.]\d+([.]\d+)?)', current_release ).group(1) ) logger.info( "Latest boost release detected as [{}]".format( as_info( current_release ) ) ) except Exception as e: logger.warn( "Cannot determine latest version of boost - [{}]. Assuming [{}].".format( str(e), current_release ) ) else: logger.info( "In offline mode. No version of boost specified so assuming [{}]".format( as_info( current_release ) ) ) return current_release
def get_branch(cls, path): branch = None remote = None # In case we have a detached head we use this result = cls.execute_command( "{git} show -s --pretty=\%d --decorate=full HEAD".format( git=cls.binary()), path) match = re.search(r'HEAD(?:(?:[^ ]* -> |[^,]*, )(?P<refs>[^)]+))?', result) if match: refs = [{ "ref": r.strip(), "type": "" } for r in match.group("refs").split(',')] logger.trace("Refs (using show) for [{}] are [{}]".format( as_notice(path), colour_items((r["ref"] for r in refs)))) if refs: for ref in refs: if ref["ref"].startswith("refs/heads/"): ref["ref"] = ref["ref"][len("refs/heads/"):] ref["type"] = "L" elif ref["ref"].startswith("refs/tags/"): ref["ref"] = ref["ref"][len("refs/tags/"):] ref["type"] = "T" elif ref["ref"].startswith("refs/remotes/"): ref["ref"] = ref["ref"][len("refs/remotes/"):] ref["type"] = "R" else: ref["type"] = "U" logger.trace( "Refs (after classification) for [{}] are [{}]".format( as_notice(path), colour_items( (":".join([r["type"], r["ref"]]) for r in refs)))) if refs[0]["type"] == "L": branch = refs[0]["ref"] elif refs[0]["type"] == "T": branch = refs[0]["ref"] elif refs[0]["type"] == "R": branch = refs[0]["ref"].split('/')[1] remote = next( (ref["ref"] for ref in refs if ref["type"] == "R"), None) logger.trace("Branch (using show) for [{}] is [{}]".format( as_notice(path), as_info(str(branch)))) else: logger.warn("No branch found from [{}]".format(result)) return branch, remote
def update_from_repository( self, location, full_url, local_dir_with_sub_dir, vc_type, vcs_backend ): url, repository, branch, remote, revision = self.get_info( location, local_dir_with_sub_dir, full_url, vc_type ) rev_options = self.get_rev_options( vc_type, vcs_backend, local_remote=remote ) version = self.ver_rev_summary( branch, revision, self._full_url.path )[0] logger.info( "Updating [{}] in [{}]{} at [{}]".format( as_info( location ), as_notice( local_dir_with_sub_dir ), ( rev_options and " on {}".format( as_notice( str(rev_options) ) ) or "" ), as_info( version ) ) ) try: update( vcs_backend, local_dir_with_sub_dir, rev_options ) logger.debug( "Successfully updated [{}]".format( as_info( location ) ) ) except pip_exceptions.PipError as error: logger.warn( "Could not update [{}] in [{}]{} due to error [{}]".format( as_warning( location ), as_warning( local_dir_with_sub_dir ), ( rev_options and " at {}".format( as_warning( str(rev_options) ) ) or "" ), as_warning( str(error) ) ) )
def get_rev_options( self, vc_type, vcs_backend, local_remote=None ): url, rev = get_url_rev( vcs_backend ) logger.debug( "make_rev_options for [{}] at url [{}] with rev [{}]/[{}]".format( as_info( vc_type ), as_notice( str(url) ), as_notice( str(rev) ), as_notice( str(local_remote) ) ) ) return make_rev_options( vc_type, vcs_backend, url, rev, local_remote )
def apply_patch_if_needed(home): patch_applied_path = os.path.join(home, "cuppa_test_patch_applied.txt") diff_file = "boost_test_patch.diff" if os.path.exists(patch_applied_path): logger.debug("[{}] already applied".format(as_info(diff_file))) return diff_path = os.path.join(os.path.split(__file__)[0], "boost", diff_file) command = "patch --batch -p1 --input={}".format(diff_path) logger.info("Applying [{}] using [{}] in [{}]".format( as_info(diff_file), as_info(command), as_info(home))) if subprocess.call(shlex.split(command), cwd=home) != 0: logger.error("Could not apply [{}]".format(diff_file)) with open(patch_applied_path, "w") as patch_applied_file: pass
def _read( cls, json_report_path, default={} ): with open( json_report_path, "r" ) as report: try: report = json.load( report ) return report except ValueError as error: logger.error( "Test Report [{}] does not contain valid JSON. Error [{}] encountered while parsing".format( as_info( json_report_path ), as_error( str(error) ) ) ) return default
def _get_location( cls, env ): import SCons.Errors location_id = cls.location_id( env ) if not location_id: return None if location_id not in cls._cached_locations: location = location_id[0] develop = location_id[1] branch = location_id[2] use_develop = location_id[3] try: cls._cached_locations[location_id] = cuppa.location.Location( env, location, develop=develop, branch=branch, extra_sub_path=cls._extra_sub_path ) logger.debug( "Adding location [{}]({}) to cached locations".format( as_notice( cls._name.title() ), as_notice( str(location_id) ) ) ) except cuppa.location.LocationException as error: logger.error( "Could not get location for [{}] at [{}] (and develop [{}], use=[{}]) with branch [{}] and extra sub path [{}]. Failed with error [{}]" .format( as_notice( cls._name.title() ), as_info( str(location) ), as_info( str(develop) ), as_notice( str(use_develop and True or False) ), as_notice( str(branch) ), as_notice( str(cls._extra_sub_path) ), as_error( str(error) ) ) ) raise SCons.Errors.StopError( error ) else: logger.debug( "Loading location [{}]({}) from cached locations".format( as_notice( cls._name.title() ), as_notice( str(location_id) ) ) ) return cls._cached_locations[location_id]
def determine_latest_boost_verion(): current_release = "1.61.0" try: html = lxml.html.parse('http://www.boost.org/users/download/') current_release = html.xpath("/html/body/div[2]/div/div[1]/div/div/div[2]/h3[1]/span")[0].text current_release = str( re.search( r'(\d[.]\d+([.]\d+)?)', current_release ).group(1) ) logger.debug( "latest boost release detected as [{}]".format( as_info( current_release ) ) ) except Exception as e: logger.warn( "cannot determine latest version of boost - [{}]. Assuming [{}].".format( str(e), current_release ) ) return current_release
def add_to_env( cls, env, add_toolchain, add_to_supported ): stdlib = None try: stdlib = env.get_option( 'clang-stdlib' ) suppress_debug_for_auto = env.get_option( 'clang-disable-debug-for-auto' ) except: pass for version in cls.supported_versions(): add_to_supported( version ) for version, clang in cls.available_versions().iteritems(): logger.debug( "Adding toolchain [{}] reported as [{}] with cxx_version [clang++{}] at [{}]".format( as_info(version), as_info(clang['version']), as_info(clang['cxx_version']), as_notice(clang['path']) ) ) add_toolchain( version, cls( version, clang['cxx_version'], clang['version'], clang['path'], stdlib, suppress_debug_for_auto ) )
def determine_latest_boost_verion( env ): current_release = "1.58.0" try: html = lxml.html.parse('http://www.boost.org/users/download/') current_release = html.xpath("/html/body/div[2]/div/div[1]/div/div/div[2]/h3[1]/span")[0].text current_release = str( re.search( r'(\d[.]\d+([.]\d+)?)', current_release ).group(1) ) print "cuppa: boost: latest boost release detected as [{}]".format( as_info( env, current_release ) ) except Exception as e: print as_warning( env, "cuppa: boost: warning: cannot determine latest version of boost - [{}]. Assuming [{}].".format( str(e), current_release ) ) return current_release
def apply_patch_if_needed( home, version_string ): patch_applied_path = os.path.join( home, "cuppa_test_patch_applied.txt" ) expected_diff_file = os.path.join( os.path.split( __file__ )[0], "boost_test_patch_{}.diff".format( version_string ) ) available_diff_files = sorted( glob.glob( os.path.join( os.path.split( __file__ )[0], "boost_test_patch_*.diff" ) ), reverse=True ) for diff_file in available_diff_files: if diff_file <= expected_diff_file: break logger.debug( "Using diff file [{}]".format( as_info( diff_file ) ) ) if os.path.exists( patch_applied_path ): logger.debug( "[{}] already applied".format( as_info( diff_file ) ) ) return command = "patch --batch -p1 --input={}".format( diff_file ) logger.info( "Applying [{}] using [{}] in [{}]".format( as_info( diff_file ), as_info( command ), as_info( home ) ) ) if subprocess.call( shlex.split( command ), cwd=home ) != 0: logger.error( "Could not apply [{}]".format( diff_file ) ) else: with open( patch_applied_path, "w" ) as patch_applied_file: pass
def apply_patch_if_needed( cls, home ): patch_applied_path = os.path.join( home, "cuppa_test_patch_applied.txt" ) diff_file = "boost_test_patch.diff" if os.path.exists( patch_applied_path ): logger.debug( "[{}] already applied".format( as_info( diff_file ) ) ) return diff_path = os.path.join( os.path.split( __file__ )[0], "boost", diff_file ) command = "patch --batch -p1 --input={}".format( diff_path ) logger.info( "Applying [{}] using [{}] in [{}]".format( as_info( diff_file ), as_info( command ), as_info( home ) ) ) if subprocess.call( shlex.split( command ), cwd=home ) != 0: logger.error( "Could not apply [{}]".format( diff_file ) ) with open( patch_applied_path, "w" ) as patch_applied_file: pass
def _set_qt4_dir( self, env ): command = "pkg-config --cflags QtCore" try: cflags = subprocess.check_output( shlex.split( command ), stderr=subprocess.STDOUT ).strip() if cflags: flags = env.ParseFlags( cflags ) if 'CPPPATH' in flags: shortest_path = flags['CPPPATH'][0] for include in flags['CPPPATH']: if len(include) < len(shortest_path): shortest_path = include env['QT4DIR'] = shortest_path logger.debug( "Q4DIR detected as [{}]".format( as_info( env['QT4DIR'] ) ) ) except: logger.debug( "In _set_qt4_dir() failed to execute [{}]".format( command ) )
def apply_patch_if_needed( self, env, home ): patch_applied_path = os.path.join( home, "cuppa_test_patch_applied.txt" ) diff_file = "boost_test_patch.diff" if os.path.exists( patch_applied_path ): print "cuppa: boost: [{}] already applied".format( as_info( env, diff_file ) ) return diff_path = os.path.join( os.path.split( __file__ )[0], "boost", diff_file ) command = "patch --batch -p1 --input={}".format( diff_path ) print "cuppa: boost: info: Applying [{}] using [{}] in [{}]".format( as_info( env, diff_file ), as_info( env, command ), as_info( env, home ) ) if subprocess.call( shlex.split( command ), cwd=home ) != 0: print as_error( env, "cuppa: boost: error: Could not apply [{}]".format( diff_file ) ) with open( patch_applied_path, "w" ) as patch_applied_file: pass
def check_current_version( offline ): installed_version = get_version() logger.info( "cuppa: version {}".format( as_info( installed_version ) ) ) if not offline: try: pypi = xmlrpclib.ServerProxy('http://pypi.python.org/pypi') latest_available = pypi.package_releases('cuppa')[0] if parse_version( installed_version ) < parse_version( latest_available ): logger.warn( "Newer version [{}] available. Upgrade using \"{}\"\n".format( as_warning( latest_available ), as_emphasised( "pip install -U cuppa" ) ) ) except: pass
def _set_qt5_dir( self, env ): command = "pkg-config --cflags Qt5Core" try: includes = subprocess.check_output( shlex.split( command ), stderr=subprocess.STDOUT ).strip().split() if includes: shortest_path = includes[0] for include in includes: if len(include) < len(shortest_path): shortest_path = include qt5dir = env.ParseFlags( shortest_path )['CPPPATH'][0] env['QT5DIR'] = qt5dir print "cuppa: qt5: Q5DIR detected as [{}]".format( as_info( env, env['QT5DIR'] ) ) except: #TODO: Warning? pass
def lazy_update_library_list( env, emitting, libraries, built_libraries, add_dependents, linktype, boost, stage_dir ): if add_dependents: if not emitting: libraries = set( build_with_library_name(l) for l in add_dependent_libraries( boost, linktype, libraries ) ) else: libraries = add_dependent_libraries( boost, linktype, libraries ) if not stage_dir in built_libraries: logger.trace( "Lazy update libraries list for [{}] to [{}]".format( as_info(stage_dir), colour_items(str(l) for l in libraries) ) ) built_libraries[ stage_dir ] = set( libraries ) else: logger.trace( "Lazy read libraries list for [{}]: libraries are [{}]".format( as_info(stage_dir), colour_items(str(l) for l in libraries) ) ) libraries = [ l for l in libraries if l not in built_libraries[ stage_dir ] ] return libraries