def __init__( self, env ):
self._flags = {}
self._flags['INCPATH'] = [ os.path.join( self._location.local(), "include" ) ]
pg_config = "pg_config"
if platform.system() == "Windows":
pg_config = pg_config + ".exe"
if not cuppa.output_processor.command_available( pg_config ):
# try to find the Postgresql install
program_files = os.environ.get( "ProgramW6432" )
postgresql_base = os.path.join( program_files, "PostgreSQL" )
if os.path.exists( postgresql_base ):
paths = glob.glob( postgresql_base + '\\*' )
if len(paths):
paths.sort()
latest = paths[-1]
pg_config = '\"' + os.path.join( latest, "bin", pg_config ) + '\"'
if cuppa.output_processor.command_available( pg_config ):
command = "{pg_config} --includedir".format( pg_config = pg_config )
libpq_include = subprocess.check_output( shlex.split( command ), stderr=subprocess.STDOUT ).strip()
self._flags['INCPATH'].append( libpq_include )
command = "{pg_config} --libdir".format( pg_config = pg_config )
libpq_libpath = subprocess.check_output( shlex.split( command ), stderr=subprocess.STDOUT ).strip()
self._flags['LIBPATH'] = [ libpq_libpath ]
else:
logger.warn( "postgresql: pg_config not available so cannot determine LIBPATH for postgres libraries" )
self._flags['LIBPATH'] = []
self._flags['DYNAMICLIBS'] = [ 'pq' ]
self._src_path = os.path.join( self._location.local(), "src" )
env.AddMethod( QuincePostgresqlLibraryMethod( self._location.local(), self._src_path ), "QuincePostgresqlLibrary" )
def info( path ):
if not path:
raise SubversionException("No working copy path specified for calling svnversion with.")
url = None
repository = None
branch = None
revision = None
try:
command = "svn info {}".format( path )
svn_info = subprocess.check_output( shlex.split( command ), stderr=subprocess.STDOUT )
url = re.search( r'URL: ([^\s]+)', svn_info ).expand(r'\1')
repository = re.search( r'Repository Root: ([^\s]+)', svn_info ).expand(r'\1')
branch = re.search( r'Relative URL: \^/([^\s]+)', svn_info ).expand(r'\1')
revision = re.search( r'Revision: (\d+)', svn_info ).expand(r'\1')
except subprocess.CalledProcessError:
raise SubversionException("Not a Subversion working copy")
try:
command = "svnversion -n {}".format( path )
revision = subprocess.check_output( shlex.split( command ), stderr=subprocess.STDOUT )
except subprocess.CalledProcessError:
pass
except OSError:
logger.warn( "The {} binary is not available. Consider installing it.".format( as_warning("svnversion") ) )
return url, repository, branch, revision
def __call__(self, env, libraries):
if not self._add_dependents:
logger.warn(
"BoostSharedLibrary() is deprecated, use BoostSharedLibs() or BoostSharedLib() instead"
)
libraries = Flatten([libraries])
if not 'boost' in env['BUILD_WITH']:
env.BuildWith('boost')
Boost = env['dependencies']['boost'](env)
for library in libraries:
if library.startswith('log'):
env.AppendUnique(CPPDEFINES='BOOST_LOG_DYN_LINK')
elif library == 'chrono':
env.AppendUnique(CPPDEFINES='BOOST_CHRONO_DYN_LINK')
elif library == 'filesystem':
env.AppendUnique(CPPDEFINES='BOOST_FILESYSTEM_DYN_LINK')
elif library == 'date_time':
env.AppendUnique(CPPDEFINES='BOOST_DATE_TIME_DYN_LINK')
elif library == 'regex':
env.AppendUnique(CPPDEFINES='BOOST_REGEX_DYN_LINK')
elif library == 'system':
env.AppendUnique(CPPDEFINES='BOOST_SYSTEM_DYN_LINK')
library = BoostLibraryBuilder(Boost,
add_dependents=self._add_dependents,
verbose_build=self._verbose_build,
verbose_config=self._verbose_config)(
env, None, None, libraries, 'shared')
if self._build_always:
return AlwaysBuild(library)
else:
return library
def get_branch(cls, path):
branch = None
remote = None
# In case we have a detached head we use this
result = as_str(
cls.execute_command(
"{git} show -s --pretty=\%d HEAD".format(git=cls.binary()),
path))
match = re.search(r'[(]HEAD[^,]*[,] (?P<branches>[^)]+)[)]', result)
if match:
branches = [b.strip() for b in match.group("branches").split(',')]
logger.trace("Branches (using show) for [{}] are [{}]".format(
as_notice(path), colour_items(branches)))
if len(branches) == 1:
# If this returns a tag: tag_name replace the ": " with "/" and then extract the tag_name
# otherwise this will simply extract the branch_name as expected
if not branches[0].startswith('tag:'):
remote = branches[0]
branch = branches[0].replace(': ', '/').split('/')[1]
else:
remote = branches[-2]
branch = remote.split('/')[1]
logger.trace("Branch (using show) for [{}] is [{}]".format(
as_notice(path), as_info(branch)))
else:
logger.warn("No branch found from [{}]".format(result))
return branch, remote
def retrieve_tool( cls, env ):
url = "hg+https://bitbucket.org/dirkbaechle/scons_qt4"
try:
return cuppa.location.Location( env, url, extra_sub_path = "qt4" )
except cuppa.location.LocationException:
logger.warn( "Could not retrieve scons_qt4 from [{}]".format( url ) )
return None
def _determine_latest_boost_verion(offline):
current_release = "1.71.0"
if not offline:
try:
boost_version_url = 'https://www.boost.org/users/download/'
logger.info("Checking current boost version from {}...".format(
as_info(boost_version_url)))
html = lxml.html.parse(urlopen(boost_version_url))
current_release = html.xpath(
"/html/body/div[2]/div/div[1]/div/div/div[2]/h3[1]/span"
)[0].text
current_release = str(
re.search(r'(\d[.]\d+([.]\d+)?)', current_release).group(1))
logger.info("Latest boost release detected as [{}]".format(
as_info(current_release)))
except Exception as e:
logger.warn(
"Cannot determine latest version of boost - [{}]. Assuming [{}]."
.format(str(e), current_release))
else:
logger.info(
"In offline mode. No version of boost specified so assuming [{}]".
format(as_info(current_release)))
return current_release
def get_branch(cls, path):
branch = None
try:
result = cls.execute_command(
"{git} symbolic-ref HEAD".format(git=cls.binary()), path)
branch = result.replace("refs/heads/", "").strip()
logger.trace("Branch (using symbolic-ref) for [{}] is [{}]".format(
as_notice(path), as_info(branch)))
return branch
except cls.Error:
pass
# In case we have a detached head we can fallback to this
result = cls.execute_command(
"{git} show -s --pretty=\%d HEAD".format(git=cls.binary()), path)
match = re.search(r'[(]HEAD[^,]*[,] (?P<branches>[^)]+)[)]', result)
if match:
branches = [b.strip() for b in match.group("branches").split(',')]
logger.trace("Branches (using show) for [{}] are [{}]".format(
as_notice(path), colour_items(branches)))
if len(branches) == 1:
# If this returns a tag: tag_name replace the ": " with "/" and then extract the tag_name
# otherwise this will simply extract the branch_name as expected
branch = branches[0].replace(': ', '/').split('/')[1]
else:
branch = branches[-2].split('/')[1]
logger.trace("Branch (using show) for [{}] is [{}]".format(
as_notice(path), as_info(branch)))
else:
logger.warn("No branch found from [{}]".format(result))
return branch
def info(path):
if not path:
raise SubversionException(
"No working copy path specified for calling svnversion with.")
url = None
repository = None
branch = None
revision = None
try:
command = "svn info {}".format(path)
svn_info = subprocess.check_output(shlex.split(command),
stderr=subprocess.STDOUT)
url = re.search(r'URL: ([^\s]+)', svn_info).expand(r'\1')
repository = re.search(r'Repository Root: ([^\s]+)',
svn_info).expand(r'\1')
branch = re.search(r'Relative URL: \^/([^\s]+)',
svn_info).expand(r'\1')
revision = re.search(r'Revision: (\d+)', svn_info).expand(r'\1')
except subprocess.CalledProcessError:
raise SubversionException("Not a Subversion working copy")
try:
command = "svnversion -n {}".format(path)
revision = subprocess.check_output(shlex.split(command),
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
pass
except OSError:
logger.warn(
"The {} binary is not available. Consider installing it.".format(
as_warning("svnversion")))
return url, repository, branch, revision
def __call__( self, env, libraries ):
if not self._add_dependents:
logger.warn( "BoostSharedLibrary() is deprecated, use BoostSharedLibs() or BoostSharedLib() instead" )
libraries = Flatten( [ libraries ] )
if not 'boost' in env['BUILD_WITH']:
env.BuildWith( 'boost' )
Boost = env['dependencies']['boost']( env )
for library in libraries:
if library.startswith('log'):
env.AppendUnique( CPPDEFINES = 'BOOST_LOG_DYN_LINK' )
elif library == 'chrono':
env.AppendUnique( CPPDEFINES = 'BOOST_CHRONO_DYN_LINK' )
elif library == 'filesystem':
env.AppendUnique( CPPDEFINES = 'BOOST_FILESYSTEM_DYN_LINK' )
elif library == 'date_time':
env.AppendUnique( CPPDEFINES = 'BOOST_DATE_TIME_DYN_LINK' )
elif library == 'regex':
env.AppendUnique( CPPDEFINES = 'BOOST_REGEX_DYN_LINK' )
elif library == 'system':
env.AppendUnique( CPPDEFINES = 'BOOST_SYSTEM_DYN_LINK' )
library = BoostLibraryBuilder(
Boost,
add_dependents = self._add_dependents,
verbose_build = self._verbose_build,
verbose_config = self._verbose_config )( env, None, None, libraries, 'shared' )
if self._build_always:
return AlwaysBuild( library )
else:
return library
def build_library_from_source( self, env, sources=None, library_name=None, linktype=None ):
from SCons.Script import Flatten
if not self._source_path and not sources:
logger.warn( "Attempting to build library when source path is None" )
return None
if not library_name:
library_name = self._name
if not linktype:
linktype = self._linktype
variant_key = env['tool_variant_dir']
prebuilt_objects = self.lazy_create_node( variant_key, self._prebuilt_objects )
prebuilt_libraries = self.lazy_create_node( variant_key, self._prebuilt_libraries )
local_dir = self._location.local()
local_folder = self._location.local_folder()
build_dir = os.path.abspath( os.path.join( env['abs_build_root'], local_folder, env['tool_variant_working_dir'] ) )
final_dir = os.path.abspath( os.path.normpath( os.path.join( build_dir, env['final_dir'] ) ) )
logger.debug( "build_dir for [{}] = [{}]".format( as_info(self._name), build_dir ) )
logger.debug( "final_dir for [{}] = [{}]".format( as_info(self._name), final_dir ) )
obj_suffix = env['OBJSUFFIX']
obj_builder = env.StaticObject
lib_builder = env.BuildStaticLib
if linktype == "shared":
obj_suffix = env['SHOBJSUFFIX']
obj_builder = env.SharedObject
lib_builder = env.BuildSharedLib
if not sources:
sources = env.RecursiveGlob( "*.cpp", start=self._source_path, exclude_dirs=[ env['build_dir'] ] )
sources.extend( env.RecursiveGlob( "*.cc", start=self._source_path, exclude_dirs=[ env['build_dir'] ] ) )
sources.extend( env.RecursiveGlob( "*.c", start=self._source_path, exclude_dirs=[ env['build_dir'] ] ) )
objects = []
for source in Flatten( [sources] ):
rel_path = os.path.relpath( str(source), local_dir )
rel_obj_path = os.path.splitext( rel_path )[0] + obj_suffix
obj_path = os.path.join( build_dir, rel_obj_path )
if not rel_obj_path in prebuilt_objects:
prebuilt_objects[rel_obj_path] = obj_builder( obj_path, source )
objects.append( prebuilt_objects[rel_obj_path] )
if not linktype in prebuilt_libraries:
library = lib_builder( library_name, objects, final_dir = final_dir )
if linktype == "shared":
library = env.Install( env['abs_final_dir'], library )
prebuilt_libraries[linktype] = library
else:
logger.trace( "using existing library = [{}]".format( str(prebuilt_libraries[linktype]) ) )
return prebuilt_libraries[linktype]
def get_branch(cls, path):
branch = None
remote = None
# In case we have a detached head we use this
result = cls.execute_command(
"{git} show -s --pretty=\%d --decorate=full HEAD".format(
git=cls.binary()), path)
match = re.search(r'HEAD(?:(?:[^ ]* -> |[^,]*, )(?P<refs>[^)]+))?',
result)
if match:
refs = [{
"ref": r.strip(),
"type": ""
} for r in match.group("refs").split(',')]
logger.trace("Refs (using show) for [{}] are [{}]".format(
as_notice(path), colour_items((r["ref"] for r in refs))))
if refs:
for ref in refs:
if ref["ref"].startswith("refs/heads/"):
ref["ref"] = ref["ref"][len("refs/heads/"):]
ref["type"] = "L"
elif ref["ref"].startswith("refs/tags/"):
ref["ref"] = ref["ref"][len("refs/tags/"):]
ref["type"] = "T"
elif ref["ref"].startswith("refs/remotes/"):
ref["ref"] = ref["ref"][len("refs/remotes/"):]
ref["type"] = "R"
else:
ref["type"] = "U"
logger.trace(
"Refs (after classification) for [{}] are [{}]".format(
as_notice(path),
colour_items(
(":".join([r["type"], r["ref"]]) for r in refs))))
if refs[0]["type"] == "L":
branch = refs[0]["ref"]
elif refs[0]["type"] == "T":
branch = refs[0]["ref"]
elif refs[0]["type"] == "R":
branch = refs[0]["ref"].split('/')[1]
remote = next(
(ref["ref"] for ref in refs if ref["type"] == "R"), None)
logger.trace("Branch (using show) for [{}] is [{}]".format(
as_notice(path), as_info(str(branch))))
else:
logger.warn("No branch found from [{}]".format(result))
return branch, remote
def coverage_tool(cls, reported_version):
gcov = "gcov"
versioned_gcov = "{gcov}-{version}".format(
gcov=gcov, version=str(reported_version['major']))
if cuppa.build_platform.where_is(versioned_gcov):
return versioned_gcov
if cuppa.build_platform.where_is(gcov):
version = cls.version_from_command(gcov, "gcc")
if version == reported_version:
return gcov
logger.warn(
"Coverage requested for current toolchain but none is available")
return None
def coverage_tool( cls, cxx_version ):
llvm_cov = "llvm-cov"
versioned_llvm_cov = None
if cxx_version:
versioned_llvm_cov = "{llvm_cov}-{version}".format( llvm_cov=llvm_cov, version=cxx_version[0] )
if cuppa.build_platform.where_is( versioned_llvm_cov ):
return versioned_llvm_cov + " gcov"
if cuppa.build_platform.where_is( llvm_cov ):
version = cls.llvm_version_from( llvm_cov )
if version == cxx_version:
return llvm_cov + " gcov"
logger.warn( "Coverage requested for current toolchain but none is available" )
return None
def log_exception( error, suppress=None ):
from cuppa.log import logger
from cuppa.colourise import as_info
if not suppress:
logger.fatal( "Cuppa terminated by exception [{}: {}]".format(
as_info( error.__class__.__name__ ),
as_info( str(error) )
) )
if not logger.isEnabledFor( logging.EXCEPTION ):
logger.warn( "Use {} (or above) to see the stack".format( as_info( "--verbosity=exception" ) ) )
logger.exception( traceback.format_exc() )
def log_exception( error, suppress=None ):
from cuppa.log import logger
from cuppa.colourise import as_info
if not suppress:
logger.fatal( "Cuppa terminated by exception [{}: {}]".format(
as_info( error.__class__.__name__ ),
as_info( str(error) )
) )
if not logger.isEnabledFor( logging.EXCEPTION ):
logger.warn( "Use {} (or above) to see the stack".format( as_info( "--verbosity=exception" ) ) )
logger.exception( traceback.format_exc() )
def check_current_version():
installed_version = get_version()
logger.info("cuppa: version {}".format(as_info(installed_version)))
try:
pypi = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
latest_available = pypi.package_releases('cuppa')[0]
if parse_version(installed_version) < parse_version(latest_available):
logger.warn(
"Newer version [{}] available. Upgrade using \"{}\"\n".format(
as_warning(latest_available),
as_emphasised("pip install -U cuppa")))
except:
pass
def determine_latest_boost_verion():
current_release = "1.61.0"
try:
html = lxml.html.parse('http://www.boost.org/users/download/')
current_release = html.xpath("/html/body/div[2]/div/div[1]/div/div/div[2]/h3[1]/span")[0].text
current_release = str( re.search( r'(\d[.]\d+([.]\d+)?)', current_release ).group(1) )
logger.debug( "latest boost release detected as [{}]".format( as_info( current_release ) ) )
except Exception as e:
logger.warn( "cannot determine latest version of boost - [{}]. Assuming [{}].".format( str(e), current_release ) )
return current_release
def coverage_tool(cls, cxx_version):
llvm_cov = "llvm-cov"
versioned_llvm_cov = None
if cxx_version:
versioned_llvm_cov = "{llvm_cov}-{version}".format(
llvm_cov=llvm_cov, version=cxx_version[0])
if cuppa.build_platform.where_is(versioned_llvm_cov):
return versioned_llvm_cov + " gcov"
if cuppa.build_platform.where_is(llvm_cov):
version = cls.llvm_version_from(llvm_cov)
if version == cxx_version:
return llvm_cov + " gcov"
logger.warn(
"Coverage requested for current toolchain but none is available")
return None
def _libc_version( self, machine, system ):
libc_file = "libc.so.6"
libc_path = "/lib/" + libc_file
if not path.exists( libc_path ):
multiarch_lib_path = '-'.join( [ machine, system.lower(), 'gnu' ] )
libc_path = "/lib/" + multiarch_lib_path + "/" + libc_file
try:
libc_version = Popen([libc_path], stdout=PIPE).communicate()[0]
return 'libc' + search( r'^GNU C Library [()a-zA-Z ]*([0-9][.0-9]+)', as_str( libc_version ), MULTILINE ).expand(r'\1').replace('.','')
except:
logger.warn( "Could not detect the version of libc installed. You might be missing some development libraries!" )
return None
def check_current_version( offline ):
installed_version = get_version()
logger.info( "cuppa: version {}".format( as_info( installed_version ) ) )
if not offline:
try:
pypi = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
latest_available = pypi.package_releases('cuppa')[0]
if parse_version( installed_version ) < parse_version( latest_available ):
logger.warn( "Newer version [{}] available. Upgrade using \"{}\"\n".format(
as_warning( latest_available ),
as_emphasised( "pip install -U cuppa" )
) )
except:
pass
def __init__(self, env):
self._flags = {}
self._flags['INCPATH'] = [
os.path.join(self._location.local(), "include")
]
pg_config = "pg_config"
if platform.system() == "Windows":
pg_config = pg_config + ".exe"
if not cuppa.output_processor.command_available(pg_config):
# try to find the Postgresql install
program_files = os.environ.get("ProgramW6432")
postgresql_base = os.path.join(program_files, "PostgreSQL")
if os.path.exists(postgresql_base):
paths = glob.glob(postgresql_base + '\\*')
if len(paths):
paths.sort()
latest = paths[-1]
pg_config = '\"' + os.path.join(
latest, "bin", pg_config) + '\"'
if cuppa.output_processor.command_available(pg_config):
command = "{pg_config} --includedir".format(pg_config=pg_config)
libpq_include = subprocess.check_output(
shlex.split(command), stderr=subprocess.STDOUT).strip()
self._flags['INCPATH'].append(libpq_include)
command = "{pg_config} --libdir".format(pg_config=pg_config)
libpq_libpath = subprocess.check_output(
shlex.split(command), stderr=subprocess.STDOUT).strip()
self._flags['LIBPATH'] = [libpq_libpath]
else:
logger.warn(
"postgresql: pg_config not available so cannot determine LIBPATH for postgres libraries"
)
self._flags['LIBPATH'] = []
self._flags['DYNAMICLIBS'] = ['pq']
self._src_path = os.path.join(self._location.local(), "src")
env.AddMethod(
QuincePostgresqlLibraryMethod(self._location.local(),
self._src_path),
"QuincePostgresqlLibrary")
def _determine_latest_boost_verion( offline ):
current_release = "1.69.0"
if not offline:
try:
boost_version_url = 'https://www.boost.org/users/download/'
logger.info( "Checking current boost version from {}...".format( as_info( boost_version_url ) ) )
html = lxml.html.parse( urllib2.urlopen( boost_version_url ) )
current_release = html.xpath("/html/body/div[2]/div/div[1]/div/div/div[2]/h3[1]/span")[0].text
current_release = str( re.search( r'(\d[.]\d+([.]\d+)?)', current_release ).group(1) )
logger.info( "Latest boost release detected as [{}]".format( as_info( current_release ) ) )
except Exception as e:
logger.warn( "Cannot determine latest version of boost - [{}]. Assuming [{}].".format( str(e), current_release ) )
else:
logger.info( "In offline mode. No version of boost specified so assuming [{}]".format( as_info( current_release ) ) )
return current_release
def update_from_repository( self, location, full_url, local_dir_with_sub_dir, vc_type, vcs_backend ):
url, repository, branch, remote, revision = self.get_info( location, local_dir_with_sub_dir, full_url, vc_type )
rev_options = self.get_rev_options( vc_type, vcs_backend, local_remote=remote )
version = self.ver_rev_summary( branch, revision, self._full_url.path )[0]
logger.info( "Updating [{}] in [{}]{} at [{}]".format(
as_info( location ),
as_notice( local_dir_with_sub_dir ),
( rev_options and " on {}".format( as_notice( str(rev_options) ) ) or "" ),
as_info( version )
) )
try:
update( vcs_backend, local_dir_with_sub_dir, rev_options )
logger.debug( "Successfully updated [{}]".format( as_info( location ) ) )
except pip_exceptions.PipError as error:
logger.warn( "Could not update [{}] in [{}]{} due to error [{}]".format(
as_warning( location ),
as_warning( local_dir_with_sub_dir ),
( rev_options and " at {}".format( as_warning( str(rev_options) ) ) or "" ),
as_warning( str(error) )
) )
def __call__(self, option, opt, value, parser):
toolchains = set()
requested = value.split(',')
for toolchain in requested:
supported = fnmatch.filter( self._supported, toolchain )
if not supported:
logger.warn( "Requested toolchain [{}] does not match any supported, skipping".format( as_info(toolchain) ) )
else:
available = fnmatch.filter( self._available, toolchain )
if not available:
logger.warn( "Requested toolchain [{}] does not match any available, skipping".format( as_info(toolchain) ) )
else:
toolchains.update( available )
if not toolchains:
logger.error( "None of the requested toolchains are available" )
parser.values.toolchains = list(toolchains)
def __call__( self, env, libraries ):
if not self._add_dependents:
logger.warn( "BoostStaticLibrary() is deprecated, use BoostStaticLibs() or BoostStaticLib() instead" )
libraries = Flatten( [ libraries ] )
if not 'boost' in env['BUILD_WITH']:
env.BuildWith( 'boost' )
Boost = env['dependencies']['boost']( env )
logger.trace( "Build static libraries [{}]".format( colour_items( libraries ) ) )
library = BoostLibraryBuilder(
Boost,
add_dependents = self._add_dependents,
verbose_build = self._verbose_build,
verbose_config = self._verbose_config )( env, None, None, libraries, 'static' )
if self._build_always:
return AlwaysBuild( library )
else:
return library
def get_branch( cls, path ):
branch = None
remote = None
# In case we have a detached head we use this
result = cls.execute_command( "{git} show -s --pretty=\%d HEAD".format( git=cls.binary() ), path )
match = re.search( r'[(]HEAD[^,]*[,] (?P<branches>[^)]+)[)]', result )
if match:
branches = [ b.strip() for b in match.group("branches").split(',') ]
logger.trace( "Branches (using show) for [{}] are [{}]".format( as_notice(path), colour_items(branches) ) )
if len(branches) == 1:
# If this returns a tag: tag_name replace the ": " with "/" and then extract the tag_name
# otherwise this will simply extract the branch_name as expected
if not branches[0].startswith('tag:'):
remote = branches[0]
branch = branches[0].replace(': ','/').split('/')[1]
else:
remote = branches[-2]
branch = remote.split('/')[1]
logger.trace( "Branch (using show) for [{}] is [{}]".format( as_notice(path), as_info(branch) ) )
else:
logger.warn( "No branch found from [{}]".format( result ) )
return branch, remote
def build(self, cuppa_env):
# cuppa.progress.NotifyProgress.register_callback( None, self.on_progress )
cuppa_env['empty_env'] = cuppa_env.create_env()
projects = cuppa_env.get_option('projects')
toolchains = cuppa_env['active_toolchains']
if not projects:
projects = cuppa_env['default_projects']
if not projects or not cuppa_env['run_from_launch_dir']:
sub_sconscripts = self.get_sub_sconscripts(
cuppa_env['launch_dir'],
[cuppa_env['build_root'], cuppa_env['download_root']])
if sub_sconscripts:
projects = sub_sconscripts
logger.info("Using sub-sconscripts [{}]".format(
colour_items(projects)))
elif projects:
logger.info("Using default_projects [{}]".format(
colour_items(projects)))
if projects:
sconscripts = []
for project in projects:
if (not os.path.exists(project)
and not cuppa_env['run_from_launch_dir']
and not os.path.isabs(project)):
path = os.path.join(cuppa_env['launch_dir'], project)
if os.path.exists(path):
if os.path.isdir(path):
sub_sconscripts = self.get_sub_sconscripts(
project, [
cuppa_env['build_root'],
cuppa_env['download_root']
])
if sub_sconscripts:
logger.info(
"Reading project folder [{}] and using sub-sconscripts [{}]"
.format(project,
colour_items(sub_sconscripts)))
sconscripts.extend(sub_sconscripts)
else:
sconscripts.append(path)
elif os.path.exists(project) and os.path.isdir(project):
sub_sconscripts = self.get_sub_sconscripts(
project,
[cuppa_env['build_root'], cuppa_env['download_root']])
if sub_sconscripts:
logger.info(
"Reading project folder [{}] and using sub-sconscripts [{}]"
.format(project, colour_items(sub_sconscripts)))
sconscripts.extend(sub_sconscripts)
else:
sconscripts.append(project)
for toolchain in toolchains:
build_envs = self.create_build_envs(toolchain, cuppa_env)
for build_env in build_envs:
for sconscript in sconscripts:
decider = cuppa_env.get_option('decider')
if decider:
build_env['env'].Decider(decider)
self.call_project_sconscript_files(
toolchain, build_env['variant'],
build_env['target_arch'], build_env['abi'],
build_env['env'], sconscript)
if cuppa_env['dump']:
print(
"cuppa: Performing dump only, so no builds will be attempted."
)
print("cuppa: Nothing to be done. Exiting.")
SCons.Script.Exit()
else:
logger.warn("No projects to build. Nothing to be done")
def __init__(self,
sconstruct_path,
base_path=os.path.abspath('.'),
branch_root=None,
default_options={},
default_projects=[],
default_variants=[],
default_dependencies=[],
default_profiles=[],
dependencies=[],
profiles=[],
default_runner=None,
configure_callback=None,
tools=[]):
cuppa.core.base_options.set_base_options()
cuppa_env = cuppa.core.environment.CuppaEnvironment()
cuppa_env.add_tools(tools)
dependencies, default_dependencies, dependencies_warning = self._normalise_with_defaults(
dependencies, default_dependencies, "dependencies")
profiles, default_profiles, profiles_warning = self._normalise_with_defaults(
profiles, default_profiles, "profiles")
self.initialise_options(cuppa_env, default_options, profiles,
dependencies)
cuppa_env['configured_options'] = {}
self._configure = cuppa.configure.Configure(
cuppa_env, callback=configure_callback)
enable_thirdparty_logging(
cuppa_env.get_option('enable-thirdparty-logging') and True
or False)
self._set_verbosity_level(cuppa_env)
cuppa_env['sconstruct_path'] = sconstruct_path
cuppa_env['sconstruct_dir'], cuppa_env[
'sconstruct_file'] = os.path.split(sconstruct_path)
self._set_output_format(cuppa_env)
self._configure.load()
cuppa_env['offline'] = cuppa_env.get_option('offline')
cuppa.version.check_current_version(cuppa_env['offline'])
if cuppa_env['offline']:
logger.info(as_info_label("Running in OFFLINE mode"))
logger.info("using sconstruct file [{}]".format(
as_notice(cuppa_env['sconstruct_file'])))
if dependencies_warning:
logger.warn(dependencies_warning)
if profiles_warning:
logger.warn(profiles_warning)
help = cuppa_env.get_option('help') and True or False
cuppa_env['minimal_output'] = cuppa_env.get_option('minimal_output')
cuppa_env['ignore_duplicates'] = cuppa_env.get_option(
'ignore_duplicates')
cuppa_env['working_dir'] = os.getcwd()
cuppa_env['launch_dir'] = os.path.relpath(SCons.Script.GetLaunchDir(),
cuppa_env['working_dir'])
cuppa_env['run_from_launch_dir'] = cuppa_env['launch_dir'] == "."
cuppa_env['launch_offset_dir'] = "."
if not cuppa_env['run_from_launch_dir']:
levels = len(cuppa_env['launch_dir'].split(os.path.sep))
cuppa_env['launch_offset_dir'] = os.path.sep.join(
['..' for i in range(levels)])
cuppa_env['base_path'] = os.path.normpath(
os.path.expanduser(base_path))
cuppa_env['branch_root'] = branch_root and os.path.normpath(
os.path.expanduser(branch_root)) or base_path
cuppa_env['branch_dir'] = cuppa_env['branch_root'] and os.path.relpath(
cuppa_env['base_path'], cuppa_env['branch_root']) or None
thirdparty = cuppa_env.get_option('thirdparty')
if thirdparty:
thirdparty = os.path.normpath(os.path.expanduser(thirdparty))
cuppa_env['thirdparty'] = thirdparty
cuppa.core.storage_options.process_storage_options(cuppa_env)
cuppa.core.location_options.process_location_options(cuppa_env)
cuppa_env['current_branch'] = ''
cuppa_env['current_revision'] = ''
if not help and not self._configure.handle_conf_only():
if cuppa_env['location_match_current_branch']:
url, repo, branch, remote, rev = cuppa.scms.scms.get_current_rev_info(
cuppa_env['sconstruct_dir'])
if branch:
cuppa_env['current_branch'] = branch
if rev:
cuppa_env['current_revision'] = rev
logger.info(
"Current build on branch [{}] at revision [{}] from remote [{}] in [{}] at [{}]"
.format(as_info(str(branch)), as_info(str(rev)),
as_info(str(remote)), as_info(str(repo)),
as_info(str(url))))
cuppa_env['default_projects'] = default_projects
cuppa_env['default_variants'] = default_variants and set(
default_variants) or set()
cuppa_env['default_dependencies'] = default_dependencies and default_dependencies or []
cuppa_env['BUILD_WITH'] = cuppa_env['default_dependencies']
cuppa_env['dependencies'] = {}
cuppa_env[
'default_profiles'] = default_profiles and default_profiles or []
cuppa_env['BUILD_PROFILE'] = cuppa_env['default_profiles']
cuppa_env['profiles'] = {}
test_runner = cuppa_env.get_option(
'runner', default=default_runner and default_runner or 'process')
cuppa_env['default_runner'] = test_runner
cuppa_env['propagate_env'] = cuppa_env.get_option(
'propagate-env') and True or False
cuppa_env['propagate_path'] = cuppa_env.get_option(
'propagate-path') and True or False
cuppa_env['merge_path'] = cuppa_env.get_option(
'merge-path') and True or False
cuppa_env['show_test_output'] = cuppa_env.get_option(
'show-test-output') and True or False
cuppa_env['suppress_process_output'] = cuppa_env.get_option(
'suppress-process-output') and True or False
cuppa_env['dump'] = cuppa_env.get_option('dump') and True or False
cuppa_env['clean'] = cuppa_env.get_option('clean') and True or False
self.add_variants(cuppa_env)
self.add_toolchains(cuppa_env)
self.add_platforms(cuppa_env)
cuppa_env['platform'] = cuppa.build_platform.Platform.current()
toolchains = cuppa_env.get_option('toolchains')
cuppa_env['target_architectures'] = None
if not help and not self._configure.handle_conf_only():
default_toolchain = cuppa_env['platform'].default_toolchain()
if not toolchains:
toolchains = [
cuppa_env[self.toolchains_key][default_toolchain]
]
else:
toolchains = [
cuppa_env[self.toolchains_key][t] for t in toolchains
]
cuppa_env['active_toolchains'] = toolchains
def add_profile(name, profile):
cuppa_env['profiles'][name] = profile
def add_dependency(name, dependency):
cuppa_env['dependencies'][name] = dependency
cuppa.modules.registration.get_options("methods", cuppa_env)
if not help and not self._configure.handle_conf_only():
cuppa_env[self.project_generators_key] = {}
cuppa.modules.registration.add_to_env("dependencies",
cuppa_env,
add_dependency)
cuppa.modules.registration.add_to_env("profiles", cuppa_env,
add_profile)
cuppa.modules.registration.add_to_env("methods", cuppa_env)
cuppa.modules.registration.add_to_env("project_generators",
cuppa_env)
for method_plugin in pkg_resources.iter_entry_points(
group='cuppa.method.plugins', name=None):
method_plugin.load().add_to_env(cuppa_env)
for profile_plugin in pkg_resources.iter_entry_points(
group='cuppa.profile.plugins', name=None):
profile_plugin.load().add_to_env(cuppa_env)
if profiles:
for profile in profiles:
profile.add_to_env(cuppa_env, add_profile)
logger.trace("available profiles are [{}]".format(
colour_items(sorted(cuppa_env["profiles"].keys()))))
logger.info("default profiles are [{}]".format(
colour_items(sorted(cuppa_env["default_profiles"]),
as_info)))
for dependency_plugin in pkg_resources.iter_entry_points(
group='cuppa.dependency.plugins', name=None):
dependency_plugin.load().add_to_env(
cuppa_env, add_dependency)
if dependencies:
for dependency in dependencies:
dependency.add_to_env(cuppa_env, add_dependency)
logger.trace("available dependencies are [{}]".format(
colour_items(sorted(cuppa_env["dependencies"].keys()))))
logger.info("default dependencies are [{}]".format(
colour_items(sorted(cuppa_env["default_dependencies"]),
as_info)))
# TODO - default_profile
if cuppa_env['dump']:
logger.info(
as_info_label(
"Running in DUMP mode, no building will be attempted"))
cuppa_env.dump()
job_count = cuppa_env.get_option('num_jobs')
parallel = cuppa_env.get_option('parallel')
parallel_mode = "manually"
if job_count == 1 and parallel:
job_count = multiprocessing.cpu_count()
if job_count > 1:
SCons.Script.SetOption('num_jobs', job_count)
parallel_mode = "automatically"
cuppa_env['job_count'] = job_count
cuppa_env['parallel'] = parallel
if job_count > 1:
logger.info(
"Running in {} with option [{}] set {} as [{}]".format(
as_emphasised("parallel mode"), as_info("jobs"),
as_emphasised(parallel_mode),
as_info(str(SCons.Script.GetOption('num_jobs')))))
if not help and self._configure.handle_conf_only():
self._configure.save()
if not help and not self._configure.handle_conf_only():
self.build(cuppa_env)
if self._configure.handle_conf_only():
print(
"cuppa: Handling configuration only, so no builds will be attempted."
)
print(
"cuppa: With the current configuration executing 'scons -D' would be equivalent to:"
)
print("")
print("scons -D {}".format(
self._command_line_from_settings(
cuppa_env['configured_options'])))
print("")
print("cuppa: Nothing to be done. Exiting.")
SCons.Script.Exit()
def get_branch(cls, path):
branch = None
remote = None
head_detached = False
command = "{git} branch".format(git=cls.binary())
branch_info = cls.execute_command(command, path)
if branch_info:
match = re.search(r'^[*] [(]HEAD detached ', branch_info)
if match:
head_detached = True
if not head_detached:
result = cls.execute_command(
"{git} status -sb".format(git=cls.binary()), path)
if result:
match = re.search(
r'## (?P<branch>[^)]+)[.][.][.](?P<remote>[^)\n]+)',
result)
if match:
branch = match.group("branch")
remote = match.group("remote")
match = re.search(r'## HEAD (no branch)', result)
# Check if we are rebasing
if match:
command = "{git} branch".format(git=cls.binary())
branch_info = cls.execute_command(command, path)
if branch_info:
match = re.search(
r'(no branch, rebasing (?P<branch>[^)]+))',
branch_info)
if match:
branch = match.group("branch")
logger.warn(
as_warning(
"Currently rebasing branch [{}]".format(
branch)))
return branch, remote
else:
result = cls.execute_command(
"{git} show -s --pretty=\%d --decorate=full HEAD".format(
git=cls.binary()), path)
match = re.search(r'HEAD(?:(?:[^ ]* -> |[^,]*, )(?P<refs>[^)]+))?',
result)
if match and match.group("refs"):
refs = [{
"ref": r.strip(),
"type": ""
} for r in match.group("refs").split(',')]
logger.trace("Refs (using show) for [{}] are [{}]".format(
as_notice(path), colour_items((r["ref"] for r in refs))))
if refs:
for ref in refs:
if ref["ref"].startswith("refs/heads/"):
ref["ref"] = ref["ref"][len("refs/heads/"):]
ref["type"] = "L"
elif ref["ref"].startswith("refs/tags/"):
ref["ref"] = ref["ref"][len("refs/tags/"):]
ref["type"] = "T"
elif ref["ref"].startswith("tag: refs/tags/"):
ref["ref"] = ref["ref"][len("tag: refs/tags/"):]
ref["type"] = "T"
elif ref["ref"].startswith("refs/remotes/"):
ref["ref"] = ref["ref"][len("refs/remotes/"):]
ref["type"] = "R"
else:
ref["type"] = "U"
logger.trace(
"Refs (after classification) for [{}] are [{}]".format(
as_notice(path),
colour_items((":".join([r["type"], r["ref"]])
for r in refs))))
if refs[0]["type"] == "L":
branch = refs[0]["ref"]
#elif refs[0]["type"] == "T":
#branch = refs[0]["ref"]
elif refs[0]["type"] == "R":
branch = refs[0]["ref"].split('/')[1]
remote = next(
(ref["ref"] for ref in refs if ref["type"] == "R"),
None)
logger.trace("Branch (using show) for [{}] is [{}]".format(
as_notice(path), as_info(str(branch))))
else:
if result == "(HEAD)":
command = "{git} branch".format(git=cls.binary())
branch_info = cls.execute_command(command)
if branch_info:
match = re.search(
r'(no branch, rebasing (?P<branch>[^)]+))',
branch_info)
if match:
branch = match.group("branch")
logger.warn(
as_warning(
"Currently rebasing branch [{}]".format(
branch)))
#if not branch:
#logger.warn( as_warning( "No branch found from [{}]".format( result ) ) )
return branch, remote
def build_library_from_source(self,
env,
sources=None,
library_name=None,
linktype=None):
from SCons.Script import Flatten
if not self._source_path and not sources:
logger.warn("Attempting to build library when source path is None")
return None
if not library_name:
library_name = self._name
if not linktype:
linktype = self._linktype
variant_key = env['tool_variant_dir']
prebuilt_objects = self.lazy_create_node(variant_key,
self._prebuilt_objects)
prebuilt_libraries = self.lazy_create_node(variant_key,
self._prebuilt_libraries)
local_dir = self._location.local()
local_folder = self._location.local_folder()
build_dir = os.path.abspath(
os.path.join(env['abs_build_root'], local_folder,
env['tool_variant_working_dir']))
final_dir = os.path.abspath(
os.path.normpath(os.path.join(build_dir, env['final_dir'])))
logger.debug("build_dir for [{}] = [{}]".format(
as_info(self._name), build_dir))
logger.debug("final_dir for [{}] = [{}]".format(
as_info(self._name), final_dir))
obj_suffix = env['OBJSUFFIX']
obj_builder = env.StaticObject
lib_builder = env.BuildStaticLib
if linktype == "shared":
obj_suffix = env['SHOBJSUFFIX']
obj_builder = env.SharedObject
lib_builder = env.BuildSharedLib
if not sources:
sources = env.RecursiveGlob("*.cpp",
start=self._source_path,
exclude_dirs=[env['build_dir']])
sources.extend(
env.RecursiveGlob("*.cc",
start=self._source_path,
exclude_dirs=[env['build_dir']]))
sources.extend(
env.RecursiveGlob("*.c",
start=self._source_path,
exclude_dirs=[env['build_dir']]))
objects = []
for source in Flatten([sources]):
rel_path = os.path.relpath(str(source), local_dir)
rel_obj_path = os.path.splitext(rel_path)[0] + obj_suffix
obj_path = os.path.join(build_dir, rel_obj_path)
if not rel_obj_path in prebuilt_objects:
prebuilt_objects[rel_obj_path] = obj_builder(obj_path, source)
objects.append(prebuilt_objects[rel_obj_path])
if not linktype in prebuilt_libraries:
library = lib_builder(library_name, objects, final_dir=final_dir)
if linktype == "shared":
library = env.Install(env['abs_final_dir'], library)
prebuilt_libraries[linktype] = library
else:
logger.trace("using existing library = [{}]".format(
str(prebuilt_libraries[linktype])))
return prebuilt_libraries[linktype]
def build( self, cuppa_env ):
# cuppa.progress.NotifyProgress.register_callback( None, self.on_progress )
cuppa_env['empty_env'] = cuppa_env.create_env()
projects = cuppa_env.get_option( 'projects' )
toolchains = cuppa_env['active_toolchains']
if not projects:
projects = cuppa_env['default_projects']
if not projects or not cuppa_env['run_from_launch_dir']:
sub_sconscripts = self.get_sub_sconscripts(
cuppa_env['launch_dir'],
[ cuppa_env['build_root'], cuppa_env['download_root'] ]
)
if sub_sconscripts:
projects = sub_sconscripts
logger.info( "Using sub-sconscripts [{}]".format( colour_items( projects ) ) )
elif projects:
logger.info( "Using default_projects [{}]".format( colour_items( projects ) ) )
if projects:
sconscripts = []
for project in projects:
if( not os.path.exists( project )
and not cuppa_env['run_from_launch_dir']
and not os.path.isabs( project ) ):
path = os.path.join( cuppa_env['launch_dir'], project )
if os.path.exists( path ):
if os.path.isdir( path ):
sub_sconscripts = self.get_sub_sconscripts(
project,
[ cuppa_env['build_root'], cuppa_env['download_root'] ]
)
if sub_sconscripts:
logger.info( "Reading project folder [{}] and using sub-sconscripts [{}]".format(
project, colour_items( sub_sconscripts )
) )
sconscripts.extend( sub_sconscripts )
else:
sconscripts.append( path )
elif os.path.exists( project ) and os.path.isdir( project ):
sub_sconscripts = self.get_sub_sconscripts(
project,
[ cuppa_env['build_root'], cuppa_env['download_root'] ]
)
if sub_sconscripts:
logger.info( "Reading project folder [{}] and using sub-sconscripts [{}]".format(
project, colour_items( sub_sconscripts )
) )
sconscripts.extend( sub_sconscripts )
else:
sconscripts.append( project )
for toolchain in toolchains:
build_envs = self.create_build_envs( toolchain, cuppa_env )
for build_env in build_envs:
for sconscript in sconscripts:
decider = cuppa_env.get_option( 'decider' )
if decider:
build_env['env'].Decider( decider )
self.call_project_sconscript_files( toolchain, build_env['variant'], build_env['target_arch'], build_env['env'], sconscript )
else:
logger.warn( "No projects to build. Nothing to be done" )
def get_local_directory( self, cuppa_env, location, sub_dir, branch, full_url ):
offline = cuppa_env['offline']
local_directory = None
base = cuppa_env['download_root']
if not os.path.isabs( base ):
base = os.path.join( cuppa_env['working_dir'], base )
if location.startswith( 'file:' ):
location = pip_download.url_to_path( location )
if not pip_download.is_url( location ):
if pip_download.is_archive_file( location ):
self._local_folder = self.folder_name_from_path( location, cuppa_env )
local_directory = os.path.join( base, self._local_folder )
local_dir_with_sub_dir = os.path.join( local_directory, sub_dir and sub_dir or "" )
if os.path.exists( local_dir_with_sub_dir ):
try:
os.rmdir( local_dir_with_sub_dir )
except:
return local_directory
self.extract( location, local_dir_with_sub_dir )
logger.debug( "(local archive) Location = [{}]".format( as_info( location ) ) )
logger.debug( "(local archive) Local folder = [{}]".format( as_info( self._local_folder ) ) )
else:
local_directory = branch and os.path.join( location, branch ) or location
self._local_folder = self.folder_name_from_path( location, cuppa_env )
logger.debug( "(local file) Location = [{}]".format( as_info( location ) ) )
logger.debug( "(local file) Local folder = [{}]".format( as_info( self._local_folder ) ) )
return local_directory
else:
self._local_folder = self.folder_name_from_path( full_url, cuppa_env )
local_directory = os.path.join( base, self._local_folder )
if full_url.scheme.startswith( 'http' ) and self.url_is_download_archive_url( full_url.path ):
logger.debug( "[{}] is an archive download".format( as_info( location ) ) )
local_dir_with_sub_dir = os.path.join( local_directory, sub_dir and sub_dir or "" )
# First we check to see if we already downloaded and extracted this archive before
if os.path.exists( local_dir_with_sub_dir ):
try:
# If not empty this will fail
os.rmdir( local_dir_with_sub_dir )
except:
# Not empty so we'll return this as the local_directory
logger.debug( "(already present) Location = [{}]".format( as_info( location ) ) )
logger.debug( "(already present) Local folder = [{}]".format( as_info( str(self._local_folder) ) ) )
return local_directory
if cuppa_env['dump'] or cuppa_env['clean']:
return local_directory
# If not we then check to see if we cached the download
cached_archive = self.get_cached_archive( cuppa_env['cache_root'], self._local_folder )
if cached_archive:
logger.debug( "Cached archive [{}] found for [{}]".format(
as_info( cached_archive ),
as_info( location )
) )
self.extract( cached_archive, local_dir_with_sub_dir )
else:
logger.info( "Downloading [{}]...".format( as_info( location ) ) )
try:
report_hook = None
if logger.isEnabledFor( logging.INFO ):
report_hook = ReportDownloadProgress()
filename, headers = urllib.urlretrieve( location, reporthook=report_hook )
name, extension = os.path.splitext( filename )
logger.info( "[{}] successfully downloaded to [{}]".format(
as_info( location ),
as_info( filename )
) )
self.extract( filename, local_dir_with_sub_dir )
if cuppa_env['cache_root']:
cached_archive = os.path.join( cuppa_env['cache_root'], self._local_folder )
logger.debug( "Caching downloaded file as [{}]".format( as_info( cached_archive ) ) )
shutil.copyfile( filename, cached_archive )
except urllib.ContentTooShortError as error:
logger.error( "Download of [{}] failed with error [{}]".format(
as_error( location ),
as_error( str(error) )
) )
raise LocationException( error )
elif '+' in full_url.scheme:
vc_type = location.split('+', 1)[0]
backend = pip_vcs.vcs.get_backend( vc_type )
if backend:
vcs_backend = backend( self.expand_secret( location ) )
local_dir_with_sub_dir = os.path.join( local_directory, sub_dir and sub_dir or "" )
if cuppa_env['dump'] or cuppa_env['clean']:
return local_directory
if os.path.exists( local_directory ):
url, repository, branch, remote, revision = self.get_info( location, local_dir_with_sub_dir, full_url, vc_type )
rev_options = self.get_rev_options( vc_type, vcs_backend, local_remote=remote )
version = self.ver_rev_summary( branch, revision, self._full_url.path )[0]
if not offline:
logger.info( "Updating [{}] in [{}]{} at [{}]".format(
as_info( location ),
as_notice( local_dir_with_sub_dir ),
( rev_options and " on {}".format( as_notice( str(rev_options) ) ) or "" ),
as_info( version )
) )
try:
update( vcs_backend, local_dir_with_sub_dir, rev_options )
logger.debug( "Successfully updated [{}]".format( as_info( location ) ) )
except pip_exceptions.PipError as error:
logger.warn( "Could not update [{}] in [{}]{} due to error [{}]".format(
as_warning( location ),
as_warning( local_dir_with_sub_dir ),
( rev_options and " at {}".format( as_warning( str(rev_options) ) ) or "" ),
as_warning( str(error) )
) )
else:
logger.debug( "Skipping update for [{}] as running in offline mode".format( as_info( location ) ) )
else:
rev_options = self.get_rev_options( vc_type, vcs_backend )
action = "Cloning"
if vc_type == "svn":
action = "Checking out"
max_attempts = 2
attempt = 1
while attempt <= max_attempts:
logger.info( "{} [{}] into [{}]{}".format(
action,
as_info( location ),
as_info( local_dir_with_sub_dir ),
attempt > 1 and "(attempt {})".format( str(attempt) ) or ""
) )
try:
vcs_backend.obtain( local_dir_with_sub_dir )
logger.debug( "Successfully retrieved [{}]".format( as_info( location ) ) )
break
except pip_exceptions.PipError as error:
attempt = attempt + 1
log_as = logger.warn
if attempt > max_attempts:
log_as = logger.error
log_as( "Could not retrieve [{}] into [{}]{} due to error [{}]".format(
as_info( location ),
as_notice( local_dir_with_sub_dir ),
( rev_options and " to {}".format( as_notice( str(rev_options) ) ) or ""),
as_error( str(error) )
) )
if attempt > max_attempts:
raise LocationException( str(error) )
logger.debug( "(url path) Location = [{}]".format( as_info( location ) ) )
logger.debug( "(url path) Local folder = [{}]".format( as_info( self._local_folder ) ) )
return local_directory
def get_local_directory( self, cuppa_env, location, sub_dir, branch, full_url ):
local_directory = None
base = cuppa_env['download_root']
if not os.path.isabs( base ):
base = os.path.join( cuppa_env['working_dir'], base )
if location.startswith( 'file:' ):
location = pip.download.url_to_path( location )
if not pip.download.is_url( location ):
if pip.download.is_archive_file( location ):
local_folder = self.folder_name_from_path( location )
local_directory = os.path.join( base, local_folder )
if os.path.exists( local_directory ):
try:
os.rmdir( local_directory )
except:
return local_directory, False
self.extract( location, local_directory )
else:
local_directory = branch and os.path.join( location, branch ) or location
return local_directory, False
else:
local_folder = self.folder_name_from_path( full_url )
local_directory = os.path.join( base, local_folder )
if full_url.scheme.startswith( 'http' ) and self.url_is_download_archive_url( full_url.path ):
logger.debug( "[{}] is an archive download".format( as_info( location ) ) )
local_dir_with_sub_dir = os.path.join( local_directory, sub_dir )
# First we check to see if we already downloaded and extracted this archive before
if os.path.exists( local_dir_with_sub_dir ):
try:
# If not empty this will fail
os.rmdir( local_dir_with_sub_dir )
except:
# Not empty so we'll return this as the local_directory
return local_directory, True
# If not we then check to see if we cached the download
cached_archive = self.get_cached_archive( cuppa_env['cache_root'], local_folder )
if cached_archive:
logger.debug( "Cached archive [{}] found for [{}]".format(
as_info( cached_archive ),
as_info( location )
) )
self.extract( cached_archive, local_dir_with_sub_dir )
else:
logger.info( "Downloading [{}]...".format( as_info( location ) ) )
try:
report_hook = None
if logger.isEnabledFor( logging.INFO ):
report_hook = ReportDownloadProgress()
filename, headers = urllib.urlretrieve( location, reporthook=report_hook )
name, extension = os.path.splitext( filename )
logger.info( "[{}] successfully downloaded to [{}]".format(
as_info( location ),
as_info( filename )
) )
self.extract( filename, local_dir_with_sub_dir )
if cuppa_env['cache_root']:
cached_archive = os.path.join( cuppa_env['cache_root'], local_folder )
logger.debug( "Caching downloaded file as [{}]".format( as_info( cached_archive ) ) )
shutil.copyfile( filename, cached_archive )
except urllib.ContentTooShortError as error:
logger.error( "Download of [{}] failed with error [{}]".format(
as_error( location ),
as_error( str(error) )
) )
raise LocationException( "Error obtaining [{}]: {}".format( location, error ) )
elif '+' in full_url.scheme:
vc_type = location.split('+', 1)[0]
backend = pip.vcs.vcs.get_backend( vc_type )
if backend:
vcs_backend = backend( location )
rev_options = self.get_rev_options( vc_type, vcs_backend )
local_dir_with_sub_dir = os.path.join( local_directory, sub_dir )
if os.path.exists( local_directory ):
url, repository, branch, revision = self.get_info( location, local_dir_with_sub_dir, full_url )
version = self.ver_rev_summary( branch, revision, self._full_url.path )[0]
logger.debug( "Updating [{}] in [{}]{} at [{}]".format(
as_info( location ),
as_notice( local_dir_with_sub_dir ),
( rev_options and " on {}".format( as_notice( str(rev_options) ) ) or "" ),
as_info( version )
) )
try:
vcs_backend.update( local_dir_with_sub_dir, rev_options )
logger.debug( "Successfully updated [{}]".format( as_info( location ) ) )
except pip.exceptions.InstallationError as error:
logger.warn( "Could not update [{}] in [{}]{} due to error [{}]".format(
as_warning( location ),
as_warning( local_dir_with_sub_dir ),
( rev_options and " at {}".format( as_warning( str(rev_options) ) ) or "" ),
as_warning( str(error) )
) )
else:
action = "Cloning"
if vc_type == "svn":
action = "Checking out"
logger.info( "{} [{}] into [{}]".format(
action, as_info( location ),
as_info( local_dir_with_sub_dir )
) )
try:
vcs_backend.obtain( local_dir_with_sub_dir )
logger.debug( "Successfully retrieved [{}]".format( as_info( location ) ) )
except pip.exceptions.InstallationError as error:
logger.error( "Could not retrieve [{}] into [{}]{} due to error [{}]".format(
as_error( location ),
as_error( local_dir_with_sub_dir ),
( rev_options and " to {}".format( as_error( str(rev_options) ) ) or ""),
as_error( str( error ) )
) )
raise LocationException( "Error obtaining [{}]: {}".format( location, error ) )
return local_directory, True
def __init__( self,
sconstruct_path,
base_path = os.path.abspath( '.' ),
branch_root = None,
default_options = {},
default_projects = [],
default_variants = [],
default_dependencies = [],
default_profiles = [],
dependencies = [],
profiles = [],
default_runner = None,
configure_callback = None,
tools = [] ):
cuppa.core.base_options.set_base_options()
cuppa_env = cuppa.core.environment.CuppaEnvironment()
cuppa_env.add_tools( tools )
dependencies, default_dependencies, dependencies_warning = self._normalise_with_defaults( dependencies, default_dependencies, "dependencies" )
profiles, default_profiles, profiles_warning = self._normalise_with_defaults( profiles, default_profiles, "profiles" )
self.initialise_options( cuppa_env, default_options, profiles, dependencies )
cuppa_env['configured_options'] = {}
self._configure = cuppa.configure.Configure( cuppa_env, callback=configure_callback )
enable_thirdparty_logging( cuppa_env.get_option( 'enable-thirdparty-logging' ) and True or False )
self._set_verbosity_level( cuppa_env )
cuppa_env['sconstruct_path'] = sconstruct_path
cuppa_env['sconstruct_dir'], cuppa_env['sconstruct_file'] = os.path.split(sconstruct_path)
self._set_output_format( cuppa_env )
self._configure.load()
cuppa_env['offline'] = cuppa_env.get_option( 'offline' )
cuppa.version.check_current_version( cuppa_env['offline'] )
if cuppa_env['offline']:
logger.info( as_info_label( "Running in OFFLINE mode" ) )
logger.info( "using sconstruct file [{}]".format( as_notice( cuppa_env['sconstruct_file'] ) ) )
if dependencies_warning:
logger.warn( dependencies_warning )
if profiles_warning:
logger.warn( profiles_warning )
help = cuppa_env.get_option( 'help' ) and True or False
cuppa_env['minimal_output'] = cuppa_env.get_option( 'minimal_output' )
cuppa_env['ignore_duplicates'] = cuppa_env.get_option( 'ignore_duplicates' )
cuppa_env['working_dir'] = os.getcwd()
cuppa_env['launch_dir'] = os.path.relpath( SCons.Script.GetLaunchDir(), cuppa_env['working_dir'] )
cuppa_env['run_from_launch_dir'] = cuppa_env['launch_dir'] == "."
cuppa_env['launch_offset_dir'] = "."
if not cuppa_env['run_from_launch_dir']:
levels = len( cuppa_env['launch_dir'].split( os.path.sep ) )
cuppa_env['launch_offset_dir'] = os.path.sep.join( ['..' for i in range(levels)] )
cuppa_env['base_path'] = os.path.normpath( os.path.expanduser( base_path ) )
cuppa_env['branch_root'] = branch_root and os.path.normpath( os.path.expanduser( branch_root ) ) or base_path
cuppa_env['branch_dir'] = cuppa_env['branch_root'] and os.path.relpath( cuppa_env['base_path'], cuppa_env['branch_root'] ) or None
thirdparty = cuppa_env.get_option( 'thirdparty' )
if thirdparty:
thirdparty = os.path.normpath( os.path.expanduser( thirdparty ) )
cuppa_env['thirdparty'] = thirdparty
cuppa.core.storage_options.process_storage_options( cuppa_env )
cuppa.core.location_options.process_location_options( cuppa_env )
cuppa_env['default_projects'] = default_projects
cuppa_env['default_variants'] = default_variants and set( default_variants ) or set()
cuppa_env['default_dependencies'] = default_dependencies and default_dependencies or []
cuppa_env['BUILD_WITH'] = cuppa_env['default_dependencies']
cuppa_env['dependencies'] = {}
cuppa_env['default_profiles'] = default_profiles and default_profiles or []
cuppa_env['BUILD_PROFILE'] = cuppa_env['default_profiles']
cuppa_env['profiles'] = {}
test_runner = cuppa_env.get_option( 'runner', default=default_runner and default_runner or 'process' )
cuppa_env['default_runner'] = test_runner
cuppa_env['propagate_env'] = cuppa_env.get_option( 'propagate-env' ) and True or False
cuppa_env['propagate_path'] = cuppa_env.get_option( 'propagate-path' ) and True or False
cuppa_env['merge_path'] = cuppa_env.get_option( 'merge-path' ) and True or False
cuppa_env['show_test_output'] = cuppa_env.get_option( 'show-test-output' ) and True or False
cuppa_env['suppress_process_output'] = cuppa_env.get_option( 'suppress-process-output' ) and True or False
cuppa_env['dump'] = cuppa_env.get_option( 'dump' ) and True or False
cuppa_env['clean'] = cuppa_env.get_option( 'clean' ) and True or False
self.add_variants ( cuppa_env )
self.add_toolchains ( cuppa_env )
self.add_platforms ( cuppa_env )
cuppa_env['platform'] = cuppa.build_platform.Platform.current()
toolchains = cuppa_env.get_option( 'toolchains' )
cuppa_env[ 'target_architectures' ] = None
if not help and not self._configure.handle_conf_only():
default_toolchain = cuppa_env['platform'].default_toolchain()
if not toolchains:
toolchains = [ cuppa_env[self.toolchains_key][default_toolchain] ]
else:
toolchains = [ cuppa_env[self.toolchains_key][t] for t in toolchains ]
cuppa_env['active_toolchains'] = toolchains
def add_profile( name, profile ):
cuppa_env['profiles'][name] = profile
def add_dependency( name, dependency ):
cuppa_env['dependencies'][name] = dependency
cuppa.modules.registration.get_options( "methods", cuppa_env )
if not help and not self._configure.handle_conf_only():
cuppa_env[self.project_generators_key] = {}
cuppa.modules.registration.add_to_env( "dependencies", cuppa_env, add_dependency )
cuppa.modules.registration.add_to_env( "profiles", cuppa_env, add_profile )
cuppa.modules.registration.add_to_env( "methods", cuppa_env )
cuppa.modules.registration.add_to_env( "project_generators", cuppa_env )
for method_plugin in pkg_resources.iter_entry_points( group='cuppa.method.plugins', name=None ):
method_plugin.load().add_to_env( cuppa_env )
for profile_plugin in pkg_resources.iter_entry_points( group='cuppa.profile.plugins', name=None ):
profile_plugin.load().add_to_env( cuppa_env )
if profiles:
for profile in profiles:
profile.add_to_env( cuppa_env, add_profile )
logger.trace( "available profiles are [{}]".format(
colour_items( sorted( cuppa_env["profiles"].keys() ) )
) )
logger.info( "default profiles are [{}]".format(
colour_items( sorted( cuppa_env["default_profiles"] ), as_info )
) )
for dependency_plugin in pkg_resources.iter_entry_points( group='cuppa.dependency.plugins', name=None ):
dependency_plugin.load().add_to_env( cuppa_env, add_dependency )
if dependencies:
for dependency in dependencies:
dependency.add_to_env( cuppa_env, add_dependency )
logger.trace( "available dependencies are [{}]".format(
colour_items( sorted( cuppa_env["dependencies"].keys() ) )
) )
logger.info( "default dependencies are [{}]".format(
colour_items( sorted( cuppa_env["default_dependencies"] ), as_info )
) )
# TODO - default_profile
if cuppa_env['dump']:
logger.info( as_info_label( "Running in DUMP mode, no building will be attempted" ) )
cuppa_env.dump()
job_count = cuppa_env.get_option( 'num_jobs' )
parallel = cuppa_env.get_option( 'parallel' )
parallel_mode = "manually"
if job_count==1 and parallel:
job_count = multiprocessing.cpu_count()
if job_count > 1:
SCons.Script.SetOption( 'num_jobs', job_count )
parallel_mode = "automatically"
cuppa_env['job_count'] = job_count
cuppa_env['parallel'] = parallel
if job_count>1:
logger.debug( "Running in {} with option [{}] set {} as [{}]".format(
as_emphasised("parallel mode"),
as_info( "jobs" ),
as_emphasised(parallel_mode),
as_info( str( SCons.Script.GetOption( 'num_jobs') ) )
) )
if not help and self._configure.handle_conf_only():
self._configure.save()
if not help and not self._configure.handle_conf_only():
self.build( cuppa_env )
if self._configure.handle_conf_only():
print "cuppa: Handling configuration only, so no builds will be attempted."
print "cuppa: With the current configuration executing 'scons -D' would be equivalent to:"
print ""
print "scons -D {}".format( self._command_line_from_settings( cuppa_env['configured_options'] ) )
print ""
print "cuppa: Nothing to be done. Exiting."
SCons.Script.Exit()
def get_local_directory(self, cuppa_env, location, sub_dir, branch,
full_url):
offline = cuppa_env['offline']
local_directory = None
base = cuppa_env['download_root']
if not os.path.isabs(base):
base = os.path.join(cuppa_env['working_dir'], base)
if location.startswith('file:'):
location = pip_download.url_to_path(location)
if not pip_is_url(location):
if pip_is_archive_file(location):
self._local_folder = self.folder_name_from_path(
location, cuppa_env)
local_directory = os.path.join(base, self._local_folder)
local_dir_with_sub_dir = os.path.join(
local_directory, sub_dir and sub_dir or "")
if os.path.exists(local_dir_with_sub_dir):
try:
os.rmdir(local_dir_with_sub_dir)
except:
return local_directory
self.extract(location, local_dir_with_sub_dir)
logger.debug("(local archive) Location = [{}]".format(
as_info(location)))
logger.debug("(local archive) Local folder = [{}]".format(
as_info(self._local_folder)))
else:
local_directory = branch and os.path.join(location,
branch) or location
self._local_folder = self.folder_name_from_path(
location, cuppa_env)
logger.debug("(local file) Location = [{}]".format(
as_info(location)))
logger.debug("(local file) Local folder = [{}]".format(
as_info(self._local_folder)))
return local_directory
else:
self._local_folder = self.folder_name_from_path(
full_url, cuppa_env)
local_directory = os.path.join(base, self._local_folder)
if full_url.scheme.startswith(
'http') and self.url_is_download_archive_url(
full_url.path):
logger.debug("[{}] is an archive download".format(
as_info(location)))
local_dir_with_sub_dir = os.path.join(
local_directory, sub_dir and sub_dir or "")
# First we check to see if we already downloaded and extracted this archive before
if os.path.exists(local_dir_with_sub_dir):
try:
# If not empty this will fail
os.rmdir(local_dir_with_sub_dir)
except:
# Not empty so we'll return this as the local_directory
logger.debug(
"(already present) Location = [{}]".format(
as_info(location)))
logger.debug(
"(already present) Local folder = [{}]".format(
as_info(str(self._local_folder))))
return local_directory
if cuppa_env['dump'] or cuppa_env['clean']:
return local_directory
# If not we then check to see if we cached the download
cached_archive = self.get_cached_archive(
cuppa_env['cache_root'], self._local_folder)
if cached_archive:
logger.debug("Cached archive [{}] found for [{}]".format(
as_info(cached_archive), as_info(location)))
self.extract(cached_archive, local_dir_with_sub_dir)
else:
logger.info("Downloading [{}]...".format(
as_info(location)))
try:
report_hook = None
if logger.isEnabledFor(logging.INFO):
report_hook = ReportDownloadProgress()
filename, headers = urlretrieve(location,
reporthook=report_hook)
name, extension = os.path.splitext(filename)
logger.info(
"[{}] successfully downloaded to [{}]".format(
as_info(location), as_info(filename)))
self.extract(filename, local_dir_with_sub_dir)
if cuppa_env['cache_root']:
cached_archive = os.path.join(
cuppa_env['cache_root'], self._local_folder)
logger.debug(
"Caching downloaded file as [{}]".format(
as_info(cached_archive)))
shutil.copyfile(filename, cached_archive)
except ContentTooShortError as error:
logger.error(
"Download of [{}] failed with error [{}]".format(
as_error(location), as_error(str(error))))
raise LocationException(error)
elif '+' in full_url.scheme:
vc_type = location.split('+', 1)[0]
backend = pip_vcs.vcs.get_backend(vc_type)
if backend:
try:
vcs_backend = backend(self.expand_secret(location))
except: # Pip version >= 19
backend.url = self.expand_secret(location)
vcs_backend = backend
local_dir_with_sub_dir = os.path.join(
local_directory, sub_dir and sub_dir or "")
if cuppa_env['dump'] or cuppa_env['clean']:
return local_directory
if os.path.exists(local_directory):
url, repository, branch, remote, revision = self.get_info(
location, local_dir_with_sub_dir, full_url,
vc_type)
rev_options = self.get_rev_options(vc_type,
vcs_backend,
local_remote=remote)
version = self.ver_rev_summary(branch, revision,
self._full_url.path)[0]
if not offline:
logger.info(
"Updating [{}] in [{}]{} at [{}]".format(
as_info(location),
as_notice(local_dir_with_sub_dir),
(rev_options and " on {}".format(
as_notice(str(rev_options))) or ""),
as_info(version)))
try:
update(vcs_backend, local_dir_with_sub_dir,
rev_options)
logger.debug(
"Successfully updated [{}]".format(
as_info(location)))
except pip_exceptions.PipError as error:
logger.warn(
"Could not update [{}] in [{}]{} due to error [{}]"
.format(as_warning(location),
as_warning(local_dir_with_sub_dir),
(rev_options and " at {}".format(
as_warning(str(rev_options)))
or ""), as_warning(str(error))))
else:
logger.debug(
"Skipping update for [{}] as running in offline mode"
.format(as_info(location)))
else:
rev_options = self.get_rev_options(
vc_type, vcs_backend)
action = "Cloning"
if vc_type == "svn":
action = "Checking out"
max_attempts = 2
attempt = 1
while attempt <= max_attempts:
logger.info("{} [{}] into [{}]{}".format(
action, as_info(location),
as_info(local_dir_with_sub_dir), attempt > 1
and "(attempt {})".format(str(attempt)) or ""))
try:
obtain(vcs_backend, local_dir_with_sub_dir,
vcs_backend.url)
logger.debug(
"Successfully retrieved [{}]".format(
as_info(location)))
break
except pip_exceptions.PipError as error:
attempt = attempt + 1
log_as = logger.warn
if attempt > max_attempts:
log_as = logger.error
log_as(
"Could not retrieve [{}] into [{}]{} due to error [{}]"
.format(as_info(location),
as_notice(local_dir_with_sub_dir),
(rev_options and " to {}".format(
as_notice(str(rev_options)))
or ""), as_error(str(error))))
if attempt > max_attempts:
raise LocationException(str(error))
logger.debug("(url path) Location = [{}]".format(
as_info(location)))
logger.debug("(url path) Local folder = [{}]".format(
as_info(self._local_folder)))
return local_directory
def __init__( self, cuppa_env, location, develop=None, branch_path=None, extra_sub_path=None, name_hint=None ):
logger.debug( "Create location using location=[{}], develop=[{}], branch_path=[{}], extra_sub_path=[{}], name_hint=[{}]".format(
as_info( location ),
as_info( str(develop) ),
as_info( str(branch_path) ),
as_info( str(extra_sub_path) ),
as_info( str(name_hint) )
) )
self._cuppa_env = cuppa_env
self._supports_relative_versioning = False
self._current_branch = self._cuppa_env['current_branch']
self._current_revision = self._cuppa_env['current_revision']
self._offline = self.option_set('offline')
offline = self._offline
self._default_branch = self._cuppa_env['location_default_branch']
location = self.replace_sconstruct_anchor( location )
if develop:
if not os.path.isabs( develop ):
develop = '#' + develop
develop = self.replace_sconstruct_anchor( develop )
logger.debug( "Develop location specified [{}]".format( as_info( develop ) ) )
if self.option_set('develop') and develop:
location = develop
logger.debug( "--develop specified so using location=develop=[{}]".format( as_info( develop ) ) )
scm_location = location
if location[-1] == '@':
self._supports_relative_versioning = True
scm_location = location[:-1]
scm_system, vc_type, repo_location, versioning = self.get_scm_system_and_info( self.expand_secret( scm_location ) )
logger.debug( "Local location and actions for [{location}] being determined in context:{offline}"
" vc_type=[{vc_type}], repo_location=[{repo_location}],"
" versioning=[{versioning}]".format(
location = as_info(location),
offline = self._offline and " " + as_info_label("OFFLINE") + "," or "",
vc_type = as_info(str(vc_type)),
repo_location = as_info(str(repo_location)),
versioning = as_info(str(versioning))
) )
if self._supports_relative_versioning:
if self.location_match_current_branch():
if not scm_system:
logger.warn( "Location [{}] specified using relative versioning, but no SCM system is available"
" that matches the version control type [{}]. Relative versioning will be ignored"
" for this location.".format( location, vc_type ) )
else:
branch_exists = False
logger.debug( "Relative branching active for [{location}] with"
" current branch [{branch}] and current revision [{revision}]".format(
location=as_info(str(location)),
branch=as_info(str(self._current_branch)),
revision=as_info(str(self._current_revision))
) )
if self._current_branch:
# Try to checkout on the explicit branch but if that fails fall back to
# to the default by stripping off the '@' from the end of the path
if not offline and scm_system.remote_branch_exists( repo_location, self._current_branch ):
scm_location = location + self._current_branch
logger.trace( "scm_location = [{scm_location}]".format(
scm_location=as_info(str(scm_location))
) )
elif self._current_revision:
# Try to checkout on the explicit branch but if that fails fall back to
# to the default by stripping off the '@' from the end of the path
if not offline and scm_system.remote_branch_exists( repo_location, self._current_revision ):
scm_location = location + self._current_revision
logger.trace( "scm_location = [{scm_location}]".format(
scm_location=as_info(str(scm_location))
) )
elif scm_system and not offline:
self._default_branch = scm_system.remote_default_branch( repo_location )
if self._default_branch:
scm_location = location + self._default_branch
elif( scm_system
and not versioning
and not offline
and self.option_set('location_explicit_default_branch')
):
self._default_branch = scm_system.remote_default_branch( repo_location )
if self._default_branch:
scm_location = location + '@' + self._default_branch
location = scm_location
self._location = os.path.expanduser( location )
self._full_url = urlparse( self._location )
self._sub_dir = None
self._name_hint = name_hint
if extra_sub_path:
if os.path.isabs( extra_sub_path ):
raise LocationException( "Error extra sub path [{}] is not relative".format(extra_sub_path) )
else:
self._sub_dir = os.path.normpath( extra_sub_path )
## Get the location for the source dependency. If the location is a URL or an Archive we'll need to
## retrieve the URL and extract the archive. get_local_directory() returns the location of the source
## once this is done
local_directory = self.get_local_directory( self._location, self._sub_dir, branch_path, self._full_url )
logger.trace( "Local Directory for [{}] returned as [{}]".format(
as_notice( self._location ),
as_notice( local_directory )
) )
self._base_local_directory = local_directory
self._local_directory = self._sub_dir and os.path.join( local_directory, self._sub_dir ) or local_directory
## Now that we have a locally accessible version of the dependency we can try to collate some information
## about it to allow us to specify what we are building with.
self._url, self._repository, self._branch, self._remote, self._revision = self.get_info( self._location, self._local_directory, self._full_url )
self._version, self._revision = self.ver_rev_summary( self._branch, self._revision, self._full_url.path )
logger.debug( "Using [{}]{}{} at [{}] stored in [{}]".format(
as_info( location ),
( self._branch and ":[{}]".format( as_info( str(self._branch) ) ) or "" ),
( self._remote and " from [{}]".format( as_info( str(self._remote) ) ) or "" ),
as_info( self._version ),
as_notice( self._local_directory )
) )
