def _createCellWidget(self, package):
Any.requireMsg( Any.isInstance( package, BSTPackage.BSTPackage ) or \
Any.isInstance( package, DebianPackage ),
'unexpected datatype: %s' % type(package) )
Any.requireIsTextNonEmpty(package.url)
cell = QTreeWidgetItem([package.url])
inSystem = self._model.isInstalled_locally
inProxy = self._model.isInstalled_proxySIT
inGlobal = self._model.isInstalled_globalSIT
ProjectProperties.requireIsCanonicalPath(SIT.strip(package.url))
if package.url.startswith('sit://'):
if self._hasProxy:
self._setCellColorCode(cell, 1, inProxy(package.url))
else:
self._setCellColorCode(cell, 1, None)
self._setCellColorCode(cell, 2, inGlobal(package.url))
self._setCellColorCode(cell, 3, None)
else:
self._setCellColorCode(cell, 1, None)
self._setCellColorCode(cell, 2, None)
self._setCellColorCode(cell, 3, inSystem(package.url))
return cell
def ensureHasDependency(content, package):
"""
Ensures if the direct dependencies/inclusions are present
or not in the provided string (= CMakeLists.txt file content).
"""
Any.requireIsTextNonEmpty(content)
ProjectProperties.requireIsCanonicalPath(package)
logging.debug('Validating CMakeLists.txt')
category, name, version = ProjectProperties.splitPath(package)
pkgNoVersion = os.path.join(category, name)
found = False
for dep in getDependencies(content):
if dep.find(pkgNoVersion) > 0:
found = True
logging.debug('%s dependency already present', pkgNoVersion)
if found:
return content
else:
logging.debug('inserting dependency to: %s', package)
return insertDependency(content, package)
def open( self, package=None ):
if self.url:
ProjectProperties.requireIsURL( self.url )
package = ProjectProperties.splitURL( self.url )[1]
else:
ProjectProperties.requireIsCanonicalPath( package )
topLevelDir = os.path.join( SIT.getRootPath(), package )
super( BSTGloballyInstalledPackage, self ).open( topLevelDir )
def printFetchCommands(packageList, asSubModule=True):
"""
Prints the VCS fetch command for each package in the list.
Duplicates will be removed, and output will be sorted for better
readability, e.g.:
asSubModule = True:
git submodule add [email protected]:TECH_Team/BPL.git
git submodule add [email protected]:ToolBOS/BasicComponents.git
git submodule add [email protected]:ToolBOS/ToolBOSLib.git
asSubModule = False:
git clone [email protected]:TECH_Team/BPL.git
git clone [email protected]:ToolBOS/BasicComponents.git
git clone [email protected]:ToolBOS/ToolBOSLib.git
"""
Any.requireIsIterable(packageList)
Any.requireIsBool(asSubModule)
sitRootPath = SIT.getRootPath()
commands = set() # avoids duplicates
for canonicalPath in packageList:
ProjectProperties.requireIsCanonicalPath(canonicalPath)
installRoot = os.path.join(sitRootPath, canonicalPath)
Any.requireIsExisting(installRoot)
detector = PackageDetector.PackageDetector(installRoot)
detector.retrieveMetaInfoFromSIT()
vcsRoot = detector.vcsRoot
if vcsRoot:
Any.requireIsTextNonEmpty(vcsRoot)
repo = VersionControl.auto(vcsRoot)
if isinstance(repo, SVN.SVNRepository):
repo.setConfiguredUserName()
command = repo.getSourceCodeCommand()
elif isinstance(repo, Git.RemoteGitRepository):
command = repo.getSourceCodeCommand(asSubModule)
else:
raise TypeError('unexpected datatype: %s', type(repo))
else:
command = '# VCS location unknown: %s' % canonicalPath
commands.add(command)
for command in sorted(commands):
print(command)
def listDependencies(canonicalPath,
reverse=False,
recursive=True,
missingOnly=False,
asList=False,
showDuplicates=False):
if missingOnly:
asList = True
reverse = False
# if '.' provided as package, read dependencies from ./pkgInfo.py,
# otherwise work on SIT level
if canonicalPath == '.' and not reverse:
try:
requireTopLevelDir()
except RuntimeError as details:
raise SystemExit(details)
package = BSTPackage.BSTSourcePackage()
package.open(os.getcwd())
else:
if canonicalPath == '.': # implies reverse == True
requireTopLevelDir()
package = BSTPackage.BSTSourcePackage()
package.open(os.getcwd())
canonicalPath = package.detector.canonicalPath
# strip-off the SIT part if provided, and convert to URL-style
ProjectProperties.requireIsCanonicalPath(canonicalPath)
packageURL = 'sit://' + canonicalPath
package = BSTPackage.BSTProxyInstalledPackage(packageURL)
try:
package.open()
except AssertionError as details:
logging.error(details)
logging.error('%s: No such package in SIT' % canonicalPath)
raise SystemExit()
if reverse:
package.retrieveReverseDependencies(recursive)
else:
package.retrieveDependencies(recursive,
normalDeps=True,
buildDeps=False)
if asList:
_showAsList(package, reverse, missingOnly)
else:
_showAsTree(package, reverse, recursive, showDuplicates)
def test_isDeprecated( self ):
canonicalPath = ToolBOSSettings.canonicalPath
logging.info( 'testing canonicalPath=%s' % canonicalPath )
ProjectProperties.requireIsCanonicalPath( canonicalPath )
# ToolBOSCore never becomes deprecated, I hope ;-)
self.assertFalse( ProjectProperties.isDeprecated( canonicalPath ) )
# check for invalid parameter types (boolean + list)
self.assertRaises( AssertionError, ProjectProperties.isDeprecated, True )
self.assertRaises( AssertionError, ProjectProperties.isDeprecated, [ 1, 2, 3 ] )
def getIndexFileName(canonicalPath):
"""
Returns the RTMaps index filename for the package.
Example: 'Horp_ToolBOS_1.0.pck'
"""
ProjectProperties.requireIsCanonicalPath(canonicalPath)
tmp = ProjectProperties.splitPath(canonicalPath)
result = '%s_%s.pck' % (tmp[1], tmp[2])
Any.requireIsTextNonEmpty(result)
return result
def printBuildCommands(orderedList):
"""
Prints the build-instruction for each package in the list, e.g.:
build src/ToolBOSCore
build src/ToolBOSLib
build src/IniConfigFile
build src/Middleware
Note: The list shall already be sorted according to build order!
"""
Any.requireIsListNonEmpty(orderedList)
sitRootPath = SIT.getRootPath()
commands = []
for canonicalPath in orderedList:
ProjectProperties.requireIsCanonicalPath(canonicalPath)
installRoot = os.path.join(sitRootPath, canonicalPath)
Any.requireIsExisting(installRoot)
detector = PackageDetector.PackageDetector(installRoot)
detector.retrieveMetaInfoFromSIT()
vcsRelPath = detector.vcsRelPath
vcsRoot = detector.vcsRoot
if vcsRoot:
Any.requireIsTextNonEmpty(vcsRoot)
repo = VersionControl.auto(vcsRoot)
repoName = repo.getRepositoryName()
if vcsRelPath:
location = os.path.join('src', repoName, vcsRelPath)
else:
location = os.path.join('src', repoName)
command = 'build %s' % location
else:
command = '# build %s# location unknown' % canonicalPath.ljust(70)
commands.append(command)
for command in commands:
print(command)
def getIndexFilePath_relHGR(canonicalPath, rtmapsVersion, platform):
"""
Returns a path to the *.pck index filename for the package,
relative to ${SIT}.
Example: package = 'Modules/BBCM/InputOutput/UltimaTest/3.1'
"""
ProjectProperties.requireIsCanonicalPath(canonicalPath)
Any.requireIsTextNonEmpty(rtmapsVersion)
Any.requireIsTextNonEmpty(platform)
result = os.path.join('Modules/Index', rtmapsVersion, platform,
getIndexFileName(canonicalPath))
Any.requireIsTextNonEmpty(result)
return result
def sourceWindowsBSP(msvc):
"""
Loads the necessary ToolBOSPluginWindows so that Wine and MSVC
will be found in PATH etc.
"""
allBSPs = ToolBOSSettings.getConfigOption('BST_crossCompileBSPs')
Any.requireIsDictNonEmpty(allBSPs)
if msvc in (2010, 2012):
canonicalPath = allBSPs['windows-amd64-vs2012']
elif msvc == 2017:
canonicalPath = allBSPs['windows-amd64-vs2017']
else:
raise RuntimeError('sourceWindowsBSP: unsupported MSVC version: %s' %
msvc)
ProjectProperties.requireIsCanonicalPath(canonicalPath)
ProcessEnv.source(canonicalPath)
def populate(self):
"""
Scans all package in SIT and stores the ground-truth pkgInfo.py
information into one giant hashtable for later fast access.
The assignment is "packageURL": { pkgInfo data }
"""
sitPath = SIT.getPath()
canonicalPaths = SIT.getCanonicalPaths(sitPath)
Any.requireIsListNonEmpty(canonicalPaths)
for canonicalPath in canonicalPaths:
ProjectProperties.requireIsCanonicalPath(canonicalPath)
packageURL = 'sit://' + canonicalPath
installRoot = os.path.join(sitPath, canonicalPath)
detector = PackageDetector(installRoot)
detector.retrieveMakefileInfo()
self._cache[packageURL] = detector
def insertDependency(content, package):
"""
Inserts the direct dependencies/inclusions if not found
in the provided string (= CMakeLists.txt file content).
"""
Any.requireIsTextNonEmpty(content)
ProjectProperties.requireIsCanonicalPath(package)
# insert after last occurrence of bst_find_package
modified = content.splitlines()
index = 0
for line in modified:
if line.startswith('bst_find_package'):
index = modified.index(line) + 1
if index == 0:
# if there was no bst_find_package() yet then append at the end
index = len(modified)
modified.insert(index, 'bst_find_package(%s)' % package)
return '\n'.join(modified)
def makeCategoryWriteable(sitPath, project, groupName='hriall', mode=0o0775):
"""
Changes the group and permissions of all directories between
'sitPath' up to the project version (the project's main directory
will also be group-writeable so that different developers could
install different versions.
Mind to provide an octal number as 'mode'!)
"""
from ToolBOSCore.Packages import ProjectProperties
Any.requireIsDir(sitPath)
Any.requireIsTextNonEmpty(project)
Any.requireIsTextNonEmpty(groupName)
Any.requireIsIntNotZero(mode) # <mode> must be an octal number
ProjectProperties.requireIsCanonicalPath(project)
# The current HRI-EU install procedure is implemented in a way that it always
# attempts to change ownership of the category INCLUDING the SIT root
# directory. Instead of fixing this there (time-consuming, error-prone) we
# always set it here as a shortcut.
FastScript.setGroupPermission(sitPath, groupName, mode)
# cut off the project version
tmp = ProjectProperties.splitPath(project)
mainDir = os.path.join(tmp[0], tmp[1])
tokens = mainDir.split(os.sep)
# the current path we are operating on with chmod+chgrp, starting from
# sitPath
curPath = sitPath
# for each token in category do a chmod+chgrp
for token in tokens:
curPath = os.path.join(curPath, token)
FastScript.setGroupPermission(curPath, groupName, mode)
def registerNormalPackage(package,
sitProxyPath=None,
indexBaseDir=None,
dryRun=False):
"""
Creates a symlink to the *.pck file of 'package' within the RTMaps
index.
RTMAPS_VERSION is taken from the dependency list of the package.
"""
if sitProxyPath is None:
sitProxyPath = SIT.getPath()
if indexBaseDir is None:
indexBaseDir = getIndexBaseDir(sitProxyPath)
ProjectProperties.requireIsCanonicalPath(package)
Any.requireIsDir(sitProxyPath)
Any.requireIsDir(indexBaseDir)
Any.requireIsBool(dryRun)
platformList = getPlatformNames()
packageName = ProjectProperties.getPackageName(package)
packageVersion = ProjectProperties.getPackageVersion(package)
versionTokens = ProjectProperties.splitVersion(packageVersion)
majorVersion = int(versionTokens[0])
minorVersion = int(versionTokens[1])
installRoot = os.path.join(sitProxyPath, package)
Any.requireIsListNonEmpty(platformList)
Any.requireIsTextNonEmpty(packageName)
Any.requireIsTextNonEmpty(packageVersion)
Any.requireIsDir(installRoot)
deps = ProjectProperties.getDependencies(package)
try:
Any.requireIsListNonEmpty(deps)
except AssertionError:
logging.debug(
'empty list of dependencies in RTMaps package is unplausible')
msg = "%s: unable to retrieve dependencies, please check SIT installation" % package
raise ValueError(msg)
expr = re.compile('^sit://External/RTMaps/(\d+\.\d+)')
# detect RTMaps version used by package
rtmapsVersion = ''
for dep in deps:
tmp = expr.match(dep)
if tmp:
rtmapsVersion = tmp.group(1)
break
Any.requireIsTextNonEmpty(rtmapsVersion)
logging.debug('%s depends on RTMaps %s', package, rtmapsVersion)
libDir = os.path.join(installRoot, 'lib')
pckFiles = FastScript.findFiles(libDir, ext='.pck')
Any.requireMsg(
len(pckFiles) > 0,
package + ": No *.pck file found, forgot to compile?")
for relPath in pckFiles:
pckFileName = os.path.basename(relPath)
pckPackage = os.path.splitext(pckFileName)[0]
pckFileExt = os.path.splitext(pckFileName)[1]
logging.debug('registering %s', pckPackage)
for platform in platformList:
pckPath = os.path.join(libDir, platform, pckFileName)
dstDir = os.path.join(indexBaseDir, rtmapsVersion, platform)
if os.path.exists(pckPath):
symlinkFile = '%s_%d.%d%s' % (pckPackage, majorVersion,
minorVersion, pckFileExt)
symlinkPath = os.path.join(dstDir, symlinkFile)
target = pckPath
FastScript.link(target, symlinkPath, dryRun)
package = SIT.strip(args['package']) # strip-off the SIT part if provided
userName = args['user']
#----------------------------------------------------------------------------
# Main program
#----------------------------------------------------------------------------
# When using shell tab-completion on SIT paths for the canonical path,
# a trailing slash might be added. Remove this before proceeding.
if package.endswith('/'):
package = package[:-1]
logging.debug('package truncated to: %s', package)
try:
ProjectProperties.requireIsCanonicalPath(package)
except AssertionError as details:
logging.error(details)
raise SystemExit
# look for repository URL
sitPath = SIT.getPath()
installRoot = os.path.join(sitPath, package)
url = None
detector = None
try:
detector = PackageDetector.PackageDetector(installRoot)
except AssertionError:
pass # package not found in SIT
def source(package):
"""
Python equivalent of "source BashSrc" from SIT, in order to setup
PATH, LD_LIBRARY_PATH,... within the Python process.
@anchor ProcessEnv_source
"""
ProjectProperties.requireIsCanonicalPath(package)
sourced = FastScript.getEnv('TOOLBOSCORE_SOURCED')
Any.requireMsg(sourced, '$TOOLBOSCORE_SOURCED must not be empty')
# avoid double-sourcing
if package in sourced:
return True
ProjectProperties.requireIsInstalled(package)
logging.debug('source %s/pkgInfo.py', package)
sourced = '%s %s' % (package, sourced)
FastScript.setEnv('TOOLBOSCORE_SOURCED', sourced)
# load pkgInfo.py (if exists)
try:
content = getPkgInfoContent(project=package)
except AssertionError:
return True # no such file, this is OK
except (IOError, OSError) as details:
logging.error(details)
return False
# setup environment of this package
try:
envVars = content['envVars']
except KeyError:
envVars = [] # no such setting, this is OK
sitPath = SIT.getPath()
for name, value in envVars:
# replace known placeholdes
value = value.replace('${INSTALL_ROOT}',
os.path.join(sitPath, package))
FastScript.setEnv_withExpansion(name, value)
# source dependent packages
try:
# TODO: eventually extend to sourcing recommended/suggested packages
depList = content['depends']
except (AssertionError, KeyError):
depList = [] # no such setting, this is OK
for dep in depList:
if not dep.startswith('deb://'):
source(SIT.strip(dep))
# special treatment of PYTHONPATH:
# After sourcing add new entries in PYTHONPATH to sys.path
_expandSysPath()
return True
def main(package):
# When using shell tab-completion on SIT paths for the canonical path,
# a trailing slash might be added. Remove this before proceeding.
if package.endswith('/'):
package = package[:-1]
logging.debug('package truncated to: %s', package)
try:
ProjectProperties.requireIsCanonicalPath(package)
except AssertionError as details:
logging.error(details)
raise SystemExit
# stop if package is maintained via SVN
svnUrl = ProjectProperties.getSVNLocationFromFilesystem(package)
if svnUrl:
logging.debug('found SVN URL: %s', svnUrl)
logging.error('')
logging.error('Package is maintained via SVN and cannot be cloned.')
logging.error(
'Use GitCheckout.py to access it in Git-style using the git-svn bridge :-)'
)
logging.error('')
sys.exit(-2)
# try to look-up repository URL from SIT, if not found do at least
# a good guess
installRoot = os.path.join(SIT.getPath(), package)
try:
detector = PackageDetector.PackageDetector(installRoot)
detector.retrieveMetaInfoFromSIT()
url = detector.gitOrigin
except AssertionError as e:
logging.error(e)
raise SystemExit()
# logging.debug( 'guessing Git location...' )
# url = ProjectProperties.guessGitLocation( package ) # TODO: implement guessing
if not url:
logging.error('unable to determine Git repository location')
sys.exit(-1)
Any.requireIsTextNonEmpty(url)
repo = Git.RemoteGitRepository(url)
# warn if cloning from unofficial host (see CIA-1062)
repo.checkIsOnMasterServer()
if userName:
repo.setUserName(userName)
Any.setDebugLevel(logging.DEBUG) # print executed command
repo.clone()
args = vars(argman.run())
canonicalPath = args['package']
quiet = args['quiet']
topLevelDir = args['topLevelDir']
if quiet:
Any.setDebugLevel(logging.CRITICAL)
try:
BuildSystemTools.requireTopLevelDir(topLevelDir)
except RuntimeError as details:
raise SystemExit(details)
try:
ProjectProperties.requireIsCanonicalPath(canonicalPath)
except AssertionError as details:
raise SystemExit(details)
#----------------------------------------------------------------------------
# Main program
#----------------------------------------------------------------------------
logging.info('retrieving dependencies...')
bstpkg_src = BSTPackage.BSTSourcePackage()
bstpkg_src.open(topLevelDir)
bstpkg_src.retrieveDependencies(True)
logging.info('retrieving reverse dependencies...')
bstpkg_global = BSTPackage.BSTGloballyInstalledPackage()