def test_findProxyInstallations(self):
# check if user has a valid proxy:
sitRootPath = SIT.getParentPath()
sitProxyPath = SIT.getPath()
Any.requireIsDir(sitRootPath)
Any.requireIsDir(sitProxyPath)
ProxyDir.requireIsProxyDir(sitProxyPath)
# create a fake package directory within the proxy...
packageName = 'UnittestABCDEF123'
goodNameDir = os.path.join(sitProxyPath, 'Libraries', packageName)
goodVersionDir = os.path.join(goodNameDir, '42.0')
badVersionDir = os.path.join(goodNameDir, '21.0')
FastScript.mkdir(goodVersionDir)
resultList = ProxyDir.findProxyInstallations()
self.assertTrue(
len(resultList) >= 1
) # may be greater if developer has real software installed in proxy
self.assertTrue(goodVersionDir in resultList)
self.assertFalse(badVersionDir in resultList)
# clean-up
FastScript.remove(goodNameDir)
def getFlatDependencies(canonicalPaths, cache=None, sitPath=None):
"""
Resolves all dependencies recursively and converts them to a flat
set of overall dependencies.
This function might be handy to determine all SIT packages that
need to transferred to another site.
"""
cache = {} if cache is None else cache
sitPath = SIT.getPath() if sitPath is None else sitPath
Any.requireIsIterable(canonicalPaths)
Any.requireIsDict(cache)
Any.requireIsTextNonEmpty(sitPath)
result = set()
for canonicalPath in canonicalPaths:
requireIsCanonicalPath(canonicalPath)
result.add('sit://' + canonicalPath)
deps = getDependencies(canonicalPath,
recursive=True,
cache=cache,
systemPackages=False,
sitPath=sitPath)
flatDeps = FastScript.flattenList(deps)
for item in flatDeps:
result.add(item)
return result
def isInstalled(url, sitPath=None):
"""
Checks if the given package is installed. Depending on the URL type
different checks will be performed, e.g.:
'sit://Libraries/Serialize/3.0' # check in current SIT
'deb://binutils' # check for O.S. packages (*.deb)
You may speed-up this function by providing 'sitPath' so that it
does not need to be queried internally every time.
"""
from ToolBOSCore.Platforms import Debian
Any.requireIsTextNonEmpty(url)
if ':' not in url:
# default to SIT packages
url = 'sit://' + url
protocol, remainder = splitURL(url)
if protocol == 'sit':
if not sitPath:
sitPath = SIT.getPath()
status = isInstalled_sitPackage(remainder, sitPath)
elif protocol == 'deb':
try:
status = Debian.isInstalled(remainder)
except OSError as details:
raise OSError(details)
else:
raise RuntimeError('invalid URL protocol or format')
return status
def makeDocumentation( self ):
from ToolBOSCore.BuildSystem.DocumentationCreator import DocumentationCreator
if self._outOfTree:
logging.warning( 'documentation creation in out-of-tree build not implemented' )
return False
if Any.getDebugLevel() <= 3:
from six import StringIO
# capture output so that it's not printed
output = StringIO()
else:
output = None
try:
dstSIT = SIT.getPath()
dc = DocumentationCreator( self._sourceTree, dstSIT, output, output )
dc.generate()
except AssertionError as e:
logging.error( 'failed to create documentation: %s', e )
return False
except subprocess.CalledProcessError:
if output:
print( output.getvalue() )
logging.error( 'failed to create documentation' )
return False
def test_proxyInstallation(self):
oldcwd = os.getcwd()
packageName = 'ExamplePackage'
packageVersion = '1.0'
category = 'Applications'
projectRoot = os.path.join('.', packageName, packageVersion)
output = StringIO() if Any.getDebugLevel() <= 3 else None
platform = Platforms.getHostPlatform()
sitPath = SIT.getPath()
# build + install package
FastScript.changeDirectory(projectRoot)
bst = BuildSystemTools()
bst.setStdOut(output)
bst.setStdErr(output)
bst.compile()
Any.requireIsDirNonEmpty('build')
Any.requireIsDirNonEmpty(os.path.join('build', platform))
Any.requireIsDirNonEmpty('examples')
Any.requireIsDirNonEmpty(os.path.join('examples', platform))
for fileName in ('ThreadingExampleAtomicOperation',
'ThreadingExampleJoin', 'ThreadingExampleTraps'):
Any.requireIsFileNonEmpty(os.path.join('bin', platform, fileName))
if not ProxyDir.isProxyDir(sitPath):
self.skip("%s: Is not a proxy directory" % sitPath)
bst.makeDocumentation()
bst.proxyInstall()
installRoot = os.path.join(sitPath, category, packageName,
packageVersion)
# check result
Any.requireIsDir(installRoot)
Any.requireIsDirNonEmpty(os.path.join(installRoot, 'bin', platform))
Any.requireIsDirNonEmpty(os.path.join(installRoot, 'doc/html'))
Any.requireIsFileNonEmpty(
os.path.join(installRoot, 'doc/html/index.html'))
Any.requireIsFileNonEmpty(os.path.join(installRoot, 'pkgInfo.py'))
Any.requireIsFileNonEmpty(os.path.join(installRoot,
'packageVar.cmake'))
for fileName in ('ThreadingExampleAtomicOperation',
'ThreadingExampleJoin', 'ThreadingExampleTraps'):
Any.requireIsFileNonEmpty(
os.path.join(installRoot, 'bin', platform, fileName))
# clean-up
bst.uninstall(cleanGlobalInstallation=False)
bst.distclean()
self.assertFalse(os.path.exists(installRoot))
FastScript.changeDirectory(oldcwd)
def open( self, package=None ):
if self.url:
ProjectProperties.requireIsURL( self.url )
package = ProjectProperties.splitURL( self.url )[1]
else:
ProjectProperties.requireIsCanonicalPath( package )
topLevelDir = os.path.join( SIT.getPath(), package )
super( BSTProxyInstalledPackage, self ).open( topLevelDir )
def getMaintainerFromFilesystem(project):
"""
Returns the username of the maintainer by reading the fileowner of
the project from the current SIT.
"""
requireIsCanonicalPath(project)
path = SIT.getPath() + os.sep + project
owner = FastScript.getFileOwner(path)
return owner
def uninstall(canonicalPath, cleanGlobalInstallation, dryRun=False):
"""
Delete a package from SIT, this includes:
* Proxy SIT directory
* Global SIT installation
* BBCM *.def file
* RTMaps index entry
If 'cleanGlobalInstallation=True' the package will also be
uninstalled from global SIT (if applicable). If False it
will only be uninstalled from the proxy SIT.
"""
from ToolBOSCore.Platforms import Platforms
from ToolBOSCore.Tools import RTMaps
requireIsCanonicalPath(canonicalPath)
Any.requireIsBool(dryRun)
sitProxyPath = SIT.getPath()
sitRootPath = SIT.getRootPath()
Any.requireIsTextNonEmpty(sitProxyPath)
Any.requireIsTextNonEmpty(sitRootPath)
installRoot_proxy = os.path.join(sitProxyPath, canonicalPath)
installRoot_root = os.path.join(sitRootPath, canonicalPath)
rtmapsVersion = FastScript.getEnv('RTMAPS_VERSION')
logging.info('uninstalling %s', canonicalPath)
logging.info('cleaning proxy-installation')
FastScript.remove(installRoot_proxy, dryRun)
if cleanGlobalInstallation:
logging.info('cleaning global-installation')
FastScript.remove(installRoot_root, dryRun)
if rtmapsVersion:
Any.requireIsTextNonEmpty(rtmapsVersion)
hostPlatform = Platforms.getHostPlatform()
symlink_relHGR = RTMaps.getIndexFilePath_relHGR(
canonicalPath, rtmapsVersion, hostPlatform)
Any.requireIsTextNonEmpty(symlink_relHGR)
symlinkPath = os.path.join(sitProxyPath, symlink_relHGR)
Any.requireIsTextNonEmpty(symlinkPath)
FastScript.remove(symlinkPath, dryRun)
else:
logging.debug(
'skipped searching for RTMaps index symlink (RTMaps not sourced)')
def findProxyInstallations(checkLinks=False):
"""
Returns a list of packages installed within the proxy SIT.
A ValueError will be raised if the current SIT is not a proxy.
If 'checkLinks' == True, each version symlink will also be tested
if it really points into the root SIT. For example, it might be
difficult to localize a manually changed link that points e.g. in
another group proxy or so.
"""
resultList = []
sit = SIT.getPath()
sitParent = SIT.getParentPath() # support cascaded proxies
excludePattern = re.compile('^(parentTree|\d+\.\d+.*)$')
criteria = re.compile("^(\d+)\.(\d+)(.*)")
Any.requireIsDir(sit)
Any.requireIsDir(sitParent)
requireIsProxyDir(sit)
# find all entries within the proxy that seem to be a version number
dirList = FastScript.getDirsInDirRecursive(
sit,
excludePattern=excludePattern,
onError=FastScript.printPermissionDenied)
# check each found entry if it is a version directory, if so we
# can expect that this is a local project installation
for project in dirList:
for entry in os.listdir(project):
joinedPath = os.path.join(project, entry)
# only perform test on version numbers
if criteria.search(entry):
# don't use os.path.isdir() as it follows symlinks!
fileType = os.lstat(joinedPath).st_mode
if checkLinks == True and stat.S_ISLNK(fileType) == True:
# test if symlink points into root SIT
target = os.readlink(joinedPath)
if target[0] == '/' and not target.startswith(sitParent):
logging.warning(joinedPath + ': points to ' + target)
elif stat.S_ISDIR(fileType):
# we found a proxy installation
resultList.append(joinedPath)
return resultList
def __init__(self,
projectRoot,
sitPath=None,
stdout=None,
stderr=None,
details=None):
Any.requireIsDir(projectRoot)
self.projectRoot = projectRoot
logging.debug('topLevelDir=%s' % projectRoot)
if details is None:
details = PackageDetector(projectRoot)
try:
details.retrieveMakefileInfo()
except EnvironmentError as e:
logging.warning(e) # e.g. $SIT not defined in environment
else:
Any.requireIsInstance(details, PackageDetector)
if sitPath is None:
sitPath = SIT.getPath()
if FastScript.getEnv('MAKEFILE_DOC') == 'FALSE':
handler = NullBackend
elif details.docTool == 'doxygen':
handler = DoxygenBackend
elif details.docTool == 'matdoc':
handler = MatlabBackend
elif details.docTool == '':
handler = NullBackend
elif details.isMatlabPackage():
# [CIA-1131] matdoc no longer works on xenial64
#
# On Ubuntu 16.04 disable documentation creation
# (do not use doxygen either to not overwrite the
# correct matdoc-output from Ubuntu 14.04 machines)
if getHostPlatform().startswith('xenial'):
handler = NullBackend
else:
handler = MatlabBackend
else:
handler = DoxygenBackend
self.backend = handler(details, sitPath, stdout, stderr)
def getIndexBaseDir(sitProxyPath=None):
"""
Returns the path to the user's index base directory.
If 'sitProxyPath' is omitted then the current Proxy-SIT is used.
"""
if sitProxyPath is None:
sitProxyPath = SIT.getPath()
ProxyDir.requireIsProxyDir(sitProxyPath)
indexBaseDir = os.path.join(sitProxyPath, 'Modules', 'Index')
return indexBaseDir
def registerNormalPackages(sitPath, dryRun=False):
"""
Searches the SIT for RTMaps packages and invokes
registerHondaPackage() for each of them.
"""
Any.requireIsDir(sitPath)
Any.requireIsBool(dryRun)
sitProxyPath = SIT.getPath()
ProxyDir.requireIsProxyDir(sitProxyPath)
searchBaseDir = os.path.join(sitPath, 'Modules', 'RTMaps')
# Any.requireIsTextNonEmpty( searchBaseDir ) # dir. might not exist
Any.requireIsTextNonEmpty(searchBaseDir)
indexBaseDir = getIndexBaseDir(sitProxyPath)
# Any.requireIsDir( indexBaseDir ) # dir. might not exist
Any.requireIsTextNonEmpty(indexBaseDir)
logging.debug('SIT path: %s', sitPath)
logging.debug('search basedir: %s', searchBaseDir)
logging.debug('index basedir: %s', indexBaseDir)
# find *.pck files
logging.debug('scanning %s...', searchBaseDir)
pckPaths = FastScript.findFiles(searchBaseDir, ext='.pck')
Any.requireIsList(pckPaths)
logging.debug('found HORP files:')
logging.debug(pckPaths)
# process each *.pck file (tokenize path and create symlink)
for pckPath in pckPaths:
# the *.pck file is located on the 3rd subdirectory level relative
# to the installRoot, so compute the installRoot based on this
tokens = pckPath.split(os.path.sep)
installRoot = os.path.sep.join(tokens[:-3])
package = SIT.strip(installRoot)
Any.requireIsDir(installRoot)
Any.requireIsTextNonEmpty(package)
try:
registerNormalPackage(package, sitProxyPath, indexBaseDir, dryRun)
except ValueError as e:
logging.error(e)
class BSTProxyInstalledPackage( BSTInstalledPackage ):
"""
Represents a software package which is installed in the
Proxy-SIT of the developer.
"""
_sitPath = SIT.getPath()
def open( self, package=None ):
if self.url:
ProjectProperties.requireIsURL( self.url )
package = ProjectProperties.splitURL( self.url )[1]
else:
ProjectProperties.requireIsCanonicalPath( package )
topLevelDir = os.path.join( SIT.getPath(), package )
super( BSTProxyInstalledPackage, self ).open( topLevelDir )
def populate(self):
"""
Scans all package in SIT and stores the ground-truth pkgInfo.py
information into one giant hashtable for later fast access.
The assignment is "packageURL": { pkgInfo data }
"""
sitPath = SIT.getPath()
canonicalPaths = SIT.getCanonicalPaths(sitPath)
Any.requireIsListNonEmpty(canonicalPaths)
for canonicalPath in canonicalPaths:
ProjectProperties.requireIsCanonicalPath(canonicalPath)
packageURL = 'sit://' + canonicalPath
installRoot = os.path.join(sitPath, canonicalPath)
detector = PackageDetector(installRoot)
detector.retrieveMakefileInfo()
self._cache[packageURL] = detector
def isDeprecated(canonicalPath):
"""
Checks from the filesystem if the specified package (canonical path)
is flagged as deprecated.
Returns True if either of these files exists:
<sitRoot>/Libraries/Spam/42.0/deprecated.txt
<sitRoot>/Libraries/Spam/deprecated.txt
or if the canonicalPath is listed in the file
<sitPath>/Temporary/CIA/1.0/deprecatedOverride.txt.
"""
requireIsCanonicalPath(canonicalPath)
filename = 'deprecated.txt'
sitRoot = SIT.getRootPath()
pkgPath = os.path.join(sitRoot, canonicalPath)
check1 = os.path.join(sitRoot, os.path.dirname(canonicalPath), filename)
check2 = os.path.join(sitRoot, canonicalPath, filename)
# if package is not present in SIT we can't give reliable information
# if it is deprecated or not
if not os.path.exists(pkgPath):
raise ValueError("%s: Package not installed in SIT" % canonicalPath)
if os.path.exists(check1) or os.path.exists(check2):
return True
# check CIA operator "sudo override"
overrideFile = os.path.join(SIT.getPath(),
'Temporary/CIA/1.0/deprecatedOverride.txt')
if os.path.exists(overrideFile):
for line in FastScript.getFileContent(overrideFile, splitLines=True):
if line.strip() == canonicalPath:
return True
return False
def updateProxyDir(removeBrokenSymlinks=True,
removeEmptyCategories=True,
linkNewPackagesIntoProxy=True,
checkProxyLinkTarget=True,
checkProxyLinkedVersion=True,
removeProxyInstallations=False,
cleanHomeDirectory=True,
updateRTMapsIndex=True,
dryRun=False):
"""
Updates the SIT proxy directory of the current user.
The user may influence which worker functions shall be called
(default: all)
removeBrokenSymlinks: remove broken symlinks to uninstalled
packages
removeEmptyCategories: remove SIT categories that became empty,
or packages without any version
linkNewPackagesIntoProxy: if there are new packages in the global
SIT, add a link to them into the proxy
checkProxyLinkTarget: verify that links are valid
checkProxyLinkedVersion: if there is a higher revision globally
installed (e.g. 1.0.100) and the user
has a link 1.0 pointing to 1.0.99 in the
proxy, the 1.0 link will get updated to
1.0.100 in order not to use an outdated
revision
removeProxyInstallations: DELETE ALL PACKAGES installed in the
proxy (if any)
cleanHomeDirectory: clean-up unused files under ~/.HRI
updateRTMapsIndex: update *.pck symlinks in ~/.HRI/RTMaps
If dryRun=True, nothing will actually be done.
"""
from ToolBOSCore.Tools import RTMaps
sitRoot = SIT.getParentPath()
sitProxy = SIT.getPath()
proxyChanged = False
sitRootPkgList = []
sitProxyPkgList = []
pluginsEnabled = []
Any.requireIsBool(removeBrokenSymlinks)
Any.requireIsBool(removeEmptyCategories)
Any.requireIsBool(linkNewPackagesIntoProxy)
Any.requireIsBool(checkProxyLinkTarget)
Any.requireIsBool(checkProxyLinkedVersion)
Any.requireIsBool(removeProxyInstallations)
Any.requireIsBool(cleanHomeDirectory)
Any.requireIsBool(updateRTMapsIndex)
Any.requireIsBool(dryRun)
Any.requireMsg(sitRoot != sitProxy,
'%s: Is not a proxy directory' % sitProxy)
# TBCORE-466: user should be able to disable particular plugins
#if removeProxyInstallations: # added later, see below
#pluginsEnabled.append( _removeProxyInstallations )
if removeBrokenSymlinks:
pluginsEnabled.append(_removeBrokenSymlinks)
if removeEmptyCategories:
pluginsEnabled.append(_removeEmptyCategories)
if linkNewPackagesIntoProxy:
pluginsEnabled.append(_linkNewPackagesIntoProxy)
if checkProxyLinkTarget:
pluginsEnabled.append(_checkProxyLinkTarget)
if checkProxyLinkedVersion:
pluginsEnabled.append(_checkProxyLinkedVersion)
if cleanHomeDirectory:
pluginsEnabled.append(_cleanHomeDirectory)
if not pluginsEnabled:
raise ValueError('Nothing to do. Please check your parameters.')
# in any case, after updating the proxy verify that there are no legacy
# *.def files laying around
pluginsEnabled.append(_checkDefFiles)
tp = ThreadPool.ThreadPool()
tp.add(SIT.getProjectsWithErrorHandling, sitRoot, sitRootPkgList)
tp.add(SIT.getProjectsWithErrorHandling, sitProxy, sitProxyPkgList)
tp.run()
if removeProxyInstallations:
changed = _removeProxyInstallations(sitRootPkgList, sitProxyPkgList,
sitRoot, sitProxy, dryRun)
if changed > 0:
sitProxyPkgList = []
SIT.getProjectsWithErrorHandling(sitProxy, sitProxyPkgList)
for func in pluginsEnabled:
proxyChanged |= func(sitRootPkgList, sitProxyPkgList, sitRoot,
sitProxy, dryRun)
if updateRTMapsIndex:
if RTMaps.isInstalled(sitRoot):
RTMaps.updateComponentIndex(sitRoot, sitProxy, dryRun)
else:
logging.debug('RTMaps not installed')
msg = 'Your proxy is up-to-date%s.' % (' now'
if proxyChanged == True else '')
logging.info('')
logging.info(msg)
def main(package):
# When using shell tab-completion on SIT paths for the canonical path,
# a trailing slash might be added. Remove this before proceeding.
if package.endswith('/'):
package = package[:-1]
logging.debug('package truncated to: %s', package)
try:
ProjectProperties.requireIsCanonicalPath(package)
except AssertionError as details:
logging.error(details)
raise SystemExit
# stop if package is maintained via SVN
svnUrl = ProjectProperties.getSVNLocationFromFilesystem(package)
if svnUrl:
logging.debug('found SVN URL: %s', svnUrl)
logging.error('')
logging.error('Package is maintained via SVN and cannot be cloned.')
logging.error(
'Use GitCheckout.py to access it in Git-style using the git-svn bridge :-)'
)
logging.error('')
sys.exit(-2)
# try to look-up repository URL from SIT, if not found do at least
# a good guess
installRoot = os.path.join(SIT.getPath(), package)
try:
detector = PackageDetector.PackageDetector(installRoot)
detector.retrieveMetaInfoFromSIT()
url = detector.gitOrigin
except AssertionError as e:
logging.error(e)
raise SystemExit()
# logging.debug( 'guessing Git location...' )
# url = ProjectProperties.guessGitLocation( package ) # TODO: implement guessing
if not url:
logging.error('unable to determine Git repository location')
sys.exit(-1)
Any.requireIsTextNonEmpty(url)
repo = Git.RemoteGitRepository(url)
# warn if cloning from unofficial host (see CIA-1062)
repo.checkIsOnMasterServer()
if userName:
repo.setUserName(userName)
Any.setDebugLevel(logging.DEBUG) # print executed command
repo.clone()
def source(package):
"""
Python equivalent of "source BashSrc" from SIT, in order to setup
PATH, LD_LIBRARY_PATH,... within the Python process.
@anchor ProcessEnv_source
"""
ProjectProperties.requireIsCanonicalPath(package)
sourced = FastScript.getEnv('TOOLBOSCORE_SOURCED')
Any.requireMsg(sourced, '$TOOLBOSCORE_SOURCED must not be empty')
# avoid double-sourcing
if package in sourced:
return True
ProjectProperties.requireIsInstalled(package)
logging.debug('source %s/pkgInfo.py', package)
sourced = '%s %s' % (package, sourced)
FastScript.setEnv('TOOLBOSCORE_SOURCED', sourced)
# load pkgInfo.py (if exists)
try:
content = getPkgInfoContent(project=package)
except AssertionError:
return True # no such file, this is OK
except (IOError, OSError) as details:
logging.error(details)
return False
# setup environment of this package
try:
envVars = content['envVars']
except KeyError:
envVars = [] # no such setting, this is OK
sitPath = SIT.getPath()
for name, value in envVars:
# replace known placeholdes
value = value.replace('${INSTALL_ROOT}',
os.path.join(sitPath, package))
FastScript.setEnv_withExpansion(name, value)
# source dependent packages
try:
# TODO: eventually extend to sourcing recommended/suggested packages
depList = content['depends']
except (AssertionError, KeyError):
depList = [] # no such setting, this is OK
for dep in depList:
if not dep.startswith('deb://'):
source(SIT.strip(dep))
# special treatment of PYTHONPATH:
# After sourcing add new entries in PYTHONPATH to sys.path
_expandSysPath()
return True
class BSTInstalledPackage( BSTPackage ):
"""
Represents a software package which is installed in the
Software Installation Tree (SIT).
"""
_sitPath = SIT.getPath()
_metaInfoCache = None
def __init__( self, url=None ):
super( BSTInstalledPackage, self ).__init__( url )
self.revDepSet = None
self.revDepTree = None
def open( self, topLevelDir ):
super( BSTInstalledPackage, self ).open( topLevelDir )
def isInstalled( self ):
# if package could not be opened then self.detector is None,
# hence accessing self.detector.topLevelDir won't work
if self.detector is None:
return False
Any.requireIsTextNonEmpty( self.detector.topLevelDir )
Any.requireIsTextNonEmpty( self._sitPath )
return os.path.exists( os.path.join( self._sitPath,
self.detector.topLevelDir ) )
def retrieveReverseDependencies( self, recursive ):
self._ensureMetaInfoCache()
self.revDepSet = set()
self.revDepTree = list()
for depURL in self._metaInfoCache.getReverseDependencies( self.url ):
ProjectProperties.requireIsURL( depURL )
# no Debian packages can appear in reverse dependencies of SIT packages
depPackage = BSTInstalledPackage( depURL )
depPackage.detector = self._metaInfoCache.getDetector( depURL )
if recursive:
depPackage.retrieveReverseDependencies( recursive )
self.revDepSet.update( depPackage.revDepSet )
self.revDepSet.add( depURL )
self.revDepTree.append( depPackage )
def _ensureMetaInfoCache( self ):
"""
Creates and populates the internal MetaInfoCache,
if not already existing.
"""
if not BSTInstalledPackage._metaInfoCache:
BSTInstalledPackage._metaInfoCache = MetaInfoCache()
BSTInstalledPackage._metaInfoCache.populate()
def setMetaInfoCache( self, metaInfoCache ):
Any.requireIsInstance( metaInfoCache, MetaInfoCache )
BSTInstalledPackage._metaInfoCache = metaInfoCache
from ToolBOSCore.Settings import ToolBOSSettings
from ToolBOSCore.Storage import SIT
from ToolBOSCore.Storage.BashSrc import BashSrcWriter
from ToolBOSCore.Storage.CmdSrc import CmdSrcWriter
from ToolBOSCore.Storage.PackageVar import PackageVarCmakeWriter
from ToolBOSCore.Storage.PkgInfoWriter import PkgInfoWriter
from ToolBOSCore.Util import Any
from ToolBOSCore.Util import FastScript
# location of core templates such as Master, C_Library etc.
templateDir_core = os.path.join(FastScript.getEnv('TOOLBOSCORE_ROOT'),
'etc/mako-templates')
# location of higher-level templates such as BBCMs etc.
templateDir = os.path.join(
SIT.getPath(), ToolBOSSettings.getConfigOption('package_pkgCreator'),
'etc/mako-templates')
class PackageCreator(object):
# some templates should not be directly visible to the users,
# e.g. the Software Quality Guideline, if False they will be hidden
# in the Package Creator GUI (TBCORE-1201)
show = True
def __init__(self, packageName, packageVersion, values=None, outputDir=''):
Any.requireIsTextNonEmpty(packageName)
Any.requireIsTextNonEmpty(packageVersion)
Any.requireIsText(outputDir)
def registerNormalPackage(package,
sitProxyPath=None,
indexBaseDir=None,
dryRun=False):
"""
Creates a symlink to the *.pck file of 'package' within the RTMaps
index.
RTMAPS_VERSION is taken from the dependency list of the package.
"""
if sitProxyPath is None:
sitProxyPath = SIT.getPath()
if indexBaseDir is None:
indexBaseDir = getIndexBaseDir(sitProxyPath)
ProjectProperties.requireIsCanonicalPath(package)
Any.requireIsDir(sitProxyPath)
Any.requireIsDir(indexBaseDir)
Any.requireIsBool(dryRun)
platformList = getPlatformNames()
packageName = ProjectProperties.getPackageName(package)
packageVersion = ProjectProperties.getPackageVersion(package)
versionTokens = ProjectProperties.splitVersion(packageVersion)
majorVersion = int(versionTokens[0])
minorVersion = int(versionTokens[1])
installRoot = os.path.join(sitProxyPath, package)
Any.requireIsListNonEmpty(platformList)
Any.requireIsTextNonEmpty(packageName)
Any.requireIsTextNonEmpty(packageVersion)
Any.requireIsDir(installRoot)
deps = ProjectProperties.getDependencies(package)
try:
Any.requireIsListNonEmpty(deps)
except AssertionError:
logging.debug(
'empty list of dependencies in RTMaps package is unplausible')
msg = "%s: unable to retrieve dependencies, please check SIT installation" % package
raise ValueError(msg)
expr = re.compile('^sit://External/RTMaps/(\d+\.\d+)')
# detect RTMaps version used by package
rtmapsVersion = ''
for dep in deps:
tmp = expr.match(dep)
if tmp:
rtmapsVersion = tmp.group(1)
break
Any.requireIsTextNonEmpty(rtmapsVersion)
logging.debug('%s depends on RTMaps %s', package, rtmapsVersion)
libDir = os.path.join(installRoot, 'lib')
pckFiles = FastScript.findFiles(libDir, ext='.pck')
Any.requireMsg(
len(pckFiles) > 0,
package + ": No *.pck file found, forgot to compile?")
for relPath in pckFiles:
pckFileName = os.path.basename(relPath)
pckPackage = os.path.splitext(pckFileName)[0]
pckFileExt = os.path.splitext(pckFileName)[1]
logging.debug('registering %s', pckPackage)
for platform in platformList:
pckPath = os.path.join(libDir, platform, pckFileName)
dstDir = os.path.join(indexBaseDir, rtmapsVersion, platform)
if os.path.exists(pckPath):
symlinkFile = '%s_%d.%d%s' % (pckPackage, majorVersion,
minorVersion, pckFileExt)
symlinkPath = os.path.join(dstDir, symlinkFile)
target = pckPath
FastScript.link(target, symlinkPath, dryRun)
class TaskButtonsWidget(QGroupBox, object):
"""
Provides a group box with the main control buttons (build, clean,
install, run unittest,...)
"""
build = pyqtSignal()
clean = pyqtSignal()
globalInstall = pyqtSignal()
proxyInstall = pyqtSignal()
quit = pyqtSignal()
test = pyqtSignal()
hasProxy = ProxyDir.isProxyDir(SIT.getPath())
def __init__(self):
super(QGroupBox, self).__init__('commands')
self._buildButton = QPushButton("&build")
self._cleanButton = QPushButton("&distclean")
self._testButton = QPushButton("run &testsuite")
self._proxyInstButton = QPushButton("pro&xy inst.")
self._globalInstButton = QPushButton("global &inst.")
self._quitButton = QPushButton("&quit")
self._cleanButton.setToolTip('delete build artefacts')
self._buildButton.setToolTip('compile for selected platforms')
self._proxyInstButton.setToolTip('install into Proxy-SIT')
self._globalInstButton.setToolTip('globally install into SIT')
self._testButton.setToolTip('start unittest (if present)')
if not self.hasProxy:
self._proxyInstButton.setToolTip('No Proxy-SIT available')
self._proxyInstButton.setEnabled(False)
self._cleanButton.pressed.connect(self.clean.emit)
self._buildButton.pressed.connect(self.build.emit)
self._proxyInstButton.pressed.connect(self.proxyInstall.emit)
self._globalInstButton.pressed.connect(self.globalInstall.emit)
self._testButton.pressed.connect(self.test.emit)
self._quitButton.pressed.connect(self.quit.emit)
# set start-up focus and run
self._buildButton.setFocus()
self._layout = QGridLayout()
self._layout.addWidget(self._cleanButton, 1, 1)
self._layout.addWidget(self._buildButton, 1, 2)
self._layout.addWidget(self._proxyInstButton, 2, 1)
self._layout.addWidget(self._globalInstButton, 2, 2)
self._layout.addWidget(self._testButton, 3, 1)
self._layout.addWidget(self._quitButton, 3, 2)
if not os.path.exists( 'unittest.sh' ) and \
not os.path.exists( 'unittest.bat' ):
self._testButton.setEnabled(False)
self._testButton.setToolTip(
'unittest.sh or unittest.bat not found')
self.setLayout(self._layout)
def setEnabled(self, status, button='all'):
Any.requireIsBool(status)
Any.requireIsTextNonEmpty(button)
if button == 'all':
self._buildButton.setEnabled(status)
self._cleanButton.setEnabled(status)
self._globalInstButton.setEnabled(status)
self._proxyInstButton.setEnabled(status and self.hasProxy)
self._testButton.setEnabled(status)
elif button == 'build':
self._buildButton.setEnabled(status)
elif button == 'clean':
self._cleanButton.setEnabled(status)
elif button == 'globalInstall':
self._globalInstButton.setEnabled(status)
elif button == 'proxyInstall':
self._proxyInstButton.setEnabled(status and self.hasProxy)
elif button == 'test':
self._testButton.setEnabled(status)
else:
raise ValueError('unknown button: %s' % button)
def __init__(self, model):
super(DependenciesDialog, self).__init__()
Any.requireIsInstance(model, QtPackageModel.BSTPackageModel)
numColumns = 4
self._model = model
self._treeMode = True
self._reverseMode = False
self._tree = None
self._treeWidget = QTreeWidget()
self._red = QColor(255, 120, 120)
self._green = QColor(220, 255, 220)
self._grey = QColor(240, 240, 240)
self._canonicalPath = self._model.getCanonicalPath()
self._sitProxyPath = SIT.getPath()
self._sitRootPath = SIT.getRootPath()
self._hasProxy = ProxyDir.isProxyDir(self._sitProxyPath)
# obtain data and create one QTreeWidgetItem for each
self._data1 = self._model.getDependencySet()
self._data2 = self._model.getDependencyTree()
self._data3 = self._model.getReverseDependencySet()
self._data4 = self._model.getReverseDependencyTree()
self._aptCmd = self._model.getDepInstallCmd_APT()
logging.debug('data1: %s', self._data1)
logging.debug('data2: %s', self._data2)
logging.debug('data3: %s', self._data3)
logging.debug('data4: %s', self._data4)
depsEnabled = self._data1 is not None and self._data2 is not None
revDepsEnabled = self._data3 is not None and self._data4 is not None
if depsEnabled:
self._tree1 = self._createTreeWidgetFromSet(self._data1, False)
self._tree2 = self._createTreeWidgetFromList(self._data2, False)
if revDepsEnabled:
self._tree3 = self._createTreeWidgetFromSet(self._data3, True)
self._tree4 = self._createTreeWidgetFromList(self._data4, True)
if not depsEnabled and not revDepsEnabled:
logging.error('unable to show dependency-dialog')
return
depsTooltip = 'low-level packages needed by the current one'
if revDepsEnabled:
revDepsTooltip = 'high-level packages which use the current one'
else:
revDepsTooltip = 'not globally installed --> no reverse dependencies'
self._depRadio_direct = QRadioButton('&dependencies')
self._depRadio_direct.setChecked(True)
self._depRadio_direct.setToolTip(depsTooltip)
self._depRadio_direct.clicked.connect(self._showDependencies)
self._depRadio_reverse = QRadioButton('&reverse dependencies')
self._depRadio_reverse.setToolTip(revDepsTooltip)
self._depRadio_reverse.setEnabled(revDepsEnabled)
self._depRadio_reverse.clicked.connect(self._showReverseDependencies)
depSelectLayout = QVBoxLayout()
depSelectLayout.addWidget(self._depRadio_direct)
depSelectLayout.addWidget(self._depRadio_reverse)
depSelectWidget = QGroupBox('direction')
depSelectWidget.setLayout(depSelectLayout)
# initial / default view: show own dependencies as tree
self._tree = self._tree2
self._updateView()
self._treeWidget.setColumnCount(numColumns)
self._treeWidget.setRootIsDecorated(
False) # top-left has no parent line
self._treeWidget.setSortingEnabled(False)
header = self._treeWidget.header()
header.setStretchLastSection(False)
header.setSectionResizeMode(
0, QHeaderView.Stretch) # stretch first column
headerWidget = QTreeWidgetItem([
'dependency', 'installed in proxy SIT', 'installed in global SIT',
'installed locally'
])
for i in range(1, numColumns):
header.setSectionResizeMode(i, QHeaderView.ResizeToContents)
headerWidget.setTextAlignment(i, Qt.AlignCenter)
self._treeWidget.setHeaderItem(headerWidget)
self._aptLabel = QLabel('Debian/Ubuntu packages needed:')
self._aptWidget = QTextEdit(self._aptCmd)
self._aptWidget.setFixedHeight(100)
self._aptWidget.setFontFamily("Courier New")
self._aptWidget.setReadOnly(True)
toggleView = QPushButton('&toggle tree/flat view')
toggleView.pressed.connect(self._toggleTree)
self._expandButton = QPushButton('&expand all')
self._expandButton.pressed.connect(self._treeWidget.expandAll)
self._collapseButton = QPushButton('&collapse all')
self._collapseButton.pressed.connect(self._treeWidget.collapseAll)
self._collapseButton.pressed.connect(
lambda: self._tree.setExpanded(True))
closeButton = QPushButton('&OK')
closeButton.pressed.connect(self.close)
buttonLayout = QHBoxLayout()
buttonLayout.setAlignment(Qt.AlignRight)
buttonLayout.addWidget(toggleView)
buttonLayout.addSpacing(1)
buttonLayout.addWidget(self._expandButton)
buttonLayout.addWidget(self._collapseButton)
buttonLayout.addWidget(closeButton)
buttonWidget = QWidget()
buttonWidget.setLayout(buttonLayout)
mainLayout = QVBoxLayout()
mainLayout.addWidget(depSelectWidget)
mainLayout.addWidget(self._treeWidget)
mainLayout.addWidget(self._aptLabel)
mainLayout.addWidget(self._aptWidget)
mainLayout.addWidget(buttonWidget)
# noinspection PyArgumentList
screen = QApplication.desktop().screenGeometry()
dialogWidth = screen.width() * 0.5
dialogHeight = screen.height() * 0.75
self.setLayout(mainLayout)
self.setWindowTitle('Dependencies of %s' % self._canonicalPath)
self.resize(dialogWidth, dialogHeight)
self.move(screen.center() - self.rect().center()) # center
self.show()
# When using shell tab-completion on SIT paths for the canonical path,
# a trailing slash might be added. Remove this before proceeding.
if package.endswith('/'):
package = package[:-1]
logging.debug('package truncated to: %s', package)
try:
ProjectProperties.requireIsCanonicalPath(package)
except AssertionError as details:
logging.error(details)
raise SystemExit
# look for repository URL
sitPath = SIT.getPath()
installRoot = os.path.join(sitPath, package)
url = None
detector = None
try:
detector = PackageDetector.PackageDetector(installRoot)
except AssertionError:
pass # package not found in SIT
if detector is not None:
detector.retrieveMetaInfoFromSIT()
if detector.gitOrigin:
def getDependencies(project,
recursive=False,
cache=None,
ignoreErrors=False,
highlightMissing=True,
systemPackages=True,
sitPath=None):
"""
Returns a list containing the direct dependencies of this package.
If "recursive=True", they will be followed and the list actually
becomes a nested list / tree. If the package does not have any
dependencies the list will be empty.
'project' must be specified in canonical form, e.g.:
- 'Libraries/Serialize/3.0'
- 'sit://Libraries/Serialize/3.0'
- 'deb://gcc'
If 'recursive=False', the list will contain the pure untreated
strings stored in the pkgInfo.py file. There is no preprocessing
done on its semantics. In recursive mode, we need to follow those
paths and need to resolve 'sit://' etc.
The 'cache' map is internally used to avoid processing already
treated packages again. You should not specify this parameter when
calling this function unless you need to query the dependencies of
multiple projects in a list. In such case providing a cache-map
will improve performance a lot.
If 'systemPackages=False', Non-SIT packages such as *.deb or *.rpm
packages will be excluded from the result list.
You may speed-up this function by providing 'sitPath' so that it
does not need to be queried internally every time.
"""
from ToolBOSCore.Storage.PkgInfo import getPkgInfoContent
if project.find('://') > 0:
projectURL = project
project = SIT.strip(project)
else:
projectURL = 'sit://' + project # default protocol
resultList = []
if cache is None:
cache = {}
if not sitPath:
sitPath = SIT.getPath()
try: # first look-up the cache
depList = cache[projectURL]
except KeyError: # if not found read from filesystem
if projectURL.startswith('sit://'):
try:
depList = getPkgInfoContent(project)['depends']
except (AssertionError, KeyError):
depList = []
else:
depList = [] # retrieving *.deb dependencies not implemented
# ensure to only use URL-style dependencies
for i, dep in enumerate(depList):
if isinstance(dep, str):
depList[i] = dep.replace('${HRI_GLOBAL_ROOT}/', 'sit://')
# store the direct dependencies into the cache map
cache[projectURL] = depList
for dep in depList:
if isinstance(dep, str):
if systemPackages == False and not dep.startswith('sit://'):
continue
resultList.append(dep)
if recursive:
# 2012-10-09 reactivated this check for isInstalled() == False,
# needed by ListDependencies.py + ExportWizard.py
try:
if not isInstalled(dep, sitPath):
if highlightMissing:
resultList.append([missingPkgMsg])
if not ignoreErrors:
msg = '%s depends on %s which is not installed' \
% ( project, dep )
raise RuntimeError(msg)
except ValueError:
raise ValueError('invalid dependency "%s" in package %s' %
(dep, project))
subDepList = getDependencies(dep, recursive, cache,
ignoreErrors, highlightMissing,
systemPackages, sitPath)
if len(subDepList) > 0:
resultList.append(subDepList)
else:
raise TypeError('dependency entry is not of type "str"')
return resultList
bstpkg_global = BSTPackage.BSTGloballyInstalledPackage()
try:
bstpkg_global.open(canonicalPath)
bstpkg_global.retrieveReverseDependencies(True)
except AssertionError as details:
# most likely: package is not globally installed
logging.info(details)
bstpkg_global.revDepSet = set()
Any.requireIsSet(bstpkg_src.depSet)
Any.requireIsSet(bstpkg_global.revDepSet)
fullSet = bstpkg_src.depSet | bstpkg_global.revDepSet
sitProxyPath = SIT.getPath()
sitRootPath = SIT.getRootPath()
installStatus = {}
installStatusLocal = {}
installStatusProxy = {}
installStatusGlobal = {}
logging.debug('retrieving install status of [reverse-]dependencies...')
for packageURL in fullSet:
protocol, remainder = ProjectProperties.splitURL(packageURL)
# _installStatus is the generic form, holding the installStatus
# for Debian packages (= locally) and SIT packages (= in proxy)
# since those are the effective locations for compiling sources
