def setupProxy(sitRootPath=None, sitProxyPath=None):
"""
Convenience-wrapper to create a default proxy directory.
WARNING: If a proxy already exists, it will be DELETED !
You may specify the absolute paths for the SIT root- and/or
proxy directories. If omitted, typical defaults will be used.
"""
from ToolBOSCore.Storage import ProxyDir
from ToolBOSCore.Storage import SIT
if not sitRootPath:
sitRootPath = SIT.getDefaultRootPath()
if not sitProxyPath:
sitProxyPath = SIT.getDefaultProxyPath()
Any.requireIsTextNonEmpty(sitRootPath)
Any.requireIsTextNonEmpty(sitProxyPath)
Any.requireIsDir(sitRootPath)
# delete existing proxy if it exists, it might be screwed up
if os.path.exists(sitProxyPath):
logging.info('cleaning existing proxy in %s' % sitProxyPath)
FastScript.remove(sitProxyPath)
logging.info('creating proxy directory... (this may take some time)')
logging.info('SIT Root: %s' % sitRootPath)
logging.info('SIT Proxy: %s' % sitProxyPath)
ProxyDir.createProxyDir(sitRootPath, sitProxyPath, verbose=False)
def test_findProxyInstallations(self):
# check if user has a valid proxy:
sitRootPath = SIT.getParentPath()
sitProxyPath = SIT.getPath()
Any.requireIsDir(sitRootPath)
Any.requireIsDir(sitProxyPath)
ProxyDir.requireIsProxyDir(sitProxyPath)
# create a fake package directory within the proxy...
packageName = 'UnittestABCDEF123'
goodNameDir = os.path.join(sitProxyPath, 'Libraries', packageName)
goodVersionDir = os.path.join(goodNameDir, '42.0')
badVersionDir = os.path.join(goodNameDir, '21.0')
FastScript.mkdir(goodVersionDir)
resultList = ProxyDir.findProxyInstallations()
self.assertTrue(
len(resultList) >= 1
) # may be greater if developer has real software installed in proxy
self.assertTrue(goodVersionDir in resultList)
self.assertFalse(badVersionDir in resultList)
# clean-up
FastScript.remove(goodNameDir)
def getPackageCategoryFromPath(projectRootDir):
"""
Returns the packageCategory (former known as "project start path") of a
module, e.g.:
getPackageCategoryFromPath( '/hri/sit/latest/Modules/BBCM/Food/FeedSnake/42.0' )
getPackageCategoryFromPath( 'Modules/BBCM/Food/FeedSnake/42.0' )
return both 'Modules/BBCM/Food'.
"""
Any.requireIsTextNonEmpty(projectRootDir)
tmp = projectRootDir
# first remove trailing project name and version from string
search = os.sep + getPackageName(tmp) + os.sep + getPackageVersion(tmp)
replace = ''
tmp = tmp.replace(search, replace)
# replace any values of ${HRI_GLOBAL_ROOT} or ${SIT}
tmp = SIT.stripHGR(tmp)
tmp = SIT.stripSIT(tmp)
# remove trailing slashes (occurs if user provides a string with trailing slashes)
while tmp.endswith('/'):
tmp = tmp[0:-1]
return tmp
def _getInstallStatus(self, package, sitPath):
try:
installed = ProjectProperties.isInstalled(package, sitPath)
except ValueError:
installed = False
if installed:
if package.startswith('sit://'):
msg = '%s: exists' % \
os.path.join( sitPath, SIT.strip( package) )
else:
msg = '%s: installed on this machine' % \
package.replace( 'deb://', '' )
else:
if package.startswith('sit://'):
msg = '%s: no such file or directory' % \
os.path.join( sitPath, SIT.strip( package ) )
else:
msg = '%s: not installed on this machine' % \
package.replace( 'deb://', '' )
return installed, msg
def uninstall(canonicalPath, cleanGlobalInstallation, dryRun=False):
"""
Delete a package from SIT, this includes:
* Proxy SIT directory
* Global SIT installation
* BBCM *.def file
* RTMaps index entry
If 'cleanGlobalInstallation=True' the package will also be
uninstalled from global SIT (if applicable). If False it
will only be uninstalled from the proxy SIT.
"""
from ToolBOSCore.Platforms import Platforms
from ToolBOSCore.Tools import RTMaps
requireIsCanonicalPath(canonicalPath)
Any.requireIsBool(dryRun)
sitProxyPath = SIT.getPath()
sitRootPath = SIT.getRootPath()
Any.requireIsTextNonEmpty(sitProxyPath)
Any.requireIsTextNonEmpty(sitRootPath)
installRoot_proxy = os.path.join(sitProxyPath, canonicalPath)
installRoot_root = os.path.join(sitRootPath, canonicalPath)
rtmapsVersion = FastScript.getEnv('RTMAPS_VERSION')
logging.info('uninstalling %s', canonicalPath)
logging.info('cleaning proxy-installation')
FastScript.remove(installRoot_proxy, dryRun)
if cleanGlobalInstallation:
logging.info('cleaning global-installation')
FastScript.remove(installRoot_root, dryRun)
if rtmapsVersion:
Any.requireIsTextNonEmpty(rtmapsVersion)
hostPlatform = Platforms.getHostPlatform()
symlink_relHGR = RTMaps.getIndexFilePath_relHGR(
canonicalPath, rtmapsVersion, hostPlatform)
Any.requireIsTextNonEmpty(symlink_relHGR)
symlinkPath = os.path.join(sitProxyPath, symlink_relHGR)
Any.requireIsTextNonEmpty(symlinkPath)
FastScript.remove(symlinkPath, dryRun)
else:
logging.debug(
'skipped searching for RTMaps index symlink (RTMaps not sourced)')
def findProxyInstallations(checkLinks=False):
"""
Returns a list of packages installed within the proxy SIT.
A ValueError will be raised if the current SIT is not a proxy.
If 'checkLinks' == True, each version symlink will also be tested
if it really points into the root SIT. For example, it might be
difficult to localize a manually changed link that points e.g. in
another group proxy or so.
"""
resultList = []
sit = SIT.getPath()
sitParent = SIT.getParentPath() # support cascaded proxies
excludePattern = re.compile('^(parentTree|\d+\.\d+.*)$')
criteria = re.compile("^(\d+)\.(\d+)(.*)")
Any.requireIsDir(sit)
Any.requireIsDir(sitParent)
requireIsProxyDir(sit)
# find all entries within the proxy that seem to be a version number
dirList = FastScript.getDirsInDirRecursive(
sit,
excludePattern=excludePattern,
onError=FastScript.printPermissionDenied)
# check each found entry if it is a version directory, if so we
# can expect that this is a local project installation
for project in dirList:
for entry in os.listdir(project):
joinedPath = os.path.join(project, entry)
# only perform test on version numbers
if criteria.search(entry):
# don't use os.path.isdir() as it follows symlinks!
fileType = os.lstat(joinedPath).st_mode
if checkLinks == True and stat.S_ISLNK(fileType) == True:
# test if symlink points into root SIT
target = os.readlink(joinedPath)
if target[0] == '/' and not target.startswith(sitParent):
logging.warning(joinedPath + ': points to ' + target)
elif stat.S_ISDIR(fileType):
# we found a proxy installation
resultList.append(joinedPath)
return resultList
def makeDocumentation( self ):
from ToolBOSCore.BuildSystem.DocumentationCreator import DocumentationCreator
if self._outOfTree:
logging.warning( 'documentation creation in out-of-tree build not implemented' )
return False
if Any.getDebugLevel() <= 3:
from six import StringIO
# capture output so that it's not printed
output = StringIO()
else:
output = None
try:
dstSIT = SIT.getPath()
dc = DocumentationCreator( self._sourceTree, dstSIT, output, output )
dc.generate()
except AssertionError as e:
logging.error( 'failed to create documentation: %s', e )
return False
except subprocess.CalledProcessError:
if output:
print( output.getvalue() )
logging.error( 'failed to create documentation' )
return False
def _removeBrokenSymlinks(sitRootPkgList, sitProxyPkgList, sitRoot, sitProxy,
dryRun):
"""
If 'dryRun' is True, it won't actually do anything to your proxy.
Returns the number of broken links that have been removed
(or that would have been removed, in case of dry run).
"""
requireIsProxyDir(sitProxy)
logging.info('searching for broken symlinks...')
brokenLinks = _findBrokenLinks(sitProxy)
for path in brokenLinks:
if dryRun:
logging.info('-- DRY RUN -- found broken symlink %s', path)
else:
if path.endswith('.py'):
logging.info('deleting %s', path)
else:
logging.info('package was uninstalled: %s', SIT.strip(path))
os.remove(path)
return len(brokenLinks)
def _createCellWidget(self, package):
Any.requireMsg( Any.isInstance( package, BSTPackage.BSTPackage ) or \
Any.isInstance( package, DebianPackage ),
'unexpected datatype: %s' % type(package) )
Any.requireIsTextNonEmpty(package.url)
cell = QTreeWidgetItem([package.url])
inSystem = self._model.isInstalled_locally
inProxy = self._model.isInstalled_proxySIT
inGlobal = self._model.isInstalled_globalSIT
ProjectProperties.requireIsCanonicalPath(SIT.strip(package.url))
if package.url.startswith('sit://'):
if self._hasProxy:
self._setCellColorCode(cell, 1, inProxy(package.url))
else:
self._setCellColorCode(cell, 1, None)
self._setCellColorCode(cell, 2, inGlobal(package.url))
self._setCellColorCode(cell, 3, None)
else:
self._setCellColorCode(cell, 1, None)
self._setCellColorCode(cell, 2, None)
self._setCellColorCode(cell, 3, inSystem(package.url))
return cell
def getFlatDependencies(canonicalPaths, cache=None, sitPath=None):
"""
Resolves all dependencies recursively and converts them to a flat
set of overall dependencies.
This function might be handy to determine all SIT packages that
need to transferred to another site.
"""
cache = {} if cache is None else cache
sitPath = SIT.getPath() if sitPath is None else sitPath
Any.requireIsIterable(canonicalPaths)
Any.requireIsDict(cache)
Any.requireIsTextNonEmpty(sitPath)
result = set()
for canonicalPath in canonicalPaths:
requireIsCanonicalPath(canonicalPath)
result.add('sit://' + canonicalPath)
deps = getDependencies(canonicalPath,
recursive=True,
cache=cache,
systemPackages=False,
sitPath=sitPath)
flatDeps = FastScript.flattenList(deps)
for item in flatDeps:
result.add(item)
return result
def getBuildDependencies(project, recursive=False):
"""
For now this simply calls getBuildDependenciesFromPkgInfo(),
ignoring the case that the pkgInfo.py might not be present.
Note: Unlike getDependencies(), the result of this function
is a flat (non-nested) list.
"""
from ToolBOSCore.Storage.PkgInfo import getPkgInfoContent
Any.requireIsTextNonEmpty(project)
try:
resultList = getPkgInfoContent(project)['buildDepends']
except (AssertionError, IOError, KeyError):
resultList = []
if recursive:
depList = getDependencies(project, recursive=True)
depList = FastScript.flattenList(depList)
for dep in depList:
if dep.startswith('sit://'):
try:
buildDepList = getPkgInfoContent(
SIT.strip(dep))['buildDepends']
except (AssertionError, IOError, KeyError):
buildDepList = []
if buildDepList:
resultList.append(buildDepList)
resultList = FastScript.reduceList(resultList)
return resultList
def isInstalled(url, sitPath=None):
"""
Checks if the given package is installed. Depending on the URL type
different checks will be performed, e.g.:
'sit://Libraries/Serialize/3.0' # check in current SIT
'deb://binutils' # check for O.S. packages (*.deb)
You may speed-up this function by providing 'sitPath' so that it
does not need to be queried internally every time.
"""
from ToolBOSCore.Platforms import Debian
Any.requireIsTextNonEmpty(url)
if ':' not in url:
# default to SIT packages
url = 'sit://' + url
protocol, remainder = splitURL(url)
if protocol == 'sit':
if not sitPath:
sitPath = SIT.getPath()
status = isInstalled_sitPackage(remainder, sitPath)
elif protocol == 'deb':
try:
status = Debian.isInstalled(remainder)
except OSError as details:
raise OSError(details)
else:
raise RuntimeError('invalid URL protocol or format')
return status
def registerNormalPackages(sitPath, dryRun=False):
"""
Searches the SIT for RTMaps packages and invokes
registerHondaPackage() for each of them.
"""
Any.requireIsDir(sitPath)
Any.requireIsBool(dryRun)
sitProxyPath = SIT.getPath()
ProxyDir.requireIsProxyDir(sitProxyPath)
searchBaseDir = os.path.join(sitPath, 'Modules', 'RTMaps')
# Any.requireIsTextNonEmpty( searchBaseDir ) # dir. might not exist
Any.requireIsTextNonEmpty(searchBaseDir)
indexBaseDir = getIndexBaseDir(sitProxyPath)
# Any.requireIsDir( indexBaseDir ) # dir. might not exist
Any.requireIsTextNonEmpty(indexBaseDir)
logging.debug('SIT path: %s', sitPath)
logging.debug('search basedir: %s', searchBaseDir)
logging.debug('index basedir: %s', indexBaseDir)
# find *.pck files
logging.debug('scanning %s...', searchBaseDir)
pckPaths = FastScript.findFiles(searchBaseDir, ext='.pck')
Any.requireIsList(pckPaths)
logging.debug('found HORP files:')
logging.debug(pckPaths)
# process each *.pck file (tokenize path and create symlink)
for pckPath in pckPaths:
# the *.pck file is located on the 3rd subdirectory level relative
# to the installRoot, so compute the installRoot based on this
tokens = pckPath.split(os.path.sep)
installRoot = os.path.sep.join(tokens[:-3])
package = SIT.strip(installRoot)
Any.requireIsDir(installRoot)
Any.requireIsTextNonEmpty(package)
try:
registerNormalPackage(package, sitProxyPath, indexBaseDir, dryRun)
except ValueError as e:
logging.error(e)
def createProxyDir(sitRoot, sitProxy, verbose=True):
"""
Creates an SIT proxy directory for the current user in the
directory <sitProxy> with symlinks pointing into <sitRoot>.
"""
sitRootPkgList = []
if not os.path.isdir(sitRoot):
msg = '%s: No such directory (please check $SIT)' % sitRoot
raise AssertionError(msg)
if os.path.exists(sitProxy):
msg = '%s: Directory exists' % sitProxy
raise AssertionError(msg)
if os.path.exists(os.path.join(sitRoot, SIT.parentLink)):
msg = '$SIT=%s already points to a proxy ' % sitRoot + \
'directory (cascades are not allowed)'
raise AssertionError(msg)
if os.path.realpath(sitRoot) == os.path.realpath(sitProxy):
msg = 'SIT proxy path must be different from parent SIT!'
raise AssertionError(msg)
SIT.getProjectsWithErrorHandling(sitRoot, sitRootPkgList)
for package in sitRootPkgList:
linkName = os.path.join(sitProxy, package)
target = os.path.join(sitRoot, package)
if verbose:
logging.info('linking %s' % package)
# create directory where the link shall be created
FastScript.mkdir(os.path.dirname(linkName))
logging.debug('linking %s --> %s', linkName, target)
os.symlink(target, linkName)
# create a symlink inside the proxy that points to the parent
linkName = os.path.join(sitProxy, SIT.parentLink)
os.symlink(sitRoot, linkName)
logging.info('')
logging.info('Proxy created in %s' % sitProxy)
def test_proxyInstallation(self):
oldcwd = os.getcwd()
packageName = 'ExamplePackage'
packageVersion = '1.0'
category = 'Applications'
projectRoot = os.path.join('.', packageName, packageVersion)
output = StringIO() if Any.getDebugLevel() <= 3 else None
platform = Platforms.getHostPlatform()
sitPath = SIT.getPath()
# build + install package
FastScript.changeDirectory(projectRoot)
bst = BuildSystemTools()
bst.setStdOut(output)
bst.setStdErr(output)
bst.compile()
Any.requireIsDirNonEmpty('build')
Any.requireIsDirNonEmpty(os.path.join('build', platform))
Any.requireIsDirNonEmpty('examples')
Any.requireIsDirNonEmpty(os.path.join('examples', platform))
for fileName in ('ThreadingExampleAtomicOperation',
'ThreadingExampleJoin', 'ThreadingExampleTraps'):
Any.requireIsFileNonEmpty(os.path.join('bin', platform, fileName))
if not ProxyDir.isProxyDir(sitPath):
self.skip("%s: Is not a proxy directory" % sitPath)
bst.makeDocumentation()
bst.proxyInstall()
installRoot = os.path.join(sitPath, category, packageName,
packageVersion)
# check result
Any.requireIsDir(installRoot)
Any.requireIsDirNonEmpty(os.path.join(installRoot, 'bin', platform))
Any.requireIsDirNonEmpty(os.path.join(installRoot, 'doc/html'))
Any.requireIsFileNonEmpty(
os.path.join(installRoot, 'doc/html/index.html'))
Any.requireIsFileNonEmpty(os.path.join(installRoot, 'pkgInfo.py'))
Any.requireIsFileNonEmpty(os.path.join(installRoot,
'packageVar.cmake'))
for fileName in ('ThreadingExampleAtomicOperation',
'ThreadingExampleJoin', 'ThreadingExampleTraps'):
Any.requireIsFileNonEmpty(
os.path.join(installRoot, 'bin', platform, fileName))
# clean-up
bst.uninstall(cleanGlobalInstallation=False)
bst.distclean()
self.assertFalse(os.path.exists(installRoot))
FastScript.changeDirectory(oldcwd)
def open( self, package=None ):
if self.url:
ProjectProperties.requireIsURL( self.url )
package = ProjectProperties.splitURL( self.url )[1]
else:
ProjectProperties.requireIsCanonicalPath( package )
topLevelDir = os.path.join( SIT.getRootPath(), package )
super( BSTGloballyInstalledPackage, self ).open( topLevelDir )
def test_bootstrapSIT(self):
basePkgList = SIT.getMinRequirements()
outputDir = tempfile.mkdtemp(prefix='test-')
# create test SIT
copyFilter = CopyTreeFilter.CopyTreeFilter([]) # skip copying binaries
copyFilter.filterDocu = True
SIT.bootstrap(outputDir, True, True, copyFilter.callback, False)
# check if all essential packages are available
for package in basePkgList:
Any.requireIsFile(
os.path.join(outputDir, package, 'packageVar.cmake'))
# check for Module-Index directory
Any.requireIsDir(os.path.join(outputDir, 'Modules', 'Index'))
# clean-up
FastScript.remove(outputDir)
def _checkDefFiles(sitRootPkgList, sitProxyPkgList, sitRoot, sitProxy, dryRun):
"""
Checks that there are no orphaned *.def files in the proxy,
f.i. leftovers from previous proxy-installations that have been
deleted in the meanwhile.
Superfluous *.def files might be a really hard-to-track source of
errors: In case they don't match the actual interface of the
(globally) installed component, it might be very difficult to find
out why certain inputs/outputs/references do / don't appear in DTBOS.
"""
from ToolBOSCore.Packages.ProjectProperties import isCanonicalPath
from ToolBOSCore.Packages.ProjectProperties import splitPath
Any.requireIsListNonEmpty(sitRootPkgList)
Any.requireIsListNonEmpty(sitProxyPkgList)
Any.requireIsDir(sitRoot)
Any.requireIsDir(sitProxy)
proxyPackages = findProxyInstallations() # list of absolute paths
Any.requireIsList(proxyPackages)
proxyPackages = map(SIT.strip, proxyPackages)
# filter-out 3-digit version numbers, as *.def files by convention
# are for 2-digit versions only
proxyPackages = filter(isCanonicalPath, proxyPackages)
# create a corresponding fake-index for each proxy installation, for easy
# string-based matching later
#
fakeDefs = []
for package in proxyPackages:
(category, packageName, packageVersion) = splitPath(package)
fakeDefs.append('%s_%s.def' % (packageName, packageVersion))
indexDir = os.path.join(sitProxy, 'Modules/Index')
defPathList = glob.glob(os.path.join(indexDir, '*.def'))
proxyChanged = False
for defPath in defPathList:
defFile = os.path.basename(defPath)
# delete superfluous *.def files
if defFile not in fakeDefs:
collapsed = SIT.collapseHGR(defPath)
if dryRun:
logging.info('-- DRY RUN -- found superfluous %s' %
collapsed)
else:
logging.info('deleting %s', defPath)
FastScript.remove(defPath)
return proxyChanged
def _findMatdoc(self):
sitPath = SIT.getRootPath()
package = getConfigOption('package_matdoc')
fileName = 'matdoc'
exePath = os.path.join(sitPath, package, 'bin', fileName)
if os.path.exists(exePath):
return exePath
else:
logging.warning('%s: No such file', exePath)
return fileName # default to 'matdoc'
def getMaintainerFromFilesystem(project):
"""
Returns the username of the maintainer by reading the fileowner of
the project from the current SIT.
"""
requireIsCanonicalPath(project)
path = SIT.getPath() + os.sep + project
owner = FastScript.getFileOwner(path)
return owner
def printFetchCommands(packageList, asSubModule=True):
"""
Prints the VCS fetch command for each package in the list.
Duplicates will be removed, and output will be sorted for better
readability, e.g.:
asSubModule = True:
git submodule add [email protected]:TECH_Team/BPL.git
git submodule add [email protected]:ToolBOS/BasicComponents.git
git submodule add [email protected]:ToolBOS/ToolBOSLib.git
asSubModule = False:
git clone [email protected]:TECH_Team/BPL.git
git clone [email protected]:ToolBOS/BasicComponents.git
git clone [email protected]:ToolBOS/ToolBOSLib.git
"""
Any.requireIsIterable(packageList)
Any.requireIsBool(asSubModule)
sitRootPath = SIT.getRootPath()
commands = set() # avoids duplicates
for canonicalPath in packageList:
ProjectProperties.requireIsCanonicalPath(canonicalPath)
installRoot = os.path.join(sitRootPath, canonicalPath)
Any.requireIsExisting(installRoot)
detector = PackageDetector.PackageDetector(installRoot)
detector.retrieveMetaInfoFromSIT()
vcsRoot = detector.vcsRoot
if vcsRoot:
Any.requireIsTextNonEmpty(vcsRoot)
repo = VersionControl.auto(vcsRoot)
if isinstance(repo, SVN.SVNRepository):
repo.setConfiguredUserName()
command = repo.getSourceCodeCommand()
elif isinstance(repo, Git.RemoteGitRepository):
command = repo.getSourceCodeCommand(asSubModule)
else:
raise TypeError('unexpected datatype: %s', type(repo))
else:
command = '# VCS location unknown: %s' % canonicalPath
commands.add(command)
for command in sorted(commands):
print(command)
def populate(self):
"""
Scans all package in SIT and stores the ground-truth pkgInfo.py
information into one giant hashtable for later fast access.
The assignment is "packageURL": { pkgInfo data }
"""
sitPath = SIT.getPath()
canonicalPaths = SIT.getCanonicalPaths(sitPath)
Any.requireIsListNonEmpty(canonicalPaths)
for canonicalPath in canonicalPaths:
ProjectProperties.requireIsCanonicalPath(canonicalPath)
packageURL = 'sit://' + canonicalPath
installRoot = os.path.join(sitPath, canonicalPath)
detector = PackageDetector(installRoot)
detector.retrieveMakefileInfo()
self._cache[packageURL] = detector
def isDeprecated(canonicalPath):
"""
Checks from the filesystem if the specified package (canonical path)
is flagged as deprecated.
Returns True if either of these files exists:
<sitRoot>/Libraries/Spam/42.0/deprecated.txt
<sitRoot>/Libraries/Spam/deprecated.txt
or if the canonicalPath is listed in the file
<sitPath>/Temporary/CIA/1.0/deprecatedOverride.txt.
"""
requireIsCanonicalPath(canonicalPath)
filename = 'deprecated.txt'
sitRoot = SIT.getRootPath()
pkgPath = os.path.join(sitRoot, canonicalPath)
check1 = os.path.join(sitRoot, os.path.dirname(canonicalPath), filename)
check2 = os.path.join(sitRoot, canonicalPath, filename)
# if package is not present in SIT we can't give reliable information
# if it is deprecated or not
if not os.path.exists(pkgPath):
raise ValueError("%s: Package not installed in SIT" % canonicalPath)
if os.path.exists(check1) or os.path.exists(check2):
return True
# check CIA operator "sudo override"
overrideFile = os.path.join(SIT.getPath(),
'Temporary/CIA/1.0/deprecatedOverride.txt')
if os.path.exists(overrideFile):
for line in FastScript.getFileContent(overrideFile, splitLines=True):
if line.strip() == canonicalPath:
return True
return False
def getBuildRequirements(projectURL,
recursive=False,
cache=None,
ignoreErrors=False):
"""
Returns a list of packages that are necessary to build the
specified package.
'projectURL' must be a canonical path starting with the "sit://" prefix.
If recursive=False, the list will contain the pure untreated strings
stored in the pkgInfo.py file. There is no preprocessing done on its
semantics. In recursive mode, we need to follow those paths and
need to resolve 'sit://' etc.
"""
Any.requireIsTextNonEmpty(projectURL)
Any.requireMsg( projectURL.startswith( 'sit://' ) == True,
"'project' parameter needs to start with sit:// " + \
"'(you passed: %s)" % projectURL )
# query dependencies of package, don't forget to add package itself
resultList = [] # projectURL ]
project = SIT.strip(projectURL)
if not isinstance(cache, dict):
cache = {}
try: # first look-up the cache
depList = cache[projectURL]
#logging.debug( 'CACHE HIT: %s' % projectURL )
except KeyError: # if not found read from filesystem
#logging.debug( 'CACHE MISS: %s' % projectURL )
tmpList = getDependencies( project, systemPackages=False ) + \
getBuildDependencies( project )
depList = FastScript.reduceList(tmpList)
# store the direct dependencies into the cache map
cache.update({projectURL: depList})
for dep in depList:
resultList.append(dep)
if recursive == True and not dep.startswith('deb://'):
subDepList = getBuildRequirements(dep, recursive, cache,
ignoreErrors)
for subDep in subDepList:
if subDep not in resultList:
resultList.append(subDep)
return resultList
def __init__(self,
projectRoot,
sitPath=None,
stdout=None,
stderr=None,
details=None):
Any.requireIsDir(projectRoot)
self.projectRoot = projectRoot
logging.debug('topLevelDir=%s' % projectRoot)
if details is None:
details = PackageDetector(projectRoot)
try:
details.retrieveMakefileInfo()
except EnvironmentError as e:
logging.warning(e) # e.g. $SIT not defined in environment
else:
Any.requireIsInstance(details, PackageDetector)
if sitPath is None:
sitPath = SIT.getPath()
if FastScript.getEnv('MAKEFILE_DOC') == 'FALSE':
handler = NullBackend
elif details.docTool == 'doxygen':
handler = DoxygenBackend
elif details.docTool == 'matdoc':
handler = MatlabBackend
elif details.docTool == '':
handler = NullBackend
elif details.isMatlabPackage():
# [CIA-1131] matdoc no longer works on xenial64
#
# On Ubuntu 16.04 disable documentation creation
# (do not use doxygen either to not overwrite the
# correct matdoc-output from Ubuntu 14.04 machines)
if getHostPlatform().startswith('xenial'):
handler = NullBackend
else:
handler = MatlabBackend
else:
handler = DoxygenBackend
self.backend = handler(details, sitPath, stdout, stderr)
def _checkProxyLinkedVersion(sitRootPkgList, sitProxyPkgList, sitRoot,
sitProxy, dryRun):
"""
Checks if the two-digit version in the proxy points to the most
recent version. Otherwise this can happen:
* Developer A installs "Serialize/3.0.100" into his proxy, the
2-digit link "3.0" points into the proxy to version 3.0.100.
* Developer B installs "Serialize/3.0.101" globally.
* Now the 3.0-symlink of developer A is outdated.
"""
Any.requireIsListNonEmpty(sitRootPkgList)
Any.requireIsListNonEmpty(sitProxyPkgList)
Any.requireIsDir(sitRoot)
Any.requireIsDir(sitProxy)
proxyChanged = False
for project in sitProxyPkgList:
localPatchlevel = SIT.getActiveRevision(sitProxy, project)
globalPatchlevel = SIT.getActiveRevision(sitRoot, project)
if localPatchlevel is None or globalPatchlevel is None:
continue
if localPatchlevel < globalPatchlevel:
logging.info('updating %s' % project)
pkgProxyPath = os.path.join(sitProxy, project)
pkgRootPath = os.path.join(sitRoot, project)
FastScript.remove(pkgProxyPath)
os.symlink(pkgRootPath, pkgProxyPath)
proxyChanged = True
return proxyChanged
def getIndexBaseDir(sitProxyPath=None):
"""
Returns the path to the user's index base directory.
If 'sitProxyPath' is omitted then the current Proxy-SIT is used.
"""
if sitProxyPath is None:
sitProxyPath = SIT.getPath()
ProxyDir.requireIsProxyDir(sitProxyPath)
indexBaseDir = os.path.join(sitProxyPath, 'Modules', 'Index')
return indexBaseDir
def setEnv():
"""
This function loads the pkgInfo.py of each dependency package.
If environment settings are found there they will be loaded into
the environment of the current Python process.
Example:
The pkgInfo.py of Matlab states MATLAB_ROOT, PATH and
LD_LIBRARY_PATH settings in its pkgInfo.py. Before compiling
such variables must be set.
On Linux, alternatively, this can be achieved by sourcing the
BashSrc files. Nevertheless this is not possible in all cases
(especially Matlab because multiple versions are available)
when used within CIA.
With some modifications this setEnv() approach conceptually
potentially could also work on Windows.
Attention: This function should only be called once (at least not
repeatedly when compiling the same package again from
within Python) to not unnecessarily increase the length
of PATH, LD_LIBRARY_PATH etc.
"""
try:
p = PackageDetector()
except AssertionError:
# XIF packages are generated on-the-fly during configure-phase.
# We don't consider such packages for now (experimental code).
return
p.retrieveMakefileInfo()
for package in p.dependencies + p.buildDependencies:
try:
envVars = PkgInfo.getPkgInfoContent( SIT.stripSIT(package) )['envVars']
except ( AssertionError, KeyError ):
envVars = [] # no envVars specified
except ( IOError, OSError, SyntaxError ) as details:
logging.error( details )
raise RuntimeError( 'unable to read pkgInfo.py of %s' % package )
if envVars:
logging.debug( 'found environment settings:' )
logging.debug( envVars )
for varName, varValue in envVars:
FastScript.setEnv_withExpansion( varName, varValue )
def printBuildCommands(orderedList):
"""
Prints the build-instruction for each package in the list, e.g.:
build src/ToolBOSCore
build src/ToolBOSLib
build src/IniConfigFile
build src/Middleware
Note: The list shall already be sorted according to build order!
"""
Any.requireIsListNonEmpty(orderedList)
sitRootPath = SIT.getRootPath()
commands = []
for canonicalPath in orderedList:
ProjectProperties.requireIsCanonicalPath(canonicalPath)
installRoot = os.path.join(sitRootPath, canonicalPath)
Any.requireIsExisting(installRoot)
detector = PackageDetector.PackageDetector(installRoot)
detector.retrieveMetaInfoFromSIT()
vcsRelPath = detector.vcsRelPath
vcsRoot = detector.vcsRoot
if vcsRoot:
Any.requireIsTextNonEmpty(vcsRoot)
repo = VersionControl.auto(vcsRoot)
repoName = repo.getRepositoryName()
if vcsRelPath:
location = os.path.join('src', repoName, vcsRelPath)
else:
location = os.path.join('src', repoName)
command = 'build %s' % location
else:
command = '# build %s# location unknown' % canonicalPath.ljust(70)
commands.append(command)
for command in commands:
print(command)
def toCanonicalPath(string):
"""
Takes a string and attempts to shrink it to a canonical path,
removing any leading SIT prefix or trailing slash.
'/hri/sit/latest/DevelopmentTools/ToolBOSCore/3.0/'
--> 'DevelopmentTools/ToolBOSCore/3.0'
"""
Any.requireIsTextNonEmpty(string)
canonicalPath = SIT.strip(string)
# remove trailing slashes (occurs if user provides a string with trailing slashes)
while canonicalPath.endswith('/'):
canonicalPath = canonicalPath[0:-1]
return canonicalPath