def _createAutoDoxyfile(self):
content = 'TAGFILES = '
logging.debug('cross-linking doxygen documentation into %s',
self.sitPath)
if self.details.inheritedProjects:
logging.debug('dependencies: %s', self.details.inheritedProjects)
else:
logging.debug('no dependencies found --> no cross-links')
for dep in self.details.inheritedProjects:
depRoot = os.path.join(self.sitPath, dep)
depTagFile = os.path.join(depRoot, 'doc', 'doxygen.tag')
depHTMLDir = os.path.join(depRoot, 'doc', 'html')
# ignore if we can't cross-link
depTagFileExists = os.path.exists(depTagFile)
depHTMLDirExists = os.path.exists(depHTMLDir)
logging.debug('processing dependency: %s', dep)
logging.debug(' %s: %s', depTagFile, depTagFileExists)
logging.debug(' %s: %s', depHTMLDir, depHTMLDirExists)
if depTagFileExists and depHTMLDirExists:
content += '%s=%s ' % (depTagFile, depHTMLDir)
logging.debug(' linking %s', depTagFile)
else:
logging.debug(' not linking')
logging.debug('')
FastScript.setFileContent(self.autoDoxyfile, content)
def _setUserConfigOptions(config):
"""
Writes the dict 'config' in ASCII yet Pythonic style to the user's
configfile, so that it can be edited manually and read-in using
FastScript.execFile().
"""
from ToolBOSCore.Packages.CopyrightHeader import getCopyrightHeader
Any.requireIsDict(config)
content = getCopyrightHeader('python', 'User-preferences for ToolBOS SDK')
for key, value in config.items():
if Any.isText(value):
value = "'%s'" % value # write Python string, not just value
content += '%s = %s\n\n' % (key, str(value))
content += '\n# EOF\n'
fileName = getSettingsFile_user()
dirName = os.path.dirname(fileName)
FastScript.mkdir(dirName)
FastScript.setFileContent(fileName, content)
def _createMainDoxyfile(self):
if self.details.isPythonPackage() and \
getConfigOption( 'BST_useDoxypy' ) == True:
fileName = 'Doxyfile-Python'
else:
fileName = 'Doxyfile'
content = ''
template = os.path.join(FastScript.getEnv('TOOLBOSCORE_ROOT'), 'etc',
fileName)
Any.requireIsFileNonEmpty(template)
if os.path.isdir(os.path.join(self.details.topLevelDir, 'examples')):
logging.debug('doxygen: indexing examples')
content += 'EXAMPLE_PATH = ../examples\n' + \
'EXAMPLE_PATTERNS =\n' + \
'EXAMPLE_RECURSIVE = YES\n'
if self.details.isComponent():
logging.debug('doxygen: indexing src/*.{c,cpp}')
content += 'FILE_PATTERNS = *.c *.cpp *.h\n'
content = FastScript.getFileContent(template) + content
FastScript.setFileContent(self.mainDoxyfile, content)
def _patchCIA1147( self, dryRun=False ):
"""
Remove obsolete BBCM_INFO_CATEGORY.
"""
if not self.details.isBBCM():
logging.debug( 'package is not a BBCM' )
return False
fileName = 'src/%s_info.c' % self.details.packageName
if not os.path.exists( fileName ):
return False
lines = FastScript.getFileContent( fileName, splitLines=True )
modified = []
newContent = ''
for line in lines:
if line.startswith( 'BBCM_INFO_CATEGORY' ):
modified.append( fileName )
else:
newContent += line
if modified and dryRun is False:
FastScript.setFileContent( fileName, newContent )
return modified
def save(self):
"""
Writes the user-settings in ASCII yet Pythonic style to the user's
configfile, so that it can be edited manually and read-in using
FastScript.execFile().
If there are no user-settings at all the fill be removed
(if present).
"""
if self._userSettings:
from ToolBOSCore.Packages.CopyrightHeader import getCopyrightHeader
content = getCopyrightHeader('python', 'User preferences')
for key, value in sorted(self._userSettings.items()):
if Any.isText(value):
value = "'%s'" % value # write Python string, not just value
content += '%s = %s\n\n' % (key, str(value))
content += '\n# EOF\n'
logging.debug('writing %s', self._userFile)
FastScript.setFileContent(self._userFile, content)
else:
# delete entire file if there are no settings left
logging.debug('deleting empty configfile')
FastScript.remove(self._userFile)
def runMakoEngine(srcFile, dstFile, values):
"""
Runs the templating engine, applying the given values
onto the template file 'srcFile', writing results into 'dstFile'.
"""
Any.requireIsFile(srcFile)
Any.requireIsText(dstFile)
Any.requireIsDict(values)
logging.info('processing %s' % dstFile)
# First determine the directory of the template file, and tell Mako
# to search there. In a second step tell Mako to search for a template
# file in this search path.
#
# This is the only solution to get Mako's "include" working.
lookup = TemplateLookup(directories=[os.path.dirname(srcFile)])
template = lookup.get_template(os.path.basename(srcFile))
dstContent = template.render(**values)
Any.requireIsText(dstContent)
FastScript.mkdir(os.path.dirname(dstFile)) # ensure dst dir. exists
FastScript.setFileContent(dstFile, dstContent)
Any.requireIsFile(dstFile)
# Mako does not set the executable-flag on the generated output file.
if os.access(srcFile, os.X_OK): # if executable
os.chmod(dstFile, os.stat(srcFile)[0]) # copy mode bits
def run(self):
if not 'category' in self.values:
self.values['category'] = 'External'
if not 'buildRules' in self.values:
self.values[
'buildRules'] = '''# This is a dummy file needed by various aux. scripts.
#
# The actual build instructions can be found in the compile.sh.
'''
self.createMainPackage()
srcDir = os.path.join(self.templateDir, 'External_without_compilation')
dstDir = self.dstDir
for fileName in FastScript.getFilesInDir(srcDir, '.php'):
srcFile = os.path.join(srcDir, fileName)
dstFile = os.path.join(dstDir, fileName)
self.copyVerbatim(srcFile, dstFile)
self.copyVerbatim(os.path.join(srcDir, 'pkgInfo.py'),
os.path.join(dstDir, 'pkgInfo.py'))
FastScript.remove(os.path.join(dstDir, 'unittest.sh'))
# create exemplarily (fake-)tarball file, and interface-symlink to it
tarball = os.path.join(dstDir, 'src',
'Example-1.0-precompiled.tar.bz2')
symlink = os.path.join(dstDir, 'src', 'package.tar.bz2')
logging.info('processing %s' % tarball)
FastScript.setFileContent(tarball, '')
logging.info('processing %s' % symlink)
os.symlink('Example-1.0-precompiled.tar.bz2', symlink)
# create basic packageVar.cmake
#
# Note: for some reason calling FastScript.changeDirectory() with rel. path failed,
# continuing with absolute path as workaround
dstDir = os.path.abspath(dstDir)
Any.requireIsDir(dstDir)
details = PackageDetector(dstDir)
details.retrieveMakefileInfo()
fileName = os.path.join(dstDir, 'packageVar.cmake')
PackageVarCmakeWriter(details).write(fileName)
def setup(self):
fileName = os.path.join(self.details.topLevelDir, 'matdoc.pm')
if os.path.isfile(fileName):
return # nothing to do, maintainer made custom one
username = FastScript.getCurrentUserFullName()
content = '# matdoc configuration file auto-generated by ToolBOSCore\n\n' + \
'# directories to exclude\n' + \
'@Exclude = qw(doc lib obj install test);\n\n' + \
'# project maintainer(s)\n' + \
( "@Maintainers = ( '%s' );\n\n" % username ) + \
'# create doc index for each subdirectory?\n' + \
'$Separate = 0;\n\n'
FastScript.setFileContent(fileName, content)
def _patchCIA1267( self, dryRun=False ):
"""
Check files for outdated CSV keywords and remove
these keywords and expanded information from the source code
"""
files = FastScript.getFilesInDirRecursive( 'bin' ) | \
FastScript.getFilesInDirRecursive( 'examples' ) | \
FastScript.getFilesInDirRecursive( 'src' ) | \
FastScript.getFilesInDirRecursive( 'test' )
modified = []
# CSV keywords
keywords = frozenset( [ '$Author', '$Date', '$Header', '$Id', '$Log',
'$Locker', '$Name', '$RCSfile', '$Revision',
'$Source', '$State' ] )
for filePath in files:
rewrite = False
try:
fileContent = FastScript.getFileContent( filename=filePath,
splitLines=True )
except UnicodeDecodeError:
# most probably we attempt to read a binary file,
# e.g. a compiled executable under bin/ or the like
continue
# check each line for CSV keyword and remove line if found
for line in fileContent:
if any( key in line for key in keywords):
rewrite = True
fileContent.remove( line )
if rewrite:
if dryRun:
logging.info( '[DRY-RUN] patching %s', filePath )
else:
logging.info( 'patching %s', filePath )
newContent = ''.join( fileContent )
FastScript.setFileContent( filePath, newContent )
modified.append( filePath )
return modified
def setupShell():
"""
Configures the user's shell environment to use the ToolBOS SDK.
It tries to detect if the shell was already once configured
(by this script or manually) and attempts to reset it to the
default state.
"""
fileName = os.path.expanduser('~/.bash_login')
content = '''if [ `basename -- "$0"` != Xsession ] ; then
source .bashrc
fi
'''
# remove it, because could be an existing file / symlink
FastScript.remove(fileName)
logging.info('creating %s' % fileName)
FastScript.setFileContent(fileName, content)
def run(self):
if not 'category' in self.values:
self.values['category'] = 'External'
self.createMainPackage()
srcDir = os.path.join(self.templateDir,
'External_CMake_out_of_tree_build')
dstDir = self.dstDir
for fileName in FastScript.getFilesInDir(srcDir, '.php'):
srcFile = os.path.join(srcDir, fileName)
dstFile = os.path.join(dstDir, fileName)
self.copyVerbatim(srcFile, dstFile)
self.copyVerbatim(os.path.join(srcDir, 'pkgInfo.py'),
os.path.join(dstDir, 'pkgInfo.py'))
FastScript.remove(os.path.join(dstDir, 'unittest.sh'))
# create exemplarily (fake-)tarball file, and interface-symlink to it
tarball = os.path.join(dstDir, 'src', 'Example-1.0-src.tar.bz2')
symlink = os.path.join(dstDir, 'src', 'sources.tar.bz2')
logging.info('processing %s' % tarball)
FastScript.setFileContent(tarball, '')
logging.info('processing %s' % symlink)
os.symlink('Example-1.0-src.tar.bz2', symlink)
# create basic packageVar.cmake
#
# Note: for some reason calling FastScript.changeDirectory() with rel. path failed,
# continuing with absolute path as workaround
dstDir = os.path.abspath(dstDir)
Any.requireIsDir(dstDir)
details = PackageDetector(dstDir)
details.retrieveMakefileInfo()
fileName = os.path.join(dstDir, 'packageVar.cmake')
PackageVarCmakeWriter(details).write(fileName)
def setupLegacyMSVC(configDir):
from ToolBOSCore.Storage.SIT import getPath
sitRootPath = getPath()
packageName = 'Data/wine.net/0.1'
handmadeDir = os.path.join(sitRootPath, packageName, 'config')
userName = FastScript.getCurrentUserName()
if not os.path.exists(handmadeDir):
raise AssertionError('%s: No such package in SIT' % packageName)
if not userName:
raise AssertionError('Unable to query username :-(')
# replace 'Program Files' and 'windows' directories in configDir by
# symlinks to handmade directories in SIT
for item in ('Program Files', 'windows'):
path = os.path.join(configDir, 'drive_c', item)
Any.requireIsDir(path)
FastScript.remove(path)
target = os.path.join(handmadeDir, 'drive_c', item)
FastScript.link(target, path)
# copy all the handmade *.reg files
regFiles = glob.glob("%s/*.reg" % handmadeDir)
Any.requireIsListNonEmpty(regFiles)
for srcFilePath in regFiles:
fileName = os.path.basename(srcFilePath)
dstFilePath = os.path.join(configDir, fileName)
logging.debug('cp %s %s', srcFilePath, dstFilePath)
shutil.copyfile(srcFilePath, dstFilePath)
Any.requireIsFileNonEmpty(dstFilePath)
# replace occurrences of 'roberto' by username
oldContent = FastScript.getFileContent(dstFilePath)
newContent = oldContent.replace('roberto', userName)
FastScript.setFileContent(dstFilePath, newContent)
def _replace( self, fileName, check, old, new, ticketID, dryRun, count=None ):
"""
Checks if <fileName> contains <check>, and if so it replaces
the string <old> by <new>.
If dryRun=True the check for patch necessity will be executed
normally, but then no files will be altered.
Returns a boolean if the file was affected or not.
Returns 'None' if the specified file was not found thus the
patch might not be applicable.
If the argument 'count' is given, only the first n occurrences
will be replaced.
"""
try:
content = FastScript.getFileContent( fileName )
except ( IOError, UnicodeDecodeError ):
# UnicodeDecodeError may happen when attempting to read a binary
# file (e.g. executable), skip those as they shall not be patched
return False
logging.debug( '%s: searching for "%s"', fileName, check.strip() )
needed = content.find( check ) > 0
logging.debug( 'patch "%s" --> "%s" needed: %s' % \
( old.strip(), new.strip(), str(needed) ) )
if needed and not dryRun:
if count is None:
content = content.replace( old, new )
else:
content = content.replace( old, new, count )
FastScript.setFileContent( fileName, content )
return needed
def _patchCIA727( self, dryRun=False ):
"""
Upgrade XIF packages to SplitterBBCMMaker 1.3
"""
# Part 1: update SplitterBBCMMaker version in pre-configure.sh
fileName1 = 'pre-configure.sh'
old = 'SplitterBBCMMaker/1.2/bin/MakeDataWrapper.sh'
new = 'SplitterBBCMMaker/1.3/bin/MakeDataWrapper.sh'
status = self._replace( fileName1, old, old, new, 'CIA-727a', dryRun )
# Part 2: update SplitterBBCMMaker version in post-install.sh
if status:
# Note: The old post-install.sh files contain a lot of
# additional code. We agreed to completely replace such scripts
# with a one-liner script
fileName2 = 'post-install.sh'
old = 'SplitterBBCMMaker/1.2/bin/MakeSplitterBBCM.sh'
new = '#!/bin/bash\n' + \
'$SIT/DevelopmentTools/SplitterBBCMMaker/1.3/bin/MakeSplitterBBCM.sh ' + \
'%s . %s\n\n' % ( self.details.packageName,
self.details.packageVersion )
# this does a replacement within the file, which is undesired...
status2 = self._replace( fileName2, old, old, new, 'CIA-727b', dryRun )
if status2 == True and dryRun == False:
# ...instead we want to get rid of extra code and reset
# the entire file content, effectively shortening the file
FastScript.setFileContent( fileName2, new )
return [ fileName1, fileName2 ]
else:
return False
def setDeprecated(canonicalPath, allVersions=False, message=''):
"""
Deprecate a version of a package. canonicalPath is used to
deprecate the package. If allVersions is True, all versions are
deprecated. message will be written into deprecated.txt.
"""
requireIsCanonicalPath(canonicalPath)
Any.requireIsText(message)
if message:
reason = message + '\n'
else:
reason = message
if allVersions:
relPath = os.path.dirname(canonicalPath)
else:
relPath = canonicalPath
absPath = os.path.join(SIT.getRootPath(), relPath)
filePath = os.path.join(absPath, "deprecated.txt")
if os.path.exists(absPath):
try:
FastScript.setFileContent(filePath, reason)
logging.info('%s deprecated', relPath)
logging.debug('Created file %s', filePath)
except (IOError, OSError) as e:
logging.warning(e)
logging.error('Could not deprecate %s', relPath)
return False
else:
logging.warning('%s is not installed!', relPath)
return False
return True
def _patchCIA1265( self, dryRun=False ):
"""
Checks BBCM package for included outdated headerfiles and
replaces them with the nowadays version.
Duplicates will be deleted.
"""
if not self.details.isBBCM():
return False
# Get a list of all files to check
files = FastScript.getFilesInDirRecursive( 'src' )
modified = []
# Check every file
for filePath in files:
logging.debug( 'processing %s', filePath )
# Only rewrite line if changed
rewrite = False
# Get file content
fileContent = FastScript.getFileContent( filename=filePath,
splitLines=True )
# Check every line
for line in fileContent:
item = line.split()
# Check for include statement
if line.find( '#include <BBDM' ) != -1:
# Replace old package names with the nowadays form
match = re.search( r"-A|-CID|-S", item[ 1 ] )
if match:
fileContent[ fileContent.index( line ) ] = re.sub(
r"-A|-CID|-S", "", line )
rewrite = True
includes = []
# Check file backwards
for line in reversed( fileContent ):
item = line.split()
if line.find( '#include <BBDM' ) != -1:
# Check for duplicates and remove existing ones
if line in includes:
rewrite = True
fileContent.remove( line )
# add to list of known includes
else:
includes.append( line )
# Overwrite file with new content
try:
if rewrite:
if dryRun:
logging.info( '[DRY-RUN] patching %s', filePath )
else:
logging.info( 'patching %s', filePath )
newContent = ''.join( fileContent )
FastScript.setFileContent( filePath, newContent )
modified.append( filePath )
except IOError as e:
logging.error( e )
return modified
def write(self, outputFile):
self._deploy()
logging.info('generating %s' %
os.path.relpath(outputFile, os.getcwd()))
FastScript.setFileContent(outputFile, self.content)
if not newsym.startswith("?"):
sym = newsym
# Win32 has the first _ (underscore) to remove
if bits == 32 and sym.startswith('_'):
sym = sym[1:]
if sym in symbols:
continue
symbols[sym] = isData
if verbose:
# print( 'dumpbin output: %s' % line.strip() )
print("found symbol: %s\n" % sym)
fd.close()
output = "EXPORTS\n"
for sym in symbols:
output += "\t%s\n" % sym
if outputFile:
logging.debug('writing %s', outputFile)
FastScript.setFileContent(outputFile, output)
else:
print(output)
# EOF
def setupMSVC2012(configDir):
"""
Configures the Microsoft Visual Compiler to be used with Wine
from the ToolBOS build system.
You may provide a path to the directory where your Wine
configuration is. If omitted, the path returned from getWineConfigDir()
will be used.
"""
from ToolBOSCore.Storage.SIT import getPath
if not os.path.exists(os.path.join(configDir, 'user.reg')):
setupWineDotNet(configDir)
Any.requireIsDir(configDir)
if not os.path.exists(os.path.join(configDir, 'dosdevices')):
setupWineDotNet(configDir)
logging.info('setting up Visual Studio...')
linkPath = os.path.join(configDir, 'dosdevices', 'c:')
linkTarget = '../drive_c'
FastScript.remove(linkPath)
FastScript.link(linkTarget, linkPath)
linkPath = os.path.join(configDir, 'dosdevices', 'z:')
linkTarget = '/'
FastScript.remove(linkPath)
FastScript.link(linkTarget, linkPath)
# create temp. directories
driveC = os.path.join(configDir, 'drive_c')
userName = FastScript.getCurrentUserName()
userTempDir = os.path.join(driveC, 'users', userName, 'Temp')
sysTempDir = os.path.join(driveC, 'temp')
logging.debug('userTempDir=%s', userTempDir)
FastScript.mkdir(userTempDir)
FastScript.mkdir(sysTempDir)
# ensure to NOT have the "h:" link, else wine would not find some links
FastScript.remove(os.path.join(configDir, 'dosdevices', 'h:'))
# replace "C:\Program Files" by symlink into SIT
FastScript.remove(os.path.join(configDir, 'drive_c', 'Program Files'))
sitPath = getPath()
linkPath = os.path.join(configDir, 'drive_c', 'msvc-sdk')
linkTarget = os.path.join(sitPath, 'External/MSVC/10.0/msvc-sdk')
FastScript.remove(linkPath)
FastScript.link(linkTarget, linkPath)
linkPath = os.path.join(configDir, 'drive_c', 'Program Files')
linkTarget = os.path.join(sitPath, 'External/MSVC/10.0/Program Files')
FastScript.remove(linkPath)
FastScript.link(linkTarget, linkPath)
# copy a hancrafted system.reg
srcFile = os.path.join(
sitPath, 'External/MSVC/10.0/otherstuff/winevs2012/system.reg')
dstFile = os.path.join(configDir, 'system.reg')
shutil.copyfile(srcFile, dstFile)
# force wine to use the MSVC native library
userReg = os.path.join(configDir, 'user.reg')
Any.requireIsFileNonEmpty(userReg)
content = FastScript.getFileContent(userReg)
if content.find('1413877490') == -1:
content += \
'''
[Software\\\\Wine\\\\DllOverrides] 1413877490
"mscoree"="native"
"msvcr110"="native"
'''
logging.debug('updating %s', userReg)
FastScript.setFileContent(userReg, content)
def _createUserDoxyfile(self):
if not os.path.exists(self.userDoxyfile):
FastScript.setFileContent(self.userDoxyfile,
'') # "NULL" design pattern
def write(self):
FastScript.setFileContent(self._filePath, self._content)
logging.debug('%s written', self._filePath)
origEnv = FastScript.getEnv()
FastScript.unsetEnv('VERBOSE')
FastScript.unsetEnv('BST_BUILD_JOBS')
pyScripts = glob.glob(os.path.join(binDirNoArch, '*.py'))
shScripts = glob.glob(os.path.join(binDirNoArch, '*.sh'))
executables = glob.glob(os.path.join(binDirNoArch, hostPlatform, '*'))
for program in pyScripts + shScripts + executables:
basename = os.path.basename(program)
Any.requireIsTextNonEmpty(basename)
logging.info('processing %s', basename)
output = StringIO()
cmd = '%s --help' % program
fileName = os.path.join('ReferenceData', '%s.txt' % basename)
Any.requireIsTextNonEmpty(cmd)
Any.requireIsTextNonEmpty(fileName)
FastScript.execProgram(cmd, stdout=output, stderr=output)
content = normalizeOutput(output.getvalue())
Any.requireIsTextNonEmpty(content)
FastScript.setFileContent(fileName, content)
Any.requireIsFileNonEmpty(fileName)
# EOF
