def getSystemPackages():
"""
Fetches the list of Debian packages installed on the system,
returning a dict mapping "packageName" => "version".
"""
if not ProcessEnv.which('dpkg'):
raise EnvironmentError('Not a Debian-based operating system')
output = StringIO()
regexp = re.compile("^ii\s+(.+?)\s+(.+?)\s.+$")
result = {}
try:
FastScript.execProgram('dpkg -l', stdout=output)
except OSError as details:
raise OSError(details)
for line in output.getvalue().splitlines():
if line.startswith('ii '):
tmp = regexp.match(line)
name = tmp.group(1)
version = tmp.group(2)
# try to cut-off the arch postfix (e.g. ":i386" or ":amd64")
name = name.split(':')[0]
result[name] = version
return result
def execute(self):
logging.info('running matdoc...')
prog = self._findMatdoc()
cmd = '%s %s' % (prog, self.details.packageName)
msg = 'matdoc not found'
if not os.path.exists(prog):
logging.error('%s: No such file')
logging.error(msg)
return
try:
FastScript.execProgram(cmd, stdout=self.stdout, stderr=self.stderr)
except OSError as details:
logging.debug(details)
logging.error(msg)
return
oldName = os.path.join(self.details.topLevelDir, 'doc', 'auto/html')
newName = os.path.join(self.details.topLevelDir, 'doc', 'html')
FastScript.remove(newName)
try:
os.rename(oldName, newName)
except OSError as details:
logging.debug(details)
logging.error(msg)
return
FastScript.remove(os.path.join(self.details.topLevelDir, 'doc',
'auto'))
def baseSetupWine(configDir, msvc, stdout, stderr, postfix):
if not configDir:
configDir = getWineConfigDir(postfix)
# safety check: exit if we are within CIA and configdir points to
# home directories!
if FastScript.getEnv('CIA') == 'TRUE' and configDir.startswith('/home'):
raise SystemExit(
'SAFETY GUARD: Do not touch home directory within CIA!')
sourceWindowsBSP(msvc)
FastScript.setEnv('WINEPREFIX', configDir)
FastScript.mkdir(configDir)
logging.info('setting up Wine in %s', configDir)
# temporarily unset the DISPLAY variable so that the 'winecfg'
# utility does not show up
oldDisplay = FastScript.getEnv('DISPLAY')
FastScript.unsetEnv('DISPLAY')
FastScript.execProgram('winecfg', stdout=stdout, stderr=stderr)
# wait for wineserver shutdown (default: 3 seconds)
waitWineServerShutdown(configDir, postfix)
if oldDisplay:
FastScript.setEnv('DISPLAY', oldDisplay)
logging.info('Microsoft .NET support enabled')
def diff(self, output=None):
"""
Invokes "svn diff" to yield changes within the local working copy.
If 'output' is a StringIO object, the command's output will be
redirected there (otherwise printed on screen).
"""
FastScript.execProgram("svn diff", stdout=output, stderr=output)
def switchToBranch(self, branch, output=None):
"""
Switches to the given branch.
"""
Any.requireIsTextNonEmpty(branch)
cmd = 'git checkout %s' % branch
FastScript.execProgram(cmd, stdout=output, stderr=output)
def clone(self, output=None):
"""
Clones the repository at the given URL.
If 'output' is a StringIO object, the command's output will be
redirected there (otherwise printed on screen).
"""
cmd = self.getSourceCodeCommand()
FastScript.execProgram(cmd, stdout=output, stderr=output)
def execute(self):
logging.info('running doxygen...')
doxygenBinary = self._findDoxygen()
try:
FastScript.execProgram(doxygenBinary,
stdout=self.stdout,
stderr=self.stderr,
workingDir=self.docDir)
except OSError:
logging.error('unable to execute doxygen (maybe not in $PATH)?')
def switchToBranch(self, branch, output=None):
"""
Switches bare repository to the given branch.
Hint: Better use WorkingTree.switchToBranch()
"""
Any.requireIsTextNonEmpty(branch)
cmd = 'git symbolic-ref HEAD refs/heads/' + branch
FastScript.execProgram(cmd, stdout=output, stderr=output)
def _cmakeCompile( self ):
requireTopLevelDir( self._sourceTree )
try:
FastScript.execProgram( self._buildCmd,
stdout=self._stdout,
stderr=self._stderr,
workingDir=self._buildDir )
return True
except subprocess.CalledProcessError:
return False
def checkExecutable(executablePath, details, stdout=None, stderr=None):
tmpFile = tempfile.mktemp()
cmd = VALGRIND % (tmpFile, executablePath)
FastScript.execProgram(cmd, stdout=stdout, stderr=stderr)
failed, numErrors = parseOutput(tmpFile, details)
FastScript.remove(tmpFile)
return failed, numErrors
def startWineServer(configDir=None, postfix=''):
"""
Start Winserver
You may provide a path to the Wine config directory and/or a
config name postfix. If omitted, the path returned from
getWineConfigDir() will be used.
"""
if not configDir:
configDir = getWineConfigDir(postfix)
FastScript.setEnv('WINEPREFIX', configDir)
FastScript.execProgram('wineserver')
def update(self, output=None):
"""
Updates the current working copy.
If 'output' is a StringIO object, the command's output will be
redirected there (otherwise printed on screen).
"""
cmd = 'svn up'
if self._dryRun:
logging.warning("DRY-RUN: won't alter working copy")
logging.debug("cmd: %s" % cmd)
else:
FastScript.execProgram(cmd, stdout=output, stderr=output)
def fetch(self, output=None):
"""
Retrieve new commits from the remote peer.
If 'output' is a StringIO object, the command's output will be
redirected there (otherwise printed on screen).
"""
cmd = 'git fetch'
if self._dryRun:
logging.warning("DRY-RUN: won't transfer anything")
logging.debug("cmd: %s" % cmd)
else:
FastScript.execProgram(cmd, stdout=output, stderr=output)
def test_execInAllProjects(self):
cmd = 'ExecInAllProjects.py "cat hello.txt"'
output = StringIO()
# run ExecInAllProjects.py
FastScript.execProgram(cmd, stdout=output, stderr=output)
# check result
returned = output.getvalue()
self.assertTrue(returned.find('Hello from "HelloWorld/1.0"') > 0)
self.assertTrue(returned.find('Hello from "HelloWorld/1.1"') > 0)
self.assertTrue(returned.find('Hello from "HelloWorld/1.2"') > 0)
self.assertTrue(returned.find('Hello from "Spam/42.0"') > 0)
def remove(self, fileList, output=None):
"""
Flags the given files and/or directories to be removed with
the next commit.
If 'output' is a StringIO object, the command's output will be
redirected there (otherwise printed on screen).
"""
if not Any.isIterable(fileList):
fileList = [fileList]
for item in fileList:
cmd = 'svn rm %s' % item
FastScript.execProgram(cmd, stdout=output, stderr=output)
def test_helpText(self):
tcRoot = FastScript.getEnv('TOOLBOSCORE_ROOT')
hostPlatform = Platforms.getHostPlatform()
binDirNoArch = os.path.join(tcRoot, 'bin')
Any.requireIsDirNonEmpty(binDirNoArch)
pyScripts = glob.glob(os.path.join(binDirNoArch, '*.py'))
shScripts = glob.glob(os.path.join(binDirNoArch, '*.sh'))
executables = glob.glob(os.path.join(binDirNoArch, hostPlatform, '*'))
# unset VERBOSE and BST_BUILD_JOBS to make output comparable
origEnv = FastScript.getEnv()
FastScript.unsetEnv('VERBOSE')
FastScript.unsetEnv('BST_BUILD_JOBS')
for program in pyScripts + shScripts + executables:
basename = os.path.basename(program)
Any.requireIsTextNonEmpty(basename)
logging.info('processing %s', basename)
output = StringIO()
cmd = '%s --help' % program
fileName = os.path.join('ReferenceData', '%s.txt' % basename)
Any.requireIsTextNonEmpty(cmd)
Any.requireIsTextNonEmpty(fileName)
FastScript.execProgram(cmd, stdout=output, stderr=output)
expected = FastScript.getFileContent(fileName)
result = normalizeOutput(output.getvalue())
Any.isTextNonEmpty(expected)
Any.isTextNonEmpty(result)
if result != expected:
logging.info('differences in output of %s:', basename)
logging.info('<result>\n%s', result)
logging.info('</result>')
logging.info('<expected>\n%s', expected)
logging.info('</expected>')
self.fail('help text of %s differs' % basename)
FastScript.setEnv(origEnv)
def _runScript( self, name ):
Any.requireIsTextNonEmpty( name )
( filename, cmd ) = self._assembleScriptCmd( name )
status = True
if os.path.exists( filename ):
try:
FastScript.execProgram( cmd )
except subprocess.CalledProcessError:
status = False
except OSError as details:
logging.error( '%s: %s' % ( cmd, str(details) ) )
status = False
return status
def checkout(self, revision='HEAD', output=None):
"""
Fetches the specified revison from the given URL (which needs to
point to a valid SVN repository).
If the username to use for connecting to the server does not match
your current username (e.g. on HRI-EU's Ext.SVN server), please
use insertUsernameIntoURL() prior to this call.
If 'output' is a StringIO object, the command's output will be
redirected there (otherwise printed on screen).
"""
# suppress interactive password prompt for Ext.SVN (if any),
# see JIRA ticket TBCORE-904
oldValue = FastScript.getEnv('SVN_SSH')
if oldValue and not '-oBatchMode' in oldValue:
# if SVN_SSH was already set then *append* our settings
newValue = oldValue + ' -oBatchMode=yes'
else:
# set SVN_SSH
newValue = 'ssh -oBatchMode=yes'
FastScript.setEnv('SVN_SSH', newValue)
cmd = self.getSourceCodeCommand(revision)
return FastScript.execProgram(cmd, stdout=output, stderr=output)
def getCurrentBranch(self):
"""
Returns the name of the current branch (e.g. "master").
May return None, for example if repo is in 'Detached HEAD'
state.
"""
output = StringIO()
cmd = 'git rev-parse --abbrev-ref HEAD'
FastScript.execProgram(cmd, stdout=output, stderr=output)
result = output.getvalue().strip()
logging.debug('found Git branch name: %s', result)
return result
def _retrieveMetaInfo(self):
"""
Invokes "svn info" within a working copy.
If 'output' is a StringIO object, the command's output will be
redirected there (otherwise printed on screen).
"""
if not self._infoOutput:
output = StringIO()
# Temporarily unset the LANG environment variable, in order to always
# get English output. Otherwise the "svn info" output might be German
# or Japanese which would break the later parsing, see TBCORE-59.
origEnv = FastScript.getEnv('LANG')
if origEnv:
FastScript.unsetEnv('LANG')
cmd = 'svn info'
FastScript.execProgram(cmd, stdout=output, stderr=output)
if origEnv:
FastScript.setEnv('LANG', origEnv)
self._infoOutput = output.getvalue()
self._repoRootFromInfo = re.search("Repository Root: (\S+)\n",
self._infoOutput).group(1)
logging.debug('found SVN repository root: %s',
self._repoRootFromInfo)
self._repoUrlFromInfo = re.search("URL: (\S+)\n",
self._infoOutput).group(1)
logging.debug('found SVN repository URL: %s',
self._repoUrlFromInfo)
self._revFromInfo = int(
re.search("Revision\s?: (\d+?)\s", self._infoOutput).group(1))
logging.debug('found SVN revision: %d', self._revFromInfo)
return self._infoOutput
def codeCheck(klocworkDir='klocwork', stdout=None, stderr=None):
"""
Performs a CLI-analysis of the project.
Note that the Klocwork-project must have been created before,
e.g. using createLocalProject().
"""
Any.requireIsDirNonEmpty(klocworkDir)
requireOutsideTmpDir()
ProcessEnv.source(ToolBOSSettings.getConfigOption('package_klocwork'))
kwlpDir = os.path.join(klocworkDir, '.kwlp')
Any.requireIsDirNonEmpty(kwlpDir)
cmd = 'kwcheck run -pd %s' % kwlpDir
FastScript.execProgram(cmd, stdout=stdout, stderr=stderr)
def test_serializeUtility_convert(self):
tcRoot = FastScript.getEnv('TOOLBOSCORE_ROOT')
platform = Platforms.getHostPlatform()
exe = os.path.join(tcRoot, 'bin', platform, 'ConvertSerializedData')
inFile = 'BBDMArrayBlockF32-binary.ser'
outFile = tempfile.mkstemp(prefix='test-')[1]
cmd = '%s -i %s -f Ascii -o %s' % (exe, inFile, outFile)
tmp = StringIO()
try:
FastScript.execProgram(cmd, stdout=tmp, stderr=tmp)
except OSError:
raise RuntimeError("Please compile this package first!")
output = tmp.getvalue()
# typical error could be:
#
# [... SerializeUtility.c:809 Error] Could not load data library
# 'BBDMArrayBlockF32' (libBBDMArrayBlockF32.so): Reason
# 'libBBDMArrayBlockF32.so: cannot open shared object file:
# No such file or directory'
#
# [... SerializeUtility.c:653 Error] The input file
# 'test/miscInputFiles/BBDMArrayBlockF32-binary.ser'
# does not contain valid data
# check result
self.assertTrue(output.find('libBBDMArrayBlockF32.so') == -1)
self.assertTrue(output.find('Could not load data library') == -1)
self.assertTrue(output.find('No such file or directory') == -1)
self.assertTrue(
output.find('Aborting serialization function') > 0) # at EOF
self.assertTrue(output.find('does not contain valid data') == -1)
Any.requireIsFileNonEmpty(outFile)
content = FastScript.getFileContent(outFile)
# there must be 3x "HRIS" in the ASCII representation of this file
matches = re.findall('HRIS', content)
self.assertTrue(len(matches) == 3)
# clean-up
FastScript.remove(outFile)
def getOrigin(self):
"""
Returns the URL of the 'origin' (fetch direction).
"""
tmp = StringIO()
FastScript.execProgram('git remote -v', stdout=tmp)
output = tmp.getvalue()
# Any.requireIsTextNonEmpty( output ) # repo may not have any remote
tmp = re.search("^origin\s+(.+)\s\(fetch\)", output)
try:
origin = tmp.group(1)
except AttributeError:
raise ValueError(
'this Git repository has no "origin" remote peer configured')
return origin
def test_serializeUtility_createSerializedData(self):
random.seed(self)
tcRoot = FastScript.getEnv('TOOLBOSCORE_ROOT')
platform = Platforms.getHostPlatform()
exe = os.path.join(tcRoot, 'bin', platform, 'CreateSerializedData')
count = random.randint(3, 20)
outFile = tempfile.mkstemp(prefix='test-')[1]
cmd = '%s -t BBDMArrayBlockF32 -c %d -r -f Ascii -o %s' % (exe, count,
outFile)
tmp = StringIO()
try:
FastScript.execProgram(cmd, stdout=tmp, stderr=tmp)
except OSError:
raise RuntimeError("Please compile this package first!")
output = tmp.getvalue()
# typical error could be:
#
# [... SerializeUtility.c:809 Error] Could not load data library
# 'BBDMBaseF32' (libBBDMBaseF32.so): Reason 'libBBDMBaseF32.so:
# cannot open shared object file: No such file or directory'
#
# SerializeUtility.c:1083 BBDMBaseF32: unsupported datatype
# (BBDMBaseF32_new() not found)
# check result
self.assertTrue(output.find('libBBDMArrayBlockF32.so') == -1)
self.assertTrue(output.find('cannot open shared object file') == -1)
self.assertTrue(output.find('No such file or directory') == -1)
self.assertTrue(output.find('unsupported datatype') == -1)
Any.requireIsFileNonEmpty(outFile)
content = FastScript.getFileContent(outFile)
# we must find "count"-times the keyword "HRIS" inside the file
matches = re.findall('HRIS', content)
self.assertTrue(len(matches) == count)
# clean-up
FastScript.remove(outFile)
def waitWineServerShutdown(configDir=None, postfix=''):
"""
Wait Winserver shutdown
You may provide a path to the Wine config directory and/or a
config name postfix. If omitted, the path returned from
getWineConfigDir() will be used.
"""
if not configDir:
configDir = getWineConfigDir(postfix)
FastScript.setEnv('WINEPREFIX', configDir)
# wait for wineserver shutdown (default: 3 seconds)
logging.debug('waiting for wineserver to shut down...')
try:
FastScript.execProgram('wineserver -k')
except subprocess.CalledProcessError:
pass
def test_serializeUtility_printSerializedData(self):
tcRoot = FastScript.getEnv('TOOLBOSCORE_ROOT')
platform = Platforms.getHostPlatform()
exe = os.path.join(tcRoot, 'bin', platform, 'PrintSerializedData')
inFile = os.path.join('BBDMArrayBlockF32-binary.ser')
outFile = os.path.join('BBDMArrayBlockF32-ascii.ser')
cmd = '%s -f %s' % (exe, inFile)
tmp = StringIO()
Any.requireIsFileNonEmpty(inFile)
Any.requireIsFileNonEmpty(outFile)
try:
# do not redirect stderr to file, otherwise we can't compare with
# the expected output (ANY_LOG timestamps would differ), there is
# at least stderr-message "Aborting serialization function" at EOF
FastScript.execProgram(cmd, stdout=tmp)
except OSError:
raise RuntimeError("Please compile this package first!")
expected = FastScript.getFileContent(outFile)
output = tmp.getvalue()
# typical error could be:
#
# [... SerializeUtility.c:809 Error] Could not load data library
# 'BBDMArrayBlockF32' (libBBDMArrayBlockF32.so): Reason
# 'libBBDMArrayBlockF32.so: cannot open shared object file:
# No such file or directory'
#
# [... SerializeUtility.c:591 Error] The input file
# 'test/miscInputFiles/BBDMArrayBlockF32-binary.ser' does not
# contain valid data
# check result
self.assertTrue(output.find('libBBDMArrayBlockF32.so') == -1)
self.assertTrue(output.find('cannot open shared object file') == -1)
self.assertTrue(output.find('No such file or directory') == -1)
self.assertTrue(output.find('does not contain valid data') == -1)
self.assertEqual(expected, output)
def execInAllProjects(command):
Any.requireIsTextNonEmpty(command)
# determine list of packages (and versions) to visit
dirList = []
oldcwd = os.getcwd()
if listfile:
Any.requireIsFileNonEmpty(listfile)
# read subdirectories from file, and remove trailing newlines
dirList = FastScript.getFileContent(listfile, splitLines=True)
dirList = map(str.strip, dirList)
else:
noSVN = re.compile("^.svn$")
for path in FastScript.getDirsInDirRecursive(excludePattern=noSVN):
if os.path.isfile(os.path.join(path, 'CMakeLists.txt')):
dirList.append(path)
# execute the task
for entry in dirList:
workingDir = os.path.realpath(os.path.join(oldcwd, entry))
logging.debug('cd %s' % workingDir)
logging.info('workingDir=%s', workingDir)
FastScript.changeDirectory(workingDir)
try:
FastScript.execProgram(command)
except subprocess.CalledProcessError as e:
if ignoreErrors:
logging.warning(e)
else:
raise
FastScript.changeDirectory(oldcwd)
def create(self):
"""
Creates a new SVN repository.
If the repository shall be on another server than 'localhost',
SSH will be used to tunnel the execution of commands.
"""
from ToolBOSCore.Packages import ProjectProperties
tmp1 = parse.urlsplit(self.url)
server = tmp1.netloc
repoDir = tmp1.path
server = None if server is '' else server
if server: # if not set create on localhost
Any.requireIsTextNonEmpty(server)
Any.requireIsTextNonEmpty(repoDir)
repoRoot, category, packageName = ProjectProperties.splitPath(repoDir)
repoParent = os.path.dirname(repoDir)
logging.debug('repoRoot: %s', repoRoot)
logging.debug('repoParent: %s', repoParent)
logging.debug('repoDir: %s', repoDir)
logging.debug('category: %s', category)
logging.debug('package name: %s', packageName)
logging.debug('server: %s', server)
logging.info('creating directory structure')
cmd = "mkdir -p %s" % repoDir
FastScript.execProgram(cmd, host=server, workingDir='/')
logging.info('creating new repository')
cmd = "svnadmin create --fs-type fsfs %s" % packageName
FastScript.execProgram(cmd, host=server, workingDir=repoParent)
# verify that it exists now
Any.requireMsg(self.exists(),
"failed to create repository (reason is unclear)")
# As decided by HRI-EU's Security Group in April 2009, new SVN
# repositories will be protected from world access by default.
# Beside the owner, only the same group can access repositories.
# Manual interference is necessary to change this upon demand.
if server:
groupName = FastScript.getCurrentGroupName()
logging.info('granting group read-permission (%s)' % groupName)
cmd = "chmod 2770 db db/transactions db/revs db/revprops && " + \
"chmod 660 db/write-lock && chmod 750 hooks && " + \
"chmod 770 locks && chmod o-r * && chmod o-w * && " + \
"chmod o-x * && chmod -R g-w db && chmod 0750 ."
FastScript.execProgram(cmd, host=server, workingDir=repoDir)
else:
logging.warning( 'setting repository permissions on local ' + \
'filesystem not implemented, yet' )
def clone(self, output=None):
"""
Clones the full content of the SVN repository into a local Git
repository.
If 'output' is a StringIO object, the command's output will be
redirected there (otherwise printed on screen).
"""
oldcwd = os.getcwd()
packageName = os.path.basename(self._svnRepo.url)
Any.requireIsTextNonEmpty(packageName)
FastScript.mkdir(packageName)
FastScript.changeDirectory(packageName)
cmd = "git svn init %s" % self._svnRepo.url
FastScript.execProgram(cmd, stdout=output, stderr=output)
cmd = "git svn fetch"
FastScript.execProgram(cmd, stdout=output, stderr=output)
FastScript.changeDirectory(oldcwd)
def startGUI(options='', blocking=False):
"""
Launches the Klocwork Desktop GUI.
The program can be opened blocking or non-blocking, f.i. in
background in order not to block the caller.
"""
requireOutsideTmpDir()
ProcessEnv.source(ToolBOSSettings.getConfigOption('package_klocwork'))
try:
cmd = 'kwgcheck %s' % options
logging.debug('executing: %s', cmd)
if blocking:
FastScript.execProgram(cmd)
else:
subprocess.Popen(shlex.split(cmd))
except (subprocess.CalledProcessError, OSError) as details:
logging.error(details)
