def getSystemPackages(): """ Fetches the list of Debian packages installed on the system, returning a dict mapping "packageName" => "version". """ if not ProcessEnv.which('dpkg'): raise EnvironmentError('Not a Debian-based operating system') output = StringIO() regexp = re.compile("^ii\s+(.+?)\s+(.+?)\s.+$") result = {} try: FastScript.execProgram('dpkg -l', stdout=output) except OSError as details: raise OSError(details) for line in output.getvalue().splitlines(): if line.startswith('ii '): tmp = regexp.match(line) name = tmp.group(1) version = tmp.group(2) # try to cut-off the arch postfix (e.g. ":i386" or ":amd64") name = name.split(':')[0] result[name] = version return result
def execute(self): logging.info('running matdoc...') prog = self._findMatdoc() cmd = '%s %s' % (prog, self.details.packageName) msg = 'matdoc not found' if not os.path.exists(prog): logging.error('%s: No such file') logging.error(msg) return try: FastScript.execProgram(cmd, stdout=self.stdout, stderr=self.stderr) except OSError as details: logging.debug(details) logging.error(msg) return oldName = os.path.join(self.details.topLevelDir, 'doc', 'auto/html') newName = os.path.join(self.details.topLevelDir, 'doc', 'html') FastScript.remove(newName) try: os.rename(oldName, newName) except OSError as details: logging.debug(details) logging.error(msg) return FastScript.remove(os.path.join(self.details.topLevelDir, 'doc', 'auto'))
def baseSetupWine(configDir, msvc, stdout, stderr, postfix): if not configDir: configDir = getWineConfigDir(postfix) # safety check: exit if we are within CIA and configdir points to # home directories! if FastScript.getEnv('CIA') == 'TRUE' and configDir.startswith('/home'): raise SystemExit( 'SAFETY GUARD: Do not touch home directory within CIA!') sourceWindowsBSP(msvc) FastScript.setEnv('WINEPREFIX', configDir) FastScript.mkdir(configDir) logging.info('setting up Wine in %s', configDir) # temporarily unset the DISPLAY variable so that the 'winecfg' # utility does not show up oldDisplay = FastScript.getEnv('DISPLAY') FastScript.unsetEnv('DISPLAY') FastScript.execProgram('winecfg', stdout=stdout, stderr=stderr) # wait for wineserver shutdown (default: 3 seconds) waitWineServerShutdown(configDir, postfix) if oldDisplay: FastScript.setEnv('DISPLAY', oldDisplay) logging.info('Microsoft .NET support enabled')
def diff(self, output=None): """ Invokes "svn diff" to yield changes within the local working copy. If 'output' is a StringIO object, the command's output will be redirected there (otherwise printed on screen). """ FastScript.execProgram("svn diff", stdout=output, stderr=output)
def switchToBranch(self, branch, output=None): """ Switches to the given branch. """ Any.requireIsTextNonEmpty(branch) cmd = 'git checkout %s' % branch FastScript.execProgram(cmd, stdout=output, stderr=output)
def clone(self, output=None): """ Clones the repository at the given URL. If 'output' is a StringIO object, the command's output will be redirected there (otherwise printed on screen). """ cmd = self.getSourceCodeCommand() FastScript.execProgram(cmd, stdout=output, stderr=output)
def execute(self): logging.info('running doxygen...') doxygenBinary = self._findDoxygen() try: FastScript.execProgram(doxygenBinary, stdout=self.stdout, stderr=self.stderr, workingDir=self.docDir) except OSError: logging.error('unable to execute doxygen (maybe not in $PATH)?')
def switchToBranch(self, branch, output=None): """ Switches bare repository to the given branch. Hint: Better use WorkingTree.switchToBranch() """ Any.requireIsTextNonEmpty(branch) cmd = 'git symbolic-ref HEAD refs/heads/' + branch FastScript.execProgram(cmd, stdout=output, stderr=output)
def _cmakeCompile( self ): requireTopLevelDir( self._sourceTree ) try: FastScript.execProgram( self._buildCmd, stdout=self._stdout, stderr=self._stderr, workingDir=self._buildDir ) return True except subprocess.CalledProcessError: return False
def checkExecutable(executablePath, details, stdout=None, stderr=None): tmpFile = tempfile.mktemp() cmd = VALGRIND % (tmpFile, executablePath) FastScript.execProgram(cmd, stdout=stdout, stderr=stderr) failed, numErrors = parseOutput(tmpFile, details) FastScript.remove(tmpFile) return failed, numErrors
def startWineServer(configDir=None, postfix=''): """ Start Winserver You may provide a path to the Wine config directory and/or a config name postfix. If omitted, the path returned from getWineConfigDir() will be used. """ if not configDir: configDir = getWineConfigDir(postfix) FastScript.setEnv('WINEPREFIX', configDir) FastScript.execProgram('wineserver')
def update(self, output=None): """ Updates the current working copy. If 'output' is a StringIO object, the command's output will be redirected there (otherwise printed on screen). """ cmd = 'svn up' if self._dryRun: logging.warning("DRY-RUN: won't alter working copy") logging.debug("cmd: %s" % cmd) else: FastScript.execProgram(cmd, stdout=output, stderr=output)
def fetch(self, output=None): """ Retrieve new commits from the remote peer. If 'output' is a StringIO object, the command's output will be redirected there (otherwise printed on screen). """ cmd = 'git fetch' if self._dryRun: logging.warning("DRY-RUN: won't transfer anything") logging.debug("cmd: %s" % cmd) else: FastScript.execProgram(cmd, stdout=output, stderr=output)
def test_execInAllProjects(self): cmd = 'ExecInAllProjects.py "cat hello.txt"' output = StringIO() # run ExecInAllProjects.py FastScript.execProgram(cmd, stdout=output, stderr=output) # check result returned = output.getvalue() self.assertTrue(returned.find('Hello from "HelloWorld/1.0"') > 0) self.assertTrue(returned.find('Hello from "HelloWorld/1.1"') > 0) self.assertTrue(returned.find('Hello from "HelloWorld/1.2"') > 0) self.assertTrue(returned.find('Hello from "Spam/42.0"') > 0)
def remove(self, fileList, output=None): """ Flags the given files and/or directories to be removed with the next commit. If 'output' is a StringIO object, the command's output will be redirected there (otherwise printed on screen). """ if not Any.isIterable(fileList): fileList = [fileList] for item in fileList: cmd = 'svn rm %s' % item FastScript.execProgram(cmd, stdout=output, stderr=output)
def test_helpText(self): tcRoot = FastScript.getEnv('TOOLBOSCORE_ROOT') hostPlatform = Platforms.getHostPlatform() binDirNoArch = os.path.join(tcRoot, 'bin') Any.requireIsDirNonEmpty(binDirNoArch) pyScripts = glob.glob(os.path.join(binDirNoArch, '*.py')) shScripts = glob.glob(os.path.join(binDirNoArch, '*.sh')) executables = glob.glob(os.path.join(binDirNoArch, hostPlatform, '*')) # unset VERBOSE and BST_BUILD_JOBS to make output comparable origEnv = FastScript.getEnv() FastScript.unsetEnv('VERBOSE') FastScript.unsetEnv('BST_BUILD_JOBS') for program in pyScripts + shScripts + executables: basename = os.path.basename(program) Any.requireIsTextNonEmpty(basename) logging.info('processing %s', basename) output = StringIO() cmd = '%s --help' % program fileName = os.path.join('ReferenceData', '%s.txt' % basename) Any.requireIsTextNonEmpty(cmd) Any.requireIsTextNonEmpty(fileName) FastScript.execProgram(cmd, stdout=output, stderr=output) expected = FastScript.getFileContent(fileName) result = normalizeOutput(output.getvalue()) Any.isTextNonEmpty(expected) Any.isTextNonEmpty(result) if result != expected: logging.info('differences in output of %s:', basename) logging.info('<result>\n%s', result) logging.info('</result>') logging.info('<expected>\n%s', expected) logging.info('</expected>') self.fail('help text of %s differs' % basename) FastScript.setEnv(origEnv)
def _runScript( self, name ): Any.requireIsTextNonEmpty( name ) ( filename, cmd ) = self._assembleScriptCmd( name ) status = True if os.path.exists( filename ): try: FastScript.execProgram( cmd ) except subprocess.CalledProcessError: status = False except OSError as details: logging.error( '%s: %s' % ( cmd, str(details) ) ) status = False return status
def checkout(self, revision='HEAD', output=None): """ Fetches the specified revison from the given URL (which needs to point to a valid SVN repository). If the username to use for connecting to the server does not match your current username (e.g. on HRI-EU's Ext.SVN server), please use insertUsernameIntoURL() prior to this call. If 'output' is a StringIO object, the command's output will be redirected there (otherwise printed on screen). """ # suppress interactive password prompt for Ext.SVN (if any), # see JIRA ticket TBCORE-904 oldValue = FastScript.getEnv('SVN_SSH') if oldValue and not '-oBatchMode' in oldValue: # if SVN_SSH was already set then *append* our settings newValue = oldValue + ' -oBatchMode=yes' else: # set SVN_SSH newValue = 'ssh -oBatchMode=yes' FastScript.setEnv('SVN_SSH', newValue) cmd = self.getSourceCodeCommand(revision) return FastScript.execProgram(cmd, stdout=output, stderr=output)
def getCurrentBranch(self): """ Returns the name of the current branch (e.g. "master"). May return None, for example if repo is in 'Detached HEAD' state. """ output = StringIO() cmd = 'git rev-parse --abbrev-ref HEAD' FastScript.execProgram(cmd, stdout=output, stderr=output) result = output.getvalue().strip() logging.debug('found Git branch name: %s', result) return result
def _retrieveMetaInfo(self): """ Invokes "svn info" within a working copy. If 'output' is a StringIO object, the command's output will be redirected there (otherwise printed on screen). """ if not self._infoOutput: output = StringIO() # Temporarily unset the LANG environment variable, in order to always # get English output. Otherwise the "svn info" output might be German # or Japanese which would break the later parsing, see TBCORE-59. origEnv = FastScript.getEnv('LANG') if origEnv: FastScript.unsetEnv('LANG') cmd = 'svn info' FastScript.execProgram(cmd, stdout=output, stderr=output) if origEnv: FastScript.setEnv('LANG', origEnv) self._infoOutput = output.getvalue() self._repoRootFromInfo = re.search("Repository Root: (\S+)\n", self._infoOutput).group(1) logging.debug('found SVN repository root: %s', self._repoRootFromInfo) self._repoUrlFromInfo = re.search("URL: (\S+)\n", self._infoOutput).group(1) logging.debug('found SVN repository URL: %s', self._repoUrlFromInfo) self._revFromInfo = int( re.search("Revision\s?: (\d+?)\s", self._infoOutput).group(1)) logging.debug('found SVN revision: %d', self._revFromInfo) return self._infoOutput
def codeCheck(klocworkDir='klocwork', stdout=None, stderr=None): """ Performs a CLI-analysis of the project. Note that the Klocwork-project must have been created before, e.g. using createLocalProject(). """ Any.requireIsDirNonEmpty(klocworkDir) requireOutsideTmpDir() ProcessEnv.source(ToolBOSSettings.getConfigOption('package_klocwork')) kwlpDir = os.path.join(klocworkDir, '.kwlp') Any.requireIsDirNonEmpty(kwlpDir) cmd = 'kwcheck run -pd %s' % kwlpDir FastScript.execProgram(cmd, stdout=stdout, stderr=stderr)
def test_serializeUtility_convert(self): tcRoot = FastScript.getEnv('TOOLBOSCORE_ROOT') platform = Platforms.getHostPlatform() exe = os.path.join(tcRoot, 'bin', platform, 'ConvertSerializedData') inFile = 'BBDMArrayBlockF32-binary.ser' outFile = tempfile.mkstemp(prefix='test-')[1] cmd = '%s -i %s -f Ascii -o %s' % (exe, inFile, outFile) tmp = StringIO() try: FastScript.execProgram(cmd, stdout=tmp, stderr=tmp) except OSError: raise RuntimeError("Please compile this package first!") output = tmp.getvalue() # typical error could be: # # [... SerializeUtility.c:809 Error] Could not load data library # 'BBDMArrayBlockF32' (libBBDMArrayBlockF32.so): Reason # 'libBBDMArrayBlockF32.so: cannot open shared object file: # No such file or directory' # # [... SerializeUtility.c:653 Error] The input file # 'test/miscInputFiles/BBDMArrayBlockF32-binary.ser' # does not contain valid data # check result self.assertTrue(output.find('libBBDMArrayBlockF32.so') == -1) self.assertTrue(output.find('Could not load data library') == -1) self.assertTrue(output.find('No such file or directory') == -1) self.assertTrue( output.find('Aborting serialization function') > 0) # at EOF self.assertTrue(output.find('does not contain valid data') == -1) Any.requireIsFileNonEmpty(outFile) content = FastScript.getFileContent(outFile) # there must be 3x "HRIS" in the ASCII representation of this file matches = re.findall('HRIS', content) self.assertTrue(len(matches) == 3) # clean-up FastScript.remove(outFile)
def getOrigin(self): """ Returns the URL of the 'origin' (fetch direction). """ tmp = StringIO() FastScript.execProgram('git remote -v', stdout=tmp) output = tmp.getvalue() # Any.requireIsTextNonEmpty( output ) # repo may not have any remote tmp = re.search("^origin\s+(.+)\s\(fetch\)", output) try: origin = tmp.group(1) except AttributeError: raise ValueError( 'this Git repository has no "origin" remote peer configured') return origin
def test_serializeUtility_createSerializedData(self): random.seed(self) tcRoot = FastScript.getEnv('TOOLBOSCORE_ROOT') platform = Platforms.getHostPlatform() exe = os.path.join(tcRoot, 'bin', platform, 'CreateSerializedData') count = random.randint(3, 20) outFile = tempfile.mkstemp(prefix='test-')[1] cmd = '%s -t BBDMArrayBlockF32 -c %d -r -f Ascii -o %s' % (exe, count, outFile) tmp = StringIO() try: FastScript.execProgram(cmd, stdout=tmp, stderr=tmp) except OSError: raise RuntimeError("Please compile this package first!") output = tmp.getvalue() # typical error could be: # # [... SerializeUtility.c:809 Error] Could not load data library # 'BBDMBaseF32' (libBBDMBaseF32.so): Reason 'libBBDMBaseF32.so: # cannot open shared object file: No such file or directory' # # SerializeUtility.c:1083 BBDMBaseF32: unsupported datatype # (BBDMBaseF32_new() not found) # check result self.assertTrue(output.find('libBBDMArrayBlockF32.so') == -1) self.assertTrue(output.find('cannot open shared object file') == -1) self.assertTrue(output.find('No such file or directory') == -1) self.assertTrue(output.find('unsupported datatype') == -1) Any.requireIsFileNonEmpty(outFile) content = FastScript.getFileContent(outFile) # we must find "count"-times the keyword "HRIS" inside the file matches = re.findall('HRIS', content) self.assertTrue(len(matches) == count) # clean-up FastScript.remove(outFile)
def waitWineServerShutdown(configDir=None, postfix=''): """ Wait Winserver shutdown You may provide a path to the Wine config directory and/or a config name postfix. If omitted, the path returned from getWineConfigDir() will be used. """ if not configDir: configDir = getWineConfigDir(postfix) FastScript.setEnv('WINEPREFIX', configDir) # wait for wineserver shutdown (default: 3 seconds) logging.debug('waiting for wineserver to shut down...') try: FastScript.execProgram('wineserver -k') except subprocess.CalledProcessError: pass
def test_serializeUtility_printSerializedData(self): tcRoot = FastScript.getEnv('TOOLBOSCORE_ROOT') platform = Platforms.getHostPlatform() exe = os.path.join(tcRoot, 'bin', platform, 'PrintSerializedData') inFile = os.path.join('BBDMArrayBlockF32-binary.ser') outFile = os.path.join('BBDMArrayBlockF32-ascii.ser') cmd = '%s -f %s' % (exe, inFile) tmp = StringIO() Any.requireIsFileNonEmpty(inFile) Any.requireIsFileNonEmpty(outFile) try: # do not redirect stderr to file, otherwise we can't compare with # the expected output (ANY_LOG timestamps would differ), there is # at least stderr-message "Aborting serialization function" at EOF FastScript.execProgram(cmd, stdout=tmp) except OSError: raise RuntimeError("Please compile this package first!") expected = FastScript.getFileContent(outFile) output = tmp.getvalue() # typical error could be: # # [... SerializeUtility.c:809 Error] Could not load data library # 'BBDMArrayBlockF32' (libBBDMArrayBlockF32.so): Reason # 'libBBDMArrayBlockF32.so: cannot open shared object file: # No such file or directory' # # [... SerializeUtility.c:591 Error] The input file # 'test/miscInputFiles/BBDMArrayBlockF32-binary.ser' does not # contain valid data # check result self.assertTrue(output.find('libBBDMArrayBlockF32.so') == -1) self.assertTrue(output.find('cannot open shared object file') == -1) self.assertTrue(output.find('No such file or directory') == -1) self.assertTrue(output.find('does not contain valid data') == -1) self.assertEqual(expected, output)
def execInAllProjects(command): Any.requireIsTextNonEmpty(command) # determine list of packages (and versions) to visit dirList = [] oldcwd = os.getcwd() if listfile: Any.requireIsFileNonEmpty(listfile) # read subdirectories from file, and remove trailing newlines dirList = FastScript.getFileContent(listfile, splitLines=True) dirList = map(str.strip, dirList) else: noSVN = re.compile("^.svn$") for path in FastScript.getDirsInDirRecursive(excludePattern=noSVN): if os.path.isfile(os.path.join(path, 'CMakeLists.txt')): dirList.append(path) # execute the task for entry in dirList: workingDir = os.path.realpath(os.path.join(oldcwd, entry)) logging.debug('cd %s' % workingDir) logging.info('workingDir=%s', workingDir) FastScript.changeDirectory(workingDir) try: FastScript.execProgram(command) except subprocess.CalledProcessError as e: if ignoreErrors: logging.warning(e) else: raise FastScript.changeDirectory(oldcwd)
def create(self): """ Creates a new SVN repository. If the repository shall be on another server than 'localhost', SSH will be used to tunnel the execution of commands. """ from ToolBOSCore.Packages import ProjectProperties tmp1 = parse.urlsplit(self.url) server = tmp1.netloc repoDir = tmp1.path server = None if server is '' else server if server: # if not set create on localhost Any.requireIsTextNonEmpty(server) Any.requireIsTextNonEmpty(repoDir) repoRoot, category, packageName = ProjectProperties.splitPath(repoDir) repoParent = os.path.dirname(repoDir) logging.debug('repoRoot: %s', repoRoot) logging.debug('repoParent: %s', repoParent) logging.debug('repoDir: %s', repoDir) logging.debug('category: %s', category) logging.debug('package name: %s', packageName) logging.debug('server: %s', server) logging.info('creating directory structure') cmd = "mkdir -p %s" % repoDir FastScript.execProgram(cmd, host=server, workingDir='/') logging.info('creating new repository') cmd = "svnadmin create --fs-type fsfs %s" % packageName FastScript.execProgram(cmd, host=server, workingDir=repoParent) # verify that it exists now Any.requireMsg(self.exists(), "failed to create repository (reason is unclear)") # As decided by HRI-EU's Security Group in April 2009, new SVN # repositories will be protected from world access by default. # Beside the owner, only the same group can access repositories. # Manual interference is necessary to change this upon demand. if server: groupName = FastScript.getCurrentGroupName() logging.info('granting group read-permission (%s)' % groupName) cmd = "chmod 2770 db db/transactions db/revs db/revprops && " + \ "chmod 660 db/write-lock && chmod 750 hooks && " + \ "chmod 770 locks && chmod o-r * && chmod o-w * && " + \ "chmod o-x * && chmod -R g-w db && chmod 0750 ." FastScript.execProgram(cmd, host=server, workingDir=repoDir) else: logging.warning( 'setting repository permissions on local ' + \ 'filesystem not implemented, yet' )
def clone(self, output=None): """ Clones the full content of the SVN repository into a local Git repository. If 'output' is a StringIO object, the command's output will be redirected there (otherwise printed on screen). """ oldcwd = os.getcwd() packageName = os.path.basename(self._svnRepo.url) Any.requireIsTextNonEmpty(packageName) FastScript.mkdir(packageName) FastScript.changeDirectory(packageName) cmd = "git svn init %s" % self._svnRepo.url FastScript.execProgram(cmd, stdout=output, stderr=output) cmd = "git svn fetch" FastScript.execProgram(cmd, stdout=output, stderr=output) FastScript.changeDirectory(oldcwd)
def startGUI(options='', blocking=False): """ Launches the Klocwork Desktop GUI. The program can be opened blocking or non-blocking, f.i. in background in order not to block the caller. """ requireOutsideTmpDir() ProcessEnv.source(ToolBOSSettings.getConfigOption('package_klocwork')) try: cmd = 'kwgcheck %s' % options logging.debug('executing: %s', cmd) if blocking: FastScript.execProgram(cmd) else: subprocess.Popen(shlex.split(cmd)) except (subprocess.CalledProcessError, OSError) as details: logging.error(details)