def _retrieveRevisionInfo( self ):
# check if specified revision is in cache. do not check for 'HEAD'
if self.getRevision() in self.__revisionInfoCache:
return self.__revisionInfoCache[self.getRevision()]
info = RevisionInfo( "SvnRevisionInfo" )
revisionParameter = ['-r', str( self.getRevision() )] if self.getRevision() else []
cmd = [ self.getCommand(), '--non-interactive', 'log', '--xml', '--limit', '1', self.getUrl() ] + revisionParameter
runner = RunCommand( cmd, searchPaths = self.getCommandSearchPaths() )
runner.run()
if runner.getReturnCode() == 0:
xmldoc = minidom.parseString( runner.getStdOut().encode( "utf-8" ) )
logentries = xmldoc.getElementsByTagName( 'logentry' )
assert len( logentries ) == 1
results = parse_log_entry( logentries[0] )
( info.committerName, info.commitMessage, info.revision, info.commitTime, info.commitTimeReadable ) = results
info.shortRevision = info.revision
if self.getSCMUidMapper():
email = self.getSCMUidMapper().getEmail( info.committerName )
mApp().debugN( self, 5, "E-Mail address for {0} from SCM uid mapper: {1}".format( info.committerName, email ) )
info.committerEmail = email
# add to cache. do not add 'HEAD'
if self.getRevision():
self.__revisionInfoCache[self.getRevision()] = info
return info
def resolveCommand( self ): # if no command specified, do not run check if not self.getCommand(): return runCommand = RunCommand( [ self.getCommand() ], searchPaths = self.__commandSearchPaths ) runCommand.checkVersion()
def testCheckCMakeVersion( self ): # cmake example cmakeCommand = RunCommand( [ "cmake" ] ) self.assertRaises( ConfigurationError, cmakeCommand.checkVersion, expectedReturnCode = -1 ) # cmake returns 0 version = cmakeCommand.checkVersion( expectedReturnCode = 0 ) self.assertTrue( "cmake version" in version )
def testCheckQMakeVersion( self ): # qmake example qmakeCommand = RunCommand( [ "qmake" ] ) self.assertRaises( ConfigurationError, qmakeCommand.checkVersion, expectedReturnCode = 1 ) # qmake returns 0 version = qmakeCommand.checkVersion( expectedReturnCode = 0 ) self.assertTrue( "QMake version" in version )
def testPrintCurrentRevision( self ): cmd = [ sys.executable, SimpleProjectTests.BuildScriptName, 'print', 'current-revision' ] runner = RunCommand( cmd ) runner.run() self.assertEquals( runner.getReturnCode(), 0 ) line = runner.getStdOutAsString().strip() # we cannot know what the current revision is, but if the return code is not zero, it should not be empty: self.assertTrue( line )
def testCheckReturnCodes( self ): '''Check that RunCommand returns the actual return value of the called process.''' for timeout in [ None, 1 ]: for captureOutput in [ False, True ]: for code in range( 3 ): cmd = [ sys.executable, RunCommandTests.EXECUTABLE, str( code ) ] runner = RunCommand( cmd, timeoutSeconds = timeout, captureOutput = captureOutput ) runner.run() self.assertEquals( runner.getReturnCode(), code )
def queryCurrentRevision( self ):
cmd = [ sys.executable, self.getBuildScript(), 'print', 'current-revision' ] + self.getParameters()
runner = RunCommand( cmd, 1800 )
runner.run()
if runner.getReturnCode() != 0:
raise MomError( 'Cannot get initial revision for build script "{0}".'.format( self.getBuildScript() ) )
revision = runner.getStdOutAsString().strip()
return revision
def __getSummarizedDiffForRevision( self, url, revision ):
previous = revision - 1
cmd = [ self.getCommand(), 'diff', '--summarize', '-r', str( previous ) + ':' + str( revision ), url ]
runner = RunCommand( cmd, 3600, searchPaths = self.getCommandSearchPaths() )
runner.run()
if runner.getReturnCode() != 0:
# maybe the location did not exist earlier on:
mApp().debugN( self, 2, 'cannot retrieve summarized diff for revision "{0}"'.format( revision ) )
return None
else:
return runner.getStdOut().encode( "utf-8" ).split( '\n' )
def executeWithArgs( self, timeout = 24 * 60 * 60, args = None, captureOutput = False ):
cmd = [ sys.executable, os.path.abspath( self.getBuildScript() ) ]
if args:
cmd.extend( args )
mApp().message( self, 'invoking build script: {0}'.format( ' '.join( cmd ) ) )
runner = RunCommand( cmd, timeoutSeconds = timeout, captureOutput = captureOutput )
with EnvironmentSaver():
extend_debug_prefix( 'script>' )
runner.run()
mApp().debugN( self, 2, 'build script finished, return code is {0}.'.format( runner.getReturnCode() ) )
return runner
def __getCurrentRevisionOfUrl( self, url ):
cmd = [ self.getCommand(), '--non-interactive', 'log', '--xml', '--limit', '1', url ]
runner = RunCommand( cmd, searchPaths = self.getCommandSearchPaths() )
runner.run()
if runner.getReturnCode() == 0:
xmldoc = minidom.parseString( runner.getStdOut().encode( "utf-8" ) )
logentries = xmldoc.getElementsByTagName( 'logentry' )
assert len( logentries ) == 1
result = parse_log_entry( logentries[0] )
return result[2]
else:
raise ConfigurationError( 'cannot get log for "{0}"'.format( url ) )
class ShellCommandAction( Action ):
"""ShellCommandAction encapsulates the execution of one command in the Step class.
It is mostly used internally, but can be of general use as well."""
def __init__( self, command = None, timeout = None, combineOutput = True, searchPaths = None ):
Action.__init__( self )
self.setCommand( command, timeout, searchPaths )
self.__combineOutput = combineOutput
self.__runner = None
def getLogDescription( self ):
"""Provide a textual description for the Action that can be added to the execution log file."""
return '{0}'.format( ' '.join( self.getCommand() ) )
def setCommand( self, command, timeOutPeriod = None, searchPaths = None ):
"""Set the shell command"""
check_for_list_of_paths( command, "The shell command must be a list of strings or paths." )
if timeOutPeriod != None:
check_for_nonnegative_int( timeOutPeriod, 'invalid timeout period, valid periods are [0..inf] or None for no timeout' )
if searchPaths is None:
searchPaths = []
check_for_list_of_paths( searchPaths, "The search paths need to be a list of strings." )
self.__command = command
self.__timeOutPeriod = timeOutPeriod
self.__searchPaths = searchPaths
def getCommand( self ):
"""Returns the command"""
return map( lambda x: str( x ) , self.__command )
def _getRunner( self ):
if self.__runner == None:
raise MomError( "The command runner was not initialized before being queried" )
return self.__runner
def run( self ):
"""Executes the shell command. Needs a command to be set."""
self.__runner = RunCommand( self.__command, self.__timeOutPeriod, self.__combineOutput, self.__searchPaths )
if self.getWorkingDirectory() != None:
self.__runner.setWorkingDir( self.getWorkingDirectory() )
self._getRunner().run()
self._setStdOut( self._getRunner().getStdOut() )
self._setStdErr( self._getRunner().getStdErr() )
return self._getRunner().getReturnCode()
def hasTimedOut( self ):
"""Returns True if the shell command process timed out, e.g., was not completed within the timeout period.
Can only be called after execution."""
if not self.__started:
raise MomError( 'timedOut() queried before the command was executed' )
return self._getRunner().getTimedOut()
def runUploadFiles( self ):
fromDir = self._makeCygwinPathForRsync( '{0}{1}'.format( self.getSourcePath(), os.sep ) )
toDir = os.path.join( self.getDestinationPath(), *self._getExtraUploadSubdirsAsString() )
args = [ '-avz', '-e', 'ssh -o BatchMode=yes', fromDir, toDir ]
if 'Windows' in platform.platform(): #On windows, fake source permissions to be 755
args = [ '--chmod=ugo=rwx' ] + args
cmd = [ "rsync" ] + args
searchPaths = [ "C:/Program Files/cwRsync/bin" ]
runner = RunCommand( cmd, timeoutSeconds = 7200, searchPaths = searchPaths )
if runner.run() != 0:
mApp().debugN( self, 1, 'Uploading from "{0}" to "{1}" failed!'.format( fromDir, toDir ) )
return runner.getReturnCode()
else:
mApp().debugN( self, 3, 'Uploaded "{0}" to "{1}".'.format( fromDir, toDir ) )
return 0
def run( self ):
"""Executes the shell command. Needs a command to be set.
\return 0 if minimum score is met, otherwise 1"""
cmd = [ self._getPyLintChecker().getCommand() ]
paths = self._getPyLintChecker().getCommandSearchPaths()
args = [ ]
if self._getPyLintChecker().getPyLintRcFile():
args.append( '--rcfile={0}'.format( self._getPyLintChecker().getPyLintRcFile() ) )
# Check if the source and output path exist, since this action will be executed even if there was an error before (since
# the source code can be checked even if, for example, a unit test failed)
targetPath = os.path.dirname( str( self._getPyLintChecker().getHtmlOutputPath() ) )
if not os.path.isdir( targetPath ) \
or not os.path.isdir( str( self.getWorkingDirectory() ) ):
self._getPyLintChecker().setReport( 'not executed because of previous failures' )
return 0
# First, run PyLint with parseable output, and retrieve the score and comment:
parseableCommand = cmd + [ '--output-format=parseable' ] + args + self._getPyLintChecker().getModules()
runner1 = RunCommand( parseableCommand, 1800, searchPaths = paths )
runner1.run()
if runner1.getReturnCode() >= 32:
mApp().debugN( self, 2, 'error running pylint to produce the parseable report' )
return 1
# parse output
self._getPyLintChecker().parsePyLintOutput( runner1.getStdOut() )
# Second step, run pylint again, to produce the full HTML report:
if self._getPyLintChecker().getHtmlOutputPath():
htmlCommand = cmd + [ '--output-format=html' ] + args + self._getPyLintChecker().getModules()
runner2 = RunCommand( htmlCommand )
runner2.run()
if runner2.getReturnCode() >= 32:
mApp().debugN( self, 2, 'error running pylint to generate the HTML report' )
return 1
path = str( self._getPyLintChecker().getHtmlOutputPath() )
try:
with open( path, 'w' ) as file:
file.write( runner2.getStdOut() )
mApp().debugN( self, 2, 'pylint html report is at "{0}"'.format( path ) )
except IOError as e:
mApp().debug( self, 'ERROR saving pylint html report to "{0}": {1}'.format( path, e ) )
return 1
return ( 0 if self._getPyLintChecker().isScoreOkay() else 1 )
def queryRevisionsSince( self, revision ):
'''Execute the build script, and return the lines it outputs for "query revisions-since"'''
cmd = [ sys.executable, self.getBuildScript(), 'print', 'revisions-since', str( revision ) ] + self.getParameters()
runner = RunCommand( cmd, 1800 )
runner.run()
if runner.getReturnCode() != 0:
msg = 'Cannot get revision list for build script "{0}", continuing with next project.'\
.format( self.getBuildScript() )
raise MomError( msg )
output = runner.getStdOut()
if not output:
return []
lines = output.decode().split( '\n' )
return lines
def run( self ): """Executes the shell command. Needs a command to be set.""" self.__runner = RunCommand( self.__command, self.__timeOutPeriod, self.__combineOutput, self.__searchPaths ) if self.getWorkingDirectory() != None: self.__runner.setWorkingDir( self.getWorkingDirectory() ) self._getRunner().run() self._setStdOut( self._getRunner().getStdOut() ) self._setStdErr( self._getRunner().getStdErr() ) return self._getRunner().getReturnCode()
def runCreateDirectories( self ):
# no need to create directories if no extra sub directories specified
if not self.getExtraUploadSubDirs():
return 0
path = os.path.join( *self._getExtraUploadSubdirsAsString() )
tempDir = tempfile.mkdtemp( prefix = 'mom_buildscript-', suffix = '-rsync-path' )
fullpath = os.path.join( tempDir, path )
os.makedirs( fullpath )
uploadLocation = self.getDestinationPath()
args = [ '-avz', '-e', 'ssh -o BatchMode=yes', tempDir + os.sep, uploadLocation ]
cmd = [ "rsync" ] + args
searchPaths = [ "C:/Program Files/cwRsync/bin" ]
runner = RunCommand( cmd, timeoutSeconds = 1200, searchPaths = searchPaths )
if runner.run() != 0:
mApp().debugN( self, 1, 'Creating extra sub directories {0} on the upload server failed!'.format( path ) )
return runner.getReturnCode()
else:
mApp().debugN( self, 3, 'Created extra sub directories {0} on the upload server.'.format( path ) )
return 0
def __getXmlSvnLog( self, url, revision, cap ):
cmd = [ self.getCommand(), '--non-interactive', 'log', '--xml' ]
if revision == 0:
cmd.extend( ['--limit', '1' ] )
cmd.extend( ['-rHEAD:{0}'.format( str( revision ).strip() ), url ] )
runner = RunCommand( cmd, 3600, searchPaths = self.getCommandSearchPaths() )
runner.run()
if runner.getReturnCode() == 0:
return minidom.parseString( runner.getStdOut().encode( "utf-8" ) )
elif runner.getTimedOut() == True:
raise ConfigurationError( 'Getting svn log for "{0}" timed out.'.format( self.getUrl() ) )
else:
msg = runner.getStdErrAsString().strip()
raise ConfigurationError( 'Getting svn log failed: "{0}"'.format( msg ) )
def update( self, folder ):
cmd = [ 'git', 'pull' ]
if self.getUseRebase():
cmd.append( '--rebase' )
runner = RunCommand( cmd )
runner.setWorkingDir( folder )
runner.run()
if runner.getReturnCode() == 0:
mApp().debugN( self, 2, 'Updated the git repository at "{0}"'.format( folder ) )
else:
# we are not raising an exception, because we do not want the master to die because of, for example, a temporary
# network outage
message = runner.getStdErrAsString()
mApp().message( self, 'Updating the git repository at "{0}" failed: "{1}"'.format( folder, message ) )
def runCommand( self, cmd, description, timeout = None, zeroReturnCode = True ):
'''Helper method to run shell commands in tests. It creates a RunCommand object, runs it,
and returns it. If the return code is not zero, it dumps the output of the command.'''
runner = RunCommand( cmd, timeout )
runner.run()
if zeroReturnCode and runner.getReturnCode() != 0:
print( '\n' )
print( 'command failed: {0}'.format( description ) )
print( 'output:' )
print( runner.getStdOutAsString() )
print( 'error output:' )
print( runner.getStdErrAsString() )
self.assertEqual( runner.getReturnCode() == 0, zeroReturnCode )
return runner
def querySetting( self, setting ):
cmd = [ sys.executable, self.getBuildScript(), 'query', setting ] + self.getParameters()
runner = RunCommand( cmd, 1800 )
runner.run()
if runner.getReturnCode() != 0:
raise MomError( 'Cannot query setting "{0}" for build script "{1}":\n {2}!'\
.format( setting, self.getBuildScript(), runner.getStdErrAsString() ) )
output = runner.getStdOutAsString()
if not output:
raise MomError( 'The build script "{0}" did not return a value! It said:\n {1}'
.format( self.getBuildScript(), runner.getStdErrAsString() ) )
line = output.strip()
groups = re.search( '^(.+?): (.+)$', line )
if not groups:
raise MomError( 'Did not understand this output: "{0}"!'.format( line ) )
variable = groups.groups()[1]
return variable
def fetchRepositoryFolder( self, remotePath ):
# FIXME Mike abstract cache location
path = tempfile.mkdtemp( prefix = 'mom_buildscript-', suffix = make_foldername_from_string( self.getUrl() ) )
if not self.getRevision():
self.setRevision( 'HEAD' )
location = self.getUrl()
if remotePath:
location += '/' + remotePath
cmd = [ self.getCommand(), 'co', '-r', self.getRevision(), location ]
runner = RunCommand( cmd, searchPaths = self.getCommandSearchPaths() )
runner.setWorkingDir( path )
runner.run()
if runner.getReturnCode() == 0:
localPath = os.path.join( path, remotePath )
if os.path.exists( localPath ):
return localPath, [ path ]
raise ConfigurationError( 'The remote path {0} was not found in the repository at revision {1}'.format(
remotePath, self.getRevision() ) )
def querySetting( self, name ): cmd = [ sys.executable, SimpleProjectTests.BuildScriptName, 'query', name ] runner = RunCommand( cmd ) runner.run() return runner