def testSendEmail( self ): # force loading settings from configuration files mApp().getSettings().evalConfigurationFiles() email = Email() email.setToAddresses( ['*****@*****.**'] ) email.setFromAddress( mApp().getSettings().get( Settings.EmailReporterSender ) ) email.setSubject( 'EmailerTest email' ) email.attachAlternativeTextPart( '''\ This is a test email sent by Make-O-Matic. Check it out at http://github.com/KDAB/Make-O-Matic ''', """\ <html> <head>Make-O-Matic Test Email</head> <body> <p>This is the HTML part of the test email<br> Check out Make-O-Matic at <a href="http://github.com/KDAB/Make-O-Matic">GitHub</a>. </p> </body> </html> """ ) attachmentText = "TEST:\n" + "\n".join( sys.path ) email.addTextAttachment( attachmentText, "testfile1.txt", False ) email.addTextAttachment( attachmentText, "testfile2.txt", True ) e = Emailer( 'Emailer' ) e.setup() e.send( email ) e.quit()
def setup( self ):
folders = self.getFolders()
if not folders:
mApp().debug( self, 'No folders specified for self-update, continuing.' )
for folder in folders:
mApp().debug( self, 'Self-updating directory "{0}"'.format( folder ) )
self.update( folder )
def _addXslTemplate( self, plugin ):
"""Adds the XSL template to stylesheet if plugin provides one
Merges templates from plugins into the stylesheets provided by XSL_STYLESHEETS."""
if not self.hasXsltSupport():
mApp().debugN( self, 5, "Cannot add XSL template, lacking support for XSLT transformations. Please install the python-lxml package." )
return
# iterate trough the dict from getXslTemplates(), add each template to the corresponding stylesheet
for destinationReportFormat, markup in plugin.getXslTemplates().items():
if not destinationReportFormat in self.__xslTemplateSnippets.keys():
continue # invalid key, no stylesheet registered for that type of XSL
# search for place to register new plugin templates
stylesheet = self.__xslTemplateSnippets[destinationReportFormat]
pluginTemplate = stylesheet.find( ".//{http://www.w3.org/1999/XSL/Transform}template[@match='plugin']" )
placeholder = pluginTemplate.find( ".//{http://www.w3.org/1999/XSL/Transform}choose" )
# create new element with markup provided from plugin
try:
element = etree.XML( """<xsl:when xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
xmlns="http://www.w3.org/1999/xhtml"
test="@name = '{0}'">{1}</xsl:when>""".format( plugin.getName(), markup ) )
except etree.XMLSyntaxError:
raise ConfigurationError( "XSL template of {0} plugin malformed.".format( plugin.getName() ) )
# insert new element in the placeholder from the stylesheet
placeholder.insert( 0, element )
def run( self ):
self.__started = True
stderrValue = subprocess.PIPE
if self.__combineOutput:
stderrValue = subprocess.STDOUT
if self._getRunner().getCaptureOutput():
self._process = subprocess.Popen ( self._getRunner().getCommand(), shell = False,
cwd = self._getRunner().getWorkingDir(), stdout = subprocess.PIPE, stderr = stderrValue )
output, error = self._process.communicate()
# override encoding for windows
if sys.platform == 'win32':
encoding = 'cp850'
else:
encoding = 'utf-8'
self._getRunner().setStdOut( to_unicode_or_bust( output, encoding ) )
self._getRunner().setStdErr( to_unicode_or_bust( error, encoding ) )
mApp().debugN( self._getRunner(), 5, u"STDOUT:\n{0}".format( self._getRunner().getStdOut() ) )
if not self.__combineOutput:
mApp().debugN( self._getRunner(), 5, u"STDERR:\n{0}".format( self._getRunner().getStdErr() ) )
self._getRunner().setReturnCode( self._process.returncode )
else:
self._process = subprocess.Popen ( self._getRunner().getCommand(), shell = False,
cwd = self._getRunner().getWorkingDir() )
self._process.wait()
returnCode = self._process.returncode
self._getRunner().setReturnCode( returnCode )
self._getRunner().setStdOut( None )
self._getRunner().setStdErr( None )
self.__finished = True
def _initEmailBody( self, email ):
# body
reporterUseCompression = mApp().getSettings().get( Settings.EmailReporterUseCompressionForAttachments, False )
report = InstructionsXmlReport( mApp() )
converter = XmlReportConverter( report )
### text and html part
# summary
email.attachAlternativeTextPart(
converter.convertToTextSummary(),
converter.convertToHtml( summaryOnly = True )
)
# report
if self.getEnableFullReport():
email.attachAlternativeTextPart(
converter.convertToText( short = True ),
converter.convertToHtml()
)
returnCode = mApp().getReturnCode()
if returnCode != 0:
email.addTextAttachment( converter.convertToFailedStepsLog(), "failed-steps.log", useCompression = reporterUseCompression )
# attachments
exception = mApp().getException()
if exception:
traceback = u"\n".join( exception[1] )
email.addTextAttachment( "{0}\n\n{1}".format( exception[0], traceback ), "exception.log", useCompression = reporterUseCompression )
return email
def createConfMakeInstallActions( self ): # Stupidly, QMake doesn't have a standard way of installing to a prefix so just disable this if self.installEnabled(): super( QMakeBuilder, self ).createConfMakeInstallActions() else: mApp().debugN( self, 3, 'Installation is not implemented by the project, not generating any actions.' ) pass
def executeAction(self, logFile=None):
with self.__timeKeeper:
with EnvironmentSaver():
if self.getWorkingDirectory():
mApp().debugN(self, 3, 'changing directory to "{0}"'.format(self.getWorkingDirectory()))
try:
os.chdir(str(self.getWorkingDirectory()))
except (OSError, IOError) as e:
raise BuildError(str(e))
self._aboutToStart()
mApp().debugN(self, 3, "executing action {0}".format(self.getLogDescription()))
try:
result = self.run()
if result == None or not isinstance(result, int):
raise MomError(
"Action {0} ({1}) did not return a valid non-negative integer return value from run()!".format(
self.getName(), self.getLogDescription()
)
)
self._setResult(int(result))
self._finished()
except MomException as e:
innerTraceback = "".join(traceback.format_tb(sys.exc_info()[2]))
self._aborted()
mApp().debug(self, 'execution failed: "{0}"'.format(str(e)))
mApp().debugN(self, 2, innerTraceback)
self._setStdErr("{0}:\n\n{1}".format(e, innerTraceback))
self._setResult(e.getReturnCode())
self._writeLog(logFile)
mApp().debugN(self, 2, "{0} duration: {1}".format(self.getLogDescription(), self.__timeKeeper.deltaString()))
return self.getResult()
def registerNewRevisions( self, buildScript ):
'''Determines new revisions committed since the last call with the same build script,
and adds those to the database.'''
iface = BuildScriptInterface( buildScript )
buildName = iface.querySetting( Settings.ScriptBuildName )
newestBuildInfo = self.getNewestBuildInfo( buildScript )
if newestBuildInfo:
revision = newestBuildInfo.getRevision()
mApp().debugN( self, 2, 'newest known revision for build script "{0}" ({1}) is "{2}"'
.format( buildScript, buildName, revision ) )
buildInfos = self.getBuildInfoForRevisionsSince( buildScript, buildName, revision )
if buildInfos:
mApp().message( self, 'build script "{0}" ({1}):'.format( buildScript, buildName ) )
for buildInfo in buildInfos:
msg = 'new revision "{0}"'.format( buildInfo.getRevision() )
mApp().message( self, msg )
self.saveBuildInfo( buildInfos )
else:
mApp().debug( self, 'no new revisions found for build script "{0}" ({1})'
.format( buildScript, buildName ) )
else:
buildInfo = self.getBuildInfoForInitialRevision( buildScript, buildName )
mApp().debug( self, 'saving initial revision "{0}" for build script "{1}" ({2})'
.format( buildInfo.getRevision(), buildScript, buildName ) )
self.saveBuildInfo( [ buildInfo ] )
def takeBuildInfoAndBuild( self, buildScripts ):
'''Take a new revision from the build job list. Mark it as pending, and build it. Mark it as done afterwards.'''
buildInfo = None
# get the build names of the build scripts:
buildNames = {}
for buildScript in buildScripts:
iface = BuildScriptInterface( buildScript )
buildName = iface.querySetting( Settings.ScriptBuildName )
if buildName:
buildNames[ buildName ] = buildScript
else:
# this should not happen, since it was checked before
mApp().debug( self, 'build script {0} is broken, ignoring.'.format( buildScript ) )
with self.getConnection() as conn:
buildInfos = self._loadBuildInfo( conn, BuildInfo.Status.NewRevision )
for build in buildInfos:
# the list is ordered by priority
if build.getProjectName() in buildNames:
build.setBuildStatus( BuildInfo.Status.Pending )
self._updateBuildInfo( conn, build )
buildInfo = build
break
if not buildInfo:
return False
try:
self.performBuild( buildInfo )
finally:
with self.getConnection() as conn:
buildInfo.setBuildStatus( BuildInfo.Status.Completed )
self._updateBuildInfo( conn, buildInfo )
return True
def applyBuildSequenceSwitches( self, buildSteps ):
msg = self.__getBuildSequenceDescription( buildSteps )
mApp().debugN( self, 3, 'build sequence before command line parameters: {0}'.format( msg ), compareTo = msg )
switches = self.getBuildSteps()
if not switches:
return
customSteps = switches.split( ',' )
for switch in customSteps:
stepName = None
enable = None
if switch.startswith( 'enable-' ):
stepName = switch[ len( 'enable-' ) : ].strip()
enable = True
elif switch.startswith( 'disable-' ):
stepName = switch[ len( 'disable-' ) : ].strip()
enable = False
else:
raise ConfigurationError( 'Build sequence switch "{0}" does not start with enable- or disable-!'
.format( switch ) )
# apply:
step = self._findStep( stepName, buildSteps )
if not step:
raise ConfigurationError( 'Undefined build step "{0}" in command line arguments!'.format( stepName ) )
step.setEnabled( enable )
msg = self.__getBuildSequenceDescription( buildSteps )
mApp().debug( self, 'build sequence: {0}'.format( msg ), compareTo = msg )
def runNotifications( self ): notificationsEnabled = mApp().getSettings().get( Settings.ScriptEnableNotifications ) if not notificationsEnabled: mApp().debug( self, "Not running notify phase, disabled by settings (Settings.ScriptEnableNotifications)" ) return self._runPhase( self.Phase.Notify )
def run( self ):
self.resolveCommand()
timeoutString = 'without a timeout'
if self.getTimeoutSeconds() != None:
timeoutString = 'with timeout of {0} seconds'.format( self.getTimeoutSeconds() )
combinedOutputString = 'and separate output for stdout and stderr'
if self.getCombineOutput():
combinedOutputString = 'and combined stdout and stderr output'
mApp().debugN( self, 4, 'executing "{0}" {1} {2}'.format( ' '.join( self.getCommand() ),
timeoutString, combinedOutputString ) )
runner = _CommandRunner ( self )
runner.setCombineOutput( self.getCombineOutput() )
runner.start()
# this sucks, but seems to be needed on Windows at least
while not runner.wasStarted():
time.sleep( 0.1 )
if not self.getTimeoutSeconds():
runner.join()
else:
runner.join( self.getTimeoutSeconds() )
if runner.isAlive():
runner.terminate()
runner.join( 5 )
self.__timedOut = True
timeoutString = "timed out" if self.getTimedOut() else "completed"
mApp().debugN( self, 3, '"{0}" {1}, return code is {2}'.format( ' '.join( self.getCommand() ),
timeoutString, str( self.getReturnCode() ) ) )
return self.getReturnCode()
def prepare( self ): '''Execute the prepare phase for builds.''' super( Build, self ).prepare() # set folder names # the build object does not have a parent, and defines the build base dir: mode = self.getSettings().get( Settings.ScriptRunMode ) if mode in ( Settings.RunMode_Build, Settings.RunMode_Describe ): # set base directory name parentBaseDir = os.getcwd() baseDirName = self._getBaseDirName() baseDir = os.path.join( parentBaseDir, baseDirName ) self._setBaseDir( baseDir ) # set the base log directory name logDirName = mApp().getSettings().get( Defaults.ProjectLogDir ) logDir = os.path.join( baseDir, logDirName ) self.setLogDir( logDir ) # set the base packages directory name packagesDirName = mApp().getSettings().get( Defaults.ProjectPackagesDir ) packagesDir = os.path.join( baseDir, packagesDirName ) self.setPackagesDir( packagesDir ) else: self._setBaseDir( os.getcwd() ) self.setLogDir( os.getcwd() ) self.setPackagesDir( os.getcwd() ) assert self.getBaseDir() for step in self.calculateBuildSequence(): self.addStep( step )
def __recurseUpwards( self, packages, remainingDependencies, folders, packagesInFolder ):
matches = []
if not packagesInFolder:
# when all the packages in the current folder have been processed, traverse the directory tree
# up one level, and continue there. Return an empty list if the root was reached.
if len( folders ) > 1 and len( packages ) > 0:
folders = folders[1:]
packagesInFolder = os.listdir( folders[0] )
else:
return [] # done
folder = folders[0]
# check if packages + package is a match
package = packagesInFolder[0]
remainingPackages = packagesInFolder[1:]
path = os.path.join( folder, package )
for dependency in remainingDependencies:
if fnmatch( package, dependency ):
if path not in self._getInstalledDependencies():
mApp().debugN( self, 4, 'dependency {0} matches, but is not enabled'.format( package ) )
continue
newPackages = list( packages )
newPackages.append( [ path, dependency ] )
newDependencies = list( remainingDependencies )
newDependencies.remove( dependency )
if not newDependencies:
# all dependencies have been found
matches.append( newPackages )
else:
matches.extend( self.__recurseUpwards( newPackages, newDependencies, folders, remainingPackages ) )
else:
mApp().debugN( self, 4, 'dependency {0} does not match {1}'.format( package, dependency ) )
# recurse with remaining packages (there may be other matches for the same dependency)
matches.extend( self.__recurseUpwards( packages, remainingDependencies, folders, remainingPackages ) )
return matches
def run( self ):
check_for_path( self.getPath(), "No directory specified!" )
mApp().debugN( self, 4, 'checking directory "{0}"'.format( self.getPath() ) )
if ( os.path.exists( str( self.getPath() ) ) ):
return 0
else:
return 1
def _retrieveRevisionInfo( self ):
# check if specified revision is in cache. do not check for 'HEAD'
if self.getRevision() in self.__revisionInfoCache:
return self.__revisionInfoCache[self.getRevision()]
info = RevisionInfo( "SvnRevisionInfo" )
revisionParameter = ['-r', str( self.getRevision() )] if self.getRevision() else []
cmd = [ self.getCommand(), '--non-interactive', 'log', '--xml', '--limit', '1', self.getUrl() ] + revisionParameter
runner = RunCommand( cmd, searchPaths = self.getCommandSearchPaths() )
runner.run()
if runner.getReturnCode() == 0:
xmldoc = minidom.parseString( runner.getStdOut().encode( "utf-8" ) )
logentries = xmldoc.getElementsByTagName( 'logentry' )
assert len( logentries ) == 1
results = parse_log_entry( logentries[0] )
( info.committerName, info.commitMessage, info.revision, info.commitTime, info.commitTimeReadable ) = results
info.shortRevision = info.revision
if self.getSCMUidMapper():
email = self.getSCMUidMapper().getEmail( info.committerName )
mApp().debugN( self, 5, "E-Mail address for {0} from SCM uid mapper: {1}".format( info.committerName, email ) )
info.committerEmail = email
# add to cache. do not add 'HEAD'
if self.getRevision():
self.__revisionInfoCache[self.getRevision()] = info
return info
def calculateBuildSequence( self ): '''Define the build sequence for this object. By the default, the build sequence is identical for every BuildInstructions object. Command line parameters that enable or disable steps are applied by this method.''' buildSteps = self._setupBuildSteps( Settings.ProjectBuildSequence ) # apply customizations passed as command line parameters: mApp().getParameters().applyBuildSequenceSwitches( buildSteps ) return buildSteps
def saveReport( self ): mApp().debug( self, "Saving unit test report" ) score, total = self.parseOutput( self.getAction()._getRunner().getStdOut() ) self._setScore( score, total ) runner = self.getAction()._getRunner() report = "tests succeeded." if runner.getReturnCode() == 0 else "tests FAILED." self._setReport( report )
def getRevisionsSinceForBranchBuilds( self, command, options, location, branch, tag ): path, tempdirs = self.fetchBuildScript() with TempFolderDeleter( tempdirs ): iface = BuildScriptInterface( path ) buildName = iface.querySetting( Settings.ScriptBuildName ) mApp().getSettings().set( Settings.ScriptBuildName, buildName ) scm = getScm( location ) scm.setParseBranchCommits( True ) scm._handlePrintCommands( command, options )
def testWithMissingRequiredValueDescribeMode( self ): var = 'UndefinedVariable_123456' defaultValue = 'defaultValue value' resolver = SettingResolver( var, required = True, defaultValue = defaultValue ) mApp().getSettings().set( Settings.ScriptRunMode, Settings.RunMode_Describe ) try: str( resolver ) except ConfigurationError: self.fail( 'resolving an undefined setting in describe mode should not raise a ConfigurationError!' )
def run( self ):
mApp().debugN( self, 3, 'Creating "{0}" from "{1}"'.format(
self._getPreprocessor().getOutputFilename(),
self._getPreprocessor().getInputFilename() ) )
self._process()
mApp().debugN( self, 2, 'Successfully created "{0}" from "{1}"'.format(
self._getPreprocessor().getOutputFilename(),
self._getPreprocessor().getInputFilename() ) )
return 0
def getInstanceDir( self ):
'''The instance directory contains all instance specific data.'''
path = self.getSettings().getUserFolder( self.getToolName() )
if not os.path.isdir( path ):
try:
os.makedirs( path )
mApp().debug( self, 'instance directory "{0}" created.'.format( path ) )
except OSError as e:
raise ConfigurationError( 'cannot create instance directory "{0}": {1}!'.format( path, e ) )
return path
def getDataDir( self ):
'''The data directory contains the build status database.'''
path = os.path.join( self.getInstanceDir(), '{0}-data'.format( self.getInstanceName() ) )
if not os.path.isdir( path ):
try:
os.makedirs( path )
mApp().debug( self, 'instance data directory "{0}" created.'.format( path ) )
except OSError as e:
raise ConfigurationError( 'cannot create instance data directory "{0}": {1}!'.format( path, e ) )
return path
def executeWithArgs( self, timeout = 24 * 60 * 60, args = None, captureOutput = False ):
cmd = [ sys.executable, os.path.abspath( self.getBuildScript() ) ]
if args:
cmd.extend( args )
mApp().message( self, 'invoking build script: {0}'.format( ' '.join( cmd ) ) )
runner = RunCommand( cmd, timeoutSeconds = timeout, captureOutput = captureOutput )
with EnvironmentSaver():
extend_debug_prefix( 'script>' )
runner.run()
mApp().debugN( self, 2, 'build script finished, return code is {0}.'.format( runner.getReturnCode() ) )
return runner
def send( self, email ):
addresses = email.getToAddresses()
if email.getFromAddress() and len( addresses ) > 0:
# for each address send out an unique mail with only one 'To' recipient
for address in addresses:
try:
self.__server.sendmail( email.getFromAddress(), address, email.getMessageText( address ) )
except SMTPRecipientsRefused:
mApp().debugN( self, 3, "Recipient refused: {0}".format( email.getFromAddress() ) )
else:
raise ConfigurationError( 'Sender/recipient addresses missing, cannot send mail!' )
def __getSummarizedDiffForRevision( self, url, revision ):
previous = revision - 1
cmd = [ self.getCommand(), 'diff', '--summarize', '-r', str( previous ) + ':' + str( revision ), url ]
runner = RunCommand( cmd, 3600, searchPaths = self.getCommandSearchPaths() )
runner.run()
if runner.getReturnCode() != 0:
# maybe the location did not exist earlier on:
mApp().debugN( self, 2, 'cannot retrieve summarized diff for revision "{0}"'.format( revision ) )
return None
else:
return runner.getStdOut().encode( "utf-8" ).split( '\n' )
def run(self):
check_for_path(self.getPath(), "No directory specified!")
mApp().debugN(self, 2, 'deleting directory "{0}"'.format(self.getPath()))
try:
rmtree(str(self.getPath()))
return 0
except (OSError, IOError) as e:
error = 'error deleting directory "{0}": {1}'.format(self.getPath(), str(e))
self._setStdErr(error.encode())
mApp().debug(self, error)
return 1
def prepare( self ):
currentMode = mApp().getSettings().get( Settings.ScriptRunMode )
if currentMode != Settings.RunMode_Build:
mApp().debugN( self, 2, "Not in build mode, not checking platform" )
return
if self._isMatch():
pass # all platforms with matching variables are selected
else:
text = 'WhiteLister aborted build because variable "{0}" does not match regex "{1}"'\
.format( self.getVariableName(), self.getValuePattern() )
raise AbortBuildException( text )
def report( self ):
if self.__finished:
return
report = InstructionsXmlReport( self.getInstructions() )
try:
self._openReportFile()
self._writeReport( report )
self._saveReportFile()
except ConfigurationError as e:
# Catch ConfigurationError, since we are in shutdown. Print warning message:
mApp().message( self, "An error occurred while creating the report: {0}".format( e ) )
def _upload( self ):
uploaderAction = self.__uploaderAction
if not uploaderAction:
return
uploaderAction.setSourcePath( self.getTemporaryLocation() )
uploaderAction.setDestinationPath( PathResolver( self._getFullUploadLocation ) )
mApp().debugN( self, 5, "Uploading report to {0}".format( uploaderAction.getDestinationPath() ) )
rc = uploaderAction.executeAction()
if rc != 0:
mApp().debug( self, "Uploading failed:", uploaderAction.getStdErr() )
