def testSendEmail( self ): # force loading settings from configuration files mApp().getSettings().evalConfigurationFiles() email = Email() email.setToAddresses( ['*****@*****.**'] ) email.setFromAddress( mApp().getSettings().get( Settings.EmailReporterSender ) ) email.setSubject( 'EmailerTest email' ) email.attachAlternativeTextPart( '''\ This is a test email sent by Make-O-Matic. Check it out at http://github.com/KDAB/Make-O-Matic ''', """\ <html> <head>Make-O-Matic Test Email</head> <body> <p>This is the HTML part of the test email<br> Check out Make-O-Matic at <a href="http://github.com/KDAB/Make-O-Matic">GitHub</a>. </p> </body> </html> """ ) attachmentText = "TEST:\n" + "\n".join( sys.path ) email.addTextAttachment( attachmentText, "testfile1.txt", False ) email.addTextAttachment( attachmentText, "testfile2.txt", True ) e = Emailer( 'Emailer' ) e.setup() e.send( email ) e.quit()
def setup( self ): folders = self.getFolders() if not folders: mApp().debug( self, 'No folders specified for self-update, continuing.' ) for folder in folders: mApp().debug( self, 'Self-updating directory "{0}"'.format( folder ) ) self.update( folder )
def _addXslTemplate( self, plugin ): """Adds the XSL template to stylesheet if plugin provides one Merges templates from plugins into the stylesheets provided by XSL_STYLESHEETS.""" if not self.hasXsltSupport(): mApp().debugN( self, 5, "Cannot add XSL template, lacking support for XSLT transformations. Please install the python-lxml package." ) return # iterate trough the dict from getXslTemplates(), add each template to the corresponding stylesheet for destinationReportFormat, markup in plugin.getXslTemplates().items(): if not destinationReportFormat in self.__xslTemplateSnippets.keys(): continue # invalid key, no stylesheet registered for that type of XSL # search for place to register new plugin templates stylesheet = self.__xslTemplateSnippets[destinationReportFormat] pluginTemplate = stylesheet.find( ".//{http://www.w3.org/1999/XSL/Transform}template[@match='plugin']" ) placeholder = pluginTemplate.find( ".//{http://www.w3.org/1999/XSL/Transform}choose" ) # create new element with markup provided from plugin try: element = etree.XML( """<xsl:when xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://www.w3.org/1999/xhtml" test="@name = '{0}'">{1}</xsl:when>""".format( plugin.getName(), markup ) ) except etree.XMLSyntaxError: raise ConfigurationError( "XSL template of {0} plugin malformed.".format( plugin.getName() ) ) # insert new element in the placeholder from the stylesheet placeholder.insert( 0, element )
def run( self ): self.__started = True stderrValue = subprocess.PIPE if self.__combineOutput: stderrValue = subprocess.STDOUT if self._getRunner().getCaptureOutput(): self._process = subprocess.Popen ( self._getRunner().getCommand(), shell = False, cwd = self._getRunner().getWorkingDir(), stdout = subprocess.PIPE, stderr = stderrValue ) output, error = self._process.communicate() # override encoding for windows if sys.platform == 'win32': encoding = 'cp850' else: encoding = 'utf-8' self._getRunner().setStdOut( to_unicode_or_bust( output, encoding ) ) self._getRunner().setStdErr( to_unicode_or_bust( error, encoding ) ) mApp().debugN( self._getRunner(), 5, u"STDOUT:\n{0}".format( self._getRunner().getStdOut() ) ) if not self.__combineOutput: mApp().debugN( self._getRunner(), 5, u"STDERR:\n{0}".format( self._getRunner().getStdErr() ) ) self._getRunner().setReturnCode( self._process.returncode ) else: self._process = subprocess.Popen ( self._getRunner().getCommand(), shell = False, cwd = self._getRunner().getWorkingDir() ) self._process.wait() returnCode = self._process.returncode self._getRunner().setReturnCode( returnCode ) self._getRunner().setStdOut( None ) self._getRunner().setStdErr( None ) self.__finished = True
def _initEmailBody( self, email ): # body reporterUseCompression = mApp().getSettings().get( Settings.EmailReporterUseCompressionForAttachments, False ) report = InstructionsXmlReport( mApp() ) converter = XmlReportConverter( report ) ### text and html part # summary email.attachAlternativeTextPart( converter.convertToTextSummary(), converter.convertToHtml( summaryOnly = True ) ) # report if self.getEnableFullReport(): email.attachAlternativeTextPart( converter.convertToText( short = True ), converter.convertToHtml() ) returnCode = mApp().getReturnCode() if returnCode != 0: email.addTextAttachment( converter.convertToFailedStepsLog(), "failed-steps.log", useCompression = reporterUseCompression ) # attachments exception = mApp().getException() if exception: traceback = u"\n".join( exception[1] ) email.addTextAttachment( "{0}\n\n{1}".format( exception[0], traceback ), "exception.log", useCompression = reporterUseCompression ) return email
def createConfMakeInstallActions( self ): # Stupidly, QMake doesn't have a standard way of installing to a prefix so just disable this if self.installEnabled(): super( QMakeBuilder, self ).createConfMakeInstallActions() else: mApp().debugN( self, 3, 'Installation is not implemented by the project, not generating any actions.' ) pass
def executeAction(self, logFile=None): with self.__timeKeeper: with EnvironmentSaver(): if self.getWorkingDirectory(): mApp().debugN(self, 3, 'changing directory to "{0}"'.format(self.getWorkingDirectory())) try: os.chdir(str(self.getWorkingDirectory())) except (OSError, IOError) as e: raise BuildError(str(e)) self._aboutToStart() mApp().debugN(self, 3, "executing action {0}".format(self.getLogDescription())) try: result = self.run() if result == None or not isinstance(result, int): raise MomError( "Action {0} ({1}) did not return a valid non-negative integer return value from run()!".format( self.getName(), self.getLogDescription() ) ) self._setResult(int(result)) self._finished() except MomException as e: innerTraceback = "".join(traceback.format_tb(sys.exc_info()[2])) self._aborted() mApp().debug(self, 'execution failed: "{0}"'.format(str(e))) mApp().debugN(self, 2, innerTraceback) self._setStdErr("{0}:\n\n{1}".format(e, innerTraceback)) self._setResult(e.getReturnCode()) self._writeLog(logFile) mApp().debugN(self, 2, "{0} duration: {1}".format(self.getLogDescription(), self.__timeKeeper.deltaString())) return self.getResult()
def registerNewRevisions( self, buildScript ): '''Determines new revisions committed since the last call with the same build script, and adds those to the database.''' iface = BuildScriptInterface( buildScript ) buildName = iface.querySetting( Settings.ScriptBuildName ) newestBuildInfo = self.getNewestBuildInfo( buildScript ) if newestBuildInfo: revision = newestBuildInfo.getRevision() mApp().debugN( self, 2, 'newest known revision for build script "{0}" ({1}) is "{2}"' .format( buildScript, buildName, revision ) ) buildInfos = self.getBuildInfoForRevisionsSince( buildScript, buildName, revision ) if buildInfos: mApp().message( self, 'build script "{0}" ({1}):'.format( buildScript, buildName ) ) for buildInfo in buildInfos: msg = 'new revision "{0}"'.format( buildInfo.getRevision() ) mApp().message( self, msg ) self.saveBuildInfo( buildInfos ) else: mApp().debug( self, 'no new revisions found for build script "{0}" ({1})' .format( buildScript, buildName ) ) else: buildInfo = self.getBuildInfoForInitialRevision( buildScript, buildName ) mApp().debug( self, 'saving initial revision "{0}" for build script "{1}" ({2})' .format( buildInfo.getRevision(), buildScript, buildName ) ) self.saveBuildInfo( [ buildInfo ] )
def takeBuildInfoAndBuild( self, buildScripts ): '''Take a new revision from the build job list. Mark it as pending, and build it. Mark it as done afterwards.''' buildInfo = None # get the build names of the build scripts: buildNames = {} for buildScript in buildScripts: iface = BuildScriptInterface( buildScript ) buildName = iface.querySetting( Settings.ScriptBuildName ) if buildName: buildNames[ buildName ] = buildScript else: # this should not happen, since it was checked before mApp().debug( self, 'build script {0} is broken, ignoring.'.format( buildScript ) ) with self.getConnection() as conn: buildInfos = self._loadBuildInfo( conn, BuildInfo.Status.NewRevision ) for build in buildInfos: # the list is ordered by priority if build.getProjectName() in buildNames: build.setBuildStatus( BuildInfo.Status.Pending ) self._updateBuildInfo( conn, build ) buildInfo = build break if not buildInfo: return False try: self.performBuild( buildInfo ) finally: with self.getConnection() as conn: buildInfo.setBuildStatus( BuildInfo.Status.Completed ) self._updateBuildInfo( conn, buildInfo ) return True
def applyBuildSequenceSwitches( self, buildSteps ): msg = self.__getBuildSequenceDescription( buildSteps ) mApp().debugN( self, 3, 'build sequence before command line parameters: {0}'.format( msg ), compareTo = msg ) switches = self.getBuildSteps() if not switches: return customSteps = switches.split( ',' ) for switch in customSteps: stepName = None enable = None if switch.startswith( 'enable-' ): stepName = switch[ len( 'enable-' ) : ].strip() enable = True elif switch.startswith( 'disable-' ): stepName = switch[ len( 'disable-' ) : ].strip() enable = False else: raise ConfigurationError( 'Build sequence switch "{0}" does not start with enable- or disable-!' .format( switch ) ) # apply: step = self._findStep( stepName, buildSteps ) if not step: raise ConfigurationError( 'Undefined build step "{0}" in command line arguments!'.format( stepName ) ) step.setEnabled( enable ) msg = self.__getBuildSequenceDescription( buildSteps ) mApp().debug( self, 'build sequence: {0}'.format( msg ), compareTo = msg )
def runNotifications( self ): notificationsEnabled = mApp().getSettings().get( Settings.ScriptEnableNotifications ) if not notificationsEnabled: mApp().debug( self, "Not running notify phase, disabled by settings (Settings.ScriptEnableNotifications)" ) return self._runPhase( self.Phase.Notify )
def run( self ): self.resolveCommand() timeoutString = 'without a timeout' if self.getTimeoutSeconds() != None: timeoutString = 'with timeout of {0} seconds'.format( self.getTimeoutSeconds() ) combinedOutputString = 'and separate output for stdout and stderr' if self.getCombineOutput(): combinedOutputString = 'and combined stdout and stderr output' mApp().debugN( self, 4, 'executing "{0}" {1} {2}'.format( ' '.join( self.getCommand() ), timeoutString, combinedOutputString ) ) runner = _CommandRunner ( self ) runner.setCombineOutput( self.getCombineOutput() ) runner.start() # this sucks, but seems to be needed on Windows at least while not runner.wasStarted(): time.sleep( 0.1 ) if not self.getTimeoutSeconds(): runner.join() else: runner.join( self.getTimeoutSeconds() ) if runner.isAlive(): runner.terminate() runner.join( 5 ) self.__timedOut = True timeoutString = "timed out" if self.getTimedOut() else "completed" mApp().debugN( self, 3, '"{0}" {1}, return code is {2}'.format( ' '.join( self.getCommand() ), timeoutString, str( self.getReturnCode() ) ) ) return self.getReturnCode()
def prepare( self ): '''Execute the prepare phase for builds.''' super( Build, self ).prepare() # set folder names # the build object does not have a parent, and defines the build base dir: mode = self.getSettings().get( Settings.ScriptRunMode ) if mode in ( Settings.RunMode_Build, Settings.RunMode_Describe ): # set base directory name parentBaseDir = os.getcwd() baseDirName = self._getBaseDirName() baseDir = os.path.join( parentBaseDir, baseDirName ) self._setBaseDir( baseDir ) # set the base log directory name logDirName = mApp().getSettings().get( Defaults.ProjectLogDir ) logDir = os.path.join( baseDir, logDirName ) self.setLogDir( logDir ) # set the base packages directory name packagesDirName = mApp().getSettings().get( Defaults.ProjectPackagesDir ) packagesDir = os.path.join( baseDir, packagesDirName ) self.setPackagesDir( packagesDir ) else: self._setBaseDir( os.getcwd() ) self.setLogDir( os.getcwd() ) self.setPackagesDir( os.getcwd() ) assert self.getBaseDir() for step in self.calculateBuildSequence(): self.addStep( step )
def __recurseUpwards( self, packages, remainingDependencies, folders, packagesInFolder ): matches = [] if not packagesInFolder: # when all the packages in the current folder have been processed, traverse the directory tree # up one level, and continue there. Return an empty list if the root was reached. if len( folders ) > 1 and len( packages ) > 0: folders = folders[1:] packagesInFolder = os.listdir( folders[0] ) else: return [] # done folder = folders[0] # check if packages + package is a match package = packagesInFolder[0] remainingPackages = packagesInFolder[1:] path = os.path.join( folder, package ) for dependency in remainingDependencies: if fnmatch( package, dependency ): if path not in self._getInstalledDependencies(): mApp().debugN( self, 4, 'dependency {0} matches, but is not enabled'.format( package ) ) continue newPackages = list( packages ) newPackages.append( [ path, dependency ] ) newDependencies = list( remainingDependencies ) newDependencies.remove( dependency ) if not newDependencies: # all dependencies have been found matches.append( newPackages ) else: matches.extend( self.__recurseUpwards( newPackages, newDependencies, folders, remainingPackages ) ) else: mApp().debugN( self, 4, 'dependency {0} does not match {1}'.format( package, dependency ) ) # recurse with remaining packages (there may be other matches for the same dependency) matches.extend( self.__recurseUpwards( packages, remainingDependencies, folders, remainingPackages ) ) return matches
def run( self ): check_for_path( self.getPath(), "No directory specified!" ) mApp().debugN( self, 4, 'checking directory "{0}"'.format( self.getPath() ) ) if ( os.path.exists( str( self.getPath() ) ) ): return 0 else: return 1
def _retrieveRevisionInfo( self ): # check if specified revision is in cache. do not check for 'HEAD' if self.getRevision() in self.__revisionInfoCache: return self.__revisionInfoCache[self.getRevision()] info = RevisionInfo( "SvnRevisionInfo" ) revisionParameter = ['-r', str( self.getRevision() )] if self.getRevision() else [] cmd = [ self.getCommand(), '--non-interactive', 'log', '--xml', '--limit', '1', self.getUrl() ] + revisionParameter runner = RunCommand( cmd, searchPaths = self.getCommandSearchPaths() ) runner.run() if runner.getReturnCode() == 0: xmldoc = minidom.parseString( runner.getStdOut().encode( "utf-8" ) ) logentries = xmldoc.getElementsByTagName( 'logentry' ) assert len( logentries ) == 1 results = parse_log_entry( logentries[0] ) ( info.committerName, info.commitMessage, info.revision, info.commitTime, info.commitTimeReadable ) = results info.shortRevision = info.revision if self.getSCMUidMapper(): email = self.getSCMUidMapper().getEmail( info.committerName ) mApp().debugN( self, 5, "E-Mail address for {0} from SCM uid mapper: {1}".format( info.committerName, email ) ) info.committerEmail = email # add to cache. do not add 'HEAD' if self.getRevision(): self.__revisionInfoCache[self.getRevision()] = info return info
def calculateBuildSequence( self ): '''Define the build sequence for this object. By the default, the build sequence is identical for every BuildInstructions object. Command line parameters that enable or disable steps are applied by this method.''' buildSteps = self._setupBuildSteps( Settings.ProjectBuildSequence ) # apply customizations passed as command line parameters: mApp().getParameters().applyBuildSequenceSwitches( buildSteps ) return buildSteps
def saveReport( self ): mApp().debug( self, "Saving unit test report" ) score, total = self.parseOutput( self.getAction()._getRunner().getStdOut() ) self._setScore( score, total ) runner = self.getAction()._getRunner() report = "tests succeeded." if runner.getReturnCode() == 0 else "tests FAILED." self._setReport( report )
def getRevisionsSinceForBranchBuilds( self, command, options, location, branch, tag ): path, tempdirs = self.fetchBuildScript() with TempFolderDeleter( tempdirs ): iface = BuildScriptInterface( path ) buildName = iface.querySetting( Settings.ScriptBuildName ) mApp().getSettings().set( Settings.ScriptBuildName, buildName ) scm = getScm( location ) scm.setParseBranchCommits( True ) scm._handlePrintCommands( command, options )
def testWithMissingRequiredValueDescribeMode( self ): var = 'UndefinedVariable_123456' defaultValue = 'defaultValue value' resolver = SettingResolver( var, required = True, defaultValue = defaultValue ) mApp().getSettings().set( Settings.ScriptRunMode, Settings.RunMode_Describe ) try: str( resolver ) except ConfigurationError: self.fail( 'resolving an undefined setting in describe mode should not raise a ConfigurationError!' )
def run( self ): mApp().debugN( self, 3, 'Creating "{0}" from "{1}"'.format( self._getPreprocessor().getOutputFilename(), self._getPreprocessor().getInputFilename() ) ) self._process() mApp().debugN( self, 2, 'Successfully created "{0}" from "{1}"'.format( self._getPreprocessor().getOutputFilename(), self._getPreprocessor().getInputFilename() ) ) return 0
def getInstanceDir( self ): '''The instance directory contains all instance specific data.''' path = self.getSettings().getUserFolder( self.getToolName() ) if not os.path.isdir( path ): try: os.makedirs( path ) mApp().debug( self, 'instance directory "{0}" created.'.format( path ) ) except OSError as e: raise ConfigurationError( 'cannot create instance directory "{0}": {1}!'.format( path, e ) ) return path
def getDataDir( self ): '''The data directory contains the build status database.''' path = os.path.join( self.getInstanceDir(), '{0}-data'.format( self.getInstanceName() ) ) if not os.path.isdir( path ): try: os.makedirs( path ) mApp().debug( self, 'instance data directory "{0}" created.'.format( path ) ) except OSError as e: raise ConfigurationError( 'cannot create instance data directory "{0}": {1}!'.format( path, e ) ) return path
def executeWithArgs( self, timeout = 24 * 60 * 60, args = None, captureOutput = False ): cmd = [ sys.executable, os.path.abspath( self.getBuildScript() ) ] if args: cmd.extend( args ) mApp().message( self, 'invoking build script: {0}'.format( ' '.join( cmd ) ) ) runner = RunCommand( cmd, timeoutSeconds = timeout, captureOutput = captureOutput ) with EnvironmentSaver(): extend_debug_prefix( 'script>' ) runner.run() mApp().debugN( self, 2, 'build script finished, return code is {0}.'.format( runner.getReturnCode() ) ) return runner
def send( self, email ): addresses = email.getToAddresses() if email.getFromAddress() and len( addresses ) > 0: # for each address send out an unique mail with only one 'To' recipient for address in addresses: try: self.__server.sendmail( email.getFromAddress(), address, email.getMessageText( address ) ) except SMTPRecipientsRefused: mApp().debugN( self, 3, "Recipient refused: {0}".format( email.getFromAddress() ) ) else: raise ConfigurationError( 'Sender/recipient addresses missing, cannot send mail!' )
def __getSummarizedDiffForRevision( self, url, revision ): previous = revision - 1 cmd = [ self.getCommand(), 'diff', '--summarize', '-r', str( previous ) + ':' + str( revision ), url ] runner = RunCommand( cmd, 3600, searchPaths = self.getCommandSearchPaths() ) runner.run() if runner.getReturnCode() != 0: # maybe the location did not exist earlier on: mApp().debugN( self, 2, 'cannot retrieve summarized diff for revision "{0}"'.format( revision ) ) return None else: return runner.getStdOut().encode( "utf-8" ).split( '\n' )
def run(self): check_for_path(self.getPath(), "No directory specified!") mApp().debugN(self, 2, 'deleting directory "{0}"'.format(self.getPath())) try: rmtree(str(self.getPath())) return 0 except (OSError, IOError) as e: error = 'error deleting directory "{0}": {1}'.format(self.getPath(), str(e)) self._setStdErr(error.encode()) mApp().debug(self, error) return 1
def prepare( self ): currentMode = mApp().getSettings().get( Settings.ScriptRunMode ) if currentMode != Settings.RunMode_Build: mApp().debugN( self, 2, "Not in build mode, not checking platform" ) return if self._isMatch(): pass # all platforms with matching variables are selected else: text = 'WhiteLister aborted build because variable "{0}" does not match regex "{1}"'\ .format( self.getVariableName(), self.getValuePattern() ) raise AbortBuildException( text )
def report( self ): if self.__finished: return report = InstructionsXmlReport( self.getInstructions() ) try: self._openReportFile() self._writeReport( report ) self._saveReportFile() except ConfigurationError as e: # Catch ConfigurationError, since we are in shutdown. Print warning message: mApp().message( self, "An error occurred while creating the report: {0}".format( e ) )
def _upload( self ): uploaderAction = self.__uploaderAction if not uploaderAction: return uploaderAction.setSourcePath( self.getTemporaryLocation() ) uploaderAction.setDestinationPath( PathResolver( self._getFullUploadLocation ) ) mApp().debugN( self, 5, "Uploading report to {0}".format( uploaderAction.getDestinationPath() ) ) rc = uploaderAction.executeAction() if rc != 0: mApp().debug( self, "Uploading failed:", uploaderAction.getStdErr() )