def generateAndUploadReleaseNotes(packageDistribution, svnPath, versionReleased): tmpDir = tempfile.mkdtemp() packageName = packageDistribution.getPackageName() gLogger.notice("Generating release notes for %s under %s" % (packageName, tmpDir)) for suffix, singleVersion in (("history", False), ("notes", True)): gLogger.notice("Generating %s rst" % suffix) rstHistory = os.path.join(tmpDir, "release%s.rst" % suffix) htmlHistory = os.path.join(tmpDir, "release%s.html" % suffix) Distribution.generateReleaseNotes(packageName, rstHistory, versionReleased, singleVersion) try: Distribution.generateHTMLReleaseNotesFromRST( rstHistory, htmlHistory) except Exception as x: print "Failed to generate html version of the notes:", str(x) # Attempt to generate pdf as well os.system('rst2pdf %s' % rstHistory) packageDistribution.queueImport( tmpDir, svnPath, 'Release notes for version %s' % versionReleased) if not packageDistribution.executeCommandQueue(): gLogger.error("Could not upload release notes") sys.exit(1) os.system("rm -rf '%s'" % tmpDir) gLogger.notice("Release notes committed")
def __generateTarball( self ): destDir = self.params.destination tarName = "%s-%s.tar.gz" % ( self.params.name, self.params.version ) tarfilePath = os.path.join( destDir, tarName ) dirToTar = os.path.join( self.params.destination, self.params.name ) result = Distribution.writeVersionToInit( dirToTar, self.params.version ) if not result[ 'OK' ]: return result result = Distribution.createTarball( tarfilePath, dirToTar ) if not result[ 'OK' ]: return S_ERROR( "Could not generate tarball: %s" % result[ 'Error' ] ) #Remove package dir shutil.rmtree( dirToTar ) gLogger.info( "Tar file %s created" % tarName ) return S_OK( tarfilePath )
def __generateTarball(self): destDir = self.params.destination tarName = "%s-%s.tar.gz" % (self.params.name, self.params.version) tarfilePath = os.path.join(destDir, tarName) dirToTar = os.path.join(self.params.destination, self.params.name) result = Distribution.writeVersionToInit(dirToTar, self.params.version) if not result['OK']: return result result = Distribution.createTarball(tarfilePath, dirToTar) if not result['OK']: return S_ERROR("Could not generate tarball: %s" % result['Error']) #Remove package dir shutil.rmtree(dirToTar) gLogger.info("Tar file %s created" % tarName) return S_OK(tarfilePath)
def generateAndUploadReleaseNotes( packageDistribution, svnPath, versionReleased ): tmpDir = tempfile.mkdtemp() packageName = packageDistribution.getPackageName() gLogger.notice( "Generating release notes for %s under %s" % ( packageName, tmpDir ) ) for suffix, singleVersion in ( ( "history", False ), ( "notes", True ) ): gLogger.notice( "Generating %s rst" % suffix ) rstHistory = os.path.join( tmpDir, "release%s.rst" % suffix ) htmlHistory = os.path.join( tmpDir, "release%s.html" % suffix ) Distribution.generateReleaseNotes( packageName, rstHistory, versionReleased, singleVersion ) try: Distribution.generateHTMLReleaseNotesFromRST( rstHistory, htmlHistory ) except Exception, x: print "Failed to generate html version of the notes:", str( x ) # Attempt to generate pdf as well os.system( 'rst2pdf %s' % rstHistory )
def __generateRSTFile(releaseData, rstFileName, pkgVersion, singleVersion): """create the rst file from the release.notes """ rstData = [] parsedPkgVersion = Distribution.parseVersionString( pkgVersion) if pkgVersion is not None else (99, 99, 99, 99) for version, verData in releaseData: if singleVersion and version != pkgVersion and pkgVersion is not None: continue if Distribution.parseVersionString(version) > parsedPkgVersion: continue versionLine = "Version %s" % version rstData.append("") rstData.append("=" * len(versionLine)) rstData.append(versionLine) rstData.append("=" * len(versionLine)) rstData.append("") if verData['comment']: rstData.append("\n".join(verData['comment'])) rstData.append("") for feature, featureData in verData['features']: if not featureData: continue rstData.append(feature) rstData.append("=" * len(feature)) rstData.append("") for key in sorted(featureData): rstData.append(key.capitalize()) rstData.append(":" * (len(key) + 5)) rstData.append("") for entry in featureData[key]: rstData.append(" - %s" % entry) rstData.append("") if pkgVersion is None and singleVersion: print "Just taking the first entry in releaseNotes" break #Write releasenotes.rst try: rstFilePath = os.path.join(os.getcwd(), "source", rstFileName) fd = open(rstFilePath, "w") fd.write("\n".join(rstData)) fd.close() except (OSError, IOError) as excp: return S_ERROR("Could not write %s: %s" % (rstFileName, excp)) return S_OK()
def __generateRSTFile( releaseData, rstFileName, pkgVersion, singleVersion ): """create the rst file from the release.notes """ rstData = [] parsedPkgVersion = Distribution.parseVersionString( pkgVersion ) if pkgVersion is not None else (99, 99, 99, 99) for version, verData in releaseData: if singleVersion and version != pkgVersion and pkgVersion is not None: continue if Distribution.parseVersionString( version ) > parsedPkgVersion: continue versionLine = "Version %s" % version rstData.append( "" ) rstData.append( "=" * len( versionLine ) ) rstData.append( versionLine ) rstData.append( "=" * len( versionLine ) ) rstData.append( "" ) if verData[ 'comment' ]: rstData.append( "\n".join( verData[ 'comment' ] ) ) rstData.append( "" ) for feature, featureData in verData[ 'features' ]: if not featureData: continue rstData.append( feature ) rstData.append( "=" * len( feature ) ) rstData.append( "" ) for key in sorted( featureData ): rstData.append( key.capitalize() ) rstData.append( ":" * ( len( key ) + 5 ) ) rstData.append( "" ) for entry in featureData[ key ]: rstData.append( " - %s" % entry ) rstData.append( "" ) if pkgVersion is None and singleVersion: print("Just taking the first entry in releaseNotes") break #Write releasenotes.rst try: rstFilePath = os.path.join( os.getcwd(), "source", rstFileName ) fd = open( rstFilePath, "w" ) fd.write( "\n".join( rstData ) ) fd.close() except (OSError,IOError) as excp: return S_ERROR( "Could not write %s: %s" % ( rstFileName, excp ) ) return S_OK()
def generateAndUploadReleaseNotes(packageDistribution, svnPath, versionReleased): tmpDir = tempfile.mkdtemp() packageName = packageDistribution.getPackageName() gLogger.notice("Generating release notes for %s under %s" % (packageName, tmpDir)) for suffix, singleVersion in (("history", False), ("notes", True)): gLogger.notice("Generating %s rst" % suffix) rstHistory = os.path.join(tmpDir, "release%s.rst" % suffix) htmlHistory = os.path.join(tmpDir, "release%s.html" % suffix) Distribution.generateReleaseNotes(packageName, rstHistory, versionReleased, singleVersion) try: Distribution.generateHTMLReleaseNotesFromRST( rstHistory, htmlHistory) except Exception, x: print "Failed to generate html version of the notes:", str(x) # Attempt to generate pdf as well os.system('rst2pdf %s' % rstHistory)
def tarExternals(self, releaseVersion): externalsVersion = self.relConf.getExtenalsVersion(releaseVersion) platform = Platform.getPlatformString() availableExternals = self.getAvailableExternals() if not externalsVersion: gLogger.notice("Externals is not defined for release %s" % releaseVersion) return False for externalType in self.cliParams.externalsBuildType: requestedExternals = (externalType, externalsVersion, platform, 'python%s' % self.cliParams.externalsPython) requestedExternalsString = "-".join(list(requestedExternals)) gLogger.notice("Trying to compile %s externals..." % requestedExternalsString) if not self.cliParams.forceExternals and requestedExternals in availableExternals: gLogger.notice( "Externals %s is already compiled, skipping..." % (requestedExternalsString)) continue compileScript = os.path.join(os.path.dirname(__file__), "dirac-compile-externals") if not os.path.isfile(compileScript): compileScript = os.path.join(os.path.dirname(__file__), "dirac-compile-externals.py") compileTarget = os.path.join(self.cliParams.destination, platform) cmdArgs = [] cmdArgs.append("-D '%s'" % compileTarget) cmdArgs.append("-t '%s'" % externalType) cmdArgs.append("-v '%s'" % externalsVersion) cmdArgs.append("-i '%s'" % self.cliParams.externalsPython) if cliParams.externalsLocation: cmdArgs.append("-e '%s'" % self.cliParams.externalsLocation) if cliParams.makeJobs: cmdArgs.append("-j '%s'" % self.cliParams.makeJobs) compileCmd = "%s %s" % (compileScript, " ".join(cmdArgs)) gLogger.info(compileCmd) if os.system(compileCmd): gLogger.error("Error while compiling externals!") sys.exit(1) tarfilePath = os.path.join( self.cliParams.destination, "Externals-%s.tar.gz" % (requestedExternalsString)) result = Distribution.createTarball( tarfilePath, compileTarget, os.path.join(self.cliParams.destination, "mysql")) if not result['OK']: gLogger.error( "Could not generate tarball for package %s" % package, result['Error']) sys.exit(1) os.system("rm -rf '%s'" % compileTarget) return True
def __generateRSTFile(self, releaseData, rstFileName, pkgVersion, singleVersion): rstData = [] parsedPkgVersion = Distribution.parseVersionString(pkgVersion) for version, verData in releaseData: if singleVersion and version != pkgVersion: continue if Distribution.parseVersionString(version) > parsedPkgVersion: continue versionLine = "Version %s" % version rstData.append("") rstData.append("=" * len(versionLine)) rstData.append(versionLine) rstData.append("=" * len(versionLine)) rstData.append("") if verData['comment']: rstData.append("\n".join(verData['comment'])) rstData.append("") for feature, featureData in verData['features']: if not featureData: continue rstData.append(feature) rstData.append("=" * len(feature)) rstData.append("") for key in sorted(featureData): rstData.append(key.capitalize()) rstData.append(":" * (len(key) + 5)) rstData.append("") for entry in featureData[key]: rstData.append(" - %s" % entry) rstData.append("") # Write releasenotes.rst try: rstFilePath = os.path.join(self.params.destination, self.params.name, rstFileName) with open(rstFilePath, "w") as fd: fd.write("\n".join(rstData)) except Exception as excp: return S_ERROR("Could not write %s: %s" % (rstFileName, excp)) return S_OK()
def __generateRSTFile( self, releaseData, rstFileName, pkgVersion, singleVersion ): rstData = [] parsedPkgVersion = Distribution.parseVersionString( pkgVersion ) for version, verData in releaseData: if singleVersion and version != pkgVersion: continue if Distribution.parseVersionString( version ) > parsedPkgVersion: continue versionLine = "Version %s" % version rstData.append( "" ) rstData.append( "=" * len( versionLine ) ) rstData.append( versionLine ) rstData.append( "=" * len( versionLine ) ) rstData.append( "" ) if verData[ 'comment' ]: rstData.append( "\n".join( verData[ 'comment' ] ) ) rstData.append( "" ) for feature, featureData in verData[ 'features' ]: if not featureData: continue rstData.append( feature ) rstData.append( "=" * len( feature ) ) rstData.append( "" ) for key in sorted( featureData ): rstData.append( key.capitalize() ) rstData.append( ":" * ( len( key ) + 5 ) ) rstData.append( "" ) for entry in featureData[ key ]: rstData.append( " - %s" % entry ) rstData.append( "" ) #Write releasenotes.rst try: rstFilePath = os.path.join( self.params.destination, self.params.name, rstFileName ) fd = open( rstFilePath, "w" ) fd.write( "\n".join( rstData ) ) fd.close() except Exception, excp: return S_ERROR( "Could not write %s: %s" % ( rstFileName, excp ) )
def generateAndUploadReleaseNotes( packageDistribution, svnPath, versionReleased ): tmpDir = tempfile.mkdtemp() packageName = packageDistribution.getPackageName() gLogger.notice( "Generating release notes for %s under %s" % ( packageName, tmpDir ) ) for suffix, singleVersion in ( ( "history", False ), ( "notes", True ) ): gLogger.notice( "Generating %s rst" % suffix ) rstHistory = os.path.join( tmpDir, "release%s.rst" % suffix ) htmlHistory = os.path.join( tmpDir, "release%s.html" % suffix ) Distribution.generateReleaseNotes( packageName, rstHistory, versionReleased, singleVersion ) try: Distribution.generateHTMLReleaseNotesFromRST( rstHistory, htmlHistory ) except Exception as x: print "Failed to generate html version of the notes:", str( x ) # Attempt to generate pdf as well os.system( 'rst2pdf %s' % rstHistory ) packageDistribution.queueImport( tmpDir, svnPath, 'Release notes for version %s' % versionReleased ) if not packageDistribution.executeCommandQueue(): gLogger.error( "Could not upload release notes" ) sys.exit( 1 ) os.system( "rm -rf '%s'" % tmpDir ) gLogger.notice( "Release notes committed" )
def tarExternals(self, releaseVersion): externalsVersion = self.relConf.getExtenalsVersion(releaseVersion) platform = Platform.getPlatformString() availableExternals = self.getAvailableExternals() if not externalsVersion: gLogger.notice("Externals is not defined for release %s" % releaseVersion) return False for externalType in self.cliParams.externalsBuildType: requestedExternals = (externalType, externalsVersion, platform, 'python%s' % self.cliParams.externalsPython) requestedExternalsString = "-".join(list(requestedExternals)) gLogger.notice("Trying to compile %s externals..." % requestedExternalsString) if not self.cliParams.forceExternals and requestedExternals in availableExternals: gLogger.notice("Externals %s is already compiled, skipping..." % (requestedExternalsString)) continue compileScript = os.path.join(os.path.dirname(__file__), "dirac-compile-externals") if not os.path.isfile(compileScript): compileScript = os.path.join(os.path.dirname(__file__), "dirac-compile-externals.py") compileTarget = os.path.join(self.cliParams.destination, platform) cmdArgs = [] cmdArgs.append("-D '%s'" % compileTarget) cmdArgs.append("-t '%s'" % externalType) cmdArgs.append("-v '%s'" % externalsVersion) cmdArgs.append("-i '%s'" % self.cliParams.externalsPython) if cliParams.externalsLocation: cmdArgs.append("-e '%s'" % self.cliParams.externalsLocation) if cliParams.makeJobs: cmdArgs.append("-j '%s'" % self.cliParams.makeJobs) compileCmd = "%s %s" % (compileScript, " ".join(cmdArgs)) gLogger.info(compileCmd) if os.system(compileCmd): gLogger.error("Error while compiling externals!") sys.exit(1) tarfilePath = os.path.join(self.cliParams.destination, "Externals-%s.tar.gz" % (requestedExternalsString)) result = Distribution.createTarball(tarfilePath, compileTarget, os.path.join(self.cliParams.destination, "mysql")) if not result['OK']: gLogger.error("Could not generate tarball for package %s" % requestedExternalsString, result['Error']) sys.exit(1) os.system("rm -rf '%s'" % compileTarget) return True
Create tarballs for a given DIRAC release """ __RCSID__ = "$Id$" from DIRAC import S_OK, S_ERROR, gLogger from DIRAC.Core.Base import Script from DIRAC.Core.Utilities import List, File, Distribution, Platform, Subprocess, CFG import sys, os, re, urllib2, tempfile, getpass, imp try: import hashlib as md5 except ImportError: import md5 globalDistribution = Distribution.Distribution() g_uploadCmd = { 'DIRAC': "( cd %OUTLOCATION% ; tar -cf - *.tar.gz *.md5 *.cfg *.pdf *.html ) | ssh [email protected] 'cd /afs/cern.ch/lhcb/distribution/DIRAC3/installSource && tar -xvf - && ls *.tar.gz > tars.list'", 'LHCb': "( cd %OUTLOCATION% ; tar -cf - *.tar.gz *.md5 *.cfg *.pdf *.html ) | ssh [email protected] 'cd /afs/cern.ch/lhcb/distribution/LHCbDirac_project && tar -xvf - && ls *.tar.gz > tars.list'", 'ILC': "( cd %OUTLOCATION% ; tar -cf - *.tar.gz *.md5 *.cfg *.pdf *.html ) | ssh [email protected] 'cd /afs/cern.ch/lhcb/distribution/DIRAC3/tars && tar -xvf - && ls *.tar.gz > tars.list'", } ### # Load release manager from dirac-install ## diracInstallLocation = os.path.join(os.path.dirname(__file__), "dirac-install") if not os.path.isfile(diracInstallLocation):
Script.parseCommandLine( ignoreErrors = False ) gLogger.notice( 'Executing: %s ' % ( ' '.join( sys.argv ) ) ) if not svnVersion: gLogger.error( "Need to specify only one version from which to spawn the branch" ) Script.showHelp() sys.exit( 1 ) if not branchPrefix: gLogger.error( "No branch type/name defined!" ) sys.exit( 1 ) if not branchName: vTuple = Distribution.parseVersionString( svnVersion ) if not vTuple: gLogger.error( "%s is not a valid version" % svnVersion ) sys.exit( 1 ) branchName = "v%d" % vTuple[0] if vTuple[1]: branchName += "r%d" % vTuple[1] #Get username if not svnUsername: svnUsername = getpass.getuser() gLogger.notice( "Using %s as username" % svnUsername ) #Start the magic! for svnPackage in List.fromChar( svnPackages ):
Script.parseCommandLine(ignoreErrors=False) gLogger.notice('Executing: %s ' % (' '.join(sys.argv))) if not svnVersion: gLogger.error( "Need to specify only one version from which to spawn the branch") Script.showHelp() sys.exit(1) if not branchPrefix: gLogger.error("No branch type/name defined!") sys.exit(1) if not branchName: vTuple = Distribution.parseVersionString(svnVersion) if not vTuple: gLogger.error("%s is not a valid version" % svnVersion) sys.exit(1) branchName = "v%d" % vTuple[0] if vTuple[1]: branchName += "r%d" % vTuple[1] #Get username if not svnUsername: svnUsername = getpass.getuser() gLogger.notice("Using %s as username" % svnUsername) #Start the magic! for svnPackage in List.fromChar(svnPackages):
gLogger.notice("Release notes committed") ## #End of helper functions ## #Get username if not svnUsername: svnUsername = getpass.getuser() gLogger.notice("Using %s as username" % svnUsername) #Start the magic! for svnPackage in List.fromChar(svnPackages): packageDistribution = Distribution.Distribution(svnPackage) packageDistribution.setSVNUser(svnUsername) buildCFG = packageDistribution.getVersionsCFG() if 'Versions' not in buildCFG.listSections(): gLogger.error( "versions.cfg file in package %s does not contain a Versions top section" % svnPackage) continue versionsRoot = '%s/tags/%s' % (svnPackage, svnPackage) if packageDistribution.getDevPath().find("https") == 0: password = getpass.getpass("Insert password for %s: " % versionsRoot) packageDistribution.setSVNPassword(password)