def generateAndUploadReleaseNotes(packageDistribution, svnPath,
versionReleased):
tmpDir = tempfile.mkdtemp()
packageName = packageDistribution.getPackageName()
gLogger.notice("Generating release notes for %s under %s" %
(packageName, tmpDir))
for suffix, singleVersion in (("history", False), ("notes", True)):
gLogger.notice("Generating %s rst" % suffix)
rstHistory = os.path.join(tmpDir, "release%s.rst" % suffix)
htmlHistory = os.path.join(tmpDir, "release%s.html" % suffix)
Distribution.generateReleaseNotes(packageName, rstHistory,
versionReleased, singleVersion)
try:
Distribution.generateHTMLReleaseNotesFromRST(
rstHistory, htmlHistory)
except Exception as x:
print "Failed to generate html version of the notes:", str(x)
# Attempt to generate pdf as well
os.system('rst2pdf %s' % rstHistory)
packageDistribution.queueImport(
tmpDir, svnPath, 'Release notes for version %s' % versionReleased)
if not packageDistribution.executeCommandQueue():
gLogger.error("Could not upload release notes")
sys.exit(1)
os.system("rm -rf '%s'" % tmpDir)
gLogger.notice("Release notes committed")
def __generateTarball( self ):
destDir = self.params.destination
tarName = "%s-%s.tar.gz" % ( self.params.name, self.params.version )
tarfilePath = os.path.join( destDir, tarName )
dirToTar = os.path.join( self.params.destination, self.params.name )
result = Distribution.writeVersionToInit( dirToTar, self.params.version )
if not result[ 'OK' ]:
return result
result = Distribution.createTarball( tarfilePath, dirToTar )
if not result[ 'OK' ]:
return S_ERROR( "Could not generate tarball: %s" % result[ 'Error' ] )
#Remove package dir
shutil.rmtree( dirToTar )
gLogger.info( "Tar file %s created" % tarName )
return S_OK( tarfilePath )
def __generateTarball(self):
destDir = self.params.destination
tarName = "%s-%s.tar.gz" % (self.params.name, self.params.version)
tarfilePath = os.path.join(destDir, tarName)
dirToTar = os.path.join(self.params.destination, self.params.name)
result = Distribution.writeVersionToInit(dirToTar, self.params.version)
if not result['OK']:
return result
result = Distribution.createTarball(tarfilePath, dirToTar)
if not result['OK']:
return S_ERROR("Could not generate tarball: %s" % result['Error'])
#Remove package dir
shutil.rmtree(dirToTar)
gLogger.info("Tar file %s created" % tarName)
return S_OK(tarfilePath)
def generateAndUploadReleaseNotes( packageDistribution, svnPath, versionReleased ):
tmpDir = tempfile.mkdtemp()
packageName = packageDistribution.getPackageName()
gLogger.notice( "Generating release notes for %s under %s" % ( packageName, tmpDir ) )
for suffix, singleVersion in ( ( "history", False ), ( "notes", True ) ):
gLogger.notice( "Generating %s rst" % suffix )
rstHistory = os.path.join( tmpDir, "release%s.rst" % suffix )
htmlHistory = os.path.join( tmpDir, "release%s.html" % suffix )
Distribution.generateReleaseNotes( packageName, rstHistory, versionReleased, singleVersion )
try:
Distribution.generateHTMLReleaseNotesFromRST( rstHistory, htmlHistory )
except Exception, x:
print "Failed to generate html version of the notes:", str( x )
# Attempt to generate pdf as well
os.system( 'rst2pdf %s' % rstHistory )
def __generateRSTFile(releaseData, rstFileName, pkgVersion, singleVersion):
"""create the rst file from the release.notes """
rstData = []
parsedPkgVersion = Distribution.parseVersionString(
pkgVersion) if pkgVersion is not None else (99, 99, 99, 99)
for version, verData in releaseData:
if singleVersion and version != pkgVersion and pkgVersion is not None:
continue
if Distribution.parseVersionString(version) > parsedPkgVersion:
continue
versionLine = "Version %s" % version
rstData.append("")
rstData.append("=" * len(versionLine))
rstData.append(versionLine)
rstData.append("=" * len(versionLine))
rstData.append("")
if verData['comment']:
rstData.append("\n".join(verData['comment']))
rstData.append("")
for feature, featureData in verData['features']:
if not featureData:
continue
rstData.append(feature)
rstData.append("=" * len(feature))
rstData.append("")
for key in sorted(featureData):
rstData.append(key.capitalize())
rstData.append(":" * (len(key) + 5))
rstData.append("")
for entry in featureData[key]:
rstData.append(" - %s" % entry)
rstData.append("")
if pkgVersion is None and singleVersion:
print "Just taking the first entry in releaseNotes"
break
#Write releasenotes.rst
try:
rstFilePath = os.path.join(os.getcwd(), "source", rstFileName)
fd = open(rstFilePath, "w")
fd.write("\n".join(rstData))
fd.close()
except (OSError, IOError) as excp:
return S_ERROR("Could not write %s: %s" % (rstFileName, excp))
return S_OK()
def __generateRSTFile( releaseData, rstFileName, pkgVersion, singleVersion ):
"""create the rst file from the release.notes """
rstData = []
parsedPkgVersion = Distribution.parseVersionString( pkgVersion ) if pkgVersion is not None else (99, 99, 99, 99)
for version, verData in releaseData:
if singleVersion and version != pkgVersion and pkgVersion is not None:
continue
if Distribution.parseVersionString( version ) > parsedPkgVersion:
continue
versionLine = "Version %s" % version
rstData.append( "" )
rstData.append( "=" * len( versionLine ) )
rstData.append( versionLine )
rstData.append( "=" * len( versionLine ) )
rstData.append( "" )
if verData[ 'comment' ]:
rstData.append( "\n".join( verData[ 'comment' ] ) )
rstData.append( "" )
for feature, featureData in verData[ 'features' ]:
if not featureData:
continue
rstData.append( feature )
rstData.append( "=" * len( feature ) )
rstData.append( "" )
for key in sorted( featureData ):
rstData.append( key.capitalize() )
rstData.append( ":" * ( len( key ) + 5 ) )
rstData.append( "" )
for entry in featureData[ key ]:
rstData.append( " - %s" % entry )
rstData.append( "" )
if pkgVersion is None and singleVersion:
print("Just taking the first entry in releaseNotes")
break
#Write releasenotes.rst
try:
rstFilePath = os.path.join( os.getcwd(), "source", rstFileName )
fd = open( rstFilePath, "w" )
fd.write( "\n".join( rstData ) )
fd.close()
except (OSError,IOError) as excp:
return S_ERROR( "Could not write %s: %s" % ( rstFileName, excp ) )
return S_OK()
def generateAndUploadReleaseNotes(packageDistribution, svnPath,
versionReleased):
tmpDir = tempfile.mkdtemp()
packageName = packageDistribution.getPackageName()
gLogger.notice("Generating release notes for %s under %s" %
(packageName, tmpDir))
for suffix, singleVersion in (("history", False), ("notes", True)):
gLogger.notice("Generating %s rst" % suffix)
rstHistory = os.path.join(tmpDir, "release%s.rst" % suffix)
htmlHistory = os.path.join(tmpDir, "release%s.html" % suffix)
Distribution.generateReleaseNotes(packageName, rstHistory,
versionReleased, singleVersion)
try:
Distribution.generateHTMLReleaseNotesFromRST(
rstHistory, htmlHistory)
except Exception, x:
print "Failed to generate html version of the notes:", str(x)
# Attempt to generate pdf as well
os.system('rst2pdf %s' % rstHistory)
def tarExternals(self, releaseVersion):
externalsVersion = self.relConf.getExtenalsVersion(releaseVersion)
platform = Platform.getPlatformString()
availableExternals = self.getAvailableExternals()
if not externalsVersion:
gLogger.notice("Externals is not defined for release %s" %
releaseVersion)
return False
for externalType in self.cliParams.externalsBuildType:
requestedExternals = (externalType, externalsVersion, platform,
'python%s' % self.cliParams.externalsPython)
requestedExternalsString = "-".join(list(requestedExternals))
gLogger.notice("Trying to compile %s externals..." %
requestedExternalsString)
if not self.cliParams.forceExternals and requestedExternals in availableExternals:
gLogger.notice(
"Externals %s is already compiled, skipping..." %
(requestedExternalsString))
continue
compileScript = os.path.join(os.path.dirname(__file__),
"dirac-compile-externals")
if not os.path.isfile(compileScript):
compileScript = os.path.join(os.path.dirname(__file__),
"dirac-compile-externals.py")
compileTarget = os.path.join(self.cliParams.destination, platform)
cmdArgs = []
cmdArgs.append("-D '%s'" % compileTarget)
cmdArgs.append("-t '%s'" % externalType)
cmdArgs.append("-v '%s'" % externalsVersion)
cmdArgs.append("-i '%s'" % self.cliParams.externalsPython)
if cliParams.externalsLocation:
cmdArgs.append("-e '%s'" % self.cliParams.externalsLocation)
if cliParams.makeJobs:
cmdArgs.append("-j '%s'" % self.cliParams.makeJobs)
compileCmd = "%s %s" % (compileScript, " ".join(cmdArgs))
gLogger.info(compileCmd)
if os.system(compileCmd):
gLogger.error("Error while compiling externals!")
sys.exit(1)
tarfilePath = os.path.join(
self.cliParams.destination,
"Externals-%s.tar.gz" % (requestedExternalsString))
result = Distribution.createTarball(
tarfilePath, compileTarget,
os.path.join(self.cliParams.destination, "mysql"))
if not result['OK']:
gLogger.error(
"Could not generate tarball for package %s" % package,
result['Error'])
sys.exit(1)
os.system("rm -rf '%s'" % compileTarget)
return True
def __generateRSTFile(self, releaseData, rstFileName, pkgVersion,
singleVersion):
rstData = []
parsedPkgVersion = Distribution.parseVersionString(pkgVersion)
for version, verData in releaseData:
if singleVersion and version != pkgVersion:
continue
if Distribution.parseVersionString(version) > parsedPkgVersion:
continue
versionLine = "Version %s" % version
rstData.append("")
rstData.append("=" * len(versionLine))
rstData.append(versionLine)
rstData.append("=" * len(versionLine))
rstData.append("")
if verData['comment']:
rstData.append("\n".join(verData['comment']))
rstData.append("")
for feature, featureData in verData['features']:
if not featureData:
continue
rstData.append(feature)
rstData.append("=" * len(feature))
rstData.append("")
for key in sorted(featureData):
rstData.append(key.capitalize())
rstData.append(":" * (len(key) + 5))
rstData.append("")
for entry in featureData[key]:
rstData.append(" - %s" % entry)
rstData.append("")
# Write releasenotes.rst
try:
rstFilePath = os.path.join(self.params.destination,
self.params.name, rstFileName)
with open(rstFilePath, "w") as fd:
fd.write("\n".join(rstData))
except Exception as excp:
return S_ERROR("Could not write %s: %s" % (rstFileName, excp))
return S_OK()
def __generateRSTFile( self, releaseData, rstFileName, pkgVersion, singleVersion ):
rstData = []
parsedPkgVersion = Distribution.parseVersionString( pkgVersion )
for version, verData in releaseData:
if singleVersion and version != pkgVersion:
continue
if Distribution.parseVersionString( version ) > parsedPkgVersion:
continue
versionLine = "Version %s" % version
rstData.append( "" )
rstData.append( "=" * len( versionLine ) )
rstData.append( versionLine )
rstData.append( "=" * len( versionLine ) )
rstData.append( "" )
if verData[ 'comment' ]:
rstData.append( "\n".join( verData[ 'comment' ] ) )
rstData.append( "" )
for feature, featureData in verData[ 'features' ]:
if not featureData:
continue
rstData.append( feature )
rstData.append( "=" * len( feature ) )
rstData.append( "" )
for key in sorted( featureData ):
rstData.append( key.capitalize() )
rstData.append( ":" * ( len( key ) + 5 ) )
rstData.append( "" )
for entry in featureData[ key ]:
rstData.append( " - %s" % entry )
rstData.append( "" )
#Write releasenotes.rst
try:
rstFilePath = os.path.join( self.params.destination, self.params.name, rstFileName )
fd = open( rstFilePath, "w" )
fd.write( "\n".join( rstData ) )
fd.close()
except Exception, excp:
return S_ERROR( "Could not write %s: %s" % ( rstFileName, excp ) )
def generateAndUploadReleaseNotes( packageDistribution, svnPath, versionReleased ):
tmpDir = tempfile.mkdtemp()
packageName = packageDistribution.getPackageName()
gLogger.notice( "Generating release notes for %s under %s" % ( packageName, tmpDir ) )
for suffix, singleVersion in ( ( "history", False ), ( "notes", True ) ):
gLogger.notice( "Generating %s rst" % suffix )
rstHistory = os.path.join( tmpDir, "release%s.rst" % suffix )
htmlHistory = os.path.join( tmpDir, "release%s.html" % suffix )
Distribution.generateReleaseNotes( packageName, rstHistory, versionReleased, singleVersion )
try:
Distribution.generateHTMLReleaseNotesFromRST( rstHistory, htmlHistory )
except Exception as x:
print "Failed to generate html version of the notes:", str( x )
# Attempt to generate pdf as well
os.system( 'rst2pdf %s' % rstHistory )
packageDistribution.queueImport( tmpDir, svnPath, 'Release notes for version %s' % versionReleased )
if not packageDistribution.executeCommandQueue():
gLogger.error( "Could not upload release notes" )
sys.exit( 1 )
os.system( "rm -rf '%s'" % tmpDir )
gLogger.notice( "Release notes committed" )
def tarExternals(self, releaseVersion):
externalsVersion = self.relConf.getExtenalsVersion(releaseVersion)
platform = Platform.getPlatformString()
availableExternals = self.getAvailableExternals()
if not externalsVersion:
gLogger.notice("Externals is not defined for release %s" % releaseVersion)
return False
for externalType in self.cliParams.externalsBuildType:
requestedExternals = (externalType, externalsVersion, platform, 'python%s' % self.cliParams.externalsPython)
requestedExternalsString = "-".join(list(requestedExternals))
gLogger.notice("Trying to compile %s externals..." % requestedExternalsString)
if not self.cliParams.forceExternals and requestedExternals in availableExternals:
gLogger.notice("Externals %s is already compiled, skipping..." % (requestedExternalsString))
continue
compileScript = os.path.join(os.path.dirname(__file__), "dirac-compile-externals")
if not os.path.isfile(compileScript):
compileScript = os.path.join(os.path.dirname(__file__), "dirac-compile-externals.py")
compileTarget = os.path.join(self.cliParams.destination, platform)
cmdArgs = []
cmdArgs.append("-D '%s'" % compileTarget)
cmdArgs.append("-t '%s'" % externalType)
cmdArgs.append("-v '%s'" % externalsVersion)
cmdArgs.append("-i '%s'" % self.cliParams.externalsPython)
if cliParams.externalsLocation:
cmdArgs.append("-e '%s'" % self.cliParams.externalsLocation)
if cliParams.makeJobs:
cmdArgs.append("-j '%s'" % self.cliParams.makeJobs)
compileCmd = "%s %s" % (compileScript, " ".join(cmdArgs))
gLogger.info(compileCmd)
if os.system(compileCmd):
gLogger.error("Error while compiling externals!")
sys.exit(1)
tarfilePath = os.path.join(self.cliParams.destination, "Externals-%s.tar.gz" % (requestedExternalsString))
result = Distribution.createTarball(tarfilePath,
compileTarget,
os.path.join(self.cliParams.destination, "mysql"))
if not result['OK']:
gLogger.error("Could not generate tarball for package %s" % requestedExternalsString, result['Error'])
sys.exit(1)
os.system("rm -rf '%s'" % compileTarget)
return True
Create tarballs for a given DIRAC release
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger
from DIRAC.Core.Base import Script
from DIRAC.Core.Utilities import List, File, Distribution, Platform, Subprocess, CFG
import sys, os, re, urllib2, tempfile, getpass, imp
try:
import hashlib as md5
except ImportError:
import md5
globalDistribution = Distribution.Distribution()
g_uploadCmd = {
'DIRAC':
"( cd %OUTLOCATION% ; tar -cf - *.tar.gz *.md5 *.cfg *.pdf *.html ) | ssh [email protected] 'cd /afs/cern.ch/lhcb/distribution/DIRAC3/installSource && tar -xvf - && ls *.tar.gz > tars.list'",
'LHCb':
"( cd %OUTLOCATION% ; tar -cf - *.tar.gz *.md5 *.cfg *.pdf *.html ) | ssh [email protected] 'cd /afs/cern.ch/lhcb/distribution/LHCbDirac_project && tar -xvf - && ls *.tar.gz > tars.list'",
'ILC':
"( cd %OUTLOCATION% ; tar -cf - *.tar.gz *.md5 *.cfg *.pdf *.html ) | ssh [email protected] 'cd /afs/cern.ch/lhcb/distribution/DIRAC3/tars && tar -xvf - && ls *.tar.gz > tars.list'",
}
###
# Load release manager from dirac-install
##
diracInstallLocation = os.path.join(os.path.dirname(__file__), "dirac-install")
if not os.path.isfile(diracInstallLocation):
Script.parseCommandLine( ignoreErrors = False )
gLogger.notice( 'Executing: %s ' % ( ' '.join( sys.argv ) ) )
if not svnVersion:
gLogger.error( "Need to specify only one version from which to spawn the branch" )
Script.showHelp()
sys.exit( 1 )
if not branchPrefix:
gLogger.error( "No branch type/name defined!" )
sys.exit( 1 )
if not branchName:
vTuple = Distribution.parseVersionString( svnVersion )
if not vTuple:
gLogger.error( "%s is not a valid version" % svnVersion )
sys.exit( 1 )
branchName = "v%d" % vTuple[0]
if vTuple[1]:
branchName += "r%d" % vTuple[1]
#Get username
if not svnUsername:
svnUsername = getpass.getuser()
gLogger.notice( "Using %s as username" % svnUsername )
#Start the magic!
for svnPackage in List.fromChar( svnPackages ):
Script.parseCommandLine(ignoreErrors=False)
gLogger.notice('Executing: %s ' % (' '.join(sys.argv)))
if not svnVersion:
gLogger.error(
"Need to specify only one version from which to spawn the branch")
Script.showHelp()
sys.exit(1)
if not branchPrefix:
gLogger.error("No branch type/name defined!")
sys.exit(1)
if not branchName:
vTuple = Distribution.parseVersionString(svnVersion)
if not vTuple:
gLogger.error("%s is not a valid version" % svnVersion)
sys.exit(1)
branchName = "v%d" % vTuple[0]
if vTuple[1]:
branchName += "r%d" % vTuple[1]
#Get username
if not svnUsername:
svnUsername = getpass.getuser()
gLogger.notice("Using %s as username" % svnUsername)
#Start the magic!
for svnPackage in List.fromChar(svnPackages):
gLogger.notice("Release notes committed")
##
#End of helper functions
##
#Get username
if not svnUsername:
svnUsername = getpass.getuser()
gLogger.notice("Using %s as username" % svnUsername)
#Start the magic!
for svnPackage in List.fromChar(svnPackages):
packageDistribution = Distribution.Distribution(svnPackage)
packageDistribution.setSVNUser(svnUsername)
buildCFG = packageDistribution.getVersionsCFG()
if 'Versions' not in buildCFG.listSections():
gLogger.error(
"versions.cfg file in package %s does not contain a Versions top section"
% svnPackage)
continue
versionsRoot = '%s/tags/%s' % (svnPackage, svnPackage)
if packageDistribution.getDevPath().find("https") == 0:
password = getpass.getpass("Insert password for %s: " % versionsRoot)
packageDistribution.setSVNPassword(password)
