def isLastReleaseFromBranch(self, version, branch): # check if the dev marked version in setup.py from the given branch # compares with our version we will guess. If so, this means no # other branch was used for release this package. branchURL = self.getBranchURL(branch) if branchURL.endswith('/'): branchURL = branchURL[:-1] pyURL = '%s/setup.py' % branchURL req = urllib2.Request(pyURL) if self.packageIndexUsername: base64string = base64.encodestring( '%s:%s' % (self.packageIndexUsername, self.packageIndexPassword))[:-1] req.add_header("Authorization", "Basic %s" % base64string) setuppy = urllib2.urlopen(req).read() nextVersion = re.search("version ?= ?'(.*)',", setuppy) if not nextVersion: logger.error("No version = found in setup.py, cannot update!") # prevent mess up, force ensure new release return False else: nextVersion = nextVersion.groups()[0] setupVersion = '%sdev' % base.guessNextVersion(version) if setupVersion == nextVersion: return True else: logger.info("Last release %s wasn't released from branch %r " % ( version, branch)) return False
def build(configFile, options): # save the time we started now = datetime.datetime.now() # Read the configuration file. logger.info('Loading configuration file: ' + configFile) config = base.NonDestructiveRawConfigParser() config.read(configFile) # Create the project config parser logger.info('Creating Project Configuration') projectParser = base.NonDestructiveRawConfigParser() template_path = None if config.has_option(base.BUILD_SECTION, 'template'): template = config.get(base.BUILD_SECTION, 'template') logger.info('Loading Project Configuration Template: ' + template) projectParser.read([template]) template_path = os.path.abspath(template) if not projectParser.has_section('versions'): projectParser.add_section('versions') # Determine all versions of the important packages pkgversions = {} pkginfos = {} for pkg in config.get(base.BUILD_SECTION, 'packages').split(): customPath = None if ':' in pkg: pkg, customPath = pkg.split(':') builder = package.PackageBuilder(pkg, options) version = builder.runCLI(configFile, askToCreateRelease=True, forceSvnAuth = options.forceSvnAuth) pkgversions[pkg] = version pkginfos[pkg] = (builder.branchUrl, builder.branchRevision) projectParser.set('versions', pkg, version) # Get upload type try: uploadType = config.get(base.BUILD_SECTION, 'buildout-upload-type') except ConfigParser.NoOptionError: uploadType = "webdav" # Stop if no buildout-server given try: config.get(base.BUILD_SECTION, 'buildout-server') except ConfigParser.NoOptionError: logger.info('No buildout-server specified in the cfg, STOPPING') logger.info('Selected package versions:\n%s' % ( '\n'.join('%s = %s' % (pkg, version) for pkg, version in pkgversions.items())) ) return # Write the new configuration file to disk projectName = config.get(base.BUILD_SECTION, 'name') defaultVersion = configVersion = config.get(base.BUILD_SECTION, 'version') projectVersions = findProjectVersions(projectName, config, options, uploadType) # Determine new project version if projectVersions: defaultVersion = projectVersions[-1] if options.nextVersion or configVersion == '+': defaultVersion = base.guessNextVersion(defaultVersion) if options.forceVersion: if options.forceVersion in projectVersions: logger.error('Forced version %s already exists' % options.forceVersion) else: defaultVersion = options.forceVersion projectVersion = base.getInput( 'Project Version', defaultVersion, options.useDefaults) # Write out the new project config -- the pinned versions projectConfigFilename = '%s-%s.cfg' % (projectName, projectVersion) logger.info('Writing project configuration file: ' + projectConfigFilename) projectParser.write(open(projectConfigFilename, 'w')) filesToUpload = [projectConfigFilename] try: hashConfigFiles = config.getboolean(base.BUILD_SECTION, 'hash-config-files') except ConfigParser.NoOptionError: hashConfigFiles = False # Process config files, check for dependent config files # we should make sure that they are on the server # by design only the projectConfigFilename will have variable dependencies if template_path: if hashConfigFiles: hashes = {} else: hashes = None dependencies = getDependentConfigFiles(os.path.dirname(template_path), projectConfigFilename, addSelf=False, outfile=projectConfigFilename, hashes=hashes) if hashConfigFiles: dependencies = addHashes(dependencies, hashes) #fix main config too addHashes([projectConfigFilename], hashes, rename=False) filesToUpload.extend(dependencies) # Dump package repo infos # do it here, projectConfigFilename might be rewritten by # getDependentConfigFiles projectFile = open(projectConfigFilename, 'a') projectFile.write('\n') projectFile.write('# package SVN infos:\n') for pkg, pkginfo in pkginfos.items(): projectFile.writelines( ('# %s\n' % pkg, '# svn URL:%s\n' % pkginfo[0], '# svn repo revision:%s\n' % pkginfo[1][0], '# svn last change revision:%s\n' % pkginfo[1][1], )) logger.info('SVN info: %s: %s %s %s', pkg, pkginfo[0], pkginfo[1][0], pkginfo[1][1]) projectFile.close() # Create deployment configurations # first define a parser for load addition variable (vars) section somewhere # in your templates chain. varsParser = base.NonDestructiveRawConfigParser() # make sure we use the right (tempalte inheritation) order varsParser.read(reversed(filesToUpload)) for section in config.sections(): if section == base.BUILD_SECTION: continue logger.info('Building deployment configuration: %s', section) template_path = config.get(section, 'template') logger.info('Loading deploy template file: %s', template_path) template = file(template_path, 'r').read() vars = dict([(name, value) for name, value in config.items(section) if name != 'template']) # apply additional vars defined in release template section if a # section name is defined as vars argument # or keep the vars argument as is if there is no section. This is # usefull if someone uses the vars attribute as argument (BBB) vName = vars.get('vars') if varsParser.has_section(vName): # apply vars from given section as additional vars for name, value in varsParser.items(vName): if name not in vars: # but only if not overriden in deployment config vars[name] = value # apply defaults vars['project-name'] = projectName vars['project-version'] = projectVersion vars['instance-name'] = section # add current time vars['current-datetime'] = now.isoformat() vars['current-date'] = now.date().isoformat() vars['current-time'] = now.time().isoformat() #handle multi-line items, ConfigParser removes leading spaces #we need to add some back otherwise it will be a parsing error for k, v in vars.items(): if '\n' in v: #add a 2 space indent vars[k] = v.replace('\n', '\n ') try: deployConfigText = template % vars except KeyError, e: logger.error("The %s deployment configuration is missing the %r setting required by %s", section, e.message, template_path) sys.exit(0) deployConfigFilename = '%s-%s-%s.cfg' %( config.get(base.BUILD_SECTION, 'name'), section, projectVersion) deployConfig = base.NonDestructiveRawConfigParser() deployConfig.readfp(StringIO.StringIO(deployConfigText)) deployConfig.set('buildout', 'extends', projectConfigFilename) logger.info('Writing deployment file: ' + deployConfigFilename) deployConfig.write(open(deployConfigFilename, 'w')) filesToUpload.append(deployConfigFilename)
def runCLI(self, configFile, askToCreateRelease=False, forceSvnAuth=False): logger.info('-' * 79) logger.info(self.pkg) logger.info('-' * 79) logger.info('Start releasing new version of ' + self.pkg) # 1. Read the configuration file. logger.info('Loading configuration file: ' + configFile) config = ConfigParser.RawConfigParser() config.read(configFile) # 1.1. Get package index info. self.packageIndexUrl = config.get( base.BUILD_SECTION, 'package-index') self.packageIndexUsername = config.get( base.BUILD_SECTION, 'package-index-username') self.packageIndexPassword = config.get( base.BUILD_SECTION, 'package-index-password') # 1.2. Get svn repository info. self.svnRepositoryUrl = config.get( base.BUILD_SECTION, 'svn-repos') if forceSvnAuth: svnRepositoryUsername = config.get( base.BUILD_SECTION, 'svn-repos-username') svnRepositoryPassword = config.get( base.BUILD_SECTION, 'svn-repos-password') self.svn = base.SVN(svnRepositoryUsername, svnRepositoryPassword, forceAuth=True) else: self.svn = base.SVN() try: self.uploadType = config.get( base.BUILD_SECTION, 'upload-type') except ConfigParser.NoOptionError: self.uploadType = 'internal' try: self.tagLayout = config.get( base.BUILD_SECTION, 'tag-layout') except ConfigParser.NoOptionError: self.tagLayout = 'flat' # 1.3. Determine the possibly custom path. for pkg in config.get(base.BUILD_SECTION, 'packages').split(): if pkg.startswith(self.pkg): if ':' in pkg: self.customPath = pkg.split(':')[1] break # 2. Find all versions. versions = self.findVersions() logger.info('Existing %s versions: %s' % ( self.pkg, ' | '.join(reversed(versions)))) # 3. Determine the default version to suggest. defaultVersion = None # 3.1 Set default version based on forceVersion forceVersion = self.options.forceVersion if forceVersion: if forceVersion in versions: logger.error('Forced version %s already exists' % forceVersion) else: defaultVersion = forceVersion if versions and not defaultVersion: if self.options.nextVersion: # 3.2. If the branch was specified, check whether it changed # since the last release or if independent is set, if the last # release is based on the current branch changed = False if self.options.branch: logger.info("Checking for changes since version %s; please " "wait...", versions[-1]) changed = self.hasChangedSince(versions[-1], self.options.branch) if self.options.independent and not changed: # only check if not already marked as changed logger.info("Checking if last release is based on " "branch %s; please wait...", self.options.branch) if not self.isLastReleaseFromBranch(versions[-1], self.options.branch): changed = True if not changed: logger.info("No changes detected.") else: logger.info("Not checking for changes since version %s " "because no -b or --use-branch was specified.", versions[-1]) # 3.3. If the branch changed and the next version should be # suggested, let's find the next version. if changed: defaultVersion = base.guessNextVersion(versions[-1]) else: defaultVersion = versions[-1] else: logger.info("Not checking for changes because -n or " "--next-version was not used") defaultVersion = versions[-1] else: logger.info( "Not checking for changes because --force-version was used") # If there's no version the package is probably non existent if defaultVersion is None and self.options.defaultPackageVersion: # avoid interactive questions (handy for automated builds) defaultVersion = self.options.defaultPackageVersion branch = self.options.branch while True: version = base.getInput( 'Version for `%s`' %self.pkg, defaultVersion, self.options.useDefaults and defaultVersion is not None) if version not in versions and not self.options.offline: if askToCreateRelease: print 'The release %s-%s does not exist.' %(pkg, version) doRelease = base.getInput( 'Do you want to create it? yes/no', 'yes', self.options.useDefaults) if doRelease == 'no': continue # 4. Now create a release for this version. if not self.options.offline: # 4.1. Determine the branch from which to base the release # on. if branch is None: print 'Available Branches:' for branch in self.getBranches(): print ' * ' + branch print ' * trunk' branch = base.getInput( 'What branch do you want to use?', 'trunk', self.options.useDefaults) # 4.2. Create the release. self.createRelease(version, branch) break # 5. Return the version number. logger.info('Chosen version: ' + version) # save the info for build.py if branch is None: branch = 'trunk' self.branchUrl = self.getBranchURL(branch) self.branchRevision = self.getRevision(self.branchUrl) return version
def createRelease(self, version, branch): logger.info('Creating release %r for %r from branch %r' %( version, self.pkg, branch)) # 0. Skip creating releases in offline mode. if self.options.offline: logger.info('Offline: Skip creating a release.') return # 1. Create Release Tag branchUrl = self.getBranchURL(branch) tagUrl = self.getTagURL(version) logger.info('Creating release tag') #TODO: destination folder might not exist... create it self.svn.cp(branchUrl, tagUrl, "Create release tag %s." % version) #base.do('svn cp -m "Create release tag %s." %s %s' %( # version, branchUrl, tagUrl)) # 2. Download tag buildDir = tempfile.mkdtemp() tagDir = os.path.join(buildDir, '%s-%s' %(self.pkg, version)) self.svn.co(tagUrl, tagDir) #base.do('svn co %s %s' %(tagUrl, tagDir)) # 3. Create release # 3.1. Remove setup.cfg logger.info("Updating tag version metadata") setupCfgPath = os.path.join(tagDir, 'setup.cfg') if os.path.exists(setupCfgPath): os.remove(setupCfgPath) # 3.2. Update the version setuppy = file(os.path.join(tagDir, 'setup.py'), 'r').read() setuppy = re.sub( "version ?= ?'(.*)',", "version = '%s'," %version, setuppy) file(os.path.join(tagDir, 'setup.py'), 'w').write(setuppy) # 3.3. Check it all in self.svn.ci(tagDir, "Prepare for release %s." % version) #base.do('svn ci -m "Prepare for release %s." %s' %(version, tagDir)) # 4. Upload the distribution if self.uploadType == 'internal': # 3.4. Create distribution logger.info("Creating release tarball") base.do('python setup.py sdist', cwd = tagDir) if is_win32: ext = 'zip' else: ext = 'tar.gz' distributionFileName = os.path.join( tagDir, 'dist', '%s-%s.%s' %(self.pkg, version, ext)) if not self.options.noUpload: logger.info("Uploading release.") base.uploadFile( distributionFileName, self.packageIndexUrl, self.packageIndexUsername, self.packageIndexPassword, self.options.offline) elif self.uploadType == 'setup.py': # 3.4. Create distribution and upload in one step logger.info("Uploading release to PyPI.") # definitely DO NOT register!!! base.do('python setup.py sdist upload', cwd = tagDir) else: logger.warn('Unknown uploadType: ' + self.uploadType) # 5. Update the start branch to the next development (dev) version if not self.options.noBranchUpdate: logger.info("Updating branch version metadata") # 5.1. Check out the branch. branchDir = os.path.join(buildDir, 'branch') self.svn.co(branchUrl, branchDir) #base.do('svn co --non-recursive %s %s' %(branchUrl, branchDir)) # 5.2. Get the current version. setuppy = file(os.path.join(branchDir, 'setup.py'), 'r').read() currVersion = re.search("version ?= ?'(.*)',", setuppy) if not currVersion: logger.error("No version = found in setup.py, cannot update!") else: currVersion = currVersion.groups()[0] # 5.3. Update setup/py to the next version of the currently # released one newVersion = base.guessNextVersion(version) + 'dev' setuppy = re.sub( "version ?= ?'(.*)',", "version = '%s'," %newVersion, setuppy) file(os.path.join(branchDir, 'setup.py'), 'w').write(setuppy) # 5.4. Check in the changes. self.svn.ci(branchDir, "Update version number to %s." % newVersion) #base.do('svn ci -m "Update version number to %s." %s' %( # newVersion, branchDir)) # 6. Cleanup rmtree(buildDir)