def verify_configs(configs_dir, revision, hghost, configs_repo, changesets, filename): """Check the release_configs and l10n-changesets against tagged revisions""" release_config_file = path.join(configs_dir, 'mozilla', filename) l10n_changesets_file = path.join(configs_dir, 'mozilla', changesets) configs_url = make_hg_url(hghost, configs_repo, revision=revision, filename=path.join('mozilla', filename)) l10n_url = make_hg_url(hghost, configs_repo, revision=revision, filename=path.join('mozilla', changesets)) success = True try: official_configs = urllib2.urlopen(configs_url) log.info("Comparing tagged revision %s to on-disk %s ..." % ( configs_url, filename)) if not compare(official_configs, release_config_file): log.error("local configs do not match tagged revisions in repo") success = False error_tally.add('verify_configs') l10n_changesets = urllib2.urlopen(l10n_url) log.info("Comparing tagged revision %s to on-disk %s ..." % ( l10n_url, changesets)) if not compare(l10n_changesets, l10n_changesets_file): log.error("local l10n-changesets do not match tagged revisions" " in repo") success = False error_tally.add('verify_configs') except urllib2.HTTPError: log.error("cannot find configs in repo %s" % configs_url) log.error("cannot find configs in repo %s" % l10n_url) success = False error_tally.add('verify_configs') return success
def tagOtherRepo(config, repo, reponame, revision, pushAttempts): remote = make_hg_url(HG, repo) mercurial(remote, reponame) def tagRepo(repo, attempt, config, revision, tags): # set totalChangesets=1 because tag() generates exactly 1 commit totalChangesets = 1 # update to the desired revision first, then to the tip of revision's # branch to avoid new head creation update(repo, revision=revision) update(repo) tag(repo, revision, tags, config['hgUsername']) outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote, ssh_username=config[ 'hgUsername'], ssh_key=config['hgSshKey'])) if len(outgoingRevs) != totalChangesets: raise Exception("Wrong number of outgoing revisions") pushRepo = make_hg_url(HG, repo, protocol='ssh') def tag_wrapper(r, n): tagRepo(r, n, config, revision, tags) def cleanup_wrapper(): cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'], config['hgSshKey']) retry(apply_and_push, cleanup=cleanup_wrapper, args=(reponame, pushRepo, tag_wrapper, pushAttempts), kwargs=dict(ssh_username=config['hgUsername'], ssh_key=config['hgSshKey']))
def tagOtherRepo(config, repo, reponame, revision, pushAttempts): remote = make_hg_url(HG, repo) mercurial(remote, reponame) def tagRepo(repo, attempt, config, revision, tags): # set totalChangesets=1 because tag() generates exactly 1 commit totalChangesets = 1 # update to the desired revision first, then to the tip of revision's # branch to avoid new head creation update(repo, revision=revision) update(repo) tag(repo, revision, tags, config['hgUsername']) outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote, ssh_username=config['hgUsername'], ssh_key=config['hgSshKey'])) if len(outgoingRevs) != totalChangesets: raise Exception("Wrong number of outgoing revisions") pushRepo = make_hg_url(HG, repo, protocol='ssh') def tag_wrapper(r, n): tagRepo(r, n, config, revision, tags) def cleanup_wrapper(): cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'], config['hgSshKey']) retry(apply_and_push, cleanup=cleanup_wrapper, args=(reponame, pushRepo, tag_wrapper, pushAttempts), kwargs=dict(ssh_username=config['hgUsername'], ssh_key=config['hgSshKey']))
def verify_configs(revision, hghost, configs_repo, changesets, filename): """Check the release_configs and l10n-changesets against tagged revisions""" configs_url = make_hg_url(hghost, configs_repo, revision=revision, filename="mozilla/%s" % filename) l10n_url = make_hg_url(hghost, configs_repo, revision=revision, filename="mozilla/%s" % changesets) success = True try: official_configs = urllib2.urlopen(configs_url) log.info("Comparing tagged revision %s to on-disk %s ..." % (configs_url, filename)) if not compare(official_configs, filename): log.error("local configs do not match tagged revisions in repo") success = False error_tally.add('verify_configs') l10n_changesets = urllib2.urlopen(l10n_url) log.info("Comparing tagged revision %s to on-disk %s ..." % (l10n_url, changesets)) if not compare(l10n_changesets, changesets): log.error( "local l10n-changesets do not match tagged revisions in repo") success = False error_tally.add('verify_configs') except urllib2.HTTPError: log.error("cannot find configs in repo %s" % configs_url) log.error("cannot find configs in repo %s" % l10n_url) success = False error_tally.add('verify_configs') return success
def testMakeHGUrlNoRevisionNoFilename(self): repo_url = make_hg_url( "hg.mozilla.org", "/build/tools" ) expected_url = "https://hg.mozilla.org/build/tools" self.assertEquals(repo_url, expected_url)
def make_hg_get_request(repo_path, revision, filename=None, hg_url='hg.mozilla.org'): """Wrapper to make a GET request for a specific URI under hg repo""" url = make_hg_url(hg_url, repo_path, revision=revision, filename=filename) return make_generic_get_request(url)
def testMakeHGUrl(self): # construct an hg url specific to revision, branch and filename and try # to pull it down file_url = make_hg_url( "hg.mozilla.org", "//build/tools/", revision="FIREFOX_3_6_12_RELEASE", filename="/lib/python/util/hg.py" ) expected_url = "https://hg.mozilla.org/build/tools/raw-file/FIREFOX_3_6_12_RELEASE/lib/python/util/hg.py" self.assertEquals(file_url, expected_url)
def testMakeHGUrlDifferentProtocol(self): repo_url = make_hg_url( "hg.mozilla.org", "/build/tools", protocol='ssh' ) expected_url = "ssh://hg.mozilla.org/build/tools" self.assertEquals(repo_url, expected_url)
def bump_configs(server, username, sshKey, repo, repoPath, configsToBump, configsToOverride): reponame = get_repo_name(repo) repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame)) pushRepo = make_hg_url(server, '%s/%s' % (repoPath, reponame), protocol='ssh') retry(mercurial, args=(repo_url, reponame)) def bump(repo, configsToBump, configsToOverride): """Process dynamic (version, buildNumber, etc.) variables in configsToBump, then append overrides files to both configsToBump and configsToOverride.""" # First pass. Bump variables in configsToBump. configs = ['%s/%s' % (repo, x) for x in configsToBump.keys()] cmd = ['python', BUMP_SCRIPT, '--bump-version', '--revision=tip'] cmd.extend(configs) run_cmd(cmd) # Second pass. Append override files to configsToBump and # configsToOverride. for config, overrides in \ configsToBump.items() + configsToOverride.items(): newContent = cat([path.join(repo, config)] + [path.join(repo, x) for x in overrides]) fh = open(path.join(repo, config), 'wb') fh.write(newContent) fh.close() run_cmd(['hg', 'commit', '-m', 'Automatic config bump'], cwd=repo) def bump_wrapper(r, n): bump(r, configsToBump, configsToOverride) def cleanup_wrapper(): cleanOutgoingRevs(reponame, pushRepo, username, sshKey) retry(apply_and_push, cleanup=cleanup_wrapper, args=(reponame, pushRepo, bump_wrapper), kwargs=dict(ssh_username=username, ssh_key=sshKey)) tags = [] for configfile in configsToBump.keys(): config = readReleaseConfig(path.join(reponame, configfile)) tags.extend( getTags(config['baseTag'], config['buildNumber'], buildTag=True)) return tags
def testMakeHGUrlNoFilename(self): file_url = make_hg_url( "hg.mozilla.org", "/build/tools", revision="default" ) expected_url = "https://hg.mozilla.org/build/tools/rev/default" self.assertEquals(file_url, expected_url)
def bump_configs(server, username, sshKey, repo, repoPath, configsToBump, configsToOverride): reponame = get_repo_name(repo) repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame)) pushRepo = make_hg_url(server, '%s/%s' % (repoPath, reponame), protocol='ssh') retry(mercurial, args=(repo_url, reponame)) def bump(repo, configsToBump, configsToOverride): """Process dynamic (version, buildNumber, etc.) variables in configsToBump, then append overrides files to both configsToBump and configsToOverride.""" # First pass. Bump variables in configsToBump. configs = ['%s/%s' % (repo, x) for x in configsToBump.keys()] cmd = ['python', BUMP_SCRIPT, '--bump-version', '--revision=tip'] cmd.extend(configs) run_cmd(cmd) # Second pass. Append override files to configsToBump and # configsToOverride. for config, overrides in \ configsToBump.items() + configsToOverride.items(): newContent = cat([path.join(repo, config)] + [path.join(repo, x) for x in overrides]) fh = open(path.join(repo, config), 'wb') fh.write(newContent) fh.close() run_cmd(['hg', 'commit', '-m', 'Automatic config bump'], cwd=repo) def bump_wrapper(r, n): bump(r, configsToBump, configsToOverride) def cleanup_wrapper(): cleanOutgoingRevs(reponame, pushRepo, username, sshKey) retry(apply_and_push, cleanup=cleanup_wrapper, args=(reponame, pushRepo, bump_wrapper), kwargs=dict(ssh_username=username, ssh_key=sshKey)) tags = [] for configfile in configsToBump.keys(): config = readReleaseConfig(path.join(reponame, configfile)) tags.extend(getTags(config['baseTag'], config['buildNumber'], buildTag=True)) return tags
def testMakeHGUrl(self): # construct an hg url specific to revision, branch and filename and try # to pull it down file_url = make_hg_url("hg.mozilla.org", '//build/tools/', revision='FIREFOX_3_6_12_RELEASE', filename="/lib/python/util/hg.py") expected_url = "https://hg.mozilla.org/build/tools/raw-file/FIREFOX_3_6_12_RELEASE/lib/python/util/hg.py" self.assertEquals(file_url, expected_url)
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch, pushAttempts, defaultBranch='default'): remote = make_hg_url(HG, repo) mercurial(remote, reponame) def bump_and_tag(repo, attempt, config, relbranch, revision, tags, defaultBranch): # set relbranchChangesets=1 because tag() generates exactly 1 commit relbranchChangesets = 1 defaultBranchChangesets = 0 if relbranch in get_branches(reponame): update(reponame, revision=relbranch) else: update(reponame, revision=revision) run_cmd(['hg', 'branch', relbranch], cwd=reponame) if len(bumpFiles) > 0: # Bump files on the relbranch, if necessary bump(reponame, bumpFiles, 'version') run_cmd(['hg', 'diff'], cwd=repo) try: get_output(['hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version'])], cwd=reponame) relbranchChangesets += 1 except subprocess.CalledProcessError, e: # We only want to ignore exceptions caused by having nothing to # commit, which are OK. We still want to raise exceptions caused # by any other thing. if e.returncode != 1 or "nothing changed" not in e.output: raise # We always want our tags pointing at the tip of the relbranch # so we need to grab the current revision after we've switched # branches and bumped versions. revision = get_revision(reponame) # Create the desired tags on the relbranch tag(repo, revision, tags, config['hgUsername']) # This is the bump of the version on the default branch # We do it after the other one in order to get the tip of the # repository back on default, thus avoiding confusion. if len(bumpFiles) > 0: update(reponame, revision=defaultBranch) bump(reponame, bumpFiles, 'nextVersion') run_cmd(['hg', 'diff'], cwd=repo) try: get_output(['hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version'])], cwd=reponame) defaultBranchChangesets += 1 except subprocess.CalledProcessError, e: if e.returncode != 1 or "nothing changed" not in e.output: raise
def tag_repo(server, username, sshKey, repo, repoPath, tags): reponame = get_repo_name(repo) repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame)) pushRepo = make_hg_url(server, '%s/%s' % (repoPath, reponame), protocol='ssh') mercurial(repo_url, reponame) def do_tag(repo, tags): cmd = ['hg', 'tag', '-f', '-m', 'Automatic preproduction tag'] + tags run_cmd(cmd, cwd=repo) def do_tag_wrapper(r, n): do_tag(r, tags) def cleanup_wrapper(): cleanOutgoingRevs(reponame, pushRepo, username, sshKey) retry(apply_and_push, cleanup=cleanup_wrapper, args=(reponame, pushRepo, do_tag_wrapper), kwargs=dict(ssh_username=username, ssh_key=sshKey))
def maybe_delete_repo(server, username, sshKey, repo, repoPath): reponame = get_repo_name(repo) repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame)) try: log.info("Trying to open %s" % repo_url) urlopen(repo_url) except URLError: log.info("%s doesn't exist, not deleting" % reponame) else: log.info('Deleting %s' % reponame) retry(run_remote_cmd, args=('edit %s delete YES' % reponame, server, username, sshKey))
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch, pushAttempts): remote = make_hg_url(HG, repo) mercurial(remote, reponame) def bump_and_tag(repo, attempt, config, relbranch, revision, tags): # set relbranchChangesets=1 because tag() generates exactly 1 commit relbranchChangesets = 1 if relbranch in get_branches(reponame): update(reponame, revision=relbranch) else: update(reponame, revision=revision) run_cmd(['hg', 'branch', relbranch], cwd=reponame) if len(bumpFiles) > 0: # Bump files on the relbranch, if necessary bump(reponame, bumpFiles, 'version') run_cmd(['hg', 'diff'], cwd=repo) try: get_output([ 'hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version']) ], cwd=reponame) relbranchChangesets += 1 except subprocess.CalledProcessError, e: # We only want to ignore exceptions caused by having nothing to # commit, which are OK. We still want to raise exceptions caused # by any other thing. if e.returncode != 1 or "nothing changed" not in e.output: raise # We always want our tags pointing at the tip of the relbranch # so we need to grab the current revision after we've switched # branches and bumped versions. revision = get_revision(reponame) # Create the desired tags on the relbranch tag(repo, revision, tags, config['hgUsername']) # Validate that the repository is only different from the remote in # ways we expect. outgoingRevs = out(src=reponame, remote=remote, ssh_username=config['hgUsername'], ssh_key=config['hgSshKey']) if len([r for r in outgoingRevs if r[BRANCH] == relbranch ]) != relbranchChangesets: raise Exception("Incorrect number of revisions on %s" % relbranch) if len(outgoingRevs) != relbranchChangesets: raise Exception("Wrong number of outgoing revisions")
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch, pushAttempts, defaultBranch='default'): remote = make_hg_url(HG, repo) retry(mercurial, args=(remote, reponame)) def bump_and_tag(repo, attempt, config, relbranch, revision, tags, defaultBranch): relbranchChangesets = len(tags) defaultBranchChangesets = 0 if relbranch in get_branches(reponame): update(reponame, revision=relbranch) else: update(reponame, revision=revision) run_cmd(['hg', 'branch', relbranch], cwd=reponame) if len(bumpFiles) > 0: # Bump files on the relbranch, if necessary bump(reponame, bumpFiles, 'version') run_cmd(['hg', 'diff'], cwd=repo) try: get_output(['hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version'])], cwd=reponame) relbranchChangesets += 1 revision = get_revision(reponame) except subprocess.CalledProcessError, e: # We only want to ignore exceptions caused by having nothing to # commit, which are OK. We still want to raise exceptions caused # by any other thing. if e.returncode != 1 or "nothing changed" not in e.output: raise # Create the desired tags on the relbranch tag(repo, revision, tags, config['hgUsername']) # This is the bump of the version on the default branch # We do it after the other one in order to get the tip of the # repository back on default, thus avoiding confusion. if len(bumpFiles) > 0: update(reponame, revision=defaultBranch) bump(reponame, bumpFiles, 'nextVersion') run_cmd(['hg', 'diff'], cwd=repo) try: get_output(['hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version'])], cwd=reponame) defaultBranchChangesets += 1 except subprocess.CalledProcessError, e: if e.returncode != 1 or "nothing changed" not in e.output: raise
def tagOtherRepo(config, repo, reponame, revision, pushAttempts): remote = make_hg_url(HG, repo) retry(mercurial, args=(remote, reponame)) def tagRepo(repo, attempt, config, revision, tags): totalChangesets = len(tags) tag(repo, revision, tags, config['hgUsername']) outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote, ssh_username=config['hgUsername'], ssh_key=config['hgSshKey'])) if len(outgoingRevs) != totalChangesets: raise Exception("Wrong number of outgoing revisions") pushRepo = make_hg_url(HG, repo, protocol='ssh') def tag_wrapper(r, n): tagRepo(r, n, config, revision, tags) def cleanup_wrapper(): cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'], config['hgSshKey']) retry(apply_and_push, cleanup=cleanup_wrapper, args=(reponame, pushRepo, tag_wrapper, pushAttempts), kwargs=dict(ssh_username=config['hgUsername'], ssh_key=config['hgSshKey']))
def verify_repo(branch, revision, hghost): """Poll the hgweb interface for a given branch and revision to make sure it exists""" repo_url = make_hg_url(hghost, branch, revision=revision) log.info("Checking for existence of %s..." % repo_url) success = True try: repo_page = urllib2.urlopen(repo_url) log.info("Got: %s !" % repo_page.geturl()) except urllib2.HTTPError: log.error("Repo does not exist with required revision. Check again, or use -b to bypass") success = False error_tally.add('verify_repo') return success
def verify_mozconfigs(branch, revision, hghost, product, mozconfigs, appName, whitelist=None): """Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences""" if whitelist: mozconfigWhitelist = readConfig(whitelist, ['whitelist']) else: mozconfigWhitelist = {} log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product) success = True types = {'+': 'release', '-': 'nightly'} for platform,mozconfig in mozconfigs.items(): urls = [] mozconfigs = [] # Create links to the two mozconfigs. releaseConfig = make_hg_url(hghost, branch, 'http', revision, mozconfig) urls.append(releaseConfig) # The nightly one is the exact same URL, with the file part changed. urls.append(releaseConfig.rstrip('release') + 'nightly') for url in urls: try: mozconfigs.append(urllib2.urlopen(url).readlines()) except urllib2.HTTPError as e: log.error("MISSING: %s - ERROR: %s" % (url, e.msg)) diffInstance = difflib.Differ() if len(mozconfigs) == 2: diffList = list(diffInstance.compare(mozconfigs[0],mozconfigs[1])) for line in diffList: clean_line = line[1:].strip() if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1: # skip comment lines if clean_line.startswith('#'): continue # compare to whitelist if line[0] == '-' and mozconfigWhitelist.get(branch, {}).has_key(platform) \ and clean_line in mozconfigWhitelist[branch][platform]: continue if line[0] == '+' and mozconfigWhitelist.get('nightly', {}).has_key(platform) \ and clean_line in mozconfigWhitelist['nightly'][platform]: continue if line[0] == '-': opposite = 'release' else: opposite = 'nightly' log.error("not in %s mozconfig's whitelist (%s/%s/%s) : %s" % (opposite, branch, platform, types[line[0]], clean_line)) success = False error_tally.add('verify_mozconfig') else: log.info("Missing mozconfigs to compare for %s" % platform) error_tally.add("verify_mozconfigs: Confirm that %s does not have release/nightly mozconfigs to compare" % platform) return success
def verify_mozconfigs(branch, version, hghost, product, platforms, whitelist=None): """Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences""" if whitelist: mozconfigWhitelist = readConfig(whitelist, ['whitelist']) else: mozconfigWhitelist = {} log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product) success = True types = {'+': 'release', '-': 'nightly'} tag = ''.join([product.upper(), "_", version.replace('.','_'), "_RELEASE"]) for platform in platforms: urls = [] mozconfigs = [] for type in types.values(): urls.append(make_hg_url(hghost, 'build/buildbot-configs', 'http', tag, os.path.join('mozilla2', platform, branch, type,'mozconfig'))) for url in urls: try: mozconfigs.append(urllib2.urlopen(url).readlines()) except urllib2.HTTPError as e: log.error("MISSING: %s - ERROR: %s" % (url, e.msg)) diffInstance = difflib.Differ() if len(mozconfigs) == 2: diffList = list(diffInstance.compare(mozconfigs[0],mozconfigs[1])) for line in diffList: clean_line = line[1:].strip() if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1: # skip comment lines if clean_line.startswith('#'): continue # compare to whitelist if line[0] == '-' and mozconfigWhitelist.get(branch, {}).has_key(platform) \ and clean_line in mozconfigWhitelist[branch][platform]: continue if line[0] == '+' and mozconfigWhitelist.get('nightly', {}).has_key(platform) \ and clean_line in mozconfigWhitelist['nightly'][platform]: continue if line[0] == '-': opposite = 'release' else: opposite = 'nightly' log.error("not in %s mozconfig's whitelist (%s/%s/%s) : %s" % (opposite, branch, platform, types[line[0]], clean_line)) success = False error_tally.add('verify_mozconfig') else: log.info("Missing mozconfigs to compare for %s" % platform) error_tally.add("verify_mozconfigs: Confirm that %s does not have release/nightly mozconfigs to compare" % platform) return success
def verify_repo(branch, revision, hghost): """Poll the hgweb interface for a given branch and revision to make sure it exists""" repo_url = make_hg_url(hghost, branch, revision=revision) log.info("Checking for existence of %s..." % repo_url) success = True try: repo_page = urllib2.urlopen(repo_url) log.info("Got: %s !" % repo_page.geturl()) except urllib2.HTTPError: log.error("Repo does not exist with required revision." " Check again, or use -b to bypass") success = False error_tally.add('verify_repo') return success
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch, pushAttempts): remote = make_hg_url(HG, repo) mercurial(remote, reponame) def bump_and_tag(repo, attempt, config, relbranch, revision, tags): # set relbranchChangesets=1 because tag() generates exactly 1 commit relbranchChangesets = 1 if relbranch in get_branches(reponame): update(reponame, revision=relbranch) else: update(reponame, revision=revision) run_cmd(['hg', 'branch', relbranch], cwd=reponame) if len(bumpFiles) > 0: # Bump files on the relbranch, if necessary bump(reponame, bumpFiles, 'version') run_cmd(['hg', 'diff'], cwd=repo) try: get_output(['hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version'])], cwd=reponame) relbranchChangesets += 1 except subprocess.CalledProcessError, e: # We only want to ignore exceptions caused by having nothing to # commit, which are OK. We still want to raise exceptions caused # by any other thing. if e.returncode != 1 or "nothing changed" not in e.output: raise # We always want our tags pointing at the tip of the relbranch # so we need to grab the current revision after we've switched # branches and bumped versions. revision = get_revision(reponame) # Create the desired tags on the relbranch tag(repo, revision, tags, config['hgUsername']) # Validate that the repository is only different from the remote in # ways we expect. outgoingRevs = out(src=reponame, remote=remote, ssh_username=config['hgUsername'], ssh_key=config['hgSshKey']) if len([r for r in outgoingRevs if r[BRANCH] == relbranch]) != relbranchChangesets: raise Exception("Incorrect number of revisions on %s" % relbranch) if len(outgoingRevs) != relbranchChangesets: raise Exception("Wrong number of outgoing revisions")
def verify_build(sourceRepo, hghost): """ Ensure that the bumpFiles match what the release config wants them to be""" success = True for filename, versions in sourceRepo['bumpFiles'].iteritems(): try: url = make_hg_url(hghost, sourceRepo['path'], revision=sourceRepo['revision'], filename=filename) found_version = urllib2.urlopen(url).read() if not findVersion(found_version, versions['version']): log.error("%s has incorrect version '%s' (expected '%s')" % \ (filename, found_version, versions['version'])) success = False error_tally.add('verify_build') except urllib2.HTTPError, inst: log.error("cannot find %s. Check again, or -b to bypass" % inst.geturl()) success = False error_tally.add('verify_build')
def verify_l10n_changesets(hgHost, l10n_changesets): """Checks for the existance of all l10n changesets""" success = True locales = query_locale_revisions(l10n_changesets) for locale in sorted(locales.keys()): revision = locales[locale] localePath = '%(repoPath)s/%(locale)s/file/%(revision)s' % { 'repoPath': releaseConfig['l10nRepoPath'].strip('/'), 'locale': locale, 'revision': revision, } locale_url = make_hg_url(hgHost, localePath, protocol='https') log.info("Checking for existence l10n changeset %s %s in repo %s ..." % (locale, revision, locale_url)) try: urllib2.urlopen(locale_url) except urllib2.HTTPError, e: log.error("error checking l10n changeset %s: %d %s" % (locale_url, e.code, e.reason)) success = False error_tally.add('verify_l10n')
def verify_l10n_changesets(hgHost, l10n_changesets): """Checks for the existance of all l10n changesets""" success = True locales = query_locale_revisions(l10n_changesets) for locale in sorted(locales.keys()): revision = locales[locale] localePath = '%(repoPath)s/%(locale)s/file/%(revision)s' % { 'repoPath': releaseConfig['l10nRepoPath'].strip('/'), 'locale': locale, 'revision': revision, } locale_url = make_hg_url(hgHost, localePath, protocol='https') log.info("Checking for existence l10n changeset %s %s in repo %s ..." % (locale, revision, locale_url)) success = retry(get_l10n_changesets, kwargs=dict(locale_url=locale_url), attempts=3, sleeptime=1, retry_exceptions=(urllib2.HTTPError,)) if not success: error_tally.add('verify_l10n') return success
def verify_l10n_changesets(hgHost, l10n_changesets): """Checks for the existance of all l10n changesets""" success = True locales = query_locale_revisions(l10n_changesets) for locale in sorted(locales.keys()): revision = locales[locale] localePath = '%(repoPath)s/%(locale)s/file/%(revision)s' % { 'repoPath': releaseConfig['l10nRepoPath'].strip('/'), 'locale': locale, 'revision': revision, } locale_url = make_hg_url(hgHost, localePath, protocol='https') log.info("Checking for existence l10n changeset %s %s in repo %s ..." % (locale, revision, locale_url)) success = retry(get_l10n_changesets, kwargs=dict(locale_url=locale_url), attempts=3, sleeptime=1, retry_exceptions=(urllib2.HTTPError, urllib2.URLError)) if not success: error_tally.add('verify_l10n') return success
from build.download import downloadNightlyBuild from build.l10n import repackLocale, l10nRepackPrep, getNightlyLocalesForChunk import build.misc from build.paths import getLatestDir from build.upload import postUploadCmdPrefix from release.info import readBranchConfig from util.commands import run_cmd from util.hg import mercurial, update, make_hg_url logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") log = logging.getLogger(__name__) HG = "hg.mozilla.org" DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url(HG, "build/buildbot-configs") class RepackError(Exception): pass def createRepacks(sourceRepo, l10nRepoDir, l10nBaseRepo, mozconfigPath, objdir, makeDirs, locales, ftpProduct, stageServer, stageUsername, stageSshKey, compareLocalesRepo, merge, platform, stage_platform, mobileDirName, en_us_binary_url): sourceRepoName = path.split(sourceRepo)[-1] localeSrcDir = path.join(sourceRepoName, objdir, mobileDirName, "locales") # Even on Windows we need to use "/" as a separator for this because # compare-locales doesn"t work any other way l10nIni = "/".join([sourceRepoName, mobileDirName, "locales", "l10n.ini"])
from build.upload import postUploadCmdPrefix from release.download import downloadReleaseBuilds, downloadUpdateIgnore404 from release.info import readReleaseConfig, readConfig, fileInfo from release.l10n import getReleaseLocalesForChunk from util.hg import mercurial, update, make_hg_url from util.retry import retry from util.commands import run_cmd from release.info import getBuildID logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") log = logging.getLogger(__name__) HG = "hg.mozilla.org" DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url(HG, "build/buildbot-configs") class RepackError(Exception): pass def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo, mozconfigPath, srcMozconfigPath, objdir, makeDirs, appName,
sys.path.append(path.join(path.dirname(__file__), "../../lib/python")) from build.download import downloadNightlyBuild from build.l10n import repackLocale, l10nRepackPrep, getNightlyLocalesForChunk import build.misc from build.paths import getLatestDir from build.upload import postUploadCmdPrefix from release.info import readBranchConfig from util.commands import run_cmd from util.hg import mercurial, update, make_hg_url logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") log = logging.getLogger(__name__) HG="hg.mozilla.org" DEFAULT_BUILDBOT_CONFIGS_REPO=make_hg_url(HG, "build/buildbot-configs") class RepackError(Exception): pass def createRepacks(sourceRepo,l10nRepoDir, l10nBaseRepo, mozconfigPath, objdir, makeDirs, locales, ftpProduct, stageServer, stageUsername, stageSshKey, compareLocalesRepo, merge, platform, stage_platform, mobileDirName, en_us_binary_url): sourceRepoName = path.split(sourceRepo)[-1] localeSrcDir = path.join(sourceRepoName, objdir, mobileDirName, "locales") # Even on Windows we need to use "/" as a separator for this because # compare-locales doesn"t work any other way l10nIni = "/".join([sourceRepoName, mobileDirName, "locales", "l10n.ini"])
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") log = logging.getLogger(__name__) site.addsitedir(path.join(path.dirname(__file__), "../../lib/python")) site.addsitedir(path.join(path.dirname(__file__), "../../lib/python/vendor")) from release.info import readReleaseConfig, readConfig from release.paths import makeCandidatesDir, makeReleasesDir from util.hg import update, make_hg_url, mercurial from util.commands import run_remote_cmd from util.transfer import scp from util.retry import retry import requests DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url("hg.mozilla.org", "build/buildbot-configs") REQUIRED_RELEASE_CONFIG = ("productName", "version", "buildNumber", "stage_product") DEFAULT_RSYNC_EXCLUDES = [ "--exclude=*tests*", "--exclude=*crashreporter*", "--exclude=*.log", "--exclude=*.txt", "--exclude=*unsigned*", "--exclude=*update-backup*", "--exclude=*partner-repacks*", "--exclude=*.checksums", "--exclude=*.checksums.asc", "--exclude=logs", "--exclude=jsshell*",
try: import json except ImportError: import simplejson as json logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") log = logging.getLogger(__name__) sys.path.append(path.join(path.dirname(__file__), "../../lib/python")) from release.info import readReleaseConfig, readBranchConfig from release.paths import makeCandidatesDir, makeReleasesDir from util.hg import update, make_hg_url, mercurial from util.commands import run_remote_cmd DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url('hg.mozilla.org', 'build/buildbot-configs') REQUIRED_BRANCH_CONFIG = ("stage_server", "stage_username", "stage_ssh_key") REQUIRED_RELEASE_CONFIG = ("productName", "version", "buildNumber") DEFAULT_RSYNC_EXCLUDES = ['--exclude=*tests*', '--exclude=*crashreporter*', '--exclude=*.log', '--exclude=*.txt', '--exclude=*unsigned*', '--exclude=*update-backup*', '--exclude=*partner-repacks*', '--exclude=*.checksums', '--exclude=logs', '--exclude=jsshell*', ]
sys.path.append(path.join(path.dirname(__file__), "../../lib/python")) from build.l10n import repackLocale, l10nRepackPrep import build.misc from build.upload import postUploadCmdPrefix from release.download import downloadReleaseBuilds, downloadUpdateIgnore404 from release.info import readReleaseConfig, readBranchConfig from release.l10n import getReleaseLocalesForChunk from util.hg import mercurial, update, make_hg_url from util.retry import retry logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") log = logging.getLogger(__name__) HG="hg.mozilla.org" DEFAULT_BUILDBOT_CONFIGS_REPO=make_hg_url(HG, "build/buildbot-configs") class RepackError(Exception): pass def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo, mozconfigPath, objdir, makeDirs, appName, locales, product, version, buildNumber, stageServer, stageUsername, stageSshKey, compareLocalesRepo, merge, platform, brand, generatePartials=False, oldVersion=None, oldBuildNumber=None, appVersion=None): sourceRepoName = path.split(sourceRepo)[-1] localeSrcDir = path.join(sourceRepoName, objdir, appName, "locales") # Even on Windows we need to use "/" as a separator for this because # compare-locales doesn"t work any other way l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])
import sys sys.path.append(path.join(path.dirname(__file__), "../../lib/python")) logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") log = logging.getLogger(__name__) from util.commands import run_cmd, get_output from util.hg import mercurial, apply_and_push, update, get_revision, \ make_hg_url, out, BRANCH, REVISION, get_branches, cleanOutgoingRevs from util.retry import retry from build.versions import bumpFile from release.info import readReleaseConfig, getTags, generateRelbranchName from release.l10n import getL10nRepositories HG="hg.mozilla.org" DEFAULT_BUILDBOT_CONFIGS_REPO=make_hg_url(HG, 'build/buildbot-configs') DEFAULT_MAX_PUSH_ATTEMPTS=10 REQUIRED_CONFIG = ('version', 'appVersion', 'appName', 'productName', 'milestone', 'buildNumber', 'hgUsername', 'hgSshKey', 'baseTag', 'l10nRepoPath', 'sourceRepositories', 'l10nRevisionFile') REQUIRED_SOURCE_REPO_KEYS = ('path', 'revision') def getBumpCommitMessage(productName, version): return 'Automated checkin: version bump for ' + productName + ' ' + \ version + ' release. CLOSED TREE a=release' def getTagCommitMessage(revision, tag): return "Added tag " + tag + " for changeset " + revision + \ ". CLOSED TREE a=release"
test_success = True buildNumber = options.buildNumber products = [] check_buildbot() if not options.dryrun and not options.skip_reconfig: from util.fabric.common import check_fabric, FabricHelper check_fabric() if options.configs_dir: configs_dir = options.configs_dir cleanup_configs = False else: cleanup_configs = True configs_dir = mkdtemp() remote = make_hg_url(HG, options.configs_repo_url) retry(mercurial, args=(remote, configs_dir), kwargs={'branch': options.configs_branch}) update(configs_dir, options.configs_branch) # https://bugzilla.mozilla.org/show_bug.cgi?id=678103#c5 # This goes through the list of config files in reverse order, which is a # hacky way of making sure that the config file that's listed first is the # one that's loaded in releaseConfig for the sendchange. for releaseConfigFile in list(reversed(options.releaseConfigFiles)): abs_release_config_file = path.join(configs_dir, 'mozilla', releaseConfigFile) releaseConfig = readReleaseConfig(abs_release_config_file) products.append(releaseConfig['productName'])
def __init__(self, rmConfig, **kwargs): BuildFactory.__init__(self, **kwargs) self.addStep( SetProperty( name='set_topdir', command=['pwd'], property='topdir', workdir='.', )) self.addStep( ShellCommand( command=['rm', '-rvf', 'tools', 'buildbot-configs'], workdir='.', )) self.addStep( ShellCommand( command=[ 'hg', 'clone', make_hg_url(rmConfig['HG_HOST'], 'build/tools'), 'tools' ], workdir='.', haltOnFailure=True, )) self.addStep( ShellCommand( command=[ rmConfig['PYTHON'], 'scripts/preproduction/repo_setup.py', '-c', 'scripts/preproduction/repo_setup_config.py' ], workdir='tools', haltOnFailure=True, )) self.addStep( SetProperty( property='previousSetupMakefile', command='ls %s/Makefile 2>/dev/null || exit 0' % rmConfig['releaseMasterDir'], flunkOnFailure=False, haltOnFailure=False, warnOnFailure=True, )) def previousSetupExists(step): return \ step.build.getProperties().has_key('previousSetupMakefile') \ and len(step.build.getProperty('previousSetupMakefile')) > 0 self.addStep( ShellCommand( command=[ rmConfig['PYTHON'], 'tools/buildfarm/maintenance/buildbot-wrangler.py', 'stop', '%s/master' % rmConfig['releaseMasterDir'] ], workdir=rmConfig['releaseMasterDir'], flunkOnFailure=False, doStepIf=previousSetupExists, )) self.addStep( ShellCommand( command=['rm', '-rvf', rmConfig['releaseMasterDir']], workdir='.', haltOnFailure=True, )) self.addStep( ShellCommand( command=['rm', '-rvf', 'buildbot-configs'], workdir='.', haltOnFailure=True, )) self.addStep( ShellCommand( command=[ 'hg', 'clone', make_hg_url(rmConfig['HG_HOST'], '%s/buildbot-configs' % rmConfig['HG_DIR']), 'buildbot-configs' ], workdir='.', haltOnFailure=True, )) self.addStep( ShellCommand( command=[ 'make', '-f', 'Makefile.setup', 'PYTHON=%s' % rmConfig['PYTHON'], 'VIRTUALENV=%s' % rmConfig['VIRTUALENV'], 'HG_DIR=%s' % rmConfig['HG_DIR'], 'MASTER_NAME=pp-release', 'MASTERS_JSON=/home/cltbld/conf/master_config.json', 'BASEDIR=%s' % rmConfig['releaseMasterDir'], 'BUILDBOTCUSTOM_BRANCH=%s' % rmConfig['BUILDBOTCUSTOM_BRANCH'], 'BUILDBOTCONFIGS_BRANCH=%s' % rmConfig['BUILDBOTCONFIGS_BRANCH'], 'virtualenv', 'deps', 'install-buildbot', 'master', 'master-makefile' ], workdir='buildbot-configs', env={ 'PIP_DOWNLOAD_CACHE': WithProperties('%(topdir)s/cache'), 'CC': None, 'CXX': None }, haltOnFailure=True, )) self.addStep( ShellCommand( command=['make', 'checkconfig'], workdir=rmConfig['releaseMasterDir'], haltOnFailure=True, )) self.addStep( ShellCommand( command=['touch', 'twistd.log'], workdir='%s/master' % rmConfig['releaseMasterDir'], )) self.addStep( ShellCommand( command=[ 'bash', '-c', 'if [ -e ~/conf/passwords.py ]; then cp -fv ~/conf/passwords.py ./; fi' ], workdir='%s/master' % rmConfig['releaseMasterDir'], )) self.addStep( ShellCommand( command=[ rmConfig['PYTHON'], 'tools/buildfarm/maintenance/buildbot-wrangler.py', 'start', '%s/master' % rmConfig['releaseMasterDir'] ], workdir=rmConfig['releaseMasterDir'], )) for release_config in rmConfig['releaseConfigs']: self.addStep( SetProperty( property='release_tag', command=[ rmConfig['PYTHON'], '-c', 'execfile("%s"); \ print releaseConfig["baseTag"] + "_RELEASE"' % release_config ], workdir='%s/buildbot-configs' % rmConfig['releaseMasterDir'], )) self.addStep( SetProperty( property='release_branch', command=[ rmConfig['PYTHON'], '-c', 'execfile("%s"); \ print releaseConfig["sourceRepositories"]["mozilla"]["path"]' % release_config ], workdir='%s/buildbot-configs' % rmConfig['releaseMasterDir'], )) self.addStep( ShellCommand( command=[ 'buildbot', 'sendchange', '--username', 'preproduction', '--master', rmConfig['releaseMasterHostPort'], '--branch', WithProperties('%(release_branch)s'), '-p', 'products:firefox', '-p', WithProperties('script_repo_revision:%(release_tag)s'), 'release_build' ], workdir='.', ))
test_success = True buildNumber = options.buildNumber products = [] check_buildbot() if not options.dryrun and not options.skip_reconfig: from util.fabric.common import check_fabric, FabricHelper check_fabric() if options.configs_dir: configs_dir = options.configs_dir cleanup_configs = False else: cleanup_configs = True configs_dir = mkdtemp() remote = make_hg_url(HG, options.configs_repo_url) retry(mercurial, args=(remote, configs_dir), kwargs={'branch': options.configs_branch}) update(configs_dir, options.configs_branch) # https://bugzilla.mozilla.org/show_bug.cgi?id=678103#c5 # This goes through the list of config files in reverse order, which is a # hacky way of making sure that the config file that's listed first is the # one that's loaded in releaseConfig for the sendchange. for releaseConfigFile in list(reversed(options.releaseConfigFiles)): abs_release_config_file = path.join(configs_dir, 'mozilla', releaseConfigFile) releaseConfig = readReleaseConfig(abs_release_config_file) products.append(releaseConfig['productName']) if not options.buildNumber:
from build.upload import postUploadCmdPrefix from release.download import downloadReleaseBuilds from release.info import readReleaseConfig, readBranchConfig from release.l10n import getReleaseLocalesFromJsonForChunk from release.paths import makeCandidatesDir from util.commands import run_cmd from util.hg import mercurial, update, make_hg_url from util.retry import retry logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") log = logging.getLogger(__name__) HG = "hg.mozilla.org" DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url(HG, "build/buildbot-configs") class RepackError(Exception): pass def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo, mozconfigPath, objdir, makeDirs, locales, ftpProduct, appName, version, appVersion, buildNumber, stageServer, stageUsername, stageSshKey, compareLocalesRepo, merge, platform, stage_platform, brand, mobileDirName): sourceRepoName = path.split(sourceRepo)[-1] nightlyDir = "candidates" localeSrcDir = path.join(sourceRepoName, objdir, mobileDirName, "locales") # Even on Windows we need to use "/" as a separator for this because
def __init__(self, rmConfig, **kwargs): BuildFactory.__init__(self, **kwargs) self.addStep(SetProperty( name='set_topdir', command=['pwd'], property='topdir', workdir='.', )) self.addStep(ShellCommand( command=['rm', '-rvf', 'tools', 'buildbot-configs'], workdir='.', )) self.addStep(ShellCommand( command=['hg', 'clone', make_hg_url(rmConfig['HG_HOST'], 'build/tools'), 'tools'], workdir='.', haltOnFailure=True, )) self.addStep(ShellCommand( command=[rmConfig['PYTHON'], 'scripts/preproduction/repo_setup.py', '-c', 'scripts/preproduction/repo_setup_config.py'], workdir='tools', haltOnFailure=True, )) self.addStep(SetProperty( property='previousSetupMakefile', command='ls %s/Makefile 2>/dev/null || exit 0' % rmConfig['releaseMasterDir'], flunkOnFailure=False, haltOnFailure=False, warnOnFailure=True, )) def previousSetupExists(step): return \ step.build.getProperties().has_key('previousSetupMakefile') \ and len(step.build.getProperty('previousSetupMakefile')) > 0 self.addStep(ShellCommand( command=[rmConfig['PYTHON'], 'tools/buildfarm/maintenance/buildbot-wrangler.py', 'stop', '%s/master' % rmConfig['releaseMasterDir']], workdir=rmConfig['releaseMasterDir'], flunkOnFailure=False, doStepIf=previousSetupExists, )) self.addStep(ShellCommand( command=['rm', '-rvf', rmConfig['releaseMasterDir']], workdir='.', haltOnFailure=True, )) self.addStep(ShellCommand( command=['rm', '-rvf', 'buildbot-configs'], workdir='.', haltOnFailure=True, )) self.addStep(ShellCommand( command=['hg', 'clone', make_hg_url(rmConfig['HG_HOST'], '%s/buildbot-configs' % rmConfig['HG_DIR']), 'buildbot-configs'], workdir='.', haltOnFailure=True, )) self.addStep(ShellCommand( command=['make', '-f', 'Makefile.setup', 'PYTHON=%s' % rmConfig['PYTHON'], 'VIRTUALENV=%s' % rmConfig['VIRTUALENV'], 'HG_DIR=%s' % rmConfig['HG_DIR'], 'MASTER_NAME=pp-release', 'MASTERS_JSON=/home/cltbld/conf/master_config.json', 'BASEDIR=%s' % rmConfig['releaseMasterDir'], 'BUILDBOTCUSTOM_BRANCH=%s' % rmConfig['BUILDBOTCUSTOM_BRANCH'], 'BUILDBOTCONFIGS_BRANCH=%s' % rmConfig['BUILDBOTCONFIGS_BRANCH'], 'UNIVERSAL=1', 'virtualenv', 'deps', 'install-buildbot', 'master', 'master-makefile'], workdir='buildbot-configs', env={'PIP_DOWNLOAD_CACHE': WithProperties('%(topdir)s/cache'), 'CC': None, 'CXX': None }, haltOnFailure=True, )) self.addStep(ShellCommand( command=['make', 'checkconfig'], workdir=rmConfig['releaseMasterDir'], haltOnFailure=True, )) self.addStep(ShellCommand( command=['touch', 'twistd.log'], workdir='%s/master' % rmConfig['releaseMasterDir'], )) self.addStep(ShellCommand( command=[ 'bash', '-c', 'if [ -e ~/conf/passwords.py ]; then cp -fv ~/conf/passwords.py ./; fi' ], workdir='%s/master' % rmConfig['releaseMasterDir'], )) self.addStep(ShellCommand( command=[rmConfig['PYTHON'], 'tools/buildfarm/maintenance/buildbot-wrangler.py', 'start', '%s/master' % rmConfig['releaseMasterDir']], workdir=rmConfig['releaseMasterDir'], )) for release_config in rmConfig['releaseConfigs']: self.addStep(SetProperty( property='release_tag', command=[rmConfig['PYTHON'], '-c', 'execfile("%s"); \ print releaseConfig["baseTag"] + "_RELEASE"' % release_config], workdir='%s/buildbot-configs' % rmConfig['releaseMasterDir'], )) self.addStep(SetProperty( property='release_branch', command=[rmConfig['PYTHON'], '-c', 'execfile("%s"); \ print releaseConfig["sourceRepositories"]["mozilla"]["path"]' % release_config], workdir='%s/buildbot-configs' % rmConfig['releaseMasterDir'], )) self.addStep(ShellCommand( command=['buildbot', 'sendchange', '--username', 'preproduction', '--master', rmConfig['releaseMasterHostPort'], '--branch', WithProperties('%(release_branch)s'), '-p', 'products:firefox', '-p', WithProperties( 'script_repo_revision:%(release_tag)s'), 'release_build'], workdir='.', ))
site.addsitedir(path.join(path.dirname(__file__), "../../lib/python")) site.addsitedir(path.join(path.dirname(__file__), "../../lib/python/vendor")) from distutils.version import LooseVersion from release.updates.patcher import PatcherConfig from release.l10n import makeReleaseRepackUrls from release.platforms import buildbot2updatePlatforms, buildbot2ftp from release.paths import makeReleasesDir, makeCandidatesDir from release.info import readReleaseConfig from util.retry import retry from util.hg import mercurial, make_hg_url, update from release.updates.verify import UpdateVerifyConfig HG = "hg.mozilla.org" DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url(HG, 'build/buildbot-configs') DEFAULT_MAX_PUSH_ATTEMPTS = 10 REQUIRED_CONFIG = ('productName', 'buildNumber', 'ausServerUrl', 'stagingServer') FTP_SERVER_TEMPLATE = 'http://%s/pub/mozilla.org' logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") log = logging.getLogger() def validate(options): err = False config = {} if not path.exists(path.join('buildbot-configs', options.release_config)): log.error("%s does not exist!" % options.release_config)
sys.path.append(path.join(path.dirname(__file__), "../../lib/python")) logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") log = logging.getLogger(__name__) from release.info import readReleaseConfig, readBranchConfig from util.hg import update, mercurial, make_hg_url from util.commands import run_remote_cmd from release.download import rsyncFilesByPattern, rsyncFiles from release.signing import generateChecksums, signFiles from release.paths import makeCandidatesDir DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url('hg.mozilla.org', 'build/buildbot-configs') REQUIRED_BRANCH_CONFIG = ("stage_server", "stage_username", "stage_ssh_key") REQUIRED_RELEASE_CONFIG = ("productName", "version", "buildNumber") def validate(options, args): if not options.configfile: log.info("Must pass --configfile") sys.exit(1) releaseConfigFile = path.join("buildbot-configs", options.releaseConfig) branchConfigFile = path.join("buildbot-configs", options.configfile) branchConfigDir = path.dirname(branchConfigFile) if not path.exists(branchConfigFile): log.info("%s does not exist!" % branchConfigFile)
def main(options): log.info('Loading config from %s' % options.config) config = load_config(options.config) if config.getboolean('release-runner', 'verbose'): log_level=logging.DEBUG else: log_level=logging.INFO logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=log_level) check_buildbot() check_fabric() # Shorthand sendchange_master = config.get('release-runner', 'sendchange_master') api_root = config.get('api', 'api_root') username = config.get('api', 'username') password = config.get('api', 'password') hg_username = config.get('release-runner', 'hg_username') hg_ssh_key = config.get('release-runner', 'hg_ssh_key') buildbot_configs = config.get('release-runner', 'buildbot_configs') buildbot_configs_branch = config.get('release-runner', 'buildbot_configs_branch') buildbotcustom = config.get('release-runner', 'buildbotcustom') buildbotcustom_branch = config.get('release-runner', 'buildbotcustom_branch') tools = config.get('release-runner', 'tools') tools_branch = config.get('release-runner', 'tools_branch') masters_json = config.get('release-runner', 'masters_json') staging = config.getboolean('release-runner', 'staging') sleeptime = config.getint('release-runner', 'sleeptime') notify_from = get_config(config, 'release-runner', 'notify_from', None) notify_to = get_config(config, 'release-runner', 'notify_to', None) ssh_username = get_config(config, 'release-runner', 'ssh_username', None) ssh_key = get_config(config, 'release-runner', 'ssh_key', None) if isinstance(notify_to, basestring): notify_to = [x.strip() for x in notify_to.split(',')] smtp_server = get_config(config, 'release-runner', 'smtp_server', 'localhost') configs_workdir = 'buildbot-configs' custom_workdir = 'buildbotcustom' tools_workdir = 'tools' configs_pushRepo = make_hg_url(HG, get_repo_path(buildbot_configs), protocol='ssh') custom_pushRepo = make_hg_url(HG, get_repo_path(buildbotcustom), protocol='ssh') tools_pushRepo = make_hg_url(HG, get_repo_path(tools), protocol='ssh') rr = ReleaseRunner(api_root=api_root, username=username, password=password) # Main loop waits for new releases, processes them and exits. while True: try: log.debug('Fetching release requests') rr.get_release_requests() if rr.new_releases: for release in rr.new_releases: log.info('Got a new release request: %s' % release) break else: log.debug('Sleeping for %d seconds before polling again' % sleeptime) time.sleep(sleeptime) except Exception, e: log.error("Caught exception when polling:", exc_info=True) sys.exit(5)
def verify_mozconfigs(branch, revision, hghost, product, mozconfigs, nightly_mozconfigs, whitelist=None): """Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences""" branch_name = get_repo_name(branch) if whitelist: mozconfigWhitelist = readConfig(whitelist, ['whitelist']) else: mozconfigWhitelist = {} log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product) success = True for platform, mozconfig in mozconfigs.items(): urls = [] mozconfigs = [] nightly_mozconfig = nightly_mozconfigs[platform] mozconfig_paths = [mozconfig, nightly_mozconfig] # Create links to the two mozconfigs. for c in mozconfig, nightly_mozconfig: urls.append(make_hg_url(hghost, branch, 'http', revision, c)) for url in urls: try: mozconfigs.append(urllib2.urlopen(url).readlines()) except urllib2.HTTPError as e: log.error("MISSING: %s - ERROR: %s" % (url, e.msg)) # Nothing to compare against return False diffInstance = difflib.Differ() if len(mozconfigs) == 2: diffList = list(diffInstance.compare(mozconfigs[0], mozconfigs[1])) for line in diffList: clean_line = line[1:].strip() if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1: # skip comment lines if clean_line.startswith('#'): continue # compare to whitelist message = "" if line[0] == '-': # handle lines that move around in diff if '+' + line[1:] in diffList: continue if platform in mozconfigWhitelist.get(branch_name, {}): if clean_line in \ mozconfigWhitelist[branch_name][platform]: continue elif line[0] == '+': if '-' + line[1:] in diffList: continue if platform in mozconfigWhitelist.get('nightly', {}): if clean_line in \ mozconfigWhitelist['nightly'][platform]: continue else: log.warning("%s not in %s %s!" % ( clean_line, platform, mozconfigWhitelist['nightly'][platform])) else: log.error("Skipping line %s!" % line) continue message = "found in %s but not in %s: %s" if line[0] == '-': log.error(message % (mozconfig_paths[0], mozconfig_paths[1], clean_line)) else: log.error(message % (mozconfig_paths[1], mozconfig_paths[0], clean_line)) success = False else: log.info("Missing mozconfigs to compare for %s" % platform) return False return success
def verify_mozconfigs(branch, revision, hghost, product, mozconfigs, nightly_mozconfigs, whitelist=None): """Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences""" branch_name = get_repo_name(branch) if whitelist: mozconfigWhitelist = readConfig(whitelist, ['whitelist']) else: mozconfigWhitelist = {} log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product) success = True for platform, mozconfig in mozconfigs.items(): urls = [] mozconfigs = [] nightly_mozconfig = nightly_mozconfigs[platform] mozconfig_paths = [mozconfig, nightly_mozconfig] # Create links to the two mozconfigs. for c in mozconfig, nightly_mozconfig: urls.append(make_hg_url(hghost, branch, 'http', revision, c)) for url in urls: try: mozconfigs.append(urllib2.urlopen(url).readlines()) except urllib2.HTTPError as e: log.error("MISSING: %s - ERROR: %s" % (url, e.msg)) # Nothing to compare against return False diffInstance = difflib.Differ() if len(mozconfigs) == 2: diffList = list(diffInstance.compare(mozconfigs[0], mozconfigs[1])) for line in diffList: clean_line = line[1:].strip() if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1: # skip comment lines if clean_line.startswith('#'): continue # compare to whitelist message = "" if line[0] == '-': # handle lines that move around in diff if '+' + line[1:] in diffList: continue if platform in mozconfigWhitelist.get(branch_name, {}): if clean_line in \ mozconfigWhitelist[branch_name][platform]: continue elif line[0] == '+': if '-' + line[1:] in diffList: continue if platform in mozconfigWhitelist.get('nightly', {}): if clean_line in \ mozconfigWhitelist['nightly'][platform]: continue else: log.warning( "%s not in %s %s!" % (clean_line, platform, mozconfigWhitelist['nightly'][platform])) else: log.error("Skipping line %s!" % line) continue message = "found in %s but not in %s: %s" if line[0] == '-': log.error(message % (mozconfig_paths[0], mozconfig_paths[1], clean_line)) else: log.error(message % (mozconfig_paths[1], mozconfig_paths[0], clean_line)) success = False else: log.info("Missing mozconfigs to compare for %s" % platform) return False return success
sys.path.append(path.join(path.dirname(__file__), "../../lib/python")) logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") log = logging.getLogger(__name__) from util.commands import run_cmd, get_output from util.hg import mercurial, apply_and_push, update, get_revision, \ make_hg_url, out, BRANCH, get_branches, cleanOutgoingRevs from util.retry import retry from build.versions import bumpFile from release.info import readReleaseConfig, getTags, generateRelbranchName from release.l10n import getL10nRepositories HG = "hg.mozilla.org" DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url(HG, 'build/buildbot-configs') DEFAULT_MAX_PUSH_ATTEMPTS = 10 REQUIRED_CONFIG = ('version', 'appVersion', 'appName', 'productName', 'buildNumber', 'hgUsername', 'hgSshKey', 'baseTag', 'l10nRepoPath', 'sourceRepositories', 'l10nRevisionFile') REQUIRED_SOURCE_REPO_KEYS = ('path', 'revision') def getBumpCommitMessage(productName, version): return 'Automated checkin: version bump for ' + productName + ' ' + \ version + ' release. DONTBUILD CLOSED TREE a=release' def getTagCommitMessage(revision, tags): return "Added " + " ".join(tags) + " tag(s) for changeset " + revision + \ ". DONTBUILD CLOSED TREE a=release"
import os from os import path import site import logging import sys site.addsitedir(path.join(path.dirname(__file__), "../../lib/python")) from balrog.submitter.cli import ReleaseCreator, ReleasePusher from release.info import readReleaseConfig from util.retry import retry from util.hg import mercurial, make_hg_url HG = "hg.mozilla.org" DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url(HG, 'build/buildbot-configs') REQUIRED_CONFIG = ('appVersion', 'productName', 'version', 'enUSPlatforms', 'testChannels', 'releaseChannel', 'baseTag', 'buildNumber', 'partialUpdates', 'stagingServer', 'bouncerServer', 'testChannelRuleIds') def validate(options): err = False config = {} if not path.exists(path.join('buildbot-configs', options.release_config)): print "%s does not exist!" % options.release_config sys.exit(1) config = readReleaseConfig(path.join('buildbot-configs', options.release_config)) for key in REQUIRED_CONFIG:
def testMakeHGUrlDifferentProtocol(self): repo_url = make_hg_url("hg.mozilla.org", "/build/tools", protocol='ssh') expected_url = "ssh://hg.mozilla.org/build/tools" self.assertEquals(repo_url, expected_url)
def testMakeHGUrlNoFilename(self): file_url = make_hg_url("hg.mozilla.org", "/build/tools", revision="default") expected_url = "https://hg.mozilla.org/build/tools/rev/default" self.assertEquals(file_url, expected_url)
def testMakeHGUrlNoRevisionNoFilename(self): repo_url = make_hg_url("hg.mozilla.org", "/build/tools") expected_url = "https://hg.mozilla.org/build/tools" self.assertEquals(repo_url, expected_url)
def main(options): log.info('Loading config from %s' % options.config) config = load_config(options.config) if config.getboolean('release-runner', 'verbose'): log_level = logging.DEBUG else: log_level = logging.INFO logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=log_level) # Suppress logging of retry(), see bug 925321 for the details logging.getLogger("util.retry").setLevel(logging.WARN) check_buildbot() check_fabric() # Shorthand sendchange_master = config.get('release-runner', 'sendchange_master') api_root = config.get('api', 'api_root') username = config.get('api', 'username') password = config.get('api', 'password') hg_host = config.get('release-runner', 'hg_host') hg_username = config.get('release-runner', 'hg_username') hg_ssh_key = config.get('release-runner', 'hg_ssh_key') buildbot_configs = config.get('release-runner', 'buildbot_configs') buildbot_configs_branch = config.get('release-runner', 'buildbot_configs_branch') buildbotcustom = config.get('release-runner', 'buildbotcustom') buildbotcustom_branch = config.get('release-runner', 'buildbotcustom_branch') tools = config.get('release-runner', 'tools') tools_branch = config.get('release-runner', 'tools_branch') masters_json = config.get('release-runner', 'masters_json') staging = config.getboolean('release-runner', 'staging') sleeptime = config.getint('release-runner', 'sleeptime') notify_from = get_config(config, 'release-runner', 'notify_from', None) notify_to = get_config(config, 'release-runner', 'notify_to', None) ssh_username = get_config(config, 'release-runner', 'ssh_username', None) ssh_key = get_config(config, 'release-runner', 'ssh_key', None) if isinstance(notify_to, basestring): notify_to = [x.strip() for x in notify_to.split(',')] smtp_server = get_config(config, 'release-runner', 'smtp_server', 'localhost') configs_workdir = 'buildbot-configs' custom_workdir = 'buildbotcustom' tools_workdir = 'tools' if "://" in buildbot_configs and not buildbot_configs.startswith("file"): configs_pushRepo = make_hg_url(hg_host, get_repo_path(buildbot_configs), protocol='ssh') else: configs_pushRepo = buildbot_configs if "://" in buildbotcustom and not buildbotcustom.startswith("file"): custom_pushRepo = make_hg_url(hg_host, get_repo_path(buildbotcustom), protocol='ssh') else: custom_pushRepo = buildbotcustom if "://" in tools and not tools.startswith("file"): tools_pushRepo = make_hg_url(hg_host, get_repo_path(tools), protocol='ssh') else: tools_pushRepo = tools rr = ReleaseRunner(api_root=api_root, username=username, password=password) # Main loop waits for new releases, processes them and exits. while True: try: log.debug('Fetching release requests') rr.get_release_requests() if rr.new_releases: for release in rr.new_releases: log.info('Got a new release request: %s' % release) break else: log.debug('Sleeping for %d seconds before polling again' % sleeptime) time.sleep(sleeptime) except Exception, e: log.error("Caught exception when polling:", exc_info=True) sys.exit(5)
def verify_mozconfigs(branch, version, hghost, product, platforms, whitelist=None): """Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences""" if whitelist: mozconfigWhitelist = readConfig(whitelist, ['whitelist']) else: mozconfigWhitelist = {} log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product) success = True types = {'+': 'release', '-': 'nightly'} tag = ''.join( [product.upper(), "_", version.replace('.', '_'), "_RELEASE"]) for platform in platforms: urls = [] mozconfigs = [] for type in types.values(): urls.append( make_hg_url( hghost, 'build/buildbot-configs', 'http', tag, os.path.join('mozilla2', platform, branch, type, 'mozconfig'))) for url in urls: try: mozconfigs.append(urllib2.urlopen(url).readlines()) except urllib2.HTTPError as e: log.error("MISSING: %s - ERROR: %s" % (url, e.msg)) diffInstance = difflib.Differ() if len(mozconfigs) == 2: diffList = list(diffInstance.compare(mozconfigs[0], mozconfigs[1])) for line in diffList: clean_line = line[1:].strip() if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1: # skip comment lines if clean_line.startswith('#'): continue # compare to whitelist if line[0] == '-' and mozconfigWhitelist.get(branch, {}).has_key(platform) \ and clean_line in mozconfigWhitelist[branch][platform]: continue if line[0] == '+' and mozconfigWhitelist.get('nightly', {}).has_key(platform) \ and clean_line in mozconfigWhitelist['nightly'][platform]: continue if line[0] == '-': opposite = 'release' else: opposite = 'nightly' log.error( "not in %s mozconfig's whitelist (%s/%s/%s) : %s" % (opposite, branch, platform, types[line[0]], clean_line)) success = False error_tally.add('verify_mozconfig') else: log.info("Missing mozconfigs to compare for %s" % platform) error_tally.add( "verify_mozconfigs: Confirm that %s does not have release/nightly mozconfigs to compare" % platform) return success