def verify_mozconfigs(branch, revision, hghost, product, mozconfigs, appName, whitelist=None): """Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences""" if whitelist: mozconfigWhitelist = readConfig(whitelist, ['whitelist']) else: mozconfigWhitelist = {} log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product) success = True types = {'+': 'release', '-': 'nightly'} for platform,mozconfig in mozconfigs.items(): urls = [] mozconfigs = [] # Create links to the two mozconfigs. releaseConfig = make_hg_url(hghost, branch, 'http', revision, mozconfig) urls.append(releaseConfig) # The nightly one is the exact same URL, with the file part changed. urls.append(releaseConfig.rstrip('release') + 'nightly') for url in urls: try: mozconfigs.append(urllib2.urlopen(url).readlines()) except urllib2.HTTPError as e: log.error("MISSING: %s - ERROR: %s" % (url, e.msg)) diffInstance = difflib.Differ() if len(mozconfigs) == 2: diffList = list(diffInstance.compare(mozconfigs[0],mozconfigs[1])) for line in diffList: clean_line = line[1:].strip() if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1: # skip comment lines if clean_line.startswith('#'): continue # compare to whitelist if line[0] == '-' and mozconfigWhitelist.get(branch, {}).has_key(platform) \ and clean_line in mozconfigWhitelist[branch][platform]: continue if line[0] == '+' and mozconfigWhitelist.get('nightly', {}).has_key(platform) \ and clean_line in mozconfigWhitelist['nightly'][platform]: continue if line[0] == '-': opposite = 'release' else: opposite = 'nightly' log.error("not in %s mozconfig's whitelist (%s/%s/%s) : %s" % (opposite, branch, platform, types[line[0]], clean_line)) success = False error_tally.add('verify_mozconfig') else: log.info("Missing mozconfigs to compare for %s" % platform) error_tally.add("verify_mozconfigs: Confirm that %s does not have release/nightly mozconfigs to compare" % platform) return success
def verify_mozconfigs(branch, version, hghost, product, platforms, whitelist=None): """Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences""" if whitelist: mozconfigWhitelist = readConfig(whitelist, ['whitelist']) else: mozconfigWhitelist = {} log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product) success = True types = {'+': 'release', '-': 'nightly'} tag = ''.join([product.upper(), "_", version.replace('.','_'), "_RELEASE"]) for platform in platforms: urls = [] mozconfigs = [] for type in types.values(): urls.append(make_hg_url(hghost, 'build/buildbot-configs', 'http', tag, os.path.join('mozilla2', platform, branch, type,'mozconfig'))) for url in urls: try: mozconfigs.append(urllib2.urlopen(url).readlines()) except urllib2.HTTPError as e: log.error("MISSING: %s - ERROR: %s" % (url, e.msg)) diffInstance = difflib.Differ() if len(mozconfigs) == 2: diffList = list(diffInstance.compare(mozconfigs[0],mozconfigs[1])) for line in diffList: clean_line = line[1:].strip() if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1: # skip comment lines if clean_line.startswith('#'): continue # compare to whitelist if line[0] == '-' and mozconfigWhitelist.get(branch, {}).has_key(platform) \ and clean_line in mozconfigWhitelist[branch][platform]: continue if line[0] == '+' and mozconfigWhitelist.get('nightly', {}).has_key(platform) \ and clean_line in mozconfigWhitelist['nightly'][platform]: continue if line[0] == '-': opposite = 'release' else: opposite = 'nightly' log.error("not in %s mozconfig's whitelist (%s/%s/%s) : %s" % (opposite, branch, platform, types[line[0]], clean_line)) success = False error_tally.add('verify_mozconfig') else: log.info("Missing mozconfigs to compare for %s" % platform) error_tally.add("verify_mozconfigs: Confirm that %s does not have release/nightly mozconfigs to compare" % platform) return success
def verify_mozconfigs(branch, revision, hghost, product, mozconfigs, nightly_mozconfigs, whitelist=None): """Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences""" branch_name = get_repo_name(branch) if whitelist: mozconfigWhitelist = readConfig(whitelist, ['whitelist']) else: mozconfigWhitelist = {} log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product) success = True for platform, mozconfig in mozconfigs.items(): urls = [] mozconfigs = [] nightly_mozconfig = nightly_mozconfigs[platform] mozconfig_paths = [mozconfig, nightly_mozconfig] # Create links to the two mozconfigs. for c in mozconfig, nightly_mozconfig: urls.append(make_hg_url(hghost, branch, 'http', revision, c)) for url in urls: try: mozconfigs.append(urllib2.urlopen(url).readlines()) except urllib2.HTTPError as e: log.error("MISSING: %s - ERROR: %s" % (url, e.msg)) # Nothing to compare against return False diffInstance = difflib.Differ() if len(mozconfigs) == 2: diffList = list(diffInstance.compare(mozconfigs[0], mozconfigs[1])) for line in diffList: clean_line = line[1:].strip() if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1: # skip comment lines if clean_line.startswith('#'): continue # compare to whitelist message = "" if line[0] == '-': # handle lines that move around in diff if '+' + line[1:] in diffList: continue if platform in mozconfigWhitelist.get(branch_name, {}): if clean_line in \ mozconfigWhitelist[branch_name][platform]: continue elif line[0] == '+': if '-' + line[1:] in diffList: continue if platform in mozconfigWhitelist.get('nightly', {}): if clean_line in \ mozconfigWhitelist['nightly'][platform]: continue else: log.warning( "%s not in %s %s!" % (clean_line, platform, mozconfigWhitelist['nightly'][platform])) else: log.error("Skipping line %s!" % line) continue message = "found in %s but not in %s: %s" if line[0] == '-': log.error(message % (mozconfig_paths[0], mozconfig_paths[1], clean_line)) else: log.error(message % (mozconfig_paths[1], mozconfig_paths[0], clean_line)) success = False else: log.info("Missing mozconfigs to compare for %s" % platform) return False return success
mozillaSrcDir = None # If mozilla_dir is defined, extend the paths in makeDirs with the prefix # of the mozilla_dir if 'mozilla_dir' in releaseConfig: for i in range(0, len(makeDirs)): makeDirs[i] = path.join(releaseConfig['mozilla_dir'], makeDirs[i]) mozillaDir = releaseConfig['mozilla_dir'] mozillaSrcDir = releaseConfig['mozilla_dir'] elif 'mozilla_srcdir' in releaseConfig: mozillaSrcDir = releaseConfig['mozilla_srcdir'] if not options.tooltool_script: options.tooltool_script = ['/tools/tooltool.py'] if options.balrog_api_root: credentials = readConfig(options.credentials_file, required=['balrog_credentials']) auth = (options.balrog_username, credentials['balrog_credentials'][options.balrog_username]) balrog_submitter = ReleaseSubmitterV3(options.balrog_api_root, auth) else: balrog_submitter = None partialUpdates = releaseConfig.get('partialUpdates', {}).copy() createRepacks( sourceRepo=make_hg_url(branchConfig["hghost"], sourceRepoInfo["path"]), revision=options.releaseTag, l10nRepoDir=l10nRepoDir, l10nBaseRepo=make_hg_url(branchConfig["hghost"], releaseConfig["l10nRepoPath"]), mozconfigPath=mozconfig, srcMozconfigPath=releaseConfig.get('l10n_mozconfigs',
# If mozilla_dir is defined, extend the paths in makeDirs with the prefix # of the mozilla_dir if 'mozilla_dir' in releaseConfig: for i in range(0, len(makeDirs)): makeDirs[i] = path.join(releaseConfig['mozilla_dir'], makeDirs[i]) mozillaDir = releaseConfig['mozilla_dir'] mozillaSrcDir = releaseConfig['mozilla_dir'] elif 'mozilla_srcdir' in releaseConfig: mozillaSrcDir = releaseConfig['mozilla_srcdir'] if not options.tooltool_script: options.tooltool_script = ['/tools/tooltool.py'] if options.balrog_api_root: credentials = readConfig(options.credentials_file, required=['balrog_credentials'] ) auth = (options.balrog_username, credentials['balrog_credentials'][options.balrog_username]) balrog_submitter = ReleaseSubmitterV3(options.balrog_api_root, auth) else: balrog_submitter = None partialUpdates = releaseConfig.get('partialUpdates', {}).copy() # FIXME: the follwong hack can be removed when win64 has the same list of # partial update as other platforms. Check mozilla-esr38 to be sure. if platform in releaseConfig.get('HACK_first_released_version', {}): partialUpdates_copy = {} for k, v in partialUpdates.iteritems(): if LooseVersion(k) >= LooseVersion(releaseConfig['HACK_first_released_version'][platform]): partialUpdates_copy[k] = v partialUpdates = partialUpdates_copy
def verify_mozconfigs(branch, version, hghost, product, platforms, whitelist=None): """Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences""" if whitelist: mozconfigWhitelist = readConfig(whitelist, ['whitelist']) else: mozconfigWhitelist = {} log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product) success = True types = {'+': 'release', '-': 'nightly'} tag = ''.join( [product.upper(), "_", version.replace('.', '_'), "_RELEASE"]) for platform in platforms: urls = [] mozconfigs = [] for type in types.values(): urls.append( make_hg_url( hghost, 'build/buildbot-configs', 'http', tag, os.path.join('mozilla2', platform, branch, type, 'mozconfig'))) for url in urls: try: mozconfigs.append(urllib2.urlopen(url).readlines()) except urllib2.HTTPError as e: log.error("MISSING: %s - ERROR: %s" % (url, e.msg)) diffInstance = difflib.Differ() if len(mozconfigs) == 2: diffList = list(diffInstance.compare(mozconfigs[0], mozconfigs[1])) for line in diffList: clean_line = line[1:].strip() if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1: # skip comment lines if clean_line.startswith('#'): continue # compare to whitelist if line[0] == '-' and mozconfigWhitelist.get(branch, {}).has_key(platform) \ and clean_line in mozconfigWhitelist[branch][platform]: continue if line[0] == '+' and mozconfigWhitelist.get('nightly', {}).has_key(platform) \ and clean_line in mozconfigWhitelist['nightly'][platform]: continue if line[0] == '-': opposite = 'release' else: opposite = 'nightly' log.error( "not in %s mozconfig's whitelist (%s/%s/%s) : %s" % (opposite, branch, platform, types[line[0]], clean_line)) success = False error_tally.add('verify_mozconfig') else: log.info("Missing mozconfigs to compare for %s" % platform) error_tally.add( "verify_mozconfigs: Confirm that %s does not have release/nightly mozconfigs to compare" % platform) return success
stageUsername=stageUsername, stageSshKey=stageSshKey, productName=productName, version=version, target=ftpSymlinkName) if syncPartnerBundles: doSyncPartnerBundles(stageServer=stageServer, stageUsername=stageUsername, stageSshKey=stageSshKey, productName=productName, version=version, buildNumber=buildNumber) if bouncer_aliases and productName != 'xulrunner': credentials_file = path.join(os.getcwd(), "oauth.txt") credentials = readConfig( credentials_file, required=["tuxedoUsername", "tuxedoPassword"]) auth = (credentials["tuxedoUsername"], credentials["tuxedoPassword"]) update_bouncer_aliases( tuxedoServerUrl=releaseConfig["tuxedoServerUrl"], auth=auth, version=version, bouncer_aliases=bouncer_aliases) if 'product-details' in args: updateProductDetails( productName, version, productDetailsRepo,
def read_repo_setup_config(configfile): log.info('Reading from %s' % configfile) return readConfig(configfile, keys=['repoSetupConfig'])
def verify_mozconfigs(branch, revision, hghost, product, mozconfigs, nightly_mozconfigs, whitelist=None): """Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences""" branch_name = get_repo_name(branch) if whitelist: mozconfigWhitelist = readConfig(whitelist, ['whitelist']) else: mozconfigWhitelist = {} log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product) success = True for platform, mozconfig in mozconfigs.items(): urls = [] mozconfigs = [] nightly_mozconfig = nightly_mozconfigs[platform] mozconfig_paths = [mozconfig, nightly_mozconfig] # Create links to the two mozconfigs. for c in mozconfig, nightly_mozconfig: urls.append(make_hg_url(hghost, branch, 'http', revision, c)) for url in urls: try: mozconfigs.append(urllib2.urlopen(url).readlines()) except urllib2.HTTPError as e: log.error("MISSING: %s - ERROR: %s" % (url, e.msg)) # Nothing to compare against return False diffInstance = difflib.Differ() if len(mozconfigs) == 2: diffList = list(diffInstance.compare(mozconfigs[0], mozconfigs[1])) for line in diffList: clean_line = line[1:].strip() if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1: # skip comment lines if clean_line.startswith('#'): continue # compare to whitelist message = "" if line[0] == '-': # handle lines that move around in diff if '+' + line[1:] in diffList: continue if platform in mozconfigWhitelist.get(branch_name, {}): if clean_line in \ mozconfigWhitelist[branch_name][platform]: continue elif line[0] == '+': if '-' + line[1:] in diffList: continue if platform in mozconfigWhitelist.get('nightly', {}): if clean_line in \ mozconfigWhitelist['nightly'][platform]: continue else: log.warning("%s not in %s %s!" % ( clean_line, platform, mozconfigWhitelist['nightly'][platform])) else: log.error("Skipping line %s!" % line) continue message = "found in %s but not in %s: %s" if line[0] == '-': log.error(message % (mozconfig_paths[0], mozconfig_paths[1], clean_line)) else: log.error(message % (mozconfig_paths[1], mozconfig_paths[0], clean_line)) success = False else: log.info("Missing mozconfigs to compare for %s" % platform) return False return success
stageUsername=stageUsername, stageSshKey=stageSshKey, cleanup_dir=makeReleasesDir(productName, version)) if ftpSymlinkName: updateSymlink(stageServer=stageServer, stageUsername=stageUsername, stageSshKey=stageSshKey, productName=productName, version=version, target=ftpSymlinkName) if syncPartnerBundles: doSyncPartnerBundles(stageServer=stageServer, stageUsername=stageUsername, stageSshKey=stageSshKey, productName=productName, version=version, buildNumber=buildNumber) if bouncer_aliases and productName != 'xulrunner': credentials_file = path.join(os.getcwd(), "oauth.txt") credentials = readConfig( credentials_file, required=["tuxedoUsername", "tuxedoPassword"]) auth = (credentials["tuxedoUsername"], credentials["tuxedoPassword"]) update_bouncer_aliases( tuxedoServerUrl=releaseConfig["tuxedoServerUrl"], auth=auth, version=version, bouncer_aliases=bouncer_aliases)