def verify_mozconfigs(branch, revision, hghost, product, mozconfigs, appName, whitelist=None):
"""Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences"""
if whitelist:
mozconfigWhitelist = readConfig(whitelist, ['whitelist'])
else:
mozconfigWhitelist = {}
log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product)
success = True
types = {'+': 'release', '-': 'nightly'}
for platform,mozconfig in mozconfigs.items():
urls = []
mozconfigs = []
# Create links to the two mozconfigs.
releaseConfig = make_hg_url(hghost, branch, 'http', revision, mozconfig)
urls.append(releaseConfig)
# The nightly one is the exact same URL, with the file part changed.
urls.append(releaseConfig.rstrip('release') + 'nightly')
for url in urls:
try:
mozconfigs.append(urllib2.urlopen(url).readlines())
except urllib2.HTTPError as e:
log.error("MISSING: %s - ERROR: %s" % (url, e.msg))
diffInstance = difflib.Differ()
if len(mozconfigs) == 2:
diffList = list(diffInstance.compare(mozconfigs[0],mozconfigs[1]))
for line in diffList:
clean_line = line[1:].strip()
if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1:
# skip comment lines
if clean_line.startswith('#'):
continue
# compare to whitelist
if line[0] == '-' and mozconfigWhitelist.get(branch, {}).has_key(platform) \
and clean_line in mozconfigWhitelist[branch][platform]:
continue
if line[0] == '+' and mozconfigWhitelist.get('nightly', {}).has_key(platform) \
and clean_line in mozconfigWhitelist['nightly'][platform]:
continue
if line[0] == '-':
opposite = 'release'
else:
opposite = 'nightly'
log.error("not in %s mozconfig's whitelist (%s/%s/%s) : %s" % (opposite, branch, platform, types[line[0]], clean_line))
success = False
error_tally.add('verify_mozconfig')
else:
log.info("Missing mozconfigs to compare for %s" % platform)
error_tally.add("verify_mozconfigs: Confirm that %s does not have release/nightly mozconfigs to compare" % platform)
return success
def verify_mozconfigs(branch, version, hghost, product, platforms, whitelist=None):
"""Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences"""
if whitelist:
mozconfigWhitelist = readConfig(whitelist, ['whitelist'])
else:
mozconfigWhitelist = {}
log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product)
success = True
types = {'+': 'release', '-': 'nightly'}
tag = ''.join([product.upper(), "_", version.replace('.','_'), "_RELEASE"])
for platform in platforms:
urls = []
mozconfigs = []
for type in types.values():
urls.append(make_hg_url(hghost, 'build/buildbot-configs', 'http',
tag, os.path.join('mozilla2', platform,
branch, type,'mozconfig')))
for url in urls:
try:
mozconfigs.append(urllib2.urlopen(url).readlines())
except urllib2.HTTPError as e:
log.error("MISSING: %s - ERROR: %s" % (url, e.msg))
diffInstance = difflib.Differ()
if len(mozconfigs) == 2:
diffList = list(diffInstance.compare(mozconfigs[0],mozconfigs[1]))
for line in diffList:
clean_line = line[1:].strip()
if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1:
# skip comment lines
if clean_line.startswith('#'):
continue
# compare to whitelist
if line[0] == '-' and mozconfigWhitelist.get(branch, {}).has_key(platform) \
and clean_line in mozconfigWhitelist[branch][platform]:
continue
if line[0] == '+' and mozconfigWhitelist.get('nightly', {}).has_key(platform) \
and clean_line in mozconfigWhitelist['nightly'][platform]:
continue
if line[0] == '-':
opposite = 'release'
else:
opposite = 'nightly'
log.error("not in %s mozconfig's whitelist (%s/%s/%s) : %s" % (opposite, branch, platform, types[line[0]], clean_line))
success = False
error_tally.add('verify_mozconfig')
else:
log.info("Missing mozconfigs to compare for %s" % platform)
error_tally.add("verify_mozconfigs: Confirm that %s does not have release/nightly mozconfigs to compare" % platform)
return success
def verify_mozconfigs(branch,
revision,
hghost,
product,
mozconfigs,
nightly_mozconfigs,
whitelist=None):
"""Compare nightly mozconfigs for branch to release mozconfigs and
compare to whitelist of known differences"""
branch_name = get_repo_name(branch)
if whitelist:
mozconfigWhitelist = readConfig(whitelist, ['whitelist'])
else:
mozconfigWhitelist = {}
log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product)
success = True
for platform, mozconfig in mozconfigs.items():
urls = []
mozconfigs = []
nightly_mozconfig = nightly_mozconfigs[platform]
mozconfig_paths = [mozconfig, nightly_mozconfig]
# Create links to the two mozconfigs.
for c in mozconfig, nightly_mozconfig:
urls.append(make_hg_url(hghost, branch, 'http', revision, c))
for url in urls:
try:
mozconfigs.append(urllib2.urlopen(url).readlines())
except urllib2.HTTPError as e:
log.error("MISSING: %s - ERROR: %s" % (url, e.msg))
# Nothing to compare against
return False
diffInstance = difflib.Differ()
if len(mozconfigs) == 2:
diffList = list(diffInstance.compare(mozconfigs[0], mozconfigs[1]))
for line in diffList:
clean_line = line[1:].strip()
if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1:
# skip comment lines
if clean_line.startswith('#'):
continue
# compare to whitelist
message = ""
if line[0] == '-':
# handle lines that move around in diff
if '+' + line[1:] in diffList:
continue
if platform in mozconfigWhitelist.get(branch_name, {}):
if clean_line in \
mozconfigWhitelist[branch_name][platform]:
continue
elif line[0] == '+':
if '-' + line[1:] in diffList:
continue
if platform in mozconfigWhitelist.get('nightly', {}):
if clean_line in \
mozconfigWhitelist['nightly'][platform]:
continue
else:
log.warning(
"%s not in %s %s!" %
(clean_line, platform,
mozconfigWhitelist['nightly'][platform]))
else:
log.error("Skipping line %s!" % line)
continue
message = "found in %s but not in %s: %s"
if line[0] == '-':
log.error(message % (mozconfig_paths[0],
mozconfig_paths[1], clean_line))
else:
log.error(message % (mozconfig_paths[1],
mozconfig_paths[0], clean_line))
success = False
else:
log.info("Missing mozconfigs to compare for %s" % platform)
return False
return success
mozillaSrcDir = None
# If mozilla_dir is defined, extend the paths in makeDirs with the prefix
# of the mozilla_dir
if 'mozilla_dir' in releaseConfig:
for i in range(0, len(makeDirs)):
makeDirs[i] = path.join(releaseConfig['mozilla_dir'], makeDirs[i])
mozillaDir = releaseConfig['mozilla_dir']
mozillaSrcDir = releaseConfig['mozilla_dir']
elif 'mozilla_srcdir' in releaseConfig:
mozillaSrcDir = releaseConfig['mozilla_srcdir']
if not options.tooltool_script:
options.tooltool_script = ['/tools/tooltool.py']
if options.balrog_api_root:
credentials = readConfig(options.credentials_file,
required=['balrog_credentials'])
auth = (options.balrog_username,
credentials['balrog_credentials'][options.balrog_username])
balrog_submitter = ReleaseSubmitterV3(options.balrog_api_root, auth)
else:
balrog_submitter = None
partialUpdates = releaseConfig.get('partialUpdates', {}).copy()
createRepacks(
sourceRepo=make_hg_url(branchConfig["hghost"], sourceRepoInfo["path"]),
revision=options.releaseTag,
l10nRepoDir=l10nRepoDir,
l10nBaseRepo=make_hg_url(branchConfig["hghost"],
releaseConfig["l10nRepoPath"]),
mozconfigPath=mozconfig,
srcMozconfigPath=releaseConfig.get('l10n_mozconfigs',
# If mozilla_dir is defined, extend the paths in makeDirs with the prefix
# of the mozilla_dir
if 'mozilla_dir' in releaseConfig:
for i in range(0, len(makeDirs)):
makeDirs[i] = path.join(releaseConfig['mozilla_dir'], makeDirs[i])
mozillaDir = releaseConfig['mozilla_dir']
mozillaSrcDir = releaseConfig['mozilla_dir']
elif 'mozilla_srcdir' in releaseConfig:
mozillaSrcDir = releaseConfig['mozilla_srcdir']
if not options.tooltool_script:
options.tooltool_script = ['/tools/tooltool.py']
if options.balrog_api_root:
credentials = readConfig(options.credentials_file,
required=['balrog_credentials']
)
auth = (options.balrog_username, credentials['balrog_credentials'][options.balrog_username])
balrog_submitter = ReleaseSubmitterV3(options.balrog_api_root, auth)
else:
balrog_submitter = None
partialUpdates = releaseConfig.get('partialUpdates', {}).copy()
# FIXME: the follwong hack can be removed when win64 has the same list of
# partial update as other platforms. Check mozilla-esr38 to be sure.
if platform in releaseConfig.get('HACK_first_released_version', {}):
partialUpdates_copy = {}
for k, v in partialUpdates.iteritems():
if LooseVersion(k) >= LooseVersion(releaseConfig['HACK_first_released_version'][platform]):
partialUpdates_copy[k] = v
partialUpdates = partialUpdates_copy
def verify_mozconfigs(branch,
version,
hghost,
product,
platforms,
whitelist=None):
"""Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences"""
if whitelist:
mozconfigWhitelist = readConfig(whitelist, ['whitelist'])
else:
mozconfigWhitelist = {}
log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product)
success = True
types = {'+': 'release', '-': 'nightly'}
tag = ''.join(
[product.upper(), "_",
version.replace('.', '_'), "_RELEASE"])
for platform in platforms:
urls = []
mozconfigs = []
for type in types.values():
urls.append(
make_hg_url(
hghost, 'build/buildbot-configs', 'http', tag,
os.path.join('mozilla2', platform, branch, type,
'mozconfig')))
for url in urls:
try:
mozconfigs.append(urllib2.urlopen(url).readlines())
except urllib2.HTTPError as e:
log.error("MISSING: %s - ERROR: %s" % (url, e.msg))
diffInstance = difflib.Differ()
if len(mozconfigs) == 2:
diffList = list(diffInstance.compare(mozconfigs[0], mozconfigs[1]))
for line in diffList:
clean_line = line[1:].strip()
if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1:
# skip comment lines
if clean_line.startswith('#'):
continue
# compare to whitelist
if line[0] == '-' and mozconfigWhitelist.get(branch, {}).has_key(platform) \
and clean_line in mozconfigWhitelist[branch][platform]:
continue
if line[0] == '+' and mozconfigWhitelist.get('nightly', {}).has_key(platform) \
and clean_line in mozconfigWhitelist['nightly'][platform]:
continue
if line[0] == '-':
opposite = 'release'
else:
opposite = 'nightly'
log.error(
"not in %s mozconfig's whitelist (%s/%s/%s) : %s" %
(opposite, branch, platform, types[line[0]],
clean_line))
success = False
error_tally.add('verify_mozconfig')
else:
log.info("Missing mozconfigs to compare for %s" % platform)
error_tally.add(
"verify_mozconfigs: Confirm that %s does not have release/nightly mozconfigs to compare"
% platform)
return success
stageUsername=stageUsername,
stageSshKey=stageSshKey,
productName=productName,
version=version,
target=ftpSymlinkName)
if syncPartnerBundles:
doSyncPartnerBundles(stageServer=stageServer,
stageUsername=stageUsername,
stageSshKey=stageSshKey,
productName=productName,
version=version,
buildNumber=buildNumber)
if bouncer_aliases and productName != 'xulrunner':
credentials_file = path.join(os.getcwd(), "oauth.txt")
credentials = readConfig(
credentials_file,
required=["tuxedoUsername", "tuxedoPassword"])
auth = (credentials["tuxedoUsername"],
credentials["tuxedoPassword"])
update_bouncer_aliases(
tuxedoServerUrl=releaseConfig["tuxedoServerUrl"],
auth=auth,
version=version,
bouncer_aliases=bouncer_aliases)
if 'product-details' in args:
updateProductDetails(
productName,
version,
productDetailsRepo,
def read_repo_setup_config(configfile):
log.info('Reading from %s' % configfile)
return readConfig(configfile, keys=['repoSetupConfig'])
def verify_mozconfigs(branch, revision, hghost, product, mozconfigs,
nightly_mozconfigs, whitelist=None):
"""Compare nightly mozconfigs for branch to release mozconfigs and
compare to whitelist of known differences"""
branch_name = get_repo_name(branch)
if whitelist:
mozconfigWhitelist = readConfig(whitelist, ['whitelist'])
else:
mozconfigWhitelist = {}
log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product)
success = True
for platform, mozconfig in mozconfigs.items():
urls = []
mozconfigs = []
nightly_mozconfig = nightly_mozconfigs[platform]
mozconfig_paths = [mozconfig, nightly_mozconfig]
# Create links to the two mozconfigs.
for c in mozconfig, nightly_mozconfig:
urls.append(make_hg_url(hghost, branch, 'http', revision, c))
for url in urls:
try:
mozconfigs.append(urllib2.urlopen(url).readlines())
except urllib2.HTTPError as e:
log.error("MISSING: %s - ERROR: %s" % (url, e.msg))
# Nothing to compare against
return False
diffInstance = difflib.Differ()
if len(mozconfigs) == 2:
diffList = list(diffInstance.compare(mozconfigs[0], mozconfigs[1]))
for line in diffList:
clean_line = line[1:].strip()
if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1:
# skip comment lines
if clean_line.startswith('#'):
continue
# compare to whitelist
message = ""
if line[0] == '-':
# handle lines that move around in diff
if '+' + line[1:] in diffList:
continue
if platform in mozconfigWhitelist.get(branch_name, {}):
if clean_line in \
mozconfigWhitelist[branch_name][platform]:
continue
elif line[0] == '+':
if '-' + line[1:] in diffList:
continue
if platform in mozconfigWhitelist.get('nightly', {}):
if clean_line in \
mozconfigWhitelist['nightly'][platform]:
continue
else:
log.warning("%s not in %s %s!" % (
clean_line, platform,
mozconfigWhitelist['nightly'][platform]))
else:
log.error("Skipping line %s!" % line)
continue
message = "found in %s but not in %s: %s"
if line[0] == '-':
log.error(message % (mozconfig_paths[0],
mozconfig_paths[1], clean_line))
else:
log.error(message % (mozconfig_paths[1],
mozconfig_paths[0], clean_line))
success = False
else:
log.info("Missing mozconfigs to compare for %s" % platform)
return False
return success
stageUsername=stageUsername,
stageSshKey=stageSshKey,
cleanup_dir=makeReleasesDir(productName, version))
if ftpSymlinkName:
updateSymlink(stageServer=stageServer,
stageUsername=stageUsername,
stageSshKey=stageSshKey,
productName=productName,
version=version,
target=ftpSymlinkName)
if syncPartnerBundles:
doSyncPartnerBundles(stageServer=stageServer,
stageUsername=stageUsername,
stageSshKey=stageSshKey,
productName=productName,
version=version,
buildNumber=buildNumber)
if bouncer_aliases and productName != 'xulrunner':
credentials_file = path.join(os.getcwd(), "oauth.txt")
credentials = readConfig(
credentials_file,
required=["tuxedoUsername", "tuxedoPassword"])
auth = (credentials["tuxedoUsername"],
credentials["tuxedoPassword"])
update_bouncer_aliases(
tuxedoServerUrl=releaseConfig["tuxedoServerUrl"],
auth=auth,
version=version,
bouncer_aliases=bouncer_aliases)
def read_repo_setup_config(configfile):
log.info('Reading from %s' % configfile)
return readConfig(configfile, keys=['repoSetupConfig'])
