def downloadReleaseBuilds(stageServer, productName, brandName, version,
buildNumber, platform, candidatesDir=None,
signed=False, usePymake=False):
if candidatesDir is None:
candidatesDir = makeCandidatesDir(productName, version, buildNumber,
protocol='http', server=stageServer)
files = makeReleaseRepackUrls(productName, brandName, version, platform,
signed=signed)
env = {}
for fileName, remoteFile in files.iteritems():
url = '/'.join([p.strip('/') for p in [candidatesDir,
urllib.quote(remoteFile)]])
log.info("Downloading %s to %s", url, fileName)
urlretrieve(url, fileName)
if fileName.endswith('exe'):
if usePymake:
env['WIN32_INSTALLER_IN'] = msys2windows(path.join(os.getcwd(),
fileName))
else:
env['WIN32_INSTALLER_IN'] = windows2msys(path.join(os.getcwd(),
fileName))
else:
if platform.startswith('win') and not usePymake:
env['ZIP_IN'] = windows2msys(path.join(os.getcwd(), fileName))
else:
env['ZIP_IN'] = msys2windows(path.join(os.getcwd(), fileName))
return env
def downloadReleaseBuilds(stageServer,
productName,
brandName,
version,
buildNumber,
platform,
candidatesDir=None,
signed=False,
usePymake=False):
if candidatesDir is None:
candidatesDir = makeCandidatesDir(productName,
version,
buildNumber,
protocol='http',
server=stageServer)
files = makeReleaseRepackUrls(productName,
brandName,
version,
platform,
signed=signed)
env = {}
for fileName, remoteFile in files.iteritems():
url = '/'.join(
[p.strip('/')
for p in [candidatesDir, urllib.quote(remoteFile)]])
log.info("Downloading %s to %s", url, fileName)
for _ in retrier():
with open(fileName, "wb") as f:
try:
r = requests.get(url, stream=True, timeout=15)
r.raise_for_status()
for chunk in r.iter_content(chunk_size=5 * 1024**2):
f.write(chunk)
r.close()
break
except (requests.HTTPError, requests.ConnectionError,
requests.Timeout):
log.exception("Caught exception downloading")
if fileName.endswith('exe'):
if usePymake:
env['WIN32_INSTALLER_IN'] = msys2windows(
path.join(os.getcwd(), fileName))
else:
env['WIN32_INSTALLER_IN'] = windows2msys(
path.join(os.getcwd(), fileName))
else:
if platform.startswith('win') and not usePymake:
env['ZIP_IN'] = windows2msys(path.join(os.getcwd(), fileName))
else:
env['ZIP_IN'] = msys2windows(path.join(os.getcwd(), fileName))
return env
def downloadReleaseBuilds(stageServer, productName, brandName, version,
buildNumber, platform, candidatesDir=None,
signed=False, usePymake=False):
if candidatesDir is None:
candidatesDir = makeCandidatesDir(productName, version, buildNumber,
protocol='http', server=stageServer)
files = makeReleaseRepackUrls(productName, brandName, version, platform,
signed=signed)
env = {}
for fileName, remoteFile in files.iteritems():
url = '/'.join([p.strip('/') for p in [candidatesDir,
urllib.quote(remoteFile)]])
log.info("Downloading %s to %s", url, fileName)
for _ in retrier():
with open(fileName, "wb") as f:
try:
r = requests.get(url, stream=True, timeout=15)
r.raise_for_status()
for chunk in r.iter_content(chunk_size=5*1024**2):
f.write(chunk)
r.close()
break
except (requests.HTTPError, requests.ConnectionError,
requests.Timeout):
log.exception("Caught exception downloading")
if fileName.endswith('exe'):
if usePymake:
env['WIN32_INSTALLER_IN'] = msys2windows(path.join(os.getcwd(),
fileName))
else:
env['WIN32_INSTALLER_IN'] = windows2msys(path.join(os.getcwd(),
fileName))
else:
if platform.startswith('win') and not usePymake:
env['ZIP_IN'] = windows2msys(path.join(os.getcwd(), fileName))
else:
env['ZIP_IN'] = msys2windows(path.join(os.getcwd(), fileName))
return env
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir,
l10nIni, compareLocalesRepo, env, merge=True,
productName=None, platform=None,
version=None, partialUpdates=None,
buildNumber=None, stageServer=None):
repo = "/".join([l10nBaseRepo, locale])
localeDir = path.join(l10nRepoDir, locale)
retry(mercurial, args=(repo, localeDir))
update(localeDir, revision=revision)
mozillaDir = ''
if 'thunderbird' in productName:
mozillaDir = 'mozilla/'
# split on \\ since we care about the absSourceRepoPath for pymake, which
# is windows.
absSourceRepoPath = os.path.join(os.getcwd(), localeSrcDir.split("\\")[0])
use_pymake = env.get("USE_PYMAKE", False)
make = getMakeCommand(use_pymake, absSourceRepoPath)
env["AB_CD"] = locale
env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged"))
if sys.platform.startswith('win'):
if use_pymake:
env["LOCALE_MERGEDIR"] = msys2windows(env["LOCALE_MERGEDIR"])
else:
env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"])
if sys.platform.startswith('darwin'):
env["MOZ_PKG_PLATFORM"] = "mac"
UPLOAD_EXTRA_FILES = []
nativeDistDir = path.normpath(path.abspath(
path.join(localeSrcDir, '../../%sdist' % mozillaDir)))
posixDistDir = windows2msys(nativeDistDir)
mar = '%s/host/bin/mar' % posixDistDir
mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir
if platform.startswith('win'):
mar += ".exe"
mbsdiff += ".exe"
current = '%s/current' % posixDistDir
previous = '%s/previous' % posixDistDir
updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale)
updateAbsDir = '%s/%s' % (posixDistDir, updateDir)
current_mar = '%s/%s-%s.complete.mar' % (
updateAbsDir, productName, version)
unwrap_full_update = '../../../tools/update-packaging/unwrap_full_update.pl'
make_incremental_update = '../../tools/update-packaging/make_incremental_update.sh'
prevMarDir = '../../../../'
if mozillaDir:
unwrap_full_update = '../../../../%stools/update-packaging/unwrap_full_update.pl' % mozillaDir
make_incremental_update = '../../../%stools/update-packaging/make_incremental_update.sh' % mozillaDir
prevMarDir = '../../../../../'
env['MAR'] = mar
env['MBSDIFF'] = mbsdiff
log.info("Download mar tools")
if stageServer:
candidates_dir = makeCandidatesDir(productName, version, buildNumber,
protocol="http", server=stageServer)
if not path.isfile(msys2windows(mar)):
marUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mar)s" % \
dict(c_dir=candidates_dir, platform=platform,
mar=path.basename(mar))
run_cmd(['mkdir', '-p', path.dirname(mar)])
log.info("Downloading %s to %s", marUrl, mar)
urlretrieve(marUrl, msys2windows(mar))
if not sys.platform.startswith('win'):
run_cmd(['chmod', '755', mar])
if not path.isfile(msys2windows(mbsdiff)):
mbsdiffUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mbsdiff)s" % \
dict(c_dir=candidates_dir, platform=platform,
mbsdiff=path.basename(mbsdiff))
run_cmd(['mkdir', '-p', path.dirname(mbsdiff)])
log.info("Downloading %s to %s", mbsdiffUrl, mbsdiff)
urlretrieve(mbsdiffUrl, msys2windows(mbsdiff))
if not sys.platform.startswith('win'):
run_cmd(['chmod', '755', mbsdiff])
else:
log.warning('stageServer not set. mar tools will *not* be downloaded.')
compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir,
l10nIni, revision=revision, merge=merge)
run_cmd(make + ["installers-%s" % locale], cwd=localeSrcDir, env=env)
run_cmd(['rm', '-rf', current])
run_cmd(['mkdir', current])
run_cmd(['perl', unwrap_full_update, current_mar],
cwd=path.join(nativeDistDir, 'current'), env=env)
for oldVersion in partialUpdates:
prevMar = partialUpdates[oldVersion]['mar']
if prevMar:
partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion,
version)
partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name)
UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name))
run_cmd(['rm', '-rf', previous])
run_cmd(['mkdir', previous])
run_cmd(
['perl', unwrap_full_update, '%s/%s' % (prevMarDir, prevMar)],
cwd=path.join(nativeDistDir, 'previous'), env=env)
run_cmd(['bash', make_incremental_update, partial_mar, previous,
current], cwd=nativeDistDir, env=env)
if os.environ.get('MOZ_SIGN_CMD'):
run_cmd(['bash', '-c',
'%s -f mar -f gpg "%s"' %
(os.environ['MOZ_SIGN_CMD'], partial_mar)],
env=env)
UPLOAD_EXTRA_FILES.append(
'%s/%s.asc' % (updateDir, partial_mar_name))
else:
log.warning(
"Skipping partial MAR creation for %s %s" % (oldVersion,
locale))
env['UPLOAD_EXTRA_FILES'] = ' '.join(UPLOAD_EXTRA_FILES)
retry(run_cmd,
args=(make + ["upload", "AB_CD=%s" % locale], ),
kwargs={'cwd': localeSrcDir, 'env': env})
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir,
l10nIni, compareLocalesRepo, env, absObjdir, merge=True,
productName=None, platform=None,
version=None, partialUpdates=None,
buildNumber=None, stageServer=None,
mozillaDir=None, mozillaSrcDir=None):
repo = "/".join([l10nBaseRepo, locale])
localeDir = path.join(l10nRepoDir, locale)
mercurial(repo, localeDir)
update(localeDir, revision=revision)
# It's a bad assumption to make, but the source dir is currently always
# one level above the objdir.
absSourceRepoPath = path.split(absObjdir)[0]
use_pymake = env.get("USE_PYMAKE", False)
make = getMakeCommand(use_pymake, absSourceRepoPath)
env["AB_CD"] = locale
env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged"))
if sys.platform.startswith('win'):
if use_pymake:
env["LOCALE_MERGEDIR"] = msys2windows(env["LOCALE_MERGEDIR"])
else:
env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"])
if sys.platform.startswith('darwin'):
env["MOZ_PKG_PLATFORM"] = "mac"
UPLOAD_EXTRA_FILES = []
if mozillaDir:
nativeDistDir = path.normpath(path.abspath(
path.join(localeSrcDir, '../../%s/dist' % mozillaDir)))
else:
nativeDistDir = path.normpath(path.abspath(
path.join(localeSrcDir, '../../dist')))
posixDistDir = windows2msys(nativeDistDir)
mar = '%s/host/bin/mar' % posixDistDir
mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir
if platform.startswith('win'):
mar += ".exe"
mbsdiff += ".exe"
current = '%s/current' % posixDistDir
previous = '%s/previous' % posixDistDir
updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale)
updateAbsDir = '%s/%s' % (posixDistDir, updateDir)
current_mar = '%s/%s-%s.complete.mar' % (
updateAbsDir, productName, version)
unwrap_full_update = '../../../tools/update-packaging/unwrap_full_update.pl'
make_incremental_update = '../../tools/update-packaging/make_incremental_update.sh'
prevMarDir = '../../../../'
if mozillaSrcDir:
# Compensate for having the objdir or not.
additionalParent = ''
if mozillaDir:
additionalParent = '../'
unwrap_full_update = '../../../%s%s/tools/update-packaging/unwrap_full_update.pl' % (additionalParent, mozillaSrcDir)
make_incremental_update = '../../%s%s/tools/update-packaging/make_incremental_update.sh' % (additionalParent, mozillaSrcDir)
prevMarDir = '../../../../%s' % additionalParent
env['MAR'] = mar
env['MBSDIFF'] = mbsdiff
log.info("Download mar tools")
if stageServer:
candidates_dir = makeCandidatesDir(productName, version, buildNumber,
protocol="http", server=stageServer)
if not path.isfile(msys2windows(mar)):
marUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mar)s" % \
dict(c_dir=candidates_dir, platform=platform,
mar=path.basename(mar))
run_cmd(['mkdir', '-p', path.dirname(mar)])
log.info("Downloading %s to %s", marUrl, mar)
urlretrieve(marUrl, msys2windows(mar))
if not sys.platform.startswith('win'):
run_cmd(['chmod', '755', mar])
if not path.isfile(msys2windows(mbsdiff)):
mbsdiffUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mbsdiff)s" % \
dict(c_dir=candidates_dir, platform=platform,
mbsdiff=path.basename(mbsdiff))
run_cmd(['mkdir', '-p', path.dirname(mbsdiff)])
log.info("Downloading %s to %s", mbsdiffUrl, mbsdiff)
urlretrieve(mbsdiffUrl, msys2windows(mbsdiff))
if not sys.platform.startswith('win'):
run_cmd(['chmod', '755', mbsdiff])
else:
log.warning('stageServer not set. mar tools will *not* be downloaded.')
compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir,
l10nIni, revision=revision, merge=merge)
run_cmd(make + ["installers-%s" % locale], cwd=localeSrcDir, env=env)
# Our Windows-native rm from bug 727551 requires Windows-style paths
run_cmd(['rm', '-rf', msys2windows(current)])
run_cmd(['mkdir', current])
run_cmd(['perl', unwrap_full_update, current_mar],
cwd=path.join(nativeDistDir, 'current'), env=env)
for oldVersion in partialUpdates:
prevMar = partialUpdates[oldVersion]['mar']
if prevMar:
partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion,
version)
partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name)
UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name))
# Our Windows-native rm from bug 727551 requires Windows-style paths
run_cmd(['rm', '-rf', msys2windows(previous)])
run_cmd(['mkdir', previous])
run_cmd(
['perl', unwrap_full_update, '%s/%s' % (prevMarDir, prevMar)],
cwd=path.join(nativeDistDir, 'previous'), env=env)
run_cmd(['bash', make_incremental_update, partial_mar, previous,
current], cwd=nativeDistDir, env=env)
if os.environ.get('MOZ_SIGN_CMD'):
run_cmd(['bash', '-c',
'%s -f mar -f gpg "%s"' %
(os.environ['MOZ_SIGN_CMD'], partial_mar)],
env=env)
UPLOAD_EXTRA_FILES.append(
'%s/%s.asc' % (updateDir, partial_mar_name))
else:
log.warning(
"Skipping partial MAR creation for %s %s" % (oldVersion,
locale))
env['UPLOAD_EXTRA_FILES'] = ' '.join(UPLOAD_EXTRA_FILES)
retry(run_cmd,
args=(make + ["upload", "AB_CD=%s" % locale], ),
kwargs={'cwd': localeSrcDir, 'env': env})
# return the location of the checksums file, because consumers may want
# some information about the files that were generated.
# Some versions of make that we use (at least pymake) imply --print-directory
# We need to turn it off to avoid getting extra output that mess up our
# parsing of the checksum file path.
curdir = os.getcwd()
try:
os.chdir(localeSrcDir)
relative_checksums = get_output(make +
["--no-print-directory", "echo-variable-CHECKSUM_FILE", "AB_CD=%s" % locale],
env=env).strip("\"'\n")
return path.normpath(path.join(localeSrcDir, relative_checksums))
finally:
os.chdir(curdir)
def repackLocale(locale,
l10nRepoDir,
l10nBaseRepo,
revision,
localeSrcDir,
l10nIni,
compareLocalesRepo,
env,
absObjdir,
merge=True,
productName=None,
platform=None,
version=None,
partialUpdates=None,
buildNumber=None,
stageServer=None):
repo = "/".join([l10nBaseRepo, locale])
localeDir = path.join(l10nRepoDir, locale)
retry(mercurial, args=(repo, localeDir))
update(localeDir, revision=revision)
mozillaDir = ''
if 'thunderbird' in productName:
mozillaDir = 'mozilla/'
# It's a bad assumption to make, but the source dir is currently always
# one level above the objdir.
absSourceRepoPath = path.split(absObjdir)[0]
use_pymake = env.get("USE_PYMAKE", False)
make = getMakeCommand(use_pymake, absSourceRepoPath)
env["AB_CD"] = locale
env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged"))
if sys.platform.startswith('win'):
if use_pymake:
env["LOCALE_MERGEDIR"] = msys2windows(env["LOCALE_MERGEDIR"])
else:
env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"])
if sys.platform.startswith('darwin'):
env["MOZ_PKG_PLATFORM"] = "mac"
UPLOAD_EXTRA_FILES = []
nativeDistDir = path.normpath(
path.abspath(path.join(localeSrcDir, '../../%sdist' % mozillaDir)))
posixDistDir = windows2msys(nativeDistDir)
mar = '%s/host/bin/mar' % posixDistDir
mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir
if platform.startswith('win'):
mar += ".exe"
mbsdiff += ".exe"
current = '%s/current' % posixDistDir
previous = '%s/previous' % posixDistDir
updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale)
updateAbsDir = '%s/%s' % (posixDistDir, updateDir)
current_mar = '%s/%s-%s.complete.mar' % (updateAbsDir, productName,
version)
unwrap_full_update = '../../../tools/update-packaging/unwrap_full_update.pl'
make_incremental_update = '../../tools/update-packaging/make_incremental_update.sh'
prevMarDir = '../../../../'
if mozillaDir:
unwrap_full_update = '../../../../%stools/update-packaging/unwrap_full_update.pl' % mozillaDir
make_incremental_update = '../../../%stools/update-packaging/make_incremental_update.sh' % mozillaDir
prevMarDir = '../../../../../'
env['MAR'] = mar
env['MBSDIFF'] = mbsdiff
log.info("Download mar tools")
if stageServer:
candidates_dir = makeCandidatesDir(productName,
version,
buildNumber,
protocol="http",
server=stageServer)
if not path.isfile(msys2windows(mar)):
marUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mar)s" % \
dict(c_dir=candidates_dir, platform=platform,
mar=path.basename(mar))
run_cmd(['mkdir', '-p', path.dirname(mar)])
log.info("Downloading %s to %s", marUrl, mar)
urlretrieve(marUrl, msys2windows(mar))
if not sys.platform.startswith('win'):
run_cmd(['chmod', '755', mar])
if not path.isfile(msys2windows(mbsdiff)):
mbsdiffUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mbsdiff)s" % \
dict(c_dir=candidates_dir, platform=platform,
mbsdiff=path.basename(mbsdiff))
run_cmd(['mkdir', '-p', path.dirname(mbsdiff)])
log.info("Downloading %s to %s", mbsdiffUrl, mbsdiff)
urlretrieve(mbsdiffUrl, msys2windows(mbsdiff))
if not sys.platform.startswith('win'):
run_cmd(['chmod', '755', mbsdiff])
else:
log.warning('stageServer not set. mar tools will *not* be downloaded.')
compareLocales(compareLocalesRepo,
locale,
l10nRepoDir,
localeSrcDir,
l10nIni,
revision=revision,
merge=merge)
run_cmd(make + ["installers-%s" % locale], cwd=localeSrcDir, env=env)
# Our Windows-native rm from bug 727551 requires Windows-style paths
run_cmd(['rm', '-rf', msys2windows(current)])
run_cmd(['mkdir', current])
run_cmd(['perl', unwrap_full_update, current_mar],
cwd=path.join(nativeDistDir, 'current'),
env=env)
for oldVersion in partialUpdates:
prevMar = partialUpdates[oldVersion]['mar']
if prevMar:
partial_mar_name = '%s-%s-%s.partial.mar' % (productName,
oldVersion, version)
partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name)
UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name))
# Our Windows-native rm from bug 727551 requires Windows-style paths
run_cmd(['rm', '-rf', msys2windows(previous)])
run_cmd(['mkdir', previous])
run_cmd(
['perl', unwrap_full_update,
'%s/%s' % (prevMarDir, prevMar)],
cwd=path.join(nativeDistDir, 'previous'),
env=env)
run_cmd([
'bash', make_incremental_update, partial_mar, previous, current
],
cwd=nativeDistDir,
env=env)
if os.environ.get('MOZ_SIGN_CMD'):
run_cmd([
'bash', '-c',
'%s -f mar -f gpg "%s"' %
(os.environ['MOZ_SIGN_CMD'], partial_mar)
],
env=env)
UPLOAD_EXTRA_FILES.append('%s/%s.asc' %
(updateDir, partial_mar_name))
else:
log.warning("Skipping partial MAR creation for %s %s" %
(oldVersion, locale))
env['UPLOAD_EXTRA_FILES'] = ' '.join(UPLOAD_EXTRA_FILES)
retry(run_cmd,
args=(make + ["upload", "AB_CD=%s" % locale], ),
kwargs={
'cwd': localeSrcDir,
'env': env
})
# return the location of the checksums file, because consumers may want
# some information about the files that were generated.
# Some versions of make that we use (at least pymake) imply --print-directory
# We need to turn it off to avoid getting extra output that mess up our
# parsing of the checksum file path.
curdir = os.getcwd()
try:
os.chdir(localeSrcDir)
relative_checksums = get_output(make + [
"--no-print-directory", "echo-variable-CHECKSUM_FILE",
"AB_CD=%s" % locale
],
env=env).strip("\"'\n")
return path.normpath(path.join(localeSrcDir, relative_checksums))
finally:
os.chdir(curdir)
