def downloadRelease(productName,
version,
buildNumber,
stageServer,
stageUsername=None,
stageSshKey=None,
stageUrlPrefix='http://stage.mozilla.org'):
candidatesDir = makeCandidatesDir(productName, version,
buildNumber).rstrip('/')
releasesDir = makeReleasesDir(productName, version).rstrip('/')
commands = [
'rm -rf %s' % candidatesDir,
'rm -rf %s' % releasesDir,
'mkdir -p %s' % candidatesDir,
'cd %(candidatesDir)s && \
wget -nv -r -np -nH --cut-dirs=6 -R index.html* \
-X %(candidatesDir)s/unsigned \
-X %(candidatesDir)s/contrib* \
-X %(candidatesDir)s/partner-repacks \
-X %(candidatesDir)s/win32-EUballot \
%(stageUrlPrefix)s%(candidatesDir)s/' % \
(dict(candidatesDir=candidatesDir, stageUrlPrefix=stageUrlPrefix)),
'ln -s %s %s' % (candidatesDir, releasesDir),
]
for command in commands:
run_remote_cmd(command,
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
def downloadRelease(productName, version, buildNumber, stageServer,
stageUsername=None, stageSshKey=None,
stageUrlPrefix='http://stage.mozilla.org'):
candidatesDir = makeCandidatesDir(productName, version,
buildNumber).rstrip('/')
releasesDir = makeReleasesDir(productName, version).rstrip('/')
commands = [
'rm -rf %s' % candidatesDir,
'rm -rf %s' % releasesDir,
'mkdir -p %s' % candidatesDir,
'cd %(candidatesDir)s && \
wget -nv -r -np -nH --cut-dirs=6 -R index.html* \
-X %(candidatesDir)s/unsigned \
-X %(candidatesDir)s/contrib* \
-X %(candidatesDir)s/partner-repacks \
-X %(candidatesDir)s/win32-EUballot \
%(stageUrlPrefix)s%(candidatesDir)s/' % \
(dict(candidatesDir=candidatesDir, stageUrlPrefix=stageUrlPrefix)),
'ln -s %s %s' % (candidatesDir, releasesDir),
]
for command in commands:
run_remote_cmd(command, server=stageServer, username=stageUsername,
sshKey=stageSshKey)
def checkStagePermissions(productName,
version,
buildNumber,
stageServer,
stageUsername=None,
stageSshKey=None):
# The following commands should return 0 lines output and exit code 0
tests = [
"find %s ! -user ffxbld ! -path '*/contrib*'",
"find %s ! -group firefox ! -path '*/contrib*'",
"find %s -type f ! -perm 644",
"find %s -mindepth 1 -type d ! -perm 755 ! -path '*/contrib*' ! -path '*/partner-repacks*'",
"find %s -maxdepth 1 -type d ! -perm 2775 -path '*/contrib*'",
]
candidates_dir = makeCandidatesDir(productName, version, buildNumber)
errors = False
for test_template in tests:
test = test_template % candidates_dir
cmd = 'test "0" = "$(%s | wc -l)"' % test
try:
run_remote_cmd(cmd,
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
except CalledProcessError:
errors = True
print 'Error while running: %s' % test
if errors:
raise
def deleteIndexFiles(cleanup_dir, stageServer, stageUsername, stageSshKey):
run_remote_cmd(
["find", cleanup_dir, "-name", "index.html", "-exec", "rm", "-v", "{}", "\\;"],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey,
)
def makeIndexFiles(productName, version, buildNumber, stageServer, stageUsername, stageSshKey):
candidates_dir = makeCandidatesDir(productName, version, buildNumber)
indexFile = NamedTemporaryFile()
indexFile.write(indexFileTemplate % {"version": version})
indexFile.flush()
scp(indexFile.name, "%s@%s:%s/index.html" % (stageUsername, stageServer, candidates_dir), sshKey=stageSshKey)
run_remote_cmd(
["chmod", "644", "%s/index.html" % candidates_dir],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey,
)
run_remote_cmd(
[
"find",
candidates_dir,
"-mindepth",
"1",
"-type",
"d",
"-not",
"-regex",
".*contrib.*",
"-exec",
"cp",
"-pv",
"%s/index.html" % candidates_dir,
"{}",
"\\;",
],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey,
)
def updateSymlink(productName, version, stageServer, stageUsername,
stageSshKey, target):
# step 1 check if we have already pushed to mirrors (bug 1083208)
push_dir = makeReleasesDir(productName, version)
try:
# check if the remote dir exists
run_remote_cmd(['test', '-d', push_dir],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
except CalledProcessError:
log.error(
'ERROR: push to mirrors directory, %s, does not exist on %s' %
(push_dir, stageServer))
log.error(
'ERROR: Did you push to mirrors before running post release?')
raise
releases_dir = makeReleasesDir(productName)
run_remote_cmd([
'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' %
dict(rd=releases_dir, version=version, target=target)
],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
def updateSymlink(productName, version, stageServer, stageUsername,
stageSshKey, target):
releases_dir = makeReleasesDir(productName)
run_remote_cmd([
'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' %
dict(rd=releases_dir, version=version, target=target)],
server=stageServer, username=stageUsername, sshKey=stageSshKey)
def deleteIndexFiles(cleanup_dir, stageServer, stageUsername, stageSshKey):
run_remote_cmd([
'find', cleanup_dir, '-name', 'index.html', '-exec', 'rm', '-v', '{}',
'\\;'
],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
def updateSymlink(productName, version, stageServer, stageUsername,
stageSshKey, target):
releases_dir = makeReleasesDir(productName)
run_remote_cmd([
'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' %
dict(rd=releases_dir, version=version, target=target)
],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
def runAntivirusCheck(productName,
version,
buildNumber,
stageServer,
stageUsername=None,
stageSshKey=None):
candidates_dir = makeCandidatesDir(productName, version, buildNumber)
cmd = VIRUS_SCAN_CMD + [candidates_dir]
run_remote_cmd(cmd,
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
def doSyncPartnerBundles(productName, version, buildNumber, stageServer,
stageUsername, stageSshKey):
candidates_dir = makeCandidatesDir(productName, version, buildNumber)
for dest, src in PARTNER_BUNDLE_MAPPINGS.iteritems():
full_dest = path.join(PARTNER_BUNDLE_DIR, dest)
full_src = path.join(candidates_dir, 'partner-repacks', src)
full_src = full_src % {'version': version}
run_remote_cmd(['cp', '-f', full_src, full_dest],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
# And fix the permissions...
run_remote_cmd([
'find', PARTNER_BUNDLE_DIR, '-type', 'd', '-exec', 'chmod', '775',
'{}', '\\;'
],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
run_remote_cmd([
'find', PARTNER_BUNDLE_DIR, '-name', '"*.exe"', '-exec', 'chmod',
'775', '{}', '\\;'
],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
run_remote_cmd([
'find', PARTNER_BUNDLE_DIR, '-name', '"*.dmg"', '-exec', 'chmod',
'775', '{}', '\\;'
],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
def doSyncPartnerBundles(productName, version, buildNumber, stageServer, stageUsername, stageSshKey):
candidates_dir = makeCandidatesDir(productName, version, buildNumber)
for dest, src in PARTNER_BUNDLE_MAPPINGS.iteritems():
full_dest = path.join(PARTNER_BUNDLE_DIR, dest)
full_src = path.join(candidates_dir, "partner-repacks", src)
full_src = full_src % {"version": version}
run_remote_cmd(
["cp", "-f", full_src, full_dest], server=stageServer, username=stageUsername, sshKey=stageSshKey
)
# And fix the permissions...
run_remote_cmd(
["find", PARTNER_BUNDLE_DIR, "-type", "d", "-exec", "chmod", "775", "{}", "\\;"],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey,
)
run_remote_cmd(
["find", PARTNER_BUNDLE_DIR, "-name", '"*.exe"', "-exec", "chmod", "775", "{}", "\\;"],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey,
)
run_remote_cmd(
["find", PARTNER_BUNDLE_DIR, "-name", '"*.dmg"', "-exec", "chmod", "775", "{}", "\\;"],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey,
)
def pushToMirrors(productName, version, buildNumber, stageServer,
stageUsername=None, stageSshKey=None, excludes=None,
extra_excludes=None, dryRun=False):
""" excludes overrides DEFAULT_RSYNC_EXCLUDES, extra_exludes will be
appended to DEFAULT_RSYNC_EXCLUDES. """
source_dir = makeCandidatesDir(productName, version, buildNumber)
target_dir = makeReleasesDir(productName, version)
if not excludes:
excludes = DEFAULT_RSYNC_EXCLUDES
if extra_excludes:
excludes += extra_excludes
# fail/warn if target directory exists depending on dry run mode
try:
run_remote_cmd(['test', '!', '-d', target_dir], server=stageServer,
username=stageUsername, sshKey=stageSshKey)
except CalledProcessError:
if not dryRun:
raise
else:
log.warning('WARN: target directory %s exists', target_dir)
if not dryRun:
run_remote_cmd(['mkdir', '-p', target_dir], server=stageServer,
username=stageUsername, sshKey=stageSshKey)
run_remote_cmd(['chmod', 'u=rwx,g=rxs,o=rx', target_dir], server=stageServer,
username=stageUsername, sshKey=stageSshKey)
rsync_cmd = ['rsync', '-av' ]
if dryRun:
rsync_cmd.append('-n')
run_remote_cmd(rsync_cmd + excludes + [source_dir, target_dir],
server=stageServer, username=stageUsername,
sshKey=stageSshKey)
def doSyncPartnerBundles(productName, version, buildNumber, stageServer,
stageUsername, stageSshKey):
candidates_dir = makeCandidatesDir(productName, version, buildNumber)
for dest, src in PARTNER_BUNDLE_MAPPINGS.iteritems():
full_dest = path.join(PARTNER_BUNDLE_DIR, dest)
full_src = path.join(candidates_dir, 'partner-repacks', src)
full_src = full_src % {'version': version}
run_remote_cmd(
['cp', '-f', full_src, full_dest],
server=stageServer, username=stageUsername, sshKey=stageSshKey
)
# And fix the permissions...
run_remote_cmd(
['find', PARTNER_BUNDLE_DIR, '-type', 'd',
'-exec', 'chmod', '775', '{}', '\\;'],
server=stageServer, username=stageUsername, sshKey=stageSshKey
)
run_remote_cmd(
['find', PARTNER_BUNDLE_DIR, '-name', '"*.exe"',
'-exec', 'chmod', '775', '{}', '\\;'],
server=stageServer, username=stageUsername, sshKey=stageSshKey
)
run_remote_cmd(
['find', PARTNER_BUNDLE_DIR, '-name', '"*.dmg"',
'-exec', 'chmod', '775', '{}', '\\;'],
server=stageServer, username=stageUsername, sshKey=stageSshKey
)
def makeIndexFiles(productName, version, buildNumber, stageServer,
stageUsername, stageSshKey):
candidates_dir = makeCandidatesDir(productName, version, buildNumber)
indexFile = NamedTemporaryFile()
indexFile.write(indexFileTemplate % {'version': version})
indexFile.flush()
scp(
indexFile.name, '%s@%s:%s/index.html' % (
stageUsername, stageServer, candidates_dir),
sshKey=stageSshKey)
run_remote_cmd(['chmod', '644', '%s/index.html' % candidates_dir],
server=stageServer, username=stageUsername, sshKey=stageSshKey)
run_remote_cmd(
['find', candidates_dir, '-mindepth', '1', '-type', 'd', '-not', '-regex', '.*contrib.*', '-exec', 'cp', '-pv', '%s/index.html' % candidates_dir, '{}', '\\;'],
server=stageServer, username=stageUsername, sshKey=stageSshKey)
def pushToMirrors(productName,
version,
buildNumber,
stageServer,
stageUsername=None,
stageSshKey=None,
excludes=None,
extra_excludes=None,
dryRun=False,
overwrite=False):
""" excludes overrides DEFAULT_RSYNC_EXCLUDES, extra_exludes will be
appended to DEFAULT_RSYNC_EXCLUDES. """
source_dir = makeCandidatesDir(productName, version, buildNumber)
target_dir = makeReleasesDir(productName, version)
if not excludes:
excludes = DEFAULT_RSYNC_EXCLUDES
if extra_excludes:
excludes += ['--exclude=%s' % ex for ex in extra_excludes]
# fail/warn if target directory exists depending on dry run mode
try:
run_remote_cmd(['test', '!', '-d', target_dir],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
except CalledProcessError:
if overwrite:
log.info(
'target directory %s exists, but overwriting files as requested'
% target_dir)
elif dryRun:
log.warning('WARN: target directory %s exists', target_dir)
else:
raise
if not dryRun:
run_remote_cmd(['mkdir', '-p', target_dir],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
run_remote_cmd(['chmod', 'u=rwx,g=rxs,o=rx', target_dir],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
rsync_cmd = ['rsync', '-av']
if dryRun:
rsync_cmd.append('-n')
# use hardlinks
rsync_cmd.append('--link-dest=%s' % source_dir)
run_remote_cmd(rsync_cmd + excludes + [source_dir, target_dir],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
def updateSymlink(productName, version, stageServer, stageUsername,
stageSshKey, target):
# step 1 check if we have already pushed to mirrors (bug 1083208)
push_dir = makeReleasesDir(productName, version)
try:
# check if the remote dir exists
run_remote_cmd(['test', '-d', push_dir],
server=stageServer, username=stageUsername,
sshKey=stageSshKey)
except CalledProcessError:
log.error('ERROR: push to mirrors directory, %s, does not exist on %s'
% (push_dir, stageServer))
log.error('ERROR: Did you push to mirrors before running post release?')
raise
releases_dir = makeReleasesDir(productName)
run_remote_cmd([
'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' %
dict(rd=releases_dir, version=version, target=target)],
server=stageServer, username=stageUsername, sshKey=stageSshKey)
def checkStagePermissions(productName, version, buildNumber, stageServer, stageUsername, stageSshKey):
# The following commands should return 0 lines output and exit code 0
tests = [
"find %%s ! -user %s ! -path '*/contrib*'" % stageUsername,
"find %%s ! -group `id -g -n %s` ! -path '*/contrib*'" % stageUsername,
"find %s -type f ! -perm 644",
"find %s -mindepth 1 -type d ! -perm 755 ! -path '*/contrib*' ! -path '*/partner-repacks*'",
"find %s -maxdepth 1 -type d ! -perm 2775 -path '*/contrib*'",
]
candidates_dir = makeCandidatesDir(productName, version, buildNumber)
errors = False
for test_template in tests:
test = test_template % candidates_dir
cmd = 'test "0" = "$(%s | wc -l)"' % test
try:
run_remote_cmd(cmd, server=stageServer, username=stageUsername, sshKey=stageSshKey)
except CalledProcessError:
errors = True
if errors:
raise
def makeIndexFiles(productName, version, buildNumber, stageServer,
stageUsername, stageSshKey):
candidates_dir = makeCandidatesDir(productName, version, buildNumber)
indexFile = NamedTemporaryFile()
indexFile.write(indexFileTemplate % {'version': version})
indexFile.flush()
scp(indexFile.name,
'%s@%s:%s/index.html' % (stageUsername, stageServer, candidates_dir),
sshKey=stageSshKey)
run_remote_cmd(
['chmod', '644', '%s/index.html' % candidates_dir],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
run_remote_cmd([
'find', candidates_dir, '-mindepth', '1', '-type', 'd', '-not',
'-regex', '.*contrib.*', '-exec', 'cp', '-pv',
'%s/index.html' % candidates_dir, '{}', '\\;'
],
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
def pushToMirrors(
productName,
version,
buildNumber,
stageServer,
stageUsername=None,
stageSshKey=None,
excludes=None,
extra_excludes=None,
dryRun=False,
overwrite=False,
):
""" excludes overrides DEFAULT_RSYNC_EXCLUDES, extra_exludes will be
appended to DEFAULT_RSYNC_EXCLUDES. """
source_dir = makeCandidatesDir(productName, version, buildNumber)
target_dir = makeReleasesDir(productName, version)
if not excludes:
excludes = DEFAULT_RSYNC_EXCLUDES
if extra_excludes:
excludes += ["--exclude=%s" % ex for ex in extra_excludes]
# fail/warn if target directory exists depending on dry run mode
try:
run_remote_cmd(["test", "!", "-d", target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey)
except CalledProcessError:
if overwrite:
log.info("target directory %s exists, but overwriting files as requested" % target_dir)
elif dryRun:
log.warning("WARN: target directory %s exists", target_dir)
else:
raise
if not dryRun:
run_remote_cmd(["mkdir", "-p", target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey)
run_remote_cmd(
["chmod", "u=rwx,g=rxs,o=rx", target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey
)
rsync_cmd = ["rsync", "-av"]
if dryRun:
rsync_cmd.append("-n")
# use hardlinks
rsync_cmd.append("--link-dest=%s" % source_dir)
run_remote_cmd(
rsync_cmd + excludes + [source_dir, target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey
)
def runAntivirusCheck(productName, version, buildNumber, stageServer,
stageUsername=None, stageSshKey=None):
candidates_dir = makeCandidatesDir(productName, version, buildNumber)
cmd = VIRUS_SCAN_CMD + [candidates_dir]
run_remote_cmd(cmd, server=stageServer, username=stageUsername,
sshKey=stageSshKey)
def ssh(host, command, user=None, key=None):
'''Abstract ssh into a python function'''
flush()
run_remote_cmd(command, host, user, key)
flush()
def doSyncPartnerBundles(productName, version, buildNumber, stageServer,
stageUsername, stageSshKey):
candidates_dir = makeCandidatesDir(productName, version, buildNumber)
# Sync the Bing packages...
bing_dir = '%s/partner-repacks/bing' % candidates_dir
mac_bing_src = '%s/mac/en-US/Firefox\\ %s.dmg' % (bing_dir, version)
mac_bing_dst = '%s/bing/mac/en-US/Firefox-Bing.dmg' % PARTNER_BUNDLE_DIR
win32_bing_src = '%s/win32/en-US/Firefox\\ Setup\\ %s.exe' % (bing_dir, version)
win32_bing_dst = '%s/bing/win32/en-US/Firefox-Bing\\ Setup.exe' % PARTNER_BUNDLE_DIR
run_remote_cmd(['cp', '-f', mac_bing_src, mac_bing_dst],
server=stageServer, username=stageUsername, sshKey=stageSshKey
)
run_remote_cmd(['cp', '-f', win32_bing_src, win32_bing_dst],
server=stageServer, username=stageUsername, sshKey=stageSshKey
)
# Sync the MSN packages...
run_remote_cmd(
['rsync', '-av', '%s/partner-repacks/msn*' % candidates_dir,
PARTNER_BUNDLE_DIR],
server=stageServer, username=stageUsername, sshKey=stageSshKey
)
# And fix the permissions...
run_remote_cmd(
['find', PARTNER_BUNDLE_DIR, '-type', 'd',
'-exec', 'chmod', '775', '{}', '\\;'],
server=stageServer, username=stageUsername, sshKey=stageSshKey
)
run_remote_cmd(
['find', PARTNER_BUNDLE_DIR, '-name', '"*.exe"',
'-exec', 'chmod', '775', '{}', '\\;'],
server=stageServer, username=stageUsername, sshKey=stageSshKey
)
run_remote_cmd(
['find', PARTNER_BUNDLE_DIR, '-name', '"*.dmg"',
'-exec', 'chmod', '775', '{}', '\\;'],
server=stageServer, username=stageUsername, sshKey=stageSshKey
)
def deleteIndexFiles(cleanup_dir, stageServer, stageUsername,
stageSshKey):
run_remote_cmd(
['find', cleanup_dir, '-name', 'index.html', '-exec', 'rm',
'-v', '{}', '\\;'],
server=stageServer, username=stageUsername, sshKey=stageSshKey)
branchConfig, releaseConfig = validate(options, args)
productName = releaseConfig['productName']
version = releaseConfig['version']
buildNumber = releaseConfig['buildNumber']
stageServer = branchConfig['stage_server']
stageUsername = branchConfig['stage_username']
stageSshKey = path.join(os.path.expanduser("~"), ".ssh",
branchConfig["stage_ssh_key"])
candidatesDir = makeCandidatesDir(productName, version, buildNumber)
rsyncFilesByPattern(server=stageServer, userName=stageUsername,
sshKey=stageSshKey, source_dir=candidatesDir,
target_dir='temp/', pattern='*.checksums')
types = {'sha1': 'SHA1SUMS', 'md5': 'MD5SUMS', 'sha512': 'SHA512SUMS'}
generateChecksums('temp', types)
files = types.values()
signFiles(files)
upload_files = files + ['%s.asc' % x for x in files] + \
[path.join(path.dirname(__file__), 'KEY')]
log.info("Fixing permissions...")
for f in upload_files:
log.info("chmod 644 %s" % f)
os.chmod(f, 0644)
rsyncFiles(files=upload_files, server=stageServer, userName=stageUsername,
sshKey=stageSshKey, target_dir=candidatesDir)
cmd = 'mkdir -v -m 2775 %s/contrib %s/contrib-localized' % \
(candidatesDir, candidatesDir)
run_remote_cmd(cmd, server=stageServer, username=stageUsername,
sshKey=stageSshKey)
buildNumber = releaseConfig['buildNumber']
stageServer = branchConfig['stage_server']
stageUsername = branchConfig['stage_username']
stageSshKey = path.join(os.path.expanduser("~"), ".ssh",
branchConfig["stage_ssh_key"])
candidatesDir = makeCandidatesDir(productName, version, buildNumber)
rsyncFilesByPattern(server=stageServer,
userName=stageUsername,
sshKey=stageSshKey,
source_dir=candidatesDir,
target_dir='temp/',
pattern='*.checksums')
types = {'sha1': 'SHA1SUMS', 'md5': 'MD5SUMS', 'sha512': 'SHA512SUMS'}
generateChecksums('temp', types)
files = types.values()
signFiles(files)
upload_files = files + ['%s.asc' % x for x in files] + \
[path.join(path.dirname(__file__), 'KEY')]
rsyncFiles(files=upload_files,
server=stageServer,
userName=stageUsername,
sshKey=stageSshKey,
target_dir=candidatesDir)
cmd = 'mkdir -v -m 2775 %s/contrib %s/contrib-localized' % \
(candidatesDir, candidatesDir)
run_remote_cmd(cmd,
server=stageServer,
username=stageUsername,
sshKey=stageSshKey)
def ssh(host, command, user=None, key=None):
'''Abstract ssh into a python function'''
flush()
run_remote_cmd(command, host, user, key)
flush()
