def downloadRelease(productName, version, buildNumber, stageServer, stageUsername=None, stageSshKey=None, stageUrlPrefix='http://stage.mozilla.org'): candidatesDir = makeCandidatesDir(productName, version, buildNumber).rstrip('/') releasesDir = makeReleasesDir(productName, version).rstrip('/') commands = [ 'rm -rf %s' % candidatesDir, 'rm -rf %s' % releasesDir, 'mkdir -p %s' % candidatesDir, 'cd %(candidatesDir)s && \ wget -nv -r -np -nH --cut-dirs=6 -R index.html* \ -X %(candidatesDir)s/unsigned \ -X %(candidatesDir)s/contrib* \ -X %(candidatesDir)s/partner-repacks \ -X %(candidatesDir)s/win32-EUballot \ %(stageUrlPrefix)s%(candidatesDir)s/' % \ (dict(candidatesDir=candidatesDir, stageUrlPrefix=stageUrlPrefix)), 'ln -s %s %s' % (candidatesDir, releasesDir), ] for command in commands: run_remote_cmd(command, server=stageServer, username=stageUsername, sshKey=stageSshKey)
def checkStagePermissions(productName, version, buildNumber, stageServer, stageUsername=None, stageSshKey=None): # The following commands should return 0 lines output and exit code 0 tests = [ "find %s ! -user ffxbld ! -path '*/contrib*'", "find %s ! -group firefox ! -path '*/contrib*'", "find %s -type f ! -perm 644", "find %s -mindepth 1 -type d ! -perm 755 ! -path '*/contrib*' ! -path '*/partner-repacks*'", "find %s -maxdepth 1 -type d ! -perm 2775 -path '*/contrib*'", ] candidates_dir = makeCandidatesDir(productName, version, buildNumber) errors = False for test_template in tests: test = test_template % candidates_dir cmd = 'test "0" = "$(%s | wc -l)"' % test try: run_remote_cmd(cmd, server=stageServer, username=stageUsername, sshKey=stageSshKey) except CalledProcessError: errors = True print 'Error while running: %s' % test if errors: raise
def deleteIndexFiles(cleanup_dir, stageServer, stageUsername, stageSshKey): run_remote_cmd( ["find", cleanup_dir, "-name", "index.html", "-exec", "rm", "-v", "{}", "\\;"], server=stageServer, username=stageUsername, sshKey=stageSshKey, )
def makeIndexFiles(productName, version, buildNumber, stageServer, stageUsername, stageSshKey): candidates_dir = makeCandidatesDir(productName, version, buildNumber) indexFile = NamedTemporaryFile() indexFile.write(indexFileTemplate % {"version": version}) indexFile.flush() scp(indexFile.name, "%s@%s:%s/index.html" % (stageUsername, stageServer, candidates_dir), sshKey=stageSshKey) run_remote_cmd( ["chmod", "644", "%s/index.html" % candidates_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey, ) run_remote_cmd( [ "find", candidates_dir, "-mindepth", "1", "-type", "d", "-not", "-regex", ".*contrib.*", "-exec", "cp", "-pv", "%s/index.html" % candidates_dir, "{}", "\\;", ], server=stageServer, username=stageUsername, sshKey=stageSshKey, )
def updateSymlink(productName, version, stageServer, stageUsername, stageSshKey, target): # step 1 check if we have already pushed to mirrors (bug 1083208) push_dir = makeReleasesDir(productName, version) try: # check if the remote dir exists run_remote_cmd(['test', '-d', push_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) except CalledProcessError: log.error( 'ERROR: push to mirrors directory, %s, does not exist on %s' % (push_dir, stageServer)) log.error( 'ERROR: Did you push to mirrors before running post release?') raise releases_dir = makeReleasesDir(productName) run_remote_cmd([ 'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' % dict(rd=releases_dir, version=version, target=target) ], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def updateSymlink(productName, version, stageServer, stageUsername, stageSshKey, target): releases_dir = makeReleasesDir(productName) run_remote_cmd([ 'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' % dict(rd=releases_dir, version=version, target=target)], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def deleteIndexFiles(cleanup_dir, stageServer, stageUsername, stageSshKey): run_remote_cmd([ 'find', cleanup_dir, '-name', 'index.html', '-exec', 'rm', '-v', '{}', '\\;' ], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def updateSymlink(productName, version, stageServer, stageUsername, stageSshKey, target): releases_dir = makeReleasesDir(productName) run_remote_cmd([ 'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' % dict(rd=releases_dir, version=version, target=target) ], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def runAntivirusCheck(productName, version, buildNumber, stageServer, stageUsername=None, stageSshKey=None): candidates_dir = makeCandidatesDir(productName, version, buildNumber) cmd = VIRUS_SCAN_CMD + [candidates_dir] run_remote_cmd(cmd, server=stageServer, username=stageUsername, sshKey=stageSshKey)
def doSyncPartnerBundles(productName, version, buildNumber, stageServer, stageUsername, stageSshKey): candidates_dir = makeCandidatesDir(productName, version, buildNumber) for dest, src in PARTNER_BUNDLE_MAPPINGS.iteritems(): full_dest = path.join(PARTNER_BUNDLE_DIR, dest) full_src = path.join(candidates_dir, 'partner-repacks', src) full_src = full_src % {'version': version} run_remote_cmd(['cp', '-f', full_src, full_dest], server=stageServer, username=stageUsername, sshKey=stageSshKey) # And fix the permissions... run_remote_cmd([ 'find', PARTNER_BUNDLE_DIR, '-type', 'd', '-exec', 'chmod', '775', '{}', '\\;' ], server=stageServer, username=stageUsername, sshKey=stageSshKey) run_remote_cmd([ 'find', PARTNER_BUNDLE_DIR, '-name', '"*.exe"', '-exec', 'chmod', '775', '{}', '\\;' ], server=stageServer, username=stageUsername, sshKey=stageSshKey) run_remote_cmd([ 'find', PARTNER_BUNDLE_DIR, '-name', '"*.dmg"', '-exec', 'chmod', '775', '{}', '\\;' ], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def doSyncPartnerBundles(productName, version, buildNumber, stageServer, stageUsername, stageSshKey): candidates_dir = makeCandidatesDir(productName, version, buildNumber) for dest, src in PARTNER_BUNDLE_MAPPINGS.iteritems(): full_dest = path.join(PARTNER_BUNDLE_DIR, dest) full_src = path.join(candidates_dir, "partner-repacks", src) full_src = full_src % {"version": version} run_remote_cmd( ["cp", "-f", full_src, full_dest], server=stageServer, username=stageUsername, sshKey=stageSshKey ) # And fix the permissions... run_remote_cmd( ["find", PARTNER_BUNDLE_DIR, "-type", "d", "-exec", "chmod", "775", "{}", "\\;"], server=stageServer, username=stageUsername, sshKey=stageSshKey, ) run_remote_cmd( ["find", PARTNER_BUNDLE_DIR, "-name", '"*.exe"', "-exec", "chmod", "775", "{}", "\\;"], server=stageServer, username=stageUsername, sshKey=stageSshKey, ) run_remote_cmd( ["find", PARTNER_BUNDLE_DIR, "-name", '"*.dmg"', "-exec", "chmod", "775", "{}", "\\;"], server=stageServer, username=stageUsername, sshKey=stageSshKey, )
def pushToMirrors(productName, version, buildNumber, stageServer, stageUsername=None, stageSshKey=None, excludes=None, extra_excludes=None, dryRun=False): """ excludes overrides DEFAULT_RSYNC_EXCLUDES, extra_exludes will be appended to DEFAULT_RSYNC_EXCLUDES. """ source_dir = makeCandidatesDir(productName, version, buildNumber) target_dir = makeReleasesDir(productName, version) if not excludes: excludes = DEFAULT_RSYNC_EXCLUDES if extra_excludes: excludes += extra_excludes # fail/warn if target directory exists depending on dry run mode try: run_remote_cmd(['test', '!', '-d', target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) except CalledProcessError: if not dryRun: raise else: log.warning('WARN: target directory %s exists', target_dir) if not dryRun: run_remote_cmd(['mkdir', '-p', target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) run_remote_cmd(['chmod', 'u=rwx,g=rxs,o=rx', target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) rsync_cmd = ['rsync', '-av' ] if dryRun: rsync_cmd.append('-n') run_remote_cmd(rsync_cmd + excludes + [source_dir, target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def doSyncPartnerBundles(productName, version, buildNumber, stageServer, stageUsername, stageSshKey): candidates_dir = makeCandidatesDir(productName, version, buildNumber) for dest, src in PARTNER_BUNDLE_MAPPINGS.iteritems(): full_dest = path.join(PARTNER_BUNDLE_DIR, dest) full_src = path.join(candidates_dir, 'partner-repacks', src) full_src = full_src % {'version': version} run_remote_cmd( ['cp', '-f', full_src, full_dest], server=stageServer, username=stageUsername, sshKey=stageSshKey ) # And fix the permissions... run_remote_cmd( ['find', PARTNER_BUNDLE_DIR, '-type', 'd', '-exec', 'chmod', '775', '{}', '\\;'], server=stageServer, username=stageUsername, sshKey=stageSshKey ) run_remote_cmd( ['find', PARTNER_BUNDLE_DIR, '-name', '"*.exe"', '-exec', 'chmod', '775', '{}', '\\;'], server=stageServer, username=stageUsername, sshKey=stageSshKey ) run_remote_cmd( ['find', PARTNER_BUNDLE_DIR, '-name', '"*.dmg"', '-exec', 'chmod', '775', '{}', '\\;'], server=stageServer, username=stageUsername, sshKey=stageSshKey )
def makeIndexFiles(productName, version, buildNumber, stageServer, stageUsername, stageSshKey): candidates_dir = makeCandidatesDir(productName, version, buildNumber) indexFile = NamedTemporaryFile() indexFile.write(indexFileTemplate % {'version': version}) indexFile.flush() scp( indexFile.name, '%s@%s:%s/index.html' % ( stageUsername, stageServer, candidates_dir), sshKey=stageSshKey) run_remote_cmd(['chmod', '644', '%s/index.html' % candidates_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) run_remote_cmd( ['find', candidates_dir, '-mindepth', '1', '-type', 'd', '-not', '-regex', '.*contrib.*', '-exec', 'cp', '-pv', '%s/index.html' % candidates_dir, '{}', '\\;'], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def pushToMirrors(productName, version, buildNumber, stageServer, stageUsername=None, stageSshKey=None, excludes=None, extra_excludes=None, dryRun=False, overwrite=False): """ excludes overrides DEFAULT_RSYNC_EXCLUDES, extra_exludes will be appended to DEFAULT_RSYNC_EXCLUDES. """ source_dir = makeCandidatesDir(productName, version, buildNumber) target_dir = makeReleasesDir(productName, version) if not excludes: excludes = DEFAULT_RSYNC_EXCLUDES if extra_excludes: excludes += ['--exclude=%s' % ex for ex in extra_excludes] # fail/warn if target directory exists depending on dry run mode try: run_remote_cmd(['test', '!', '-d', target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) except CalledProcessError: if overwrite: log.info( 'target directory %s exists, but overwriting files as requested' % target_dir) elif dryRun: log.warning('WARN: target directory %s exists', target_dir) else: raise if not dryRun: run_remote_cmd(['mkdir', '-p', target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) run_remote_cmd(['chmod', 'u=rwx,g=rxs,o=rx', target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) rsync_cmd = ['rsync', '-av'] if dryRun: rsync_cmd.append('-n') # use hardlinks rsync_cmd.append('--link-dest=%s' % source_dir) run_remote_cmd(rsync_cmd + excludes + [source_dir, target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def updateSymlink(productName, version, stageServer, stageUsername, stageSshKey, target): # step 1 check if we have already pushed to mirrors (bug 1083208) push_dir = makeReleasesDir(productName, version) try: # check if the remote dir exists run_remote_cmd(['test', '-d', push_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) except CalledProcessError: log.error('ERROR: push to mirrors directory, %s, does not exist on %s' % (push_dir, stageServer)) log.error('ERROR: Did you push to mirrors before running post release?') raise releases_dir = makeReleasesDir(productName) run_remote_cmd([ 'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' % dict(rd=releases_dir, version=version, target=target)], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def checkStagePermissions(productName, version, buildNumber, stageServer, stageUsername, stageSshKey): # The following commands should return 0 lines output and exit code 0 tests = [ "find %%s ! -user %s ! -path '*/contrib*'" % stageUsername, "find %%s ! -group `id -g -n %s` ! -path '*/contrib*'" % stageUsername, "find %s -type f ! -perm 644", "find %s -mindepth 1 -type d ! -perm 755 ! -path '*/contrib*' ! -path '*/partner-repacks*'", "find %s -maxdepth 1 -type d ! -perm 2775 -path '*/contrib*'", ] candidates_dir = makeCandidatesDir(productName, version, buildNumber) errors = False for test_template in tests: test = test_template % candidates_dir cmd = 'test "0" = "$(%s | wc -l)"' % test try: run_remote_cmd(cmd, server=stageServer, username=stageUsername, sshKey=stageSshKey) except CalledProcessError: errors = True if errors: raise
def makeIndexFiles(productName, version, buildNumber, stageServer, stageUsername, stageSshKey): candidates_dir = makeCandidatesDir(productName, version, buildNumber) indexFile = NamedTemporaryFile() indexFile.write(indexFileTemplate % {'version': version}) indexFile.flush() scp(indexFile.name, '%s@%s:%s/index.html' % (stageUsername, stageServer, candidates_dir), sshKey=stageSshKey) run_remote_cmd( ['chmod', '644', '%s/index.html' % candidates_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) run_remote_cmd([ 'find', candidates_dir, '-mindepth', '1', '-type', 'd', '-not', '-regex', '.*contrib.*', '-exec', 'cp', '-pv', '%s/index.html' % candidates_dir, '{}', '\\;' ], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def pushToMirrors( productName, version, buildNumber, stageServer, stageUsername=None, stageSshKey=None, excludes=None, extra_excludes=None, dryRun=False, overwrite=False, ): """ excludes overrides DEFAULT_RSYNC_EXCLUDES, extra_exludes will be appended to DEFAULT_RSYNC_EXCLUDES. """ source_dir = makeCandidatesDir(productName, version, buildNumber) target_dir = makeReleasesDir(productName, version) if not excludes: excludes = DEFAULT_RSYNC_EXCLUDES if extra_excludes: excludes += ["--exclude=%s" % ex for ex in extra_excludes] # fail/warn if target directory exists depending on dry run mode try: run_remote_cmd(["test", "!", "-d", target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) except CalledProcessError: if overwrite: log.info("target directory %s exists, but overwriting files as requested" % target_dir) elif dryRun: log.warning("WARN: target directory %s exists", target_dir) else: raise if not dryRun: run_remote_cmd(["mkdir", "-p", target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) run_remote_cmd( ["chmod", "u=rwx,g=rxs,o=rx", target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey ) rsync_cmd = ["rsync", "-av"] if dryRun: rsync_cmd.append("-n") # use hardlinks rsync_cmd.append("--link-dest=%s" % source_dir) run_remote_cmd( rsync_cmd + excludes + [source_dir, target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey )
def ssh(host, command, user=None, key=None): '''Abstract ssh into a python function''' flush() run_remote_cmd(command, host, user, key) flush()
def doSyncPartnerBundles(productName, version, buildNumber, stageServer, stageUsername, stageSshKey): candidates_dir = makeCandidatesDir(productName, version, buildNumber) # Sync the Bing packages... bing_dir = '%s/partner-repacks/bing' % candidates_dir mac_bing_src = '%s/mac/en-US/Firefox\\ %s.dmg' % (bing_dir, version) mac_bing_dst = '%s/bing/mac/en-US/Firefox-Bing.dmg' % PARTNER_BUNDLE_DIR win32_bing_src = '%s/win32/en-US/Firefox\\ Setup\\ %s.exe' % (bing_dir, version) win32_bing_dst = '%s/bing/win32/en-US/Firefox-Bing\\ Setup.exe' % PARTNER_BUNDLE_DIR run_remote_cmd(['cp', '-f', mac_bing_src, mac_bing_dst], server=stageServer, username=stageUsername, sshKey=stageSshKey ) run_remote_cmd(['cp', '-f', win32_bing_src, win32_bing_dst], server=stageServer, username=stageUsername, sshKey=stageSshKey ) # Sync the MSN packages... run_remote_cmd( ['rsync', '-av', '%s/partner-repacks/msn*' % candidates_dir, PARTNER_BUNDLE_DIR], server=stageServer, username=stageUsername, sshKey=stageSshKey ) # And fix the permissions... run_remote_cmd( ['find', PARTNER_BUNDLE_DIR, '-type', 'd', '-exec', 'chmod', '775', '{}', '\\;'], server=stageServer, username=stageUsername, sshKey=stageSshKey ) run_remote_cmd( ['find', PARTNER_BUNDLE_DIR, '-name', '"*.exe"', '-exec', 'chmod', '775', '{}', '\\;'], server=stageServer, username=stageUsername, sshKey=stageSshKey ) run_remote_cmd( ['find', PARTNER_BUNDLE_DIR, '-name', '"*.dmg"', '-exec', 'chmod', '775', '{}', '\\;'], server=stageServer, username=stageUsername, sshKey=stageSshKey )
def deleteIndexFiles(cleanup_dir, stageServer, stageUsername, stageSshKey): run_remote_cmd( ['find', cleanup_dir, '-name', 'index.html', '-exec', 'rm', '-v', '{}', '\\;'], server=stageServer, username=stageUsername, sshKey=stageSshKey)
branchConfig, releaseConfig = validate(options, args) productName = releaseConfig['productName'] version = releaseConfig['version'] buildNumber = releaseConfig['buildNumber'] stageServer = branchConfig['stage_server'] stageUsername = branchConfig['stage_username'] stageSshKey = path.join(os.path.expanduser("~"), ".ssh", branchConfig["stage_ssh_key"]) candidatesDir = makeCandidatesDir(productName, version, buildNumber) rsyncFilesByPattern(server=stageServer, userName=stageUsername, sshKey=stageSshKey, source_dir=candidatesDir, target_dir='temp/', pattern='*.checksums') types = {'sha1': 'SHA1SUMS', 'md5': 'MD5SUMS', 'sha512': 'SHA512SUMS'} generateChecksums('temp', types) files = types.values() signFiles(files) upload_files = files + ['%s.asc' % x for x in files] + \ [path.join(path.dirname(__file__), 'KEY')] log.info("Fixing permissions...") for f in upload_files: log.info("chmod 644 %s" % f) os.chmod(f, 0644) rsyncFiles(files=upload_files, server=stageServer, userName=stageUsername, sshKey=stageSshKey, target_dir=candidatesDir) cmd = 'mkdir -v -m 2775 %s/contrib %s/contrib-localized' % \ (candidatesDir, candidatesDir) run_remote_cmd(cmd, server=stageServer, username=stageUsername, sshKey=stageSshKey)
buildNumber = releaseConfig['buildNumber'] stageServer = branchConfig['stage_server'] stageUsername = branchConfig['stage_username'] stageSshKey = path.join(os.path.expanduser("~"), ".ssh", branchConfig["stage_ssh_key"]) candidatesDir = makeCandidatesDir(productName, version, buildNumber) rsyncFilesByPattern(server=stageServer, userName=stageUsername, sshKey=stageSshKey, source_dir=candidatesDir, target_dir='temp/', pattern='*.checksums') types = {'sha1': 'SHA1SUMS', 'md5': 'MD5SUMS', 'sha512': 'SHA512SUMS'} generateChecksums('temp', types) files = types.values() signFiles(files) upload_files = files + ['%s.asc' % x for x in files] + \ [path.join(path.dirname(__file__), 'KEY')] rsyncFiles(files=upload_files, server=stageServer, userName=stageUsername, sshKey=stageSshKey, target_dir=candidatesDir) cmd = 'mkdir -v -m 2775 %s/contrib %s/contrib-localized' % \ (candidatesDir, candidatesDir) run_remote_cmd(cmd, server=stageServer, username=stageUsername, sshKey=stageSshKey)