def updateSymlink(productName, version, stageServer, stageUsername, stageSshKey, target): # step 1 check if we have already pushed to mirrors (bug 1083208) push_dir = makeReleasesDir(productName, version) try: # check if the remote dir exists run_remote_cmd(['test', '-d', push_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) except CalledProcessError: log.error( 'ERROR: push to mirrors directory, %s, does not exist on %s' % (push_dir, stageServer)) log.error( 'ERROR: Did you push to mirrors before running post release?') raise releases_dir = makeReleasesDir(productName) run_remote_cmd([ 'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' % dict(rd=releases_dir, version=version, target=target) ], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def downloadRelease(productName, version, buildNumber, stageServer, stageUsername=None, stageSshKey=None, stageUrlPrefix='http://stage.mozilla.org'): candidatesDir = makeCandidatesDir(productName, version, buildNumber).rstrip('/') releasesDir = makeReleasesDir(productName, version).rstrip('/') commands = [ 'rm -rf %s' % candidatesDir, 'rm -rf %s' % releasesDir, 'mkdir -p %s' % candidatesDir, 'cd %(candidatesDir)s && \ wget -nv -r -np -nH --cut-dirs=6 -R index.html* \ -X %(candidatesDir)s/unsigned \ -X %(candidatesDir)s/contrib* \ -X %(candidatesDir)s/partner-repacks \ -X %(candidatesDir)s/win32-EUballot \ %(stageUrlPrefix)s%(candidatesDir)s/' % \ (dict(candidatesDir=candidatesDir, stageUrlPrefix=stageUrlPrefix)), 'ln -s %s %s' % (candidatesDir, releasesDir), ] for command in commands: run_remote_cmd(command, server=stageServer, username=stageUsername, sshKey=stageSshKey)
def pushToMirrors(productName, version, buildNumber, stageServer, stageUsername=None, stageSshKey=None, excludes=None, extra_excludes=None, dryRun=False): """ excludes overrides DEFAULT_RSYNC_EXCLUDES, extra_exludes will be appended to DEFAULT_RSYNC_EXCLUDES. """ source_dir = makeCandidatesDir(productName, version, buildNumber) target_dir = makeReleasesDir(productName, version) if not excludes: excludes = DEFAULT_RSYNC_EXCLUDES if extra_excludes: excludes += extra_excludes # fail/warn if target directory exists depending on dry run mode try: run_remote_cmd(['test', '!', '-d', target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) except CalledProcessError: if not dryRun: raise else: log.warning('WARN: target directory %s exists', target_dir) if not dryRun: run_remote_cmd(['mkdir', '-p', target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) run_remote_cmd(['chmod', 'u=rwx,g=rxs,o=rx', target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) rsync_cmd = ['rsync', '-av' ] if dryRun: rsync_cmd.append('-n') run_remote_cmd(rsync_cmd + excludes + [source_dir, target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def updateSymlink(productName, version, stageServer, stageUsername, stageSshKey, target): releases_dir = makeReleasesDir(productName) run_remote_cmd([ 'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' % dict(rd=releases_dir, version=version, target=target)], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def pushToMirrors(productName, version, buildNumber, stageServer, stageUsername=None, stageSshKey=None, excludes=None, extra_excludes=None, dryRun=False, overwrite=False): """ excludes overrides DEFAULT_RSYNC_EXCLUDES, extra_exludes will be appended to DEFAULT_RSYNC_EXCLUDES. """ source_dir = makeCandidatesDir(productName, version, buildNumber) target_dir = makeReleasesDir(productName, version) if not excludes: excludes = DEFAULT_RSYNC_EXCLUDES if extra_excludes: excludes += ['--exclude=%s' % ex for ex in extra_excludes] # fail/warn if target directory exists depending on dry run mode try: run_remote_cmd(['test', '!', '-d', target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) except CalledProcessError: if overwrite: log.info( 'target directory %s exists, but overwriting files as requested' % target_dir) elif dryRun: log.warning('WARN: target directory %s exists', target_dir) else: raise if not dryRun: run_remote_cmd(['mkdir', '-p', target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) run_remote_cmd(['chmod', 'u=rwx,g=rxs,o=rx', target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) rsync_cmd = ['rsync', '-av'] if dryRun: rsync_cmd.append('-n') # use hardlinks rsync_cmd.append('--link-dest=%s' % source_dir) run_remote_cmd(rsync_cmd + excludes + [source_dir, target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def updateSymlink(productName, version, stageServer, stageUsername, stageSshKey, target): # step 1 check if we have already pushed to mirrors (bug 1083208) push_dir = makeReleasesDir(productName, version) try: # check if the remote dir exists run_remote_cmd(['test', '-d', push_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) except CalledProcessError: log.error('ERROR: push to mirrors directory, %s, does not exist on %s' % (push_dir, stageServer)) log.error('ERROR: Did you push to mirrors before running post release?') raise releases_dir = makeReleasesDir(productName) run_remote_cmd([ 'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' % dict(rd=releases_dir, version=version, target=target)], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def updateSymlink(productName, version, stageServer, stageUsername, stageSshKey, target): releases_dir = makeReleasesDir(productName) run_remote_cmd([ 'cd %(rd)s && rm -f %(target)s && ln -s %(version)s %(target)s' % dict(rd=releases_dir, version=version, target=target) ], server=stageServer, username=stageUsername, sshKey=stageSshKey)
def pushToMirrors( productName, version, buildNumber, stageServer, stageUsername=None, stageSshKey=None, excludes=None, extra_excludes=None, dryRun=False, overwrite=False, ): """ excludes overrides DEFAULT_RSYNC_EXCLUDES, extra_exludes will be appended to DEFAULT_RSYNC_EXCLUDES. """ source_dir = makeCandidatesDir(productName, version, buildNumber) target_dir = makeReleasesDir(productName, version) if not excludes: excludes = DEFAULT_RSYNC_EXCLUDES if extra_excludes: excludes += ["--exclude=%s" % ex for ex in extra_excludes] # fail/warn if target directory exists depending on dry run mode try: run_remote_cmd(["test", "!", "-d", target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) except CalledProcessError: if overwrite: log.info("target directory %s exists, but overwriting files as requested" % target_dir) elif dryRun: log.warning("WARN: target directory %s exists", target_dir) else: raise if not dryRun: run_remote_cmd(["mkdir", "-p", target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey) run_remote_cmd( ["chmod", "u=rwx,g=rxs,o=rx", target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey ) rsync_cmd = ["rsync", "-av"] if dryRun: rsync_cmd.append("-n") # use hardlinks rsync_cmd.append("--link-dest=%s" % source_dir) run_remote_cmd( rsync_cmd + excludes + [source_dir, target_dir], server=stageServer, username=stageUsername, sshKey=stageSshKey )
def testRemoteAndVersioned(self): got = makeReleasesDir('yx', '1.0', protocol='https', server='cee.dee') self.assertEquals('https://cee.dee/pub/yx/releases/1.0/', got)
def testRemote(self): got = makeReleasesDir('yy', protocol='http', server='foo.bar') self.assertEquals('http://foo.bar/pub/yy/releases/', got)
def testVersioned(self): got = makeReleasesDir('aa', '15.1') self.assertEquals('/pub/aa/releases/15.1/', got)
def testRemoteAndVersioned(self): got = makeReleasesDir('yx', '1.0', protocol='https', server='cee.dee') self.assertEquals( 'https://cee.dee/pub/mozilla.org/yx/releases/1.0/', got)
def testVersioned(self): got = makeReleasesDir('aa', '15.1') self.assertEquals('/pub/mozilla.org/aa/releases/15.1/', got)
extra_excludes=args.extra_excludes, buildNumber=buildNumber, overwrite=args.overwrite) if createIndexFiles: deleteIndexFiles(stageServer=stageServer, stageUsername=stageUsername, stageSshKey=stageSshKey, cleanup_dir=makeCandidatesDir( productName, version, buildNumber)) if 'postrelease' in actions: if createIndexFiles: deleteIndexFiles(stageServer=stageServer, stageUsername=stageUsername, stageSshKey=stageSshKey, cleanup_dir=makeReleasesDir(productName, version)) if ftpSymlinkName: updateSymlink(stageServer=stageServer, stageUsername=stageUsername, stageSshKey=stageSshKey, productName=productName, version=version, target=ftpSymlinkName) if syncPartnerBundles: doSyncPartnerBundles(stageServer=stageServer, stageUsername=stageUsername, stageSshKey=stageSshKey, productName=productName, version=version, buildNumber=buildNumber) if bouncer_aliases and productName != 'xulrunner':
def testBaseReleases(self): got = makeReleasesDir('bbb') self.assertEquals('/pub/bbb/releases/', got)
version=version, extra_excludes=options.extra_excludes, buildNumber=buildNumber, overwrite=options.overwrite) if createIndexFiles: deleteIndexFiles(stageServer=stageServer, stageUsername=stageUsername, stageSshKey=stageSshKey, cleanup_dir=makeCandidatesDir(productName, version, buildNumber)) if 'postrelease' in args: if createIndexFiles: deleteIndexFiles(stageServer=stageServer, stageUsername=stageUsername, stageSshKey=stageSshKey, cleanup_dir=makeReleasesDir(productName, version)) if ftpSymlinkName: updateSymlink(stageServer=stageServer, stageUsername=stageUsername, stageSshKey=stageSshKey, productName=productName, version=version, target=ftpSymlinkName) if syncPartnerBundles: doSyncPartnerBundles(stageServer=stageServer, stageUsername=stageUsername, stageSshKey=stageSshKey, productName=productName, version=version, buildNumber=buildNumber)
if ftp_platform not in platforms and locale in locales: log.info("Removing %s locale from %s platform for %s" % ( locale, ftp_platform, v)) locales.remove(locale) # Exclude locales being full checked quick_check_locales = [l for l in locales if l not in full_check_locales] # Get the intersection of from and to full_check_locales this_full_check_locales = [l for l in full_check_locales if l in locales] from_ = makeReleaseRepackUrls( product_name, app_name, v, options.platform, locale='%locale%', signed=True, exclude_secondary=True ).values()[0] release_dir = makeReleasesDir(product_name, v, ftp_root='/') from_path = "%s%s" % (release_dir, from_) if v in partials: # Full test for all locales # "from" and "to" to be downloaded from the same staging # server in dev environment if len(locales) > 0: log.info("Generating configs for partial update checks for %s" % v) uvc.addRelease(release=appVersion, build_id=build_id, locales=locales, patch_types=['complete', 'partial'], from_path=from_path, ftp_server_from=staging_server, ftp_server_to=staging_server, mar_channel_IDs=mar_channel_IDs) else:
build_id = from_["platforms"][ftp_platform] mar_channel_IDs = from_.get('mar-channel-ids') # Use new build targets for Windows, but only on compatible # versions (42+). See bug 1185456 for additional context. if args.platform not in ("win32", "win64") or \ LooseVersion(fromVersion) < LooseVersion("42.0"): update_platform = buildbot2updatePlatforms(args.platform)[0] else: update_platform = buildbot2updatePlatforms(args.platform)[1] path_ = makeReleaseRepackUrls( product_name, app_name, fromVersion, args.platform, locale='%locale%', signed=True, exclude_secondary=True ).values()[0] release_dir = makeReleasesDir(stage_product_name, fromVersion, ftp_root='/') from_path = "%s%s" % (release_dir, path_) # Exclude locales being full checked quick_check_locales = [l for l in locales if l not in full_check_locales] # Get the intersection of from and to full_check_locales this_full_check_locales = [l for l in full_check_locales if l in locales] if fromVersion in partials: log.info("Generating configs for partial update checks for %s" % fromVersion) uvc.addRelease(release=appVersion, build_id=build_id, locales=locales, patch_types=["complete", "partial"],
def testRemote(self): got = makeReleasesDir('yy', protocol='http', server='foo.bar') self.assertEquals('http://foo.bar/pub/mozilla.org/yy/releases/', got)
# See bug 1185456 for additional context. if options.platform not in ( "win32", "win64") or LooseVersion(fromVersion) < LooseVersion("42.0"): update_platform = buildbot2updatePlatforms(options.platform)[0] else: update_platform = buildbot2updatePlatforms(options.platform)[1] path_ = makeReleaseRepackUrls(product_name, app_name, fromVersion, options.platform, locale='%locale%', signed=True, exclude_secondary=True).values()[0] release_dir = makeReleasesDir(product_name, fromVersion, ftp_root='/') from_path = "%s%s" % (release_dir, path_) # Exclude locales being full checked quick_check_locales = [ l for l in locales if l not in full_check_locales ] # Get the intersection of from and to full_check_locales this_full_check_locales = [ l for l in full_check_locales if l in locales ] if fromVersion in partials: log.info("Generating configs for partial update checks for %s" % fromVersion) uvc.addRelease(release=appVersion,
def testBaseReleases(self): got = makeReleasesDir('bbb') self.assertEquals('/pub/mozilla.org/bbb/releases/', got)
# See bug 1185456 for additional context. if options.platform not in ("win32", "win64") or LooseVersion(fromVersion) < LooseVersion("42.0"): update_platform = buildbot2updatePlatforms(options.platform)[0] else: update_platform = buildbot2updatePlatforms(options.platform)[1] path_ = makeReleaseRepackUrls( product_name, app_name, fromVersion, options.platform, locale="%locale%", signed=True, exclude_secondary=True, ).values()[0] release_dir = makeReleasesDir(product_name, fromVersion, ftp_root="/") from_path = "%s%s" % (release_dir, path_) # Exclude locales being full checked quick_check_locales = [l for l in locales if l not in full_check_locales] # Get the intersection of from and to full_check_locales this_full_check_locales = [l for l in full_check_locales if l in locales] if fromVersion in partials: log.info("Generating configs for partial update checks for %s" % fromVersion) uvc.addRelease( release=appVersion, build_id=build_id, locales=locales, patch_types=["complete", "partial"], from_path=from_path,