def add_locations(self): for platform in self.platforms: template_dict = { 'product': self.productName, 'brandName': self.brandName, 'bouncer_product': self.bouncerProductName, 'version': self.version, 'milestone': self.milestone, 'prettyVersion': getPrettyVersion(self.version), 'old_version': self.oldVersion, 'ftp_platform': buildbot2ftp(platform), 'locale': ':lang' } # Full product path = self.full_product_template[platform] % template_dict self.location_add(self.bouncer_product_name, platform, path) # Complete MAR product if self.addMARs: path = self.complete_mar_template[platform] % template_dict self.location_add(self.complete_mar_bouncer_product_name, platform, path) # Partial MAR product if self.oldVersion: path = self.partial_mar_template[platform] % template_dict self.location_add(self.partial_mar_bouncer_product_name, platform, path)
def generate_data(self, appVersion, productName, version, buildNumber, updateChannels, ftpServer, bouncerServer, enUSPlatforms, schemaVersion, openURL=None, **updateKwargs): assert schemaVersion in ( 3, 4), 'Unhandled schema version %s' % schemaVersion details_product = productName.lower() if details_product == "devedition": details_product = "firefox" data = { 'detailsUrl': getProductDetails(details_product, appVersion), 'platforms': {}, 'fileUrls': {}, 'appVersion': appVersion, 'platformVersion': appVersion, 'displayVersion': getPrettyVersion(version) } actions = [] if openURL: actions.append("showURL") data["openURL"] = openURL if actions: data["actions"] = " ".join(actions) fileUrls = self._getFileUrls(productName, version, buildNumber, updateChannels, ftpServer, bouncerServer, **updateKwargs) if fileUrls: data.update(fileUrls) updateData = self._get_update_data(productName, version, **updateKwargs) if updateData: data.update(updateData) for platform in enUSPlatforms: updatePlatforms = buildbot2updatePlatforms(platform) bouncerPlatform = buildbot2bouncer(platform) ftpPlatform = buildbot2ftp(platform) data['platforms'][updatePlatforms[0]] = { 'OS_BOUNCER': bouncerPlatform, 'OS_FTP': ftpPlatform } for aliasedPlatform in updatePlatforms[1:]: data['platforms'][aliasedPlatform] = { 'alias': updatePlatforms[0] } return data
def downloadUpdate(stageServer, productName, version, buildNumber, platform, locale, candidatesDir=None): if candidatesDir is None: candidatesDir = makeCandidatesDir(productName, version, buildNumber, protocol='http', server=stageServer) fileName = '%s-%s.complete.mar' % (productName, version) destFileName = '%s-%s.%s.complete.mar' % (productName, version, locale) platformDir = buildbot2ftp(platform) url = '/'.join([ p.strip('/') for p in [candidatesDir, 'update', platformDir, locale, fileName] ]) log.info("Downloading %s to %s", url, destFileName) remote_f = urlopen(url) local_f = open(destFileName, "wb") local_f.write(remote_f.read()) local_f.close() return destFileName
def generate_data(self, appVersion, productName, version, buildNumber, updateChannels, stagingServer, bouncerServer, enUSPlatforms, schemaVersion, **updateKwargs): assert schemaVersion in (2, 3), 'Unhandled schema version %s' % schemaVersion self.name = get_release_blob_name(productName, version, buildNumber) data = { 'name': self.name, 'detailsUrl': getProductDetails(productName.lower(), appVersion), 'platforms': {}, 'fileUrls': {}, 'ftpFilenames': {}, 'bouncerProducts': {}, } data['appVersion'] = appVersion data['platformVersion'] = appVersion data['displayVersion'] = getPrettyVersion(version) # XXX: This is a hack for bug 1045583. We should remove it, and always # use "candidates" for nightlyDir after the switch to Balrog is complete. if productName.lower() == "mobile": nightlyDir = "candidates" else: nightlyDir = "nightly" for channel in updateChannels: if channel in ('betatest', 'esrtest') or "localtest" in channel: dir_ = makeCandidatesDir(productName.lower(), version, buildNumber, server=stagingServer, protocol='http', nightlyDir=nightlyDir) data['fileUrls'][channel] = '%supdate/%%OS_FTP%%/%%LOCALE%%/%%FILENAME%%' % dir_ else: url = 'http://%s/?product=%%PRODUCT%%&os=%%OS_BOUNCER%%&lang=%%LOCALE%%' % bouncerServer data['fileUrls'][channel] = url # XXX: quick hack for bug 1021026. We should be using Bouncer for this # after we implement better solution talked about in comments 2 through 4 if channel == 'release': dir_ = makeCandidatesDir(productName.lower(), version, buildNumber, server='download.cdn.mozilla.net', protocol='http', nightlyDir=nightlyDir) url = '%supdate/%%OS_FTP%%/%%LOCALE%%/%%FILENAME%%' % dir_ data['fileUrls']['beta'] = url data['fileUrls']['beta-cdntest'] = url data.update(self._get_update_data(productName, version, **updateKwargs)) for platform in enUSPlatforms: updatePlatforms = buildbot2updatePlatforms(platform) bouncerPlatform = buildbot2bouncer(platform) ftpPlatform = buildbot2ftp(platform) data['platforms'][updatePlatforms[0]] = { 'OS_BOUNCER': bouncerPlatform, 'OS_FTP': ftpPlatform } for aliasedPlatform in updatePlatforms[1:]: data['platforms'][aliasedPlatform] = { 'alias': updatePlatforms[0] } return data
def add_locations(self): for platform in self.platforms: template_dict = {'product': self.productName, 'brandName': self.brandName, 'bouncer_product': self.bouncerProductName, 'version': self.version, 'milestone': self.milestone, 'prettyVersion': getPrettyVersion(self.version), 'ftp_platform': buildbot2ftp(platform), 'locale': ':lang'} # Full product path = self.full_product_template[platform] % template_dict self.location_add(self.bouncer_product_name, platform, path) # Complete MAR product if self.addMARs: path = self.complete_mar_template[platform] % template_dict self.location_add(self.complete_mar_bouncer_product_name, platform, path) # Partial MAR product for previousVersion in self.partialVersions: template_dict['previous_version'] = previousVersion path = self.partial_mar_template[platform] % template_dict name = self.partial_mar_bouncer_product_names[ previousVersion] self.location_add(name, platform, path)
def downloadUpdate(stageServer, productName, version, buildNumber, platform, locale, candidatesDir=None): if candidatesDir is None: candidatesDir = makeCandidatesDir(productName, version, buildNumber, protocol='http', server=stageServer) fileName = '%s-%s.complete.mar' % (productName, version) destFileName = '%s-%s.%s.complete.mar' % (productName, version, locale) platformDir = buildbot2ftp(platform) url = '/'.join([ p.strip('/') for p in [candidatesDir, 'update', platformDir, locale, fileName] ]) log.info("Downloading %s to %s", url, destFileName) remote_f = urlopen(url, timeout=20) local_f = open(destFileName, "wb") local_f.write(remote_f.read()) local_f.close() expected_size = int(remote_f.info()['Content-Length']) actual_size = os.path.getsize(destFileName) if expected_size != actual_size: log.info("File is truncated, got %s of %s bytes" % (actual_size, expected_size)) raise HTTPError return destFileName
def makeReleaseRepackUrls(productName, brandName, version, platform, locale='en-US', signed=False): longVersion = version builds = {} platformDir = buildbot2ftp(platform) if productName not in ('fennec',): if platform.startswith('linux'): filename = '%s.tar.bz2' % productName builds[filename] = '/'.join([p.strip('/') for p in [ platformDir, locale, '%s-%s.tar.bz2' % (productName, version)]]) elif platform.startswith('macosx'): filename = '%s.dmg' % productName builds[filename] = '/'.join([p.strip('/') for p in [ platformDir, locale, '%s %s.dmg' % (brandName, longVersion)]]) elif platform.startswith('win'): filename = '%s.zip' % productName instname = '%s.exe' % productName prefix = [] if not signed: prefix.append('unsigned') prefix.extend([platformDir, locale]) builds[filename] = '/'.join( [p.strip('/') for p in prefix + ['%s-%s.zip' % (productName, version)]] ) builds[instname] = '/'.join( [p.strip('/') for p in prefix + ['%s Setup %s.exe' % (brandName, longVersion)]] ) else: raise "Unsupported platform" else: if platform.startswith('android'): filename = '%s-%s.%s.android-arm.apk' % (productName, version, locale) prefix = [] if not signed: prefix.append('unsigned') prefix.extend([platformDir, locale]) builds[filename] = '/'.join( [p.strip('/') for p in prefix + [filename]] ) elif platform == 'linux': filename = '%s.tar.bz2' % productName builds[filename] = '/'.join([p.strip('/') for p in [ platform, locale, '%s-%s.%s.linux-i686.tar.bz2' % (productName, version, locale)]]) elif platform == 'macosx': filename = '%s.dmg' % productName builds[filename] = '/'.join([p.strip('/') for p in [ platform, locale, '%s-%s.%s.mac.dmg' % (brandName, version, locale)]]) elif platform == 'win32': filename = '%s.zip' % productName builds[filename] = '/'.join([p.strip('/') for p in [ platform, locale, '%s-%s.%s.win32.zip' % (productName, version, locale)]]) else: raise "Unsupported platform" return builds
def testPartialCompleteMarUrl(self): for product in PRODUCTS: for platform in PLATFORMS: buildNumber = 1 p = Partial(product, VERSION, buildNumber, PROTOCOL, SERVER) complete_mar_name = p.complete_mar_name() url = '%s://%s/pub/mozilla.org/%s/candidates/%s-candidates/build%s/update/%s/en-US/%s' % ( PROTOCOL, SERVER, product, VERSION, buildNumber, buildbot2ftp(platform), complete_mar_name) self.assertEqual(p.complete_mar_url(platform), url) buildNumber = None p = Partial(product, VERSION, buildNumber, PROTOCOL, SERVER) url = '%s://%s/pub/mozilla.org/%s/releases/%s/update/%s/en-US/%s' % ( PROTOCOL, SERVER, product, VERSION, buildbot2ftp(platform), complete_mar_name) self.assertEqual(p.complete_mar_url(platform), url)
def testPartialCompleteMarUrl(self): for product in PRODUCTS: for platform in PLATFORMS: buildNumber = 1 p = Partial(product, VERSION, buildNumber, PROTOCOL, SERVER) complete_mar_name = p.complete_mar_name() url = '%s://%s/pub/mozilla.org/%s/candidates/%s-candidates/build%s/update/%s/en-US/%s' % ( PROTOCOL, SERVER, product, VERSION, buildNumber, buildbot2ftp(platform), complete_mar_name) self.assertEqual(p.complete_mar_url(platform), url) buildNumber = None p = Partial(product, VERSION, buildNumber, PROTOCOL, SERVER) url = '%s://%s/pub/mozilla.org/%s/releases/%s/update/%s/en-US/%s' % ( PROTOCOL, SERVER, product, VERSION, buildbot2ftp(platform), complete_mar_name) self.assertEqual(p.complete_mar_url(platform), url)
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir, l10nIni, compareLocalesRepo, env, merge=True, prevMar=None, productName=None, platform=None, version=None, oldVersion=None): repo = "/".join([l10nBaseRepo, locale]) localeDir = path.join(l10nRepoDir, locale) retry(mercurial, args=(repo, localeDir)) update(localeDir, revision=revision) compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir, l10nIni, revision=revision, merge=merge) env["AB_CD"] = locale env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged")) if sys.platform.startswith('win'): env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"]) if sys.platform.startswith('darwin'): env["MOZ_PKG_PLATFORM"] = "mac" run_cmd(["make", "installers-%s" % locale], cwd=localeSrcDir, env=env) UPLOAD_EXTRA_FILES = [] if prevMar: nativeDistDir = path.normpath(path.abspath(path.join(localeSrcDir, '../../dist'))) posixDistDir = windows2msys(nativeDistDir) mar = '%s/host/bin/mar' % posixDistDir mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir current = '%s/current' % posixDistDir previous = '%s/previous' % posixDistDir updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale) updateAbsDir = '%s/%s' % (posixDistDir, updateDir) current_mar = '%s/%s-%s.complete.mar' % (updateAbsDir, productName, version) partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion, version) partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name) UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name)) env['MAR'] = mar env['MBSDIFF'] = mbsdiff run_cmd(['rm', '-rf', previous, current]) run_cmd(['mkdir', previous, current]) run_cmd(['perl', '../../../tools/update-packaging/unwrap_full_update.pl', '../../../../%s' % prevMar], cwd=path.join(nativeDistDir, 'previous'), env=env) run_cmd(['perl', '../../../tools/update-packaging/unwrap_full_update.pl', current_mar], cwd=path.join(nativeDistDir, 'current'), env=env) run_cmd( ['bash', '../../tools/update-packaging/make_incremental_update.sh', partial_mar, previous, current], cwd=nativeDistDir, env=env) if os.environ.get('MOZ_SIGN_CMD'): run_cmd(['bash', '-c', '%s -f gpg -f mar "%s"' % (os.environ['MOZ_SIGN_CMD'], partial_mar)], env=env) UPLOAD_EXTRA_FILES.append('%s/%s.asc' % (updateDir, partial_mar_name)) retry(run_cmd, args=(["make", "upload", "AB_CD=%s" % locale, 'UPLOAD_EXTRA_FILES=%s' % ' '.join(UPLOAD_EXTRA_FILES)],), kwargs={'cwd': localeSrcDir, 'env': env})
def generate_data(self, appVersion, productName, version, buildNumber, partialUpdates, updateChannels, stagingServer, bouncerServer, enUSPlatforms, schemaVersion): # TODO: Multiple partial support. Probably as a part of bug 797033. previousVersion = str(max(StrictVersion(v) for v in partialUpdates)) self.name = get_release_blob_name(productName, version, buildNumber) data = { 'name': self.name, 'detailsUrl': getProductDetails(productName.lower(), appVersion), 'platforms': {}, 'fileUrls': {}, 'ftpFilenames': {}, 'bouncerProducts': {}, } assert schemaVersion in (1, 2), 'Unhandled schema version %s' % schemaVersion if schemaVersion == 1: data['appv'] = appVersion data['extv'] = appVersion elif schemaVersion == 2: data['appVersion'] = appVersion data['platformVersion'] = appVersion data['displayVersion'] = getPrettyVersion(version) for channel in updateChannels: if channel in ('betatest', 'esrtest'): dir_ = makeCandidatesDir(productName.lower(), version, buildNumber, server=stagingServer, protocol='http') data['fileUrls'][channel] = '%supdate/%%OS_FTP%%/%%LOCALE%%/%%FILENAME%%' % dir_ else: url = 'http://%s/?product=%%PRODUCT%%&os=%%OS_BOUNCER%%&lang=%%LOCALE%%' % bouncerServer data['fileUrls'][channel] = url data['ftpFilenames']['complete'] = '%s-%s.complete.mar' % (productName.lower(), version) data['ftpFilenames']['partial'] = '%s-%s-%s.partial.mar' % (productName.lower(), previousVersion, version) data['bouncerProducts']['complete'] = '%s-%s-Complete' % (productName.capitalize(), version) data['bouncerProducts']['partial'] = '%s-%s-Partial-%s' % (productName.capitalize(), version, previousVersion) for platform in enUSPlatforms: updatePlatforms = buildbot2updatePlatforms(platform) bouncerPlatform = buildbot2bouncer(platform) ftpPlatform = buildbot2ftp(platform) data['platforms'][updatePlatforms[0]] = { 'OS_BOUNCER': bouncerPlatform, 'OS_FTP': ftpPlatform } for aliasedPlatform in updatePlatforms[1:]: data['platforms'][aliasedPlatform] = { 'alias': updatePlatforms[0] } return data
def generate_data(self, appVersion, productName, version, buildNumber, updateChannels, ftpServer, bouncerServer, enUSPlatforms, **updateKwargs): details_product = productName.lower() if details_product == "devedition": details_product = "firefox" data = { 'platforms': {}, 'fileUrls': {}, 'appVersion': appVersion, 'displayVersion': getPrettyVersion(version), 'updateLine': [ { 'for': {}, 'fields': { 'detailsURL': getProductDetails(details_product, appVersion), 'type': 'minor', }, }, ] } actions = [] fileUrls = self._getFileUrls(productName, version, buildNumber, updateChannels, ftpServer, bouncerServer, **updateKwargs) if fileUrls: data.update(fileUrls) for platform in enUSPlatforms: updatePlatforms = buildbot2updatePlatforms(platform) bouncerPlatform = buildbot2bouncer(platform) ftpPlatform = buildbot2ftp(platform) data['platforms'][updatePlatforms[0]] = { 'OS_BOUNCER': bouncerPlatform, 'OS_FTP': ftpPlatform } for aliasedPlatform in updatePlatforms[1:]: data['platforms'][aliasedPlatform] = { 'alias': updatePlatforms[0] } return data
def generate_data(self, appVersion, productName, version, buildNumber, updateChannels, ftpServer, bouncerServer, enUSPlatforms, schemaVersion, openURL=None, **updateKwargs): assert schemaVersion in ( 3, 4), 'Unhandled schema version %s' % schemaVersion data = { 'detailsUrl': getProductDetails(productName.lower(), appVersion), 'platforms': {}, 'fileUrls': {}, 'appVersion': appVersion, 'platformVersion': appVersion, 'displayVersion': getPrettyVersion(version) } actions = [] if openURL: actions.append("showURL") data["openURL"] = openURL if actions: data["actions"] = " ".join(actions) fileUrls = self._getFileUrls(productName, version, buildNumber, updateChannels, ftpServer, bouncerServer, **updateKwargs) if fileUrls: data.update(fileUrls) updateData = self._get_update_data( productName, version, **updateKwargs) if updateData: data.update(updateData) for platform in enUSPlatforms: updatePlatforms = buildbot2updatePlatforms(platform) bouncerPlatform = buildbot2bouncer(platform) ftpPlatform = buildbot2ftp(platform) data['platforms'][updatePlatforms[0]] = { 'OS_BOUNCER': bouncerPlatform, 'OS_FTP': ftpPlatform } for aliasedPlatform in updatePlatforms[1:]: data['platforms'][aliasedPlatform] = { 'alias': updatePlatforms[0] } return data
def downloadUpdate(stageServer, productName, version, buildNumber, platform, locale, candidatesDir=None): if candidatesDir is None: candidatesDir = makeCandidatesDir(productName, version, buildNumber, protocol='http', server=stageServer) fileName = '%s-%s.complete.mar' % (productName, version) destFileName = '%s-%s.%s.complete.mar' % (productName, version, locale) platformDir = buildbot2ftp(platform) url = '/'.join([p.strip('/') for p in [ candidatesDir, 'update', platformDir, locale, fileName]]) log.info("Downloading %s to %s", url, destFileName) remote_f = urlopen(url) local_f = open(destFileName, "wb") local_f.write(remote_f.read()) local_f.close() return destFileName
def generate_data(self, appVersion, productName, version, buildNumber, updateChannels, stagingServer, bouncerServer, enUSPlatforms, schemaVersion, **updateKwargs): assert schemaVersion in (2, 3), 'Unhandled schema version %s' % schemaVersion self.name = get_release_blob_name(productName, version, buildNumber) data = { 'name': self.name, 'detailsUrl': getProductDetails(productName.lower(), appVersion), 'platforms': {}, 'fileUrls': {}, 'ftpFilenames': {}, 'bouncerProducts': {}, } data['appVersion'] = appVersion data['platformVersion'] = appVersion data['displayVersion'] = getPrettyVersion(version) for channel in updateChannels: if channel in ('betatest', 'esrtest'): dir_ = makeCandidatesDir(productName.lower(), version, buildNumber, server=stagingServer, protocol='http') data['fileUrls'][channel] = '%supdate/%%OS_FTP%%/%%LOCALE%%/%%FILENAME%%' % dir_ else: url = 'http://%s/?product=%%PRODUCT%%&os=%%OS_BOUNCER%%&lang=%%LOCALE%%' % bouncerServer data['fileUrls'][channel] = url data.update(self._get_update_data(productName, version, **updateKwargs)) for platform in enUSPlatforms: updatePlatforms = buildbot2updatePlatforms(platform) bouncerPlatform = buildbot2bouncer(platform) ftpPlatform = buildbot2ftp(platform) data['platforms'][updatePlatforms[0]] = { 'OS_BOUNCER': bouncerPlatform, 'OS_FTP': ftpPlatform } for aliasedPlatform in updatePlatforms[1:]: data['platforms'][aliasedPlatform] = { 'alias': updatePlatforms[0] } return data
def makeReleaseRepackUrls(productName, brandName, version, platform, locale='en-US', signed=False, exclude_secondary=False): longVersion = version builds = {} platformDir = buildbot2ftp(platform) if productName not in ('fennec', ): if platform.startswith('linux'): filename = '%s.tar.bz2' % productName builds[filename] = '/'.join([ p.strip('/') for p in [ platformDir, locale, '%s-%s.tar.bz2' % (productName, version) ] ]) elif platform.startswith('macosx'): filename = '%s.dmg' % productName builds[filename] = '/'.join([ p.strip('/') for p in [platformDir, locale, '%s %s.dmg' % (brandName, longVersion)] ]) elif platform.startswith('win'): filename = '%s.zip' % productName instname = '%s.exe' % productName prefix = [] if not signed: prefix.append('unsigned') prefix.extend([platformDir, locale]) if not exclude_secondary: builds[filename] = '/'.join([ p.strip('/') for p in prefix + ['%s-%s.zip' % (productName, version)] ]) builds[instname] = '/'.join([ p.strip('/') for p in prefix + ['%s Setup %s.exe' % (brandName, longVersion)] ]) else: raise "Unsupported platform" else: if platform.startswith('android'): filename = '%s-%s.%s.android-arm.apk' % (productName, version, locale) prefix = [] if not signed: prefix.append('unsigned') prefix.extend([platformDir, locale]) builds[filename] = '/'.join( [p.strip('/') for p in prefix + [filename]]) elif platform == 'linux': filename = '%s.tar.bz2' % productName builds[filename] = '/'.join([ p.strip('/') for p in [ platform, locale, '%s-%s.%s.linux-i686.tar.bz2' % (productName, version, locale) ] ]) elif platform == 'macosx': filename = '%s.dmg' % productName builds[filename] = '/'.join([ p.strip('/') for p in [ platform, locale, '%s-%s.%s.mac.dmg' % (brandName, version, locale) ] ]) elif platform == 'win32': filename = '%s.zip' % productName builds[filename] = '/'.join([ p.strip('/') for p in [ platform, locale, '%s-%s.%s.win32.zip' % (productName, version, locale) ] ]) else: raise "Unsupported platform" return builds
required_options = [ 'config', 'platform', 'release_config', 'buildbot_configs', 'release_tag' ] options_dict = vars(options) for opt in required_options: if not options_dict[opt]: parser.error("Required option %s not present" % opt) if options.verbose: log.setLevel(logging.DEBUG) else: log.setLevel(logging.INFO) ftp_platform = buildbot2ftp(options.platform) full_check_locales = options.full_check_locales # Variables from release config retry(mercurial, args=(options.buildbot_configs, 'buildbot-configs')) update('buildbot-configs', revision=options.release_tag) release_config = validate(options) product_name = release_config['productName'] staging_server = FTP_SERVER_TEMPLATE % release_config['stagingServer'] aus_server_url = release_config['ausServerUrl'] build_number = release_config['buildNumber'] previous_releases_staging_server = FTP_SERVER_TEMPLATE % \ release_config.get('previousReleasesStagingServer', release_config['stagingServer']) # Current version data
def test_platform_in_command(self): for platform, task in self.tasks.iteritems(): command = task['task']['payload']['command'] self.assertTrue("--platform {}".format(buildbot2ftp(platform)) in "".join(command))
required_options = ['config', 'platform', 'release_config', 'buildbot_configs', 'release_tag'] options_dict = vars(options) for opt in required_options: if not options_dict[opt]: parser.error("Required option %s not present" % opt) if options.verbose: log.setLevel(logging.DEBUG) else: log.setLevel(logging.INFO) update_platform = buildbot2updatePlatforms(options.platform)[0] ftp_platform = buildbot2ftp(options.platform) full_check_locales = options.full_check_locales # Variables from release config retry(mercurial, args=(options.buildbot_configs, 'buildbot-configs')) update('buildbot-configs', revision=options.release_tag) release_config = validate(options) product_name = release_config['productName'] staging_server = FTP_SERVER_TEMPLATE % release_config['stagingServer'] aus_server_url = release_config['ausServerUrl'] build_number = release_config['buildNumber'] previous_releases_staging_server = FTP_SERVER_TEMPLATE % \ release_config.get('previousReleasesStagingServer', release_config['stagingServer']) # Current version data
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir, l10nIni, compareLocalesRepo, env, absObjdir, merge=True, productName=None, platform=None, version=None, partialUpdates=None, buildNumber=None, stageServer=None, mozillaDir=None, mozillaSrcDir=None): repo = "/".join([l10nBaseRepo, locale]) localeDir = path.join(l10nRepoDir, locale) mercurial(repo, localeDir) update(localeDir, revision=revision) # It's a bad assumption to make, but the source dir is currently always # one level above the objdir. absSourceRepoPath = path.split(absObjdir)[0] use_pymake = env.get("USE_PYMAKE", False) make = getMakeCommand(use_pymake, absSourceRepoPath) env["AB_CD"] = locale env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged")) if sys.platform.startswith('win'): if use_pymake: env["LOCALE_MERGEDIR"] = msys2windows(env["LOCALE_MERGEDIR"]) else: env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"]) if sys.platform.startswith('darwin'): env["MOZ_PKG_PLATFORM"] = "mac" UPLOAD_EXTRA_FILES = [] if mozillaDir: nativeDistDir = path.normpath(path.abspath( path.join(localeSrcDir, '../../%s/dist' % mozillaDir))) else: nativeDistDir = path.normpath(path.abspath( path.join(localeSrcDir, '../../dist'))) posixDistDir = windows2msys(nativeDistDir) mar = '%s/host/bin/mar' % posixDistDir mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir if platform.startswith('win'): mar += ".exe" mbsdiff += ".exe" current = '%s/current' % posixDistDir previous = '%s/previous' % posixDistDir updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale) updateAbsDir = '%s/%s' % (posixDistDir, updateDir) current_mar = '%s/%s-%s.complete.mar' % ( updateAbsDir, productName, version) unwrap_full_update = '../../../tools/update-packaging/unwrap_full_update.pl' make_incremental_update = '../../tools/update-packaging/make_incremental_update.sh' prevMarDir = '../../../../' if mozillaSrcDir: # Compensate for having the objdir or not. additionalParent = '' if mozillaDir: additionalParent = '../' unwrap_full_update = '../../../%s%s/tools/update-packaging/unwrap_full_update.pl' % (additionalParent, mozillaSrcDir) make_incremental_update = '../../%s%s/tools/update-packaging/make_incremental_update.sh' % (additionalParent, mozillaSrcDir) prevMarDir = '../../../../%s' % additionalParent env['MAR'] = mar env['MBSDIFF'] = mbsdiff log.info("Download mar tools") if stageServer: candidates_dir = makeCandidatesDir(productName, version, buildNumber, protocol="http", server=stageServer) if not path.isfile(msys2windows(mar)): marUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mar)s" % \ dict(c_dir=candidates_dir, platform=platform, mar=path.basename(mar)) run_cmd(['mkdir', '-p', path.dirname(mar)]) log.info("Downloading %s to %s", marUrl, mar) urlretrieve(marUrl, msys2windows(mar)) if not sys.platform.startswith('win'): run_cmd(['chmod', '755', mar]) if not path.isfile(msys2windows(mbsdiff)): mbsdiffUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mbsdiff)s" % \ dict(c_dir=candidates_dir, platform=platform, mbsdiff=path.basename(mbsdiff)) run_cmd(['mkdir', '-p', path.dirname(mbsdiff)]) log.info("Downloading %s to %s", mbsdiffUrl, mbsdiff) urlretrieve(mbsdiffUrl, msys2windows(mbsdiff)) if not sys.platform.startswith('win'): run_cmd(['chmod', '755', mbsdiff]) else: log.warning('stageServer not set. mar tools will *not* be downloaded.') compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir, l10nIni, revision=revision, merge=merge) run_cmd(make + ["installers-%s" % locale], cwd=localeSrcDir, env=env) # Our Windows-native rm from bug 727551 requires Windows-style paths run_cmd(['rm', '-rf', msys2windows(current)]) run_cmd(['mkdir', current]) run_cmd(['perl', unwrap_full_update, current_mar], cwd=path.join(nativeDistDir, 'current'), env=env) for oldVersion in partialUpdates: prevMar = partialUpdates[oldVersion]['mar'] if prevMar: partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion, version) partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name) UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name)) # Our Windows-native rm from bug 727551 requires Windows-style paths run_cmd(['rm', '-rf', msys2windows(previous)]) run_cmd(['mkdir', previous]) run_cmd( ['perl', unwrap_full_update, '%s/%s' % (prevMarDir, prevMar)], cwd=path.join(nativeDistDir, 'previous'), env=env) run_cmd(['bash', make_incremental_update, partial_mar, previous, current], cwd=nativeDistDir, env=env) if os.environ.get('MOZ_SIGN_CMD'): run_cmd(['bash', '-c', '%s -f mar -f gpg "%s"' % (os.environ['MOZ_SIGN_CMD'], partial_mar)], env=env) UPLOAD_EXTRA_FILES.append( '%s/%s.asc' % (updateDir, partial_mar_name)) else: log.warning( "Skipping partial MAR creation for %s %s" % (oldVersion, locale)) env['UPLOAD_EXTRA_FILES'] = ' '.join(UPLOAD_EXTRA_FILES) retry(run_cmd, args=(make + ["upload", "AB_CD=%s" % locale], ), kwargs={'cwd': localeSrcDir, 'env': env}) # return the location of the checksums file, because consumers may want # some information about the files that were generated. # Some versions of make that we use (at least pymake) imply --print-directory # We need to turn it off to avoid getting extra output that mess up our # parsing of the checksum file path. curdir = os.getcwd() try: os.chdir(localeSrcDir) relative_checksums = get_output(make + ["--no-print-directory", "echo-variable-CHECKSUM_FILE", "AB_CD=%s" % locale], env=env).strip("\"'\n") return path.normpath(path.join(localeSrcDir, relative_checksums)) finally: os.chdir(curdir)
parser.add_argument("--output", dest="output", required=True, type=argparse.FileType('w')) parser.add_argument("-v", "--verbose", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO, help="Increase output verbosity") parser.add_argument("--product", required=True, help="Product name") parser.add_argument("--stage-product", help="Stage product name") parser.add_argument("--archive-prefix", required=True) parser.add_argument("--previous-archive-prefix") parser.add_argument("--balrog-url", required=True) parser.add_argument("--build-number", required=True) args = parser.parse_args() logging.basicConfig(format="%(message)s", level=args.loglevel) ftp_platform = buildbot2ftp(args.platform) full_check_locales = args.full_check_locales product_name = args.product stage_product_name = args.stage_product or product_name prev_archive_prefix = args.previous_archive_prefix or args.archive_prefix aus_server_url = args.balrog_url build_number = args.build_number # Current version data pc = PatcherConfig(args.config.read()) partials = pc['current-update']['partials'].keys() app_name = pc['appName'] to_version = pc['current-update']['to'] to_release = pc['release'][to_version] to_ = makeReleaseRepackUrls( product_name, app_name, to_version, args.platform,
const=logging.DEBUG, default=logging.INFO, help="Increase output verbosity") parser.add_argument("--product", required=True, help="Product name") parser.add_argument("--stage-product", help="Stage product name") parser.add_argument("--archive-prefix", required=True) parser.add_argument("--previous-archive-prefix") parser.add_argument("--balrog-url", required=True) parser.add_argument("--build-number", required=True) parser.add_argument("--updater-platform", dest="updater_platform", default=None) args = parser.parse_args() logging.basicConfig(format="%(message)s", level=args.loglevel) ftp_platform = buildbot2ftp(args.platform) full_check_locales = args.full_check_locales product_name = args.product stage_product_name = args.stage_product or product_name prev_archive_prefix = args.previous_archive_prefix or args.archive_prefix aus_server_url = args.balrog_url build_number = args.build_number updater_platform = args.updater_platform # Current version data pc = PatcherConfig(args.config.read()) partials = pc['current-update']['partials'].keys() app_name = pc['appName'] to_version = pc['current-update']['to'] to_release = pc['release'][to_version] to_ = makeReleaseRepackUrls(product_name,
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir, l10nIni, compareLocalesRepo, env, merge=True, productName=None, platform=None, version=None, partialUpdates=None, buildNumber=None, stageServer=None): repo = "/".join([l10nBaseRepo, locale]) localeDir = path.join(l10nRepoDir, locale) retry(mercurial, args=(repo, localeDir)) update(localeDir, revision=revision) mozillaDir = '' if 'thunderbird' in productName: mozillaDir = 'mozilla/' # split on \\ since we care about the absSourceRepoPath for pymake, which # is windows. absSourceRepoPath = os.path.join(os.getcwd(), localeSrcDir.split("\\")[0]) use_pymake = env.get("USE_PYMAKE", False) make = getMakeCommand(use_pymake, absSourceRepoPath) env["AB_CD"] = locale env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged")) if sys.platform.startswith('win'): if use_pymake: env["LOCALE_MERGEDIR"] = msys2windows(env["LOCALE_MERGEDIR"]) else: env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"]) if sys.platform.startswith('darwin'): env["MOZ_PKG_PLATFORM"] = "mac" UPLOAD_EXTRA_FILES = [] nativeDistDir = path.normpath(path.abspath( path.join(localeSrcDir, '../../%sdist' % mozillaDir))) posixDistDir = windows2msys(nativeDistDir) mar = '%s/host/bin/mar' % posixDistDir mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir if platform.startswith('win'): mar += ".exe" mbsdiff += ".exe" current = '%s/current' % posixDistDir previous = '%s/previous' % posixDistDir updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale) updateAbsDir = '%s/%s' % (posixDistDir, updateDir) current_mar = '%s/%s-%s.complete.mar' % ( updateAbsDir, productName, version) unwrap_full_update = '../../../tools/update-packaging/unwrap_full_update.pl' make_incremental_update = '../../tools/update-packaging/make_incremental_update.sh' prevMarDir = '../../../../' if mozillaDir: unwrap_full_update = '../../../../%stools/update-packaging/unwrap_full_update.pl' % mozillaDir make_incremental_update = '../../../%stools/update-packaging/make_incremental_update.sh' % mozillaDir prevMarDir = '../../../../../' env['MAR'] = mar env['MBSDIFF'] = mbsdiff log.info("Download mar tools") if stageServer: candidates_dir = makeCandidatesDir(productName, version, buildNumber, protocol="http", server=stageServer) if not path.isfile(msys2windows(mar)): marUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mar)s" % \ dict(c_dir=candidates_dir, platform=platform, mar=path.basename(mar)) run_cmd(['mkdir', '-p', path.dirname(mar)]) log.info("Downloading %s to %s", marUrl, mar) urlretrieve(marUrl, msys2windows(mar)) if not sys.platform.startswith('win'): run_cmd(['chmod', '755', mar]) if not path.isfile(msys2windows(mbsdiff)): mbsdiffUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mbsdiff)s" % \ dict(c_dir=candidates_dir, platform=platform, mbsdiff=path.basename(mbsdiff)) run_cmd(['mkdir', '-p', path.dirname(mbsdiff)]) log.info("Downloading %s to %s", mbsdiffUrl, mbsdiff) urlretrieve(mbsdiffUrl, msys2windows(mbsdiff)) if not sys.platform.startswith('win'): run_cmd(['chmod', '755', mbsdiff]) else: log.warning('stageServer not set. mar tools will *not* be downloaded.') compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir, l10nIni, revision=revision, merge=merge) run_cmd(make + ["installers-%s" % locale], cwd=localeSrcDir, env=env) run_cmd(['rm', '-rf', current]) run_cmd(['mkdir', current]) run_cmd(['perl', unwrap_full_update, current_mar], cwd=path.join(nativeDistDir, 'current'), env=env) for oldVersion in partialUpdates: prevMar = partialUpdates[oldVersion]['mar'] if prevMar: partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion, version) partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name) UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name)) run_cmd(['rm', '-rf', previous]) run_cmd(['mkdir', previous]) run_cmd( ['perl', unwrap_full_update, '%s/%s' % (prevMarDir, prevMar)], cwd=path.join(nativeDistDir, 'previous'), env=env) run_cmd(['bash', make_incremental_update, partial_mar, previous, current], cwd=nativeDistDir, env=env) if os.environ.get('MOZ_SIGN_CMD'): run_cmd(['bash', '-c', '%s -f mar -f gpg "%s"' % (os.environ['MOZ_SIGN_CMD'], partial_mar)], env=env) UPLOAD_EXTRA_FILES.append( '%s/%s.asc' % (updateDir, partial_mar_name)) else: log.warning( "Skipping partial MAR creation for %s %s" % (oldVersion, locale)) env['UPLOAD_EXTRA_FILES'] = ' '.join(UPLOAD_EXTRA_FILES) retry(run_cmd, args=(make + ["upload", "AB_CD=%s" % locale], ), kwargs={'cwd': localeSrcDir, 'env': env})
def generate_data(self, appVersion, productName, version, buildNumber, updateChannels, stagingServer, bouncerServer, enUSPlatforms, schemaVersion, openURL=None, **updateKwargs): assert schemaVersion in (2, 3), 'Unhandled schema version %s' % schemaVersion self.name = get_release_blob_name(productName, version, buildNumber) data = { 'name': self.name, 'detailsUrl': getProductDetails(productName.lower(), appVersion), 'platforms': {}, 'fileUrls': {}, 'ftpFilenames': {}, 'bouncerProducts': {}, } data['appVersion'] = appVersion data['platformVersion'] = appVersion data['displayVersion'] = getPrettyVersion(version) actions = [] if openURL: actions.append("showURL") data["openURL"] = openURL if actions: data["actions"] = " ".join(actions) # XXX: This is a hack for bug 1045583. We should remove it, and always # use "candidates" for nightlyDir after the switch to Balrog is complete. if productName.lower() == "mobile": nightlyDir = "candidates" else: nightlyDir = "nightly" for channel in updateChannels: if channel in ('betatest', 'esrtest') or "localtest" in channel: dir_ = makeCandidatesDir(productName.lower(), version, buildNumber, server=stagingServer, protocol='http', nightlyDir=nightlyDir) data['fileUrls'][channel] = '%supdate/%%OS_FTP%%/%%LOCALE%%/%%FILENAME%%' % dir_ else: url = 'http://%s/?product=%%PRODUCT%%&os=%%OS_BOUNCER%%&lang=%%LOCALE%%' % bouncerServer data['fileUrls'][channel] = url # XXX: quick hack for bug 1021026. We should be using Bouncer for this # after we implement better solution talked about in comments 2 through 4 if channel == 'release': dir_ = makeCandidatesDir(productName.lower(), version, buildNumber, server='download.cdn.mozilla.net', protocol='http', nightlyDir=nightlyDir) url = '%supdate/%%OS_FTP%%/%%LOCALE%%/%%FILENAME%%' % dir_ data['fileUrls']['beta'] = url data['fileUrls']['beta-cdntest'] = url data.update(self._get_update_data(productName, version, **updateKwargs)) for platform in enUSPlatforms: updatePlatforms = buildbot2updatePlatforms(platform) bouncerPlatform = buildbot2bouncer(platform) ftpPlatform = buildbot2ftp(platform) data['platforms'][updatePlatforms[0]] = { 'OS_BOUNCER': bouncerPlatform, 'OS_FTP': ftpPlatform } for aliasedPlatform in updatePlatforms[1:]: data['platforms'][aliasedPlatform] = { 'alias': updatePlatforms[0] } return data
def test_platform_in_command(self): for platform, task in self.tasks.iteritems(): command = task['task']['payload']['command'] self.assertTrue("--platform {}".format(buildbot2ftp(platform)) in "".join(command))
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir, l10nIni, compareLocalesRepo, env, absObjdir, merge=True, productName=None, platform=None, version=None, partialUpdates=None, buildNumber=None, stageServer=None): repo = "/".join([l10nBaseRepo, locale]) localeDir = path.join(l10nRepoDir, locale) retry(mercurial, args=(repo, localeDir)) update(localeDir, revision=revision) mozillaDir = '' if 'thunderbird' in productName: mozillaDir = 'mozilla/' # It's a bad assumption to make, but the source dir is currently always # one level above the objdir. absSourceRepoPath = path.split(absObjdir)[0] use_pymake = env.get("USE_PYMAKE", False) make = getMakeCommand(use_pymake, absSourceRepoPath) env["AB_CD"] = locale env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged")) if sys.platform.startswith('win'): if use_pymake: env["LOCALE_MERGEDIR"] = msys2windows(env["LOCALE_MERGEDIR"]) else: env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"]) if sys.platform.startswith('darwin'): env["MOZ_PKG_PLATFORM"] = "mac" UPLOAD_EXTRA_FILES = [] nativeDistDir = path.normpath( path.abspath(path.join(localeSrcDir, '../../%sdist' % mozillaDir))) posixDistDir = windows2msys(nativeDistDir) mar = '%s/host/bin/mar' % posixDistDir mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir if platform.startswith('win'): mar += ".exe" mbsdiff += ".exe" current = '%s/current' % posixDistDir previous = '%s/previous' % posixDistDir updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale) updateAbsDir = '%s/%s' % (posixDistDir, updateDir) current_mar = '%s/%s-%s.complete.mar' % (updateAbsDir, productName, version) unwrap_full_update = '../../../tools/update-packaging/unwrap_full_update.pl' make_incremental_update = '../../tools/update-packaging/make_incremental_update.sh' prevMarDir = '../../../../' if mozillaDir: unwrap_full_update = '../../../../%stools/update-packaging/unwrap_full_update.pl' % mozillaDir make_incremental_update = '../../../%stools/update-packaging/make_incremental_update.sh' % mozillaDir prevMarDir = '../../../../../' env['MAR'] = mar env['MBSDIFF'] = mbsdiff log.info("Download mar tools") if stageServer: candidates_dir = makeCandidatesDir(productName, version, buildNumber, protocol="http", server=stageServer) if not path.isfile(msys2windows(mar)): marUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mar)s" % \ dict(c_dir=candidates_dir, platform=platform, mar=path.basename(mar)) run_cmd(['mkdir', '-p', path.dirname(mar)]) log.info("Downloading %s to %s", marUrl, mar) urlretrieve(marUrl, msys2windows(mar)) if not sys.platform.startswith('win'): run_cmd(['chmod', '755', mar]) if not path.isfile(msys2windows(mbsdiff)): mbsdiffUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mbsdiff)s" % \ dict(c_dir=candidates_dir, platform=platform, mbsdiff=path.basename(mbsdiff)) run_cmd(['mkdir', '-p', path.dirname(mbsdiff)]) log.info("Downloading %s to %s", mbsdiffUrl, mbsdiff) urlretrieve(mbsdiffUrl, msys2windows(mbsdiff)) if not sys.platform.startswith('win'): run_cmd(['chmod', '755', mbsdiff]) else: log.warning('stageServer not set. mar tools will *not* be downloaded.') compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir, l10nIni, revision=revision, merge=merge) run_cmd(make + ["installers-%s" % locale], cwd=localeSrcDir, env=env) # Our Windows-native rm from bug 727551 requires Windows-style paths run_cmd(['rm', '-rf', msys2windows(current)]) run_cmd(['mkdir', current]) run_cmd(['perl', unwrap_full_update, current_mar], cwd=path.join(nativeDistDir, 'current'), env=env) for oldVersion in partialUpdates: prevMar = partialUpdates[oldVersion]['mar'] if prevMar: partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion, version) partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name) UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name)) # Our Windows-native rm from bug 727551 requires Windows-style paths run_cmd(['rm', '-rf', msys2windows(previous)]) run_cmd(['mkdir', previous]) run_cmd( ['perl', unwrap_full_update, '%s/%s' % (prevMarDir, prevMar)], cwd=path.join(nativeDistDir, 'previous'), env=env) run_cmd([ 'bash', make_incremental_update, partial_mar, previous, current ], cwd=nativeDistDir, env=env) if os.environ.get('MOZ_SIGN_CMD'): run_cmd([ 'bash', '-c', '%s -f mar -f gpg "%s"' % (os.environ['MOZ_SIGN_CMD'], partial_mar) ], env=env) UPLOAD_EXTRA_FILES.append('%s/%s.asc' % (updateDir, partial_mar_name)) else: log.warning("Skipping partial MAR creation for %s %s" % (oldVersion, locale)) env['UPLOAD_EXTRA_FILES'] = ' '.join(UPLOAD_EXTRA_FILES) retry(run_cmd, args=(make + ["upload", "AB_CD=%s" % locale], ), kwargs={ 'cwd': localeSrcDir, 'env': env }) # return the location of the checksums file, because consumers may want # some information about the files that were generated. # Some versions of make that we use (at least pymake) imply --print-directory # We need to turn it off to avoid getting extra output that mess up our # parsing of the checksum file path. curdir = os.getcwd() try: os.chdir(localeSrcDir) relative_checksums = get_output(make + [ "--no-print-directory", "echo-variable-CHECKSUM_FILE", "AB_CD=%s" % locale ], env=env).strip("\"'\n") return path.normpath(path.join(localeSrcDir, relative_checksums)) finally: os.chdir(curdir)