def _get_update_paths(self): from mozrelease.l10n import getPlatformLocales from mozrelease.paths import getCandidatesDir from mozrelease.platforms import ftp2infoFile from mozrelease.versions import MozillaVersion self.update_paths = {} ret = self._retry_download( "{}/1.0/{}.json".format( self.config["product_details_server"], self.config["stage_product"], ), "WARNING", ) releases = json.load(ret)["releases"] for release_name, release_info in reversed(sorted(releases.items())): product, version = release_name.split("-", 1) tag = "{}_{}_RELEASE".format(product.upper(), version.replace(".", "_")) # Product details has a "category" for releases that we can use to # determine the repo path. This will fail if any previous releases # were built from a project branch - but that's not something we do # at the time of writing. branch = None if release_info["category"] == "dev": branch = "releases/{}-beta".format(self.config['branch_prefix']) elif release_info["category"] == "esr": branch = "releases/{}-esr{}".format(self.config['branch_prefix'], version[:2]) elif release_info["category"] in ("major", "stability"): branch = "releases/{}-release".format(self.config['branch_prefix']) if not branch: raise Exception("Cannot determine branch, cannot continue!") # Exclude any releases that don't match one of our include version # regexes. This is generally to avoid including versions from other # channels. Eg: including betas when testing releases for v in self.config["include_versions"]: if re.match(v, version): break else: self.log("Skipping release whose version doesn't match any " "include_version pattern: %s" % release_name, level=INFO) continue # We also have to trim out previous releases that aren't in the same # product line, too old, etc. if self.config["stage_product"] != product: self.log("Skipping release that doesn't match product name: %s" % release_name, level=INFO) continue if MozillaVersion(version) < MozillaVersion(self.config["last_watershed"]): self.log("Skipping release that's behind the last watershed: %s" % release_name, level=INFO) continue if version == self.config["to_version"]: self.log("Skipping release that is the same as to version: %s" % release_name, level=INFO) continue if MozillaVersion(version) > MozillaVersion(self.config["to_version"]): self.log("Skipping release that's newer than to version: %s" % release_name, level=INFO) continue if version in self.update_paths: raise Exception("Found duplicate release for version: %s", version) # This is a crappy place to get buildids from, but we don't have a better one. # This will start to fail if old info files are deleted. info_file_url = "{}{}/{}_info.txt".format( self.config["previous_archive_prefix"], getCandidatesDir( self.config["stage_product"], version, release_info["build_number"], ), ftp2infoFile(self.config["platform"]) ) self.log("Retrieving buildid from info file: %s" % info_file_url, level=DEBUG) ret = self._retry_download(info_file_url, "WARNING") buildID = ret.read().split("=")[1].strip() shipped_locales_url = urljoin( self.config["hg_server"], "{}/raw-file/{}/{}/locales/shipped-locales".format( branch, tag, self.config["app_name"], ), ) ret = self._retry_download(shipped_locales_url, "WARNING") shipped_locales = ret.read().strip() app_version_url = urljoin( self.config["hg_server"], "{}/raw-file/{}/{}/config/version.txt".format( branch, tag, self.config["app_name"], ), ) app_version = self._retry_download(app_version_url, "WARNING").read().strip() self.log("Adding {} to update paths".format(version), level=INFO) self.update_paths[version] = { "appVersion": app_version, "locales": getPlatformLocales(shipped_locales, self.config["platform"]), "buildID": buildID, } for pattern, mar_channel_ids in self.config["mar_channel_id_overrides"].items(): if re.match(pattern, version): self.update_paths[version]["marChannelIds"] = mar_channel_ids
def create_config(self): from mozrelease.l10n import getPlatformLocales from mozrelease.platforms import ftp2updatePlatforms from mozrelease.update_verify import UpdateVerifyConfig from mozrelease.paths import getCandidatesDir, getReleasesDir, getReleaseInstallerPath from mozrelease.versions import getPrettyVersion candidates_dir = getCandidatesDir( self.config["stage_product"], self.config["to_version"], self.config["to_build_number"], ) to_ = getReleaseInstallerPath( self.config["product"], self.config["product"].title(), self.config["to_version"], self.config["platform"], locale="%locale%" ) to_path = "{}/{}".format(candidates_dir, to_) to_display_version = self.config.get("to_display_version") if not to_display_version: to_display_version = getPrettyVersion(self.config["to_version"]) self.update_verify_config = UpdateVerifyConfig( product=self.config["product"].title(), channel=self.config["channel"], aus_server=self.config["aus_server"], to=to_path, to_build_id=self.config["to_buildid"], to_app_version=self.config["to_app_version"], to_display_version=to_display_version, override_certs=self.config.get("override_certs"), ) to_shipped_locales_url = urljoin( self.config["hg_server"], "{}/raw-file/{}/{}/locales/shipped-locales".format( self.config["repo_path"], self.config["to_revision"], self.config["app_name"], ), ) to_shipped_locales = self._retry_download(to_shipped_locales_url, "WARNING").read().strip() to_locales = set(getPlatformLocales(to_shipped_locales, self.config["platform"])) completes_only_index = 0 for fromVersion in reversed(sorted(self.update_paths, key=LooseVersion)): from_ = self.update_paths[fromVersion] locales = sorted(list(set(from_["locales"]).intersection(to_locales))) appVersion = from_["appVersion"] build_id = from_["buildID"] mar_channel_IDs = from_.get('marChannelIds') # Use new build targets for Windows, but only on compatible # versions (42+). See bug 1185456 for additional context. if self.config["platform"] not in ("win32", "win64") or \ LooseVersion(fromVersion) < LooseVersion("42.0"): update_platform = ftp2updatePlatforms(self.config["platform"])[0] else: update_platform = ftp2updatePlatforms(self.config["platform"])[1] release_dir = getReleasesDir( self.config["stage_product"], fromVersion ) path_ = getReleaseInstallerPath( self.config["product"], self.config["product"].title(), fromVersion, self.config["platform"], locale="%locale%", ) from_path = "{}/{}".format(release_dir, path_) updater_package = "{}/{}".format( release_dir, getReleaseInstallerPath( self.config["product"], self.config["product"].title(), fromVersion, self.config["updater_platform"], locale="%locale%", ) ) # Exclude locales being full checked quick_check_locales = [l for l in locales if l not in self.config["full_check_locales"]] # Get the intersection of from and to full_check_locales this_full_check_locales = [l for l in self.config["full_check_locales"] if l in locales] if fromVersion in self.config["partial_versions"]: self.info("Generating configs for partial update checks for %s" % fromVersion) self.update_verify_config.addRelease( release=appVersion, build_id=build_id, locales=locales, patch_types=["complete", "partial"], from_path=from_path, ftp_server_from=self.config["previous_archive_prefix"], ftp_server_to=self.config["archive_prefix"], mar_channel_IDs=mar_channel_IDs, platform=update_platform, updater_package=updater_package ) else: if this_full_check_locales and is_triangualar(completes_only_index): self.info("Generating full check configs for %s" % fromVersion) self.update_verify_config.addRelease( release=appVersion, build_id=build_id, locales=this_full_check_locales, from_path=from_path, ftp_server_from=self.config["previous_archive_prefix"], ftp_server_to=self.config["archive_prefix"], mar_channel_IDs=mar_channel_IDs, platform=update_platform, updater_package=updater_package ) # Quick test for other locales, no download if len(quick_check_locales) > 0: self.info("Generating quick check configs for %s" % fromVersion) if not is_triangualar(completes_only_index): # Assuming we skipped full check locales, using all locales _locales = locales else: # Excluding full check locales from the quick check _locales = quick_check_locales self.update_verify_config.addRelease( release=appVersion, build_id=build_id, locales=_locales, platform=update_platform ) completes_only_index += 1
def _get_update_paths(self): from mozrelease.l10n import getPlatformLocales from mozrelease.paths import getCandidatesDir from mozrelease.platforms import ftp2infoFile from mozrelease.versions import MozillaVersion self.update_paths = {} ret = self._retry_download( "{}/1.0/{}.json".format( self.config["product_details_server"], self.config["stage_product"], ), "WARNING", ) releases = json.load(ret)["releases"] for release_name, release_info in \ reversed(sorted(releases.items(), key=lambda x: MozillaVersion(x[1]['version']))): # we need to use releases_name instead of release_info since esr # string is included in the name. later we rely on this. product, version = release_name.split('-', 1) tag = "{}_{}_RELEASE".format(product.upper(), version.replace(".", "_")) # Exclude any releases that don't match one of our include version # regexes. This is generally to avoid including versions from other # channels. Eg: including betas when testing releases for v in self.config["include_versions"]: if re.match(v, version): break else: self.log("Skipping release whose version doesn't match any " "include_version pattern: %s" % release_name, level=INFO) continue # We also have to trim out previous releases that aren't in the same # product line, too old, etc. if self.config["stage_product"] != product: self.log( "Skipping release that doesn't match product name: %s" % release_name, level=INFO) continue if MozillaVersion(version) < MozillaVersion( self.config["last_watershed"]): self.log( "Skipping release that's behind the last watershed: %s" % release_name, level=INFO) continue if version == self.config["to_version"]: self.log( "Skipping release that is the same as to version: %s" % release_name, level=INFO) continue if MozillaVersion(version) > MozillaVersion( self.config["to_version"]): self.log("Skipping release that's newer than to version: %s" % release_name, level=INFO) continue if version in self.update_paths: raise Exception("Found duplicate release for version: %s", version) # This is a crappy place to get buildids from, but we don't have a better one. # This will start to fail if old info files are deleted. info_file_url = "{}{}/{}_info.txt".format( self.config["previous_archive_prefix"], getCandidatesDir( self.config["stage_product"], version, release_info["build_number"], ), ftp2infoFile(self.config["platform"])) self.log("Retrieving buildid from info file: %s" % info_file_url, level=DEBUG) ret = self._retry_download(info_file_url, "WARNING") buildID = ret.read().split(b"=")[1].strip().decode("utf-8") branch = self._get_branch_url(self.config["branch_prefix"], version) shipped_locales_url = urljoin( self.config["hg_server"], "{}/raw-file/{}/{}/locales/shipped-locales".format( branch, tag, self.config["app_name"], ), ) ret = self._retry_download(shipped_locales_url, "WARNING") shipped_locales = ret.read().strip().decode("utf-8") app_version_url = urljoin( self.config["hg_server"], "{}/raw-file/{}/{}/config/version.txt".format( branch, tag, self.config["app_name"], ), ) app_version = self._retry_download(app_version_url, "WARNING").read() \ .strip().decode("utf-8") self.log("Adding {} to update paths".format(version), level=INFO) self.update_paths[version] = { "appVersion": app_version, "locales": getPlatformLocales(shipped_locales, self.config["platform"]), "buildID": buildID, } for pattern, mar_channel_ids in self.config[ "mar_channel_id_overrides"].items(): if re.match(pattern, version): self.update_paths[version][ "marChannelIds"] = mar_channel_ids