def _get_update_paths(self):
from mozrelease.l10n import getPlatformLocales
from mozrelease.paths import getCandidatesDir
from mozrelease.platforms import ftp2infoFile
from mozrelease.versions import MozillaVersion
self.update_paths = {}
ret = self._retry_download(
"{}/1.0/{}.json".format(
self.config["product_details_server"],
self.config["stage_product"],
),
"WARNING",
)
releases = json.load(ret)["releases"]
for release_name, release_info in reversed(sorted(releases.items())):
product, version = release_name.split("-", 1)
tag = "{}_{}_RELEASE".format(product.upper(), version.replace(".", "_"))
# Product details has a "category" for releases that we can use to
# determine the repo path. This will fail if any previous releases
# were built from a project branch - but that's not something we do
# at the time of writing.
branch = None
if release_info["category"] == "dev":
branch = "releases/{}-beta".format(self.config['branch_prefix'])
elif release_info["category"] == "esr":
branch = "releases/{}-esr{}".format(self.config['branch_prefix'], version[:2])
elif release_info["category"] in ("major", "stability"):
branch = "releases/{}-release".format(self.config['branch_prefix'])
if not branch:
raise Exception("Cannot determine branch, cannot continue!")
# Exclude any releases that don't match one of our include version
# regexes. This is generally to avoid including versions from other
# channels. Eg: including betas when testing releases
for v in self.config["include_versions"]:
if re.match(v, version):
break
else:
self.log("Skipping release whose version doesn't match any "
"include_version pattern: %s" % release_name,
level=INFO)
continue
# We also have to trim out previous releases that aren't in the same
# product line, too old, etc.
if self.config["stage_product"] != product:
self.log("Skipping release that doesn't match product name: %s" % release_name,
level=INFO)
continue
if MozillaVersion(version) < MozillaVersion(self.config["last_watershed"]):
self.log("Skipping release that's behind the last watershed: %s" % release_name,
level=INFO)
continue
if version == self.config["to_version"]:
self.log("Skipping release that is the same as to version: %s" % release_name,
level=INFO)
continue
if MozillaVersion(version) > MozillaVersion(self.config["to_version"]):
self.log("Skipping release that's newer than to version: %s" % release_name,
level=INFO)
continue
if version in self.update_paths:
raise Exception("Found duplicate release for version: %s", version)
# This is a crappy place to get buildids from, but we don't have a better one.
# This will start to fail if old info files are deleted.
info_file_url = "{}{}/{}_info.txt".format(
self.config["previous_archive_prefix"],
getCandidatesDir(
self.config["stage_product"],
version,
release_info["build_number"],
),
ftp2infoFile(self.config["platform"])
)
self.log("Retrieving buildid from info file: %s" % info_file_url, level=DEBUG)
ret = self._retry_download(info_file_url, "WARNING")
buildID = ret.read().split("=")[1].strip()
shipped_locales_url = urljoin(
self.config["hg_server"],
"{}/raw-file/{}/{}/locales/shipped-locales".format(
branch,
tag,
self.config["app_name"],
),
)
ret = self._retry_download(shipped_locales_url, "WARNING")
shipped_locales = ret.read().strip()
app_version_url = urljoin(
self.config["hg_server"],
"{}/raw-file/{}/{}/config/version.txt".format(
branch,
tag,
self.config["app_name"],
),
)
app_version = self._retry_download(app_version_url, "WARNING").read().strip()
self.log("Adding {} to update paths".format(version), level=INFO)
self.update_paths[version] = {
"appVersion": app_version,
"locales": getPlatformLocales(shipped_locales, self.config["platform"]),
"buildID": buildID,
}
for pattern, mar_channel_ids in self.config["mar_channel_id_overrides"].items():
if re.match(pattern, version):
self.update_paths[version]["marChannelIds"] = mar_channel_ids
def create_config(self):
from mozrelease.l10n import getPlatformLocales
from mozrelease.platforms import ftp2updatePlatforms
from mozrelease.update_verify import UpdateVerifyConfig
from mozrelease.paths import getCandidatesDir, getReleasesDir, getReleaseInstallerPath
from mozrelease.versions import getPrettyVersion
candidates_dir = getCandidatesDir(
self.config["stage_product"], self.config["to_version"],
self.config["to_build_number"],
)
to_ = getReleaseInstallerPath(
self.config["product"], self.config["product"].title(),
self.config["to_version"], self.config["platform"],
locale="%locale%"
)
to_path = "{}/{}".format(candidates_dir, to_)
to_display_version = self.config.get("to_display_version")
if not to_display_version:
to_display_version = getPrettyVersion(self.config["to_version"])
self.update_verify_config = UpdateVerifyConfig(
product=self.config["product"].title(), channel=self.config["channel"],
aus_server=self.config["aus_server"], to=to_path,
to_build_id=self.config["to_buildid"],
to_app_version=self.config["to_app_version"],
to_display_version=to_display_version,
override_certs=self.config.get("override_certs"),
)
to_shipped_locales_url = urljoin(
self.config["hg_server"],
"{}/raw-file/{}/{}/locales/shipped-locales".format(
self.config["repo_path"],
self.config["to_revision"],
self.config["app_name"],
),
)
to_shipped_locales = self._retry_download(to_shipped_locales_url, "WARNING").read().strip()
to_locales = set(getPlatformLocales(to_shipped_locales, self.config["platform"]))
completes_only_index = 0
for fromVersion in reversed(sorted(self.update_paths, key=LooseVersion)):
from_ = self.update_paths[fromVersion]
locales = sorted(list(set(from_["locales"]).intersection(to_locales)))
appVersion = from_["appVersion"]
build_id = from_["buildID"]
mar_channel_IDs = from_.get('marChannelIds')
# Use new build targets for Windows, but only on compatible
# versions (42+). See bug 1185456 for additional context.
if self.config["platform"] not in ("win32", "win64") or \
LooseVersion(fromVersion) < LooseVersion("42.0"):
update_platform = ftp2updatePlatforms(self.config["platform"])[0]
else:
update_platform = ftp2updatePlatforms(self.config["platform"])[1]
release_dir = getReleasesDir(
self.config["stage_product"], fromVersion
)
path_ = getReleaseInstallerPath(
self.config["product"], self.config["product"].title(),
fromVersion, self.config["platform"], locale="%locale%",
)
from_path = "{}/{}".format(release_dir, path_)
updater_package = "{}/{}".format(
release_dir,
getReleaseInstallerPath(
self.config["product"], self.config["product"].title(),
fromVersion, self.config["updater_platform"],
locale="%locale%",
)
)
# Exclude locales being full checked
quick_check_locales = [l for l in locales
if l not in self.config["full_check_locales"]]
# Get the intersection of from and to full_check_locales
this_full_check_locales = [l for l in self.config["full_check_locales"]
if l in locales]
if fromVersion in self.config["partial_versions"]:
self.info("Generating configs for partial update checks for %s" % fromVersion)
self.update_verify_config.addRelease(
release=appVersion, build_id=build_id, locales=locales,
patch_types=["complete", "partial"], from_path=from_path,
ftp_server_from=self.config["previous_archive_prefix"],
ftp_server_to=self.config["archive_prefix"],
mar_channel_IDs=mar_channel_IDs, platform=update_platform,
updater_package=updater_package
)
else:
if this_full_check_locales and is_triangualar(completes_only_index):
self.info("Generating full check configs for %s" % fromVersion)
self.update_verify_config.addRelease(
release=appVersion, build_id=build_id, locales=this_full_check_locales,
from_path=from_path,
ftp_server_from=self.config["previous_archive_prefix"],
ftp_server_to=self.config["archive_prefix"],
mar_channel_IDs=mar_channel_IDs, platform=update_platform,
updater_package=updater_package
)
# Quick test for other locales, no download
if len(quick_check_locales) > 0:
self.info("Generating quick check configs for %s" % fromVersion)
if not is_triangualar(completes_only_index):
# Assuming we skipped full check locales, using all locales
_locales = locales
else:
# Excluding full check locales from the quick check
_locales = quick_check_locales
self.update_verify_config.addRelease(
release=appVersion, build_id=build_id,
locales=_locales, platform=update_platform
)
completes_only_index += 1
def _get_update_paths(self):
from mozrelease.l10n import getPlatformLocales
from mozrelease.paths import getCandidatesDir
from mozrelease.platforms import ftp2infoFile
from mozrelease.versions import MozillaVersion
self.update_paths = {}
ret = self._retry_download(
"{}/1.0/{}.json".format(
self.config["product_details_server"],
self.config["stage_product"],
),
"WARNING",
)
releases = json.load(ret)["releases"]
for release_name, release_info in \
reversed(sorted(releases.items(),
key=lambda x: MozillaVersion(x[1]['version']))):
# we need to use releases_name instead of release_info since esr
# string is included in the name. later we rely on this.
product, version = release_name.split('-', 1)
tag = "{}_{}_RELEASE".format(product.upper(),
version.replace(".", "_"))
# Exclude any releases that don't match one of our include version
# regexes. This is generally to avoid including versions from other
# channels. Eg: including betas when testing releases
for v in self.config["include_versions"]:
if re.match(v, version):
break
else:
self.log("Skipping release whose version doesn't match any "
"include_version pattern: %s" % release_name,
level=INFO)
continue
# We also have to trim out previous releases that aren't in the same
# product line, too old, etc.
if self.config["stage_product"] != product:
self.log(
"Skipping release that doesn't match product name: %s" %
release_name,
level=INFO)
continue
if MozillaVersion(version) < MozillaVersion(
self.config["last_watershed"]):
self.log(
"Skipping release that's behind the last watershed: %s" %
release_name,
level=INFO)
continue
if version == self.config["to_version"]:
self.log(
"Skipping release that is the same as to version: %s" %
release_name,
level=INFO)
continue
if MozillaVersion(version) > MozillaVersion(
self.config["to_version"]):
self.log("Skipping release that's newer than to version: %s" %
release_name,
level=INFO)
continue
if version in self.update_paths:
raise Exception("Found duplicate release for version: %s",
version)
# This is a crappy place to get buildids from, but we don't have a better one.
# This will start to fail if old info files are deleted.
info_file_url = "{}{}/{}_info.txt".format(
self.config["previous_archive_prefix"],
getCandidatesDir(
self.config["stage_product"],
version,
release_info["build_number"],
), ftp2infoFile(self.config["platform"]))
self.log("Retrieving buildid from info file: %s" % info_file_url,
level=DEBUG)
ret = self._retry_download(info_file_url, "WARNING")
buildID = ret.read().split(b"=")[1].strip().decode("utf-8")
branch = self._get_branch_url(self.config["branch_prefix"],
version)
shipped_locales_url = urljoin(
self.config["hg_server"],
"{}/raw-file/{}/{}/locales/shipped-locales".format(
branch,
tag,
self.config["app_name"],
),
)
ret = self._retry_download(shipped_locales_url, "WARNING")
shipped_locales = ret.read().strip().decode("utf-8")
app_version_url = urljoin(
self.config["hg_server"],
"{}/raw-file/{}/{}/config/version.txt".format(
branch,
tag,
self.config["app_name"],
),
)
app_version = self._retry_download(app_version_url, "WARNING").read() \
.strip().decode("utf-8")
self.log("Adding {} to update paths".format(version), level=INFO)
self.update_paths[version] = {
"appVersion":
app_version,
"locales":
getPlatformLocales(shipped_locales, self.config["platform"]),
"buildID":
buildID,
}
for pattern, mar_channel_ids in self.config[
"mar_channel_id_overrides"].items():
if re.match(pattern, version):
self.update_paths[version][
"marChannelIds"] = mar_channel_ids
