def containsForbiddenDomain(self, product, whitelistedDomains):
"""Returns True if the blob contains any file URLs that contain a
domain that we're not allowed to serve updates to."""
# Check the top level URLs, if the exist.
for c in self.get('fileUrls', {}).values():
# New-style
if isinstance(c, dict):
for from_ in c.values():
for url in from_.values():
if isForbiddenUrl(url, product, whitelistedDomains):
return True
# Old-style
else:
if isForbiddenUrl(c, product, whitelistedDomains):
return True
# And also the locale-level URLs.
for platform in self.get('platforms', {}).values():
for locale in platform.get('locales', {}).values():
for type_ in ('partial', 'complete'):
if type_ in locale and 'fileUrl' in locale[type_]:
if isForbiddenUrl(locale[type_]['fileUrl'], product,
whitelistedDomains):
return True
for type_ in ('partials', 'completes'):
for update in locale.get(type_, {}):
if 'fileUrl' in update:
if isForbiddenUrl(update["fileUrl"], product,
whitelistedDomains):
return True
return False
def containsForbiddenDomain(self, product, whitelistedDomains):
"""Returns True if the blob contains any file URLs that contain a
domain that we're not allowed to serve updates to."""
# Check the top level URLs, if the exist.
for c in self.get('fileUrls', {}).values():
# New-style
if isinstance(c, dict):
for from_ in c.values():
for url in from_.values():
if isForbiddenUrl(url, product, whitelistedDomains):
return True
# Old-style
else:
if isForbiddenUrl(c, product, whitelistedDomains):
return True
# And also the locale-level URLs.
for platform in self.get('platforms', {}).values():
for locale in platform.get('locales', {}).values():
for type_ in ('partial', 'complete'):
if type_ in locale and 'fileUrl' in locale[type_]:
if isForbiddenUrl(locale[type_]['fileUrl'], product,
whitelistedDomains):
return True
for type_ in ('partials', 'completes'):
for update in locale.get(type_, {}):
if 'fileUrl' in update:
if isForbiddenUrl(update["fileUrl"], product,
whitelistedDomains):
return True
return False
def _getSpecificPatchXML(self, patchKey, patchType, patch, updateQuery, whitelistedDomains, specialForceHosts):
fromRelease = self._getFromRelease(patch)
# don't return an update if we don't match the from restriction
if fromRelease and not fromRelease.matchesUpdateQuery(updateQuery):
return None
# don't return an update if an older release isn't in the DB for some reason
if patch['from'] != '*' and fromRelease is None:
return None
try:
url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts)
except ValueError:
# Sometimes we may not be able to find a partial update even though
# we've told to. Because there should be a complete to fall back on,
# this is non-fatal, but is note-worthy.
# https://bugzilla.mozilla.org/show_bug.cgi?id=1312562 has more
# details on this.
if patchType == "partial":
self.log.exception("Caught non-fatal exception finding fileUrl for partial update:")
return None
# If we happen to find no _complete_ update, this is more serious,
# and we need to re-raise the exception.
else:
raise
# TODO: should be raising a bigger alarm here, or aborting
# the update entirely? Right now, another patch type could still
# return an update. Eg, the partial could contain a forbidden domain
# but the complete could still return an update from an accepted one.
if isForbiddenUrl(url, updateQuery['product'], whitelistedDomains):
return None
return ' <patch type="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s"/>' % \
(patchType, url, self["hashFunction"], patch["hashValue"], patch["filesize"])
def createSnippets(self, updateQuery, update_type, whitelistedDomains,
specialForceHosts):
snippets = {}
buildTarget = updateQuery["buildTarget"]
locale = updateQuery["locale"]
localeData = self.getLocaleData(buildTarget, locale)
for patchKey in ("partial", "complete"):
patch = localeData.get(patchKey)
if not patch:
continue
fromRelease = self._getFromRelease(patch)
if fromRelease and not fromRelease.matchesUpdateQuery(updateQuery):
continue
url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts)
if isForbiddenUrl(url, updateQuery['product'], whitelistedDomains):
break
snippet = [
"version=2",
"type=%s" % patchKey,
"url=%s" % url,
"hashFunction=%s" % self["hashFunction"],
"hashValue=%s" % patch["hashValue"],
"size=%s" % patch["filesize"],
"build=%s" % self.getBuildID(buildTarget, locale),
"displayVersion=%s" %
self.getDisplayVersion(buildTarget, locale),
"appVersion=%s" % self.getAppVersion(buildTarget, locale),
"platformVersion=%s" %
self.getPlatformVersion(buildTarget, locale),
]
if "detailsUrl" in self:
details = self["detailsUrl"].replace("%LOCALE%",
updateQuery["locale"])
details = details.replace("%locale%", updateQuery["locale"])
snippet.append("detailsUrl=%s" % details)
if "licenseUrl" in self:
license = self["licenseUrl"].replace("%LOCALE%",
updateQuery["locale"])
license = license.replace("%locale%", updateQuery["locale"])
snippet.append("licenseUrl=%s" % license)
if update_type == "major":
snippet.append("updateType=major")
for attr in self.optional_:
if attr in self:
if attr in self.interpolable_:
snippetToAppend = self[attr].replace(
"%LOCALE%", updateQuery["locale"])
snippetToAppend = snippetToAppend.replace(
"%locale%", updateQuery["locale"])
snippet.append("%s=%s" % (attr, snippetToAppend))
else:
snippet.append("%s=%s" % (attr, self[attr]))
snippets[patchKey] = "\n".join(snippet) + "\n"
for s in snippets.keys():
self.log.debug('%s\n%s' % (s, snippets[s].rstrip()))
return snippets
def createXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
buildTarget = updateQuery["buildTarget"]
vendorXML = []
for vendor in sorted(self.getVendorsForPlatform(buildTarget)):
vendorInfo = self["vendors"][vendor]
platformData = self.getPlatformData(vendor, buildTarget)
url = platformData["fileUrl"]
if isForbiddenUrl(url, whitelistedDomains):
continue
vendorXML.append(
' <addon id="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s" version="%s"/>'
% (
vendor,
url,
self["hashFunction"],
platformData["hashValue"],
platformData["filesize"],
vendorInfo["version"],
)
)
xml = ['<?xml version="1.0"?>']
xml.append("<updates>")
if vendorXML:
xml.append(" <addons>")
xml.extend(vendorXML)
xml.append(" </addons>")
xml.append("</updates>")
# ensure valid xml by using the right entity for ampersand
return re.sub("&(?!amp;)", "&", "\n".join(xml))
def createXML(self, updateQuery, update_type, whitelistedDomains,
specialForceHosts):
buildTarget = updateQuery["buildTarget"]
vendorXML = []
for vendor in self.getVendorsForPlatform(buildTarget):
vendorInfo = self["vendors"][vendor]
platformData = self.getPlatformData(vendor, buildTarget)
url = platformData["fileUrl"]
if isForbiddenUrl(url, whitelistedDomains):
continue
vendorXML.append(' <addon id="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s" version="%s"/>' % \
(vendor, url, self["hashFunction"], platformData["hashValue"],
platformData["filesize"], vendorInfo["version"]))
xml = ['<?xml version="1.0"?>']
xml.append('<updates>')
if vendorXML:
xml.append(' <addons>')
xml.extend(vendorXML)
xml.append(' </addons>')
xml.append('</updates>')
# ensure valid xml by using the right entity for ampersand
return re.sub('&(?!amp;)', '&', '\n'.join(xml))
def containsForbiddenDomain(self, product, whitelistedDomains):
"""Returns True if the blob contains any file URLs that contain a
domain that we're not allowed to serve updates to."""
for vendor in self.get('vendors', {}).values():
for platform in vendor.get('platforms', {}).values():
if 'fileUrl' in platform:
if isForbiddenUrl(platform["fileUrl"], product, whitelistedDomains):
return True
return False
def getResponse(self, updateQuery, whitelistedDomains):
url = self.get("platforms", {}).get(updateQuery["buildTarget"], {}).get("fileUrl")
hashValue = self.get("platforms", {}).get(updateQuery["buildTarget"], {}).get("hashValue")
if not url:
return {}
if isForbiddenUrl(url, updateQuery["product"], whitelistedDomains):
return {}
return {"version": self["version"], "url": url, "required": self["required"], "hashFunction": self["hashFunction"], "hashValue": hashValue}
def hasUpdates(self, updateQuery, whitelistedDomains):
buildTarget = updateQuery["buildTarget"]
for addon in sorted(self.getAddonsForPlatform(buildTarget)):
# Checking if the addon update is to be served
platformData = self.getPlatformData(addon, buildTarget)
url = platformData["fileUrl"]
# There might be no updates even if we have response products if
# they are not served from whitelisted domains
if isForbiddenUrl(url, updateQuery["product"], whitelistedDomains):
continue
return True
return False
def createSnippets(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
snippets = {}
buildTarget = updateQuery["buildTarget"]
locale = updateQuery["locale"]
localeData = self.getLocaleData(buildTarget, locale)
for patchKey in ("partial", "complete"):
patch = localeData.get(patchKey)
if not patch:
continue
fromRelease = self._getFromRelease(patch)
if fromRelease and not fromRelease.matchesUpdateQuery(updateQuery):
continue
url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts)
if isForbiddenUrl(url, updateQuery['product'], whitelistedDomains):
break
snippet = [
"version=2",
"type=%s" % patchKey,
"url=%s" % url,
"hashFunction=%s" % self["hashFunction"],
"hashValue=%s" % patch["hashValue"],
"size=%s" % patch["filesize"],
"build=%s" % self.getBuildID(buildTarget, locale),
"displayVersion=%s" % self.getDisplayVersion(buildTarget, locale),
"appVersion=%s" % self.getAppVersion(buildTarget, locale),
"platformVersion=%s" % self.getPlatformVersion(buildTarget, locale),
]
if "detailsUrl" in self:
details = self["detailsUrl"].replace("%LOCALE%", updateQuery["locale"])
details = details.replace("%locale%", updateQuery["locale"])
snippet.append("detailsUrl=%s" % details)
if "licenseUrl" in self:
license = self["licenseUrl"].replace("%LOCALE%", updateQuery["locale"])
license = license.replace("%locale%", updateQuery["locale"])
snippet.append("licenseUrl=%s" % license)
if update_type == "major":
snippet.append("updateType=major")
for attr in self.optional_:
if attr in self:
if attr in self.interpolable_:
snippetToAppend = self[attr].replace("%LOCALE%", updateQuery["locale"])
snippetToAppend = snippetToAppend.replace("%locale%", updateQuery["locale"])
snippet.append("%s=%s" % (attr, snippetToAppend))
else:
snippet.append("%s=%s" % (attr, self[attr]))
snippets[patchKey] = "\n".join(snippet) + "\n"
for s in snippets.keys():
self.log.debug('%s\n%s' % (s, snippets[s].rstrip()))
return snippets
def createSnippets(self, updateQuery, update_type, whitelistedDomains,
specialForceHosts):
snippets = {}
buildTarget = updateQuery["buildTarget"]
locale = updateQuery["locale"]
localeData = self.getLocaleData(buildTarget, locale)
for patchKey in ("partial", "complete"):
patch = localeData.get(patchKey)
if not patch:
continue
fromRelease = self._getFromRelease(patch)
if fromRelease and not fromRelease.matchesUpdateQuery(updateQuery):
continue
url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts)
if isForbiddenUrl(url, updateQuery['product'], whitelistedDomains):
break
snippet = [
"version=1",
"type=%s" % patchKey,
"url=%s" % url,
"hashFunction=%s" % self["hashFunction"],
"hashValue=%s" % patch["hashValue"],
"size=%s" % patch["filesize"],
"build=%s" % self.getBuildID(buildTarget, locale),
"appv=%s" % self.getAppv(buildTarget, locale),
"extv=%s" % self.getExtv(buildTarget, locale),
]
if "detailsUrl" in self:
details = self["detailsUrl"].replace("%LOCALE%",
updateQuery["locale"])
details = details.replace("%locale%", updateQuery["locale"])
snippet.append("detailsUrl=%s" % details)
if "licenseUrl" in self:
license = self["licenseUrl"].replace("%LOCALE%",
updateQuery["locale"])
license = license.replace("%locale%", updateQuery["locale"])
snippet.append("licenseUrl=%s" % license)
if update_type == "major":
snippet.append("updateType=major")
snippets[patchKey] = "\n".join(snippet) + "\n"
if self.get("fakePartials"
) and "complete" in snippets and "partial" not in snippets:
partial = snippets["complete"]
partial = partial.replace("type=complete", "type=partial")
snippets["partial"] = partial
for s in snippets.keys():
self.log.debug('%s\n%s' % (s, snippets[s].rstrip()))
return snippets
def createSnippets(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
snippets = {}
buildTarget = updateQuery["buildTarget"]
locale = updateQuery["locale"]
localeData = self.getLocaleData(buildTarget, locale)
for patchKey in ("partial", "complete"):
patch = localeData.get(patchKey)
if not patch:
continue
try:
fromRelease = dbo.releases.getReleaseBlob(name=patch["from"])
except KeyError:
fromRelease = None
if patch["from"] != "*" and fromRelease and not fromRelease.matchesUpdateQuery(updateQuery):
continue
url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts)
if isForbiddenUrl(url, whitelistedDomains):
break
snippet = [
"version=1",
"type=%s" % patchKey,
"url=%s" % url,
"hashFunction=%s" % self["hashFunction"],
"hashValue=%s" % patch["hashValue"],
"size=%s" % patch["filesize"],
"build=%s" % self.getBuildID(buildTarget, locale),
"appv=%s" % self.getAppv(buildTarget, locale),
"extv=%s" % self.getExtv(buildTarget, locale),
]
if "detailsUrl" in self:
details = self["detailsUrl"].replace("%LOCALE%", updateQuery["locale"])
snippet.append("detailsUrl=%s" % details)
if "licenseUrl" in self:
license = self["licenseUrl"].replace("%LOCALE%", updateQuery["locale"])
snippet.append("licenseUrl=%s" % license)
if update_type == "major":
snippet.append("updateType=major")
snippets[patchKey] = "\n".join(snippet) + "\n"
if self.get("fakePartials") and "complete" in snippets and "partial" not in snippets:
partial = snippets["complete"]
partial = partial.replace("type=complete", "type=partial")
snippets["partial"] = partial
for s in snippets.keys():
self.log.debug('%s\n%s' % (s, snippets[s].rstrip()))
return snippets
def _getSpecificPatchXML(self, patchKey, patchType, patch, updateQuery, whitelistedDomains, specialForceHosts):
fromRelease = self._getFromRelease(patch)
if fromRelease and not fromRelease.matchesUpdateQuery(updateQuery):
return None
url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts)
# TODO: should be raising a bigger alarm here, or aborting
# the update entirely? Right now, another patch type could still
# return an update. Eg, the partial could contain a forbidden domain
# but the complete could still return an update from an accepted one.
if isForbiddenUrl(url, updateQuery['product'], whitelistedDomains):
return None
return ' <patch type="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s"/>' % \
(patchType, url, self["hashFunction"], patch["hashValue"], patch["filesize"])
def _getSpecificPatchXML(self, patchKey, patchType, patch, updateQuery, whitelistedDomains, specialForceHosts):
fromRelease = self._getFromRelease(patch)
if fromRelease and not fromRelease.matchesUpdateQuery(updateQuery):
return None
url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts)
# TODO: should be raising a bigger alarm here, or aborting
# the update entirely? Right now, another patch type could still
# return an update. Eg, the partial could contain a forbidden domain
# but the complete could still return an update from an accepted one.
if isForbiddenUrl(url, whitelistedDomains):
return None
return ' <patch type="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s"/>' % \
(patchType, url, self["hashFunction"], patch["hashValue"], patch["filesize"])
def getInnerXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
buildTarget = updateQuery["buildTarget"]
vendorXML = []
for vendor in sorted(self.getVendorsForPlatform(buildTarget)):
vendorInfo = self["vendors"][vendor]
platformData = self.getPlatformData(vendor, buildTarget)
url = platformData["fileUrl"]
if isForbiddenUrl(url, updateQuery["product"], whitelistedDomains):
continue
vendorXML.append(' <addon id="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s" version="%s"/>' %
(vendor, url, self["hashFunction"], platformData["hashValue"],
platformData["filesize"], vendorInfo["version"]))
return vendorXML
def getInnerXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
# In case we have an uninstall blob, we won't have the addons section
if self.get('addons') is None:
return []
buildTarget = updateQuery["buildTarget"]
addonXML = []
for addon in sorted(self.getAddonsForPlatform(buildTarget)):
addonInfo = self["addons"][addon]
platformData = self.getPlatformData(addon, buildTarget)
url = platformData["fileUrl"]
if isForbiddenUrl(url, updateQuery["product"], whitelistedDomains):
continue
addonXML.append(' <addon id="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s" version="%s"/>' %
(addon, url, self["hashFunction"], platformData["hashValue"],
platformData["filesize"], addonInfo["version"]))
return addonXML
def _getSpecificPatchXML(self, patchKey, patchType, patch, updateQuery,
whitelistedDomains, specialForceHosts):
fromRelease = self._getFromRelease(patch)
# don't return an update if we don't match the from restriction
if fromRelease and not fromRelease.matchesUpdateQuery(updateQuery):
return None
# don't return an update if an older release isn't in the DB for some reason
if patch['from'] != '*' and fromRelease is None:
return None
try:
url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts)
except ValueError:
# Sometimes we may not be able to find a partial update even though
# we've told to. Because there should be a complete to fall back on,
# this is non-fatal, but is note-worthy.
# https://bugzilla.mozilla.org/show_bug.cgi?id=1312562 has more
# details on this.
if patchType == "partial":
self.log.exception(
"Caught non-fatal exception finding fileUrl for partial update:"
)
return None
# If we happen to find no _complete_ update, this is more serious,
# and we need to re-raise the exception.
else:
raise
# TODO: should be raising a bigger alarm here, or aborting
# the update entirely? Right now, another patch type could still
# return an update. Eg, the partial could contain a forbidden domain
# but the complete could still return an update from an accepted one.
if isForbiddenUrl(url, updateQuery['product'], whitelistedDomains):
return None
patchXML = ' <patch type="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s"' % \
(patchType, url, self["hashFunction"], patch["hashValue"], patch["filesize"])
additionalPatchAttributes = self._getAdditionalPatchAttributes(patch)
for attribute in additionalPatchAttributes:
patchXML += ' %s="%s"' % (attribute,
additionalPatchAttributes[attribute])
patchXML += '/>'
return patchXML
def containsForbiddenDomain(self, product, whitelistedDomains):
urls = [p["fileUrl"] for p in self["platforms"].values()]
return any([isForbiddenUrl(url, product, whitelistedDomains) for url in urls])
