def containsForbiddenDomain(self, product, whitelistedDomains): """Returns True if the blob contains any file URLs that contain a domain that we're not allowed to serve updates to.""" # Check the top level URLs, if the exist. for c in self.get('fileUrls', {}).values(): # New-style if isinstance(c, dict): for from_ in c.values(): for url in from_.values(): if isForbiddenUrl(url, product, whitelistedDomains): return True # Old-style else: if isForbiddenUrl(c, product, whitelistedDomains): return True # And also the locale-level URLs. for platform in self.get('platforms', {}).values(): for locale in platform.get('locales', {}).values(): for type_ in ('partial', 'complete'): if type_ in locale and 'fileUrl' in locale[type_]: if isForbiddenUrl(locale[type_]['fileUrl'], product, whitelistedDomains): return True for type_ in ('partials', 'completes'): for update in locale.get(type_, {}): if 'fileUrl' in update: if isForbiddenUrl(update["fileUrl"], product, whitelistedDomains): return True return False
def _getSpecificPatchXML(self, patchKey, patchType, patch, updateQuery, whitelistedDomains, specialForceHosts): fromRelease = self._getFromRelease(patch) # don't return an update if we don't match the from restriction if fromRelease and not fromRelease.matchesUpdateQuery(updateQuery): return None # don't return an update if an older release isn't in the DB for some reason if patch['from'] != '*' and fromRelease is None: return None try: url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts) except ValueError: # Sometimes we may not be able to find a partial update even though # we've told to. Because there should be a complete to fall back on, # this is non-fatal, but is note-worthy. # https://bugzilla.mozilla.org/show_bug.cgi?id=1312562 has more # details on this. if patchType == "partial": self.log.exception("Caught non-fatal exception finding fileUrl for partial update:") return None # If we happen to find no _complete_ update, this is more serious, # and we need to re-raise the exception. else: raise # TODO: should be raising a bigger alarm here, or aborting # the update entirely? Right now, another patch type could still # return an update. Eg, the partial could contain a forbidden domain # but the complete could still return an update from an accepted one. if isForbiddenUrl(url, updateQuery['product'], whitelistedDomains): return None return ' <patch type="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s"/>' % \ (patchType, url, self["hashFunction"], patch["hashValue"], patch["filesize"])
def createSnippets(self, updateQuery, update_type, whitelistedDomains, specialForceHosts): snippets = {} buildTarget = updateQuery["buildTarget"] locale = updateQuery["locale"] localeData = self.getLocaleData(buildTarget, locale) for patchKey in ("partial", "complete"): patch = localeData.get(patchKey) if not patch: continue fromRelease = self._getFromRelease(patch) if fromRelease and not fromRelease.matchesUpdateQuery(updateQuery): continue url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts) if isForbiddenUrl(url, updateQuery['product'], whitelistedDomains): break snippet = [ "version=2", "type=%s" % patchKey, "url=%s" % url, "hashFunction=%s" % self["hashFunction"], "hashValue=%s" % patch["hashValue"], "size=%s" % patch["filesize"], "build=%s" % self.getBuildID(buildTarget, locale), "displayVersion=%s" % self.getDisplayVersion(buildTarget, locale), "appVersion=%s" % self.getAppVersion(buildTarget, locale), "platformVersion=%s" % self.getPlatformVersion(buildTarget, locale), ] if "detailsUrl" in self: details = self["detailsUrl"].replace("%LOCALE%", updateQuery["locale"]) details = details.replace("%locale%", updateQuery["locale"]) snippet.append("detailsUrl=%s" % details) if "licenseUrl" in self: license = self["licenseUrl"].replace("%LOCALE%", updateQuery["locale"]) license = license.replace("%locale%", updateQuery["locale"]) snippet.append("licenseUrl=%s" % license) if update_type == "major": snippet.append("updateType=major") for attr in self.optional_: if attr in self: if attr in self.interpolable_: snippetToAppend = self[attr].replace( "%LOCALE%", updateQuery["locale"]) snippetToAppend = snippetToAppend.replace( "%locale%", updateQuery["locale"]) snippet.append("%s=%s" % (attr, snippetToAppend)) else: snippet.append("%s=%s" % (attr, self[attr])) snippets[patchKey] = "\n".join(snippet) + "\n" for s in snippets.keys(): self.log.debug('%s\n%s' % (s, snippets[s].rstrip())) return snippets
def createXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts): buildTarget = updateQuery["buildTarget"] vendorXML = [] for vendor in sorted(self.getVendorsForPlatform(buildTarget)): vendorInfo = self["vendors"][vendor] platformData = self.getPlatformData(vendor, buildTarget) url = platformData["fileUrl"] if isForbiddenUrl(url, whitelistedDomains): continue vendorXML.append( ' <addon id="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s" version="%s"/>' % ( vendor, url, self["hashFunction"], platformData["hashValue"], platformData["filesize"], vendorInfo["version"], ) ) xml = ['<?xml version="1.0"?>'] xml.append("<updates>") if vendorXML: xml.append(" <addons>") xml.extend(vendorXML) xml.append(" </addons>") xml.append("</updates>") # ensure valid xml by using the right entity for ampersand return re.sub("&(?!amp;)", "&", "\n".join(xml))
def createXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts): buildTarget = updateQuery["buildTarget"] vendorXML = [] for vendor in self.getVendorsForPlatform(buildTarget): vendorInfo = self["vendors"][vendor] platformData = self.getPlatformData(vendor, buildTarget) url = platformData["fileUrl"] if isForbiddenUrl(url, whitelistedDomains): continue vendorXML.append(' <addon id="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s" version="%s"/>' % \ (vendor, url, self["hashFunction"], platformData["hashValue"], platformData["filesize"], vendorInfo["version"])) xml = ['<?xml version="1.0"?>'] xml.append('<updates>') if vendorXML: xml.append(' <addons>') xml.extend(vendorXML) xml.append(' </addons>') xml.append('</updates>') # ensure valid xml by using the right entity for ampersand return re.sub('&(?!amp;)', '&', '\n'.join(xml))
def containsForbiddenDomain(self, product, whitelistedDomains): """Returns True if the blob contains any file URLs that contain a domain that we're not allowed to serve updates to.""" for vendor in self.get('vendors', {}).values(): for platform in vendor.get('platforms', {}).values(): if 'fileUrl' in platform: if isForbiddenUrl(platform["fileUrl"], product, whitelistedDomains): return True return False
def getResponse(self, updateQuery, whitelistedDomains): url = self.get("platforms", {}).get(updateQuery["buildTarget"], {}).get("fileUrl") hashValue = self.get("platforms", {}).get(updateQuery["buildTarget"], {}).get("hashValue") if not url: return {} if isForbiddenUrl(url, updateQuery["product"], whitelistedDomains): return {} return {"version": self["version"], "url": url, "required": self["required"], "hashFunction": self["hashFunction"], "hashValue": hashValue}
def hasUpdates(self, updateQuery, whitelistedDomains): buildTarget = updateQuery["buildTarget"] for addon in sorted(self.getAddonsForPlatform(buildTarget)): # Checking if the addon update is to be served platformData = self.getPlatformData(addon, buildTarget) url = platformData["fileUrl"] # There might be no updates even if we have response products if # they are not served from whitelisted domains if isForbiddenUrl(url, updateQuery["product"], whitelistedDomains): continue return True return False
def createSnippets(self, updateQuery, update_type, whitelistedDomains, specialForceHosts): snippets = {} buildTarget = updateQuery["buildTarget"] locale = updateQuery["locale"] localeData = self.getLocaleData(buildTarget, locale) for patchKey in ("partial", "complete"): patch = localeData.get(patchKey) if not patch: continue fromRelease = self._getFromRelease(patch) if fromRelease and not fromRelease.matchesUpdateQuery(updateQuery): continue url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts) if isForbiddenUrl(url, updateQuery['product'], whitelistedDomains): break snippet = [ "version=2", "type=%s" % patchKey, "url=%s" % url, "hashFunction=%s" % self["hashFunction"], "hashValue=%s" % patch["hashValue"], "size=%s" % patch["filesize"], "build=%s" % self.getBuildID(buildTarget, locale), "displayVersion=%s" % self.getDisplayVersion(buildTarget, locale), "appVersion=%s" % self.getAppVersion(buildTarget, locale), "platformVersion=%s" % self.getPlatformVersion(buildTarget, locale), ] if "detailsUrl" in self: details = self["detailsUrl"].replace("%LOCALE%", updateQuery["locale"]) details = details.replace("%locale%", updateQuery["locale"]) snippet.append("detailsUrl=%s" % details) if "licenseUrl" in self: license = self["licenseUrl"].replace("%LOCALE%", updateQuery["locale"]) license = license.replace("%locale%", updateQuery["locale"]) snippet.append("licenseUrl=%s" % license) if update_type == "major": snippet.append("updateType=major") for attr in self.optional_: if attr in self: if attr in self.interpolable_: snippetToAppend = self[attr].replace("%LOCALE%", updateQuery["locale"]) snippetToAppend = snippetToAppend.replace("%locale%", updateQuery["locale"]) snippet.append("%s=%s" % (attr, snippetToAppend)) else: snippet.append("%s=%s" % (attr, self[attr])) snippets[patchKey] = "\n".join(snippet) + "\n" for s in snippets.keys(): self.log.debug('%s\n%s' % (s, snippets[s].rstrip())) return snippets
def createSnippets(self, updateQuery, update_type, whitelistedDomains, specialForceHosts): snippets = {} buildTarget = updateQuery["buildTarget"] locale = updateQuery["locale"] localeData = self.getLocaleData(buildTarget, locale) for patchKey in ("partial", "complete"): patch = localeData.get(patchKey) if not patch: continue fromRelease = self._getFromRelease(patch) if fromRelease and not fromRelease.matchesUpdateQuery(updateQuery): continue url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts) if isForbiddenUrl(url, updateQuery['product'], whitelistedDomains): break snippet = [ "version=1", "type=%s" % patchKey, "url=%s" % url, "hashFunction=%s" % self["hashFunction"], "hashValue=%s" % patch["hashValue"], "size=%s" % patch["filesize"], "build=%s" % self.getBuildID(buildTarget, locale), "appv=%s" % self.getAppv(buildTarget, locale), "extv=%s" % self.getExtv(buildTarget, locale), ] if "detailsUrl" in self: details = self["detailsUrl"].replace("%LOCALE%", updateQuery["locale"]) details = details.replace("%locale%", updateQuery["locale"]) snippet.append("detailsUrl=%s" % details) if "licenseUrl" in self: license = self["licenseUrl"].replace("%LOCALE%", updateQuery["locale"]) license = license.replace("%locale%", updateQuery["locale"]) snippet.append("licenseUrl=%s" % license) if update_type == "major": snippet.append("updateType=major") snippets[patchKey] = "\n".join(snippet) + "\n" if self.get("fakePartials" ) and "complete" in snippets and "partial" not in snippets: partial = snippets["complete"] partial = partial.replace("type=complete", "type=partial") snippets["partial"] = partial for s in snippets.keys(): self.log.debug('%s\n%s' % (s, snippets[s].rstrip())) return snippets
def createSnippets(self, updateQuery, update_type, whitelistedDomains, specialForceHosts): snippets = {} buildTarget = updateQuery["buildTarget"] locale = updateQuery["locale"] localeData = self.getLocaleData(buildTarget, locale) for patchKey in ("partial", "complete"): patch = localeData.get(patchKey) if not patch: continue try: fromRelease = dbo.releases.getReleaseBlob(name=patch["from"]) except KeyError: fromRelease = None if patch["from"] != "*" and fromRelease and not fromRelease.matchesUpdateQuery(updateQuery): continue url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts) if isForbiddenUrl(url, whitelistedDomains): break snippet = [ "version=1", "type=%s" % patchKey, "url=%s" % url, "hashFunction=%s" % self["hashFunction"], "hashValue=%s" % patch["hashValue"], "size=%s" % patch["filesize"], "build=%s" % self.getBuildID(buildTarget, locale), "appv=%s" % self.getAppv(buildTarget, locale), "extv=%s" % self.getExtv(buildTarget, locale), ] if "detailsUrl" in self: details = self["detailsUrl"].replace("%LOCALE%", updateQuery["locale"]) snippet.append("detailsUrl=%s" % details) if "licenseUrl" in self: license = self["licenseUrl"].replace("%LOCALE%", updateQuery["locale"]) snippet.append("licenseUrl=%s" % license) if update_type == "major": snippet.append("updateType=major") snippets[patchKey] = "\n".join(snippet) + "\n" if self.get("fakePartials") and "complete" in snippets and "partial" not in snippets: partial = snippets["complete"] partial = partial.replace("type=complete", "type=partial") snippets["partial"] = partial for s in snippets.keys(): self.log.debug('%s\n%s' % (s, snippets[s].rstrip())) return snippets
def _getSpecificPatchXML(self, patchKey, patchType, patch, updateQuery, whitelistedDomains, specialForceHosts): fromRelease = self._getFromRelease(patch) if fromRelease and not fromRelease.matchesUpdateQuery(updateQuery): return None url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts) # TODO: should be raising a bigger alarm here, or aborting # the update entirely? Right now, another patch type could still # return an update. Eg, the partial could contain a forbidden domain # but the complete could still return an update from an accepted one. if isForbiddenUrl(url, updateQuery['product'], whitelistedDomains): return None return ' <patch type="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s"/>' % \ (patchType, url, self["hashFunction"], patch["hashValue"], patch["filesize"])
def _getSpecificPatchXML(self, patchKey, patchType, patch, updateQuery, whitelistedDomains, specialForceHosts): fromRelease = self._getFromRelease(patch) if fromRelease and not fromRelease.matchesUpdateQuery(updateQuery): return None url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts) # TODO: should be raising a bigger alarm here, or aborting # the update entirely? Right now, another patch type could still # return an update. Eg, the partial could contain a forbidden domain # but the complete could still return an update from an accepted one. if isForbiddenUrl(url, whitelistedDomains): return None return ' <patch type="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s"/>' % \ (patchType, url, self["hashFunction"], patch["hashValue"], patch["filesize"])
def getInnerXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts): buildTarget = updateQuery["buildTarget"] vendorXML = [] for vendor in sorted(self.getVendorsForPlatform(buildTarget)): vendorInfo = self["vendors"][vendor] platformData = self.getPlatformData(vendor, buildTarget) url = platformData["fileUrl"] if isForbiddenUrl(url, updateQuery["product"], whitelistedDomains): continue vendorXML.append(' <addon id="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s" version="%s"/>' % (vendor, url, self["hashFunction"], platformData["hashValue"], platformData["filesize"], vendorInfo["version"])) return vendorXML
def getInnerXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts): # In case we have an uninstall blob, we won't have the addons section if self.get('addons') is None: return [] buildTarget = updateQuery["buildTarget"] addonXML = [] for addon in sorted(self.getAddonsForPlatform(buildTarget)): addonInfo = self["addons"][addon] platformData = self.getPlatformData(addon, buildTarget) url = platformData["fileUrl"] if isForbiddenUrl(url, updateQuery["product"], whitelistedDomains): continue addonXML.append(' <addon id="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s" version="%s"/>' % (addon, url, self["hashFunction"], platformData["hashValue"], platformData["filesize"], addonInfo["version"])) return addonXML
def _getSpecificPatchXML(self, patchKey, patchType, patch, updateQuery, whitelistedDomains, specialForceHosts): fromRelease = self._getFromRelease(patch) # don't return an update if we don't match the from restriction if fromRelease and not fromRelease.matchesUpdateQuery(updateQuery): return None # don't return an update if an older release isn't in the DB for some reason if patch['from'] != '*' and fromRelease is None: return None try: url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts) except ValueError: # Sometimes we may not be able to find a partial update even though # we've told to. Because there should be a complete to fall back on, # this is non-fatal, but is note-worthy. # https://bugzilla.mozilla.org/show_bug.cgi?id=1312562 has more # details on this. if patchType == "partial": self.log.exception( "Caught non-fatal exception finding fileUrl for partial update:" ) return None # If we happen to find no _complete_ update, this is more serious, # and we need to re-raise the exception. else: raise # TODO: should be raising a bigger alarm here, or aborting # the update entirely? Right now, another patch type could still # return an update. Eg, the partial could contain a forbidden domain # but the complete could still return an update from an accepted one. if isForbiddenUrl(url, updateQuery['product'], whitelistedDomains): return None patchXML = ' <patch type="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s"' % \ (patchType, url, self["hashFunction"], patch["hashValue"], patch["filesize"]) additionalPatchAttributes = self._getAdditionalPatchAttributes(patch) for attribute in additionalPatchAttributes: patchXML += ' %s="%s"' % (attribute, additionalPatchAttributes[attribute]) patchXML += '/>' return patchXML
def containsForbiddenDomain(self, product, whitelistedDomains): urls = [p["fileUrl"] for p in self["platforms"].values()] return any([isForbiddenUrl(url, product, whitelistedDomains) for url in urls])