def test_flush(self, mock_utc_now): now = utc_now() now_plus_10 = now + datetime.timedelta(seconds=10) now_plus_20 = now + datetime.timedelta(seconds=20) mock_utc_now.return_value = now cache = ExpiringCache(default_ttl=100) # At time now cache["foo"] = "bar" # At time now + 10 mock_utc_now.return_value = now_plus_10 cache["foo10"] = "bar" # At time now + 20 mock_utc_now.return_value = now_plus_20 cache["foo20"] = "bar" assert cache._data == { "foo": [now + cache._default_ttl, "bar"], "foo10": [now_plus_10 + cache._default_ttl, "bar"], "foo20": [now_plus_20 + cache._default_ttl, "bar"], } # Set to now + 105 which expires the first, but not the other two mock_utc_now.return_value = now + datetime.timedelta(seconds=105) cache.flush() # We don't want to trigger eviction or anything like that, so check # the contents of the internal data structure directly assert cache._data == { "foo10": [now_plus_10 + cache._default_ttl, "bar"], "foo20": [now_plus_20 + cache._default_ttl, "bar"], }
def test_expiration(self, mock_utc_now): # Mock utc_now to return the current time so we can set the expiry for # the key now = utc_now() mock_utc_now.return_value = now cache = ExpiringCache(default_ttl=100) cache['foo'] = 'bar' assert len(cache) == 1 cache.set('long_foo', value='bar2', ttl=1000) assert len(cache) == 2 # default ttl is 100, so 99 seconds into the future, we should get back # both cached values mock_utc_now.return_value = now + datetime.timedelta(seconds=99) assert cache['foo'] == 'bar' assert cache['long_foo'] == 'bar2' assert len(cache) == 2 # ttl is 100, so 101 seconds into the future, we should get a KeyError # for one cached key and the other should be fine mock_utc_now.return_value = now + datetime.timedelta(seconds=101) with pytest.raises(KeyError): cache['foo'] assert cache['long_foo'] == 'bar2' assert len(cache) == 1
def test_flush(self, mock_utc_now): now = utc_now() now_plus_10 = now + datetime.timedelta(seconds=10) now_plus_20 = now + datetime.timedelta(seconds=20) mock_utc_now.return_value = now cache = ExpiringCache(default_ttl=100) # At time now cache['foo'] = 'bar' # At time now + 10 mock_utc_now.return_value = now_plus_10 cache['foo10'] = 'bar' # At time now + 20 mock_utc_now.return_value = now_plus_20 cache['foo20'] = 'bar' assert (cache._data == { 'foo': [now + cache._default_ttl, 'bar'], 'foo10': [now_plus_10 + cache._default_ttl, 'bar'], 'foo20': [now_plus_20 + cache._default_ttl, 'bar'], }) # Set to now + 105 which expires the first, but not the other two mock_utc_now.return_value = now + datetime.timedelta(seconds=105) cache.flush() assert (cache._data == { 'foo10': [now_plus_10 + cache._default_ttl, 'bar'], 'foo20': [now_plus_20 + cache._default_ttl, 'bar'], })
def __init__(self, config): super().__init__(config) self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL) self.metrics = markus.get_metrics('processor.betaversionrule') # For looking up version strings self.version_string_api = config.version_string_api self.session = session_with_retries()
def __init__(self, config): super(BetaVersionRule, self).__init__(config) self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL) self.metrics = markus.get_metrics('processor.betaversionrule') # NOTE(willkg): These config values come from Processor2015 instance and are # used for lookup in product_versions self.conn_context = config.database_class(config)
def test_max_size(self): cache = ExpiringCache(max_size=5) cache['foo1'] = 1 cache['foo2'] = 1 cache['foo3'] = 1 cache['foo4'] = 1 cache['foo5'] = 1 assert list(cache.keys()) == ['foo1', 'foo2', 'foo3', 'foo4', 'foo5'] cache['foo6'] = 1 assert list(cache.keys()) == ['foo2', 'foo3', 'foo4', 'foo5', 'foo6']
def test_max_size(self): cache = ExpiringCache(max_size=5) cache["foo1"] = 1 cache["foo2"] = 1 cache["foo3"] = 1 cache["foo4"] = 1 cache["foo5"] = 1 assert list(cache.keys()) == ["foo1", "foo2", "foo3", "foo4", "foo5"] cache["foo6"] = 1 assert list(cache.keys()) == ["foo2", "foo3", "foo4", "foo5", "foo6"]
def test_get_set(self): cache = ExpiringCache(default_ttl=600) with pytest.raises(KeyError): cache['foo'] cache['foo'] = 'bar' assert cache['foo'] == 'bar'
def test_get_set(self): cache = ExpiringCache(default_ttl=600) with pytest.raises(KeyError): cache["foo"] cache["foo"] = "bar" assert cache["foo"] == "bar"
def __init__(self, config): super().__init__(config) self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL) self.metrics = markus.get_metrics('processor.betaversionrule') # For looking up version strings self.version_string_api = config.version_string_api self.session = session_with_retries()
class BetaVersionRule(Rule): #: Hold at most this many items in cache; items are a key and a value #: both of which are short strings, so this doesn't take much memory CACHE_MAX_SIZE = 5000 #: Items in cache expire after 30 minutes by default SHORT_CACHE_TTL = 60 * 30 #: If we know it's good, cache it for 24 hours because it won't change LONG_CACHE_TTL = 60 * 60 * 24 #: List of products to do lookups for SUPPORTED_PRODUCTS = ['firefox', 'fennec', 'fennecandroid'] def __init__(self, config): super().__init__(config) self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL) self.metrics = markus.get_metrics('processor.betaversionrule') # For looking up version strings self.version_string_api = config.version_string_api self.session = session_with_retries() def _get_real_version(self, product, channel, build_id): """Return real version number from crashstats_productversion table :arg str product: the product :arg str channel: the release channel :arg int build_id: the build id as a string :returns: ``None`` or the version string that should be used """ # Fix the product so it matches the data in the table if (product, channel) == ('firefox', 'aurora') and build_id > '20170601': product = 'DevEdition' elif product == 'firefox': product = 'Firefox' elif product in ('fennec', 'fennecandroid'): product = 'Fennec' key = '%s:%s:%s' % (product, channel, build_id) if key in self.cache: self.metrics.incr('cache', tags=['result:hit']) return self.cache[key] self.metrics.incr('cache', tags=['result:miss']) resp = self.session.get(self.version_string_api, params={ 'product': product, 'channel': channel, 'build_id': build_id }) if resp.status_code != 200: versions = [] else: versions = resp.json()['hits'] if not versions: # We didn't get an answer which could mean that this is a weird # build and there is no answer or it could mean that Buildhub # doesn't know, yet. Maybe in the future we get a better answer so # we use the short ttl. self.metrics.incr('lookup', tags=['result:fail']) self.cache.set(key, value=None, ttl=self.SHORT_CACHE_TTL) return None # If we got an answer we should keep it around for a while because it's # a real answer and it's not going to change so use the long ttl plus # a fudge factor. real_version = versions[0]['version_string'] self.metrics.incr('lookup', tags=['result:success']) self.cache.set(key, value=real_version, ttl=self.LONG_CACHE_TTL) return real_version def predicate(self, raw_crash, raw_dumps, processed_crash, proc_meta): # Beta and aurora versions send the wrong version in the crash report, # so we need to fix them return processed_crash.get('release_channel', '').lower() in ('beta', 'aurora') def action(self, raw_crash, raw_dumps, processed_crash, processor_meta): product = processed_crash.get('product', '').strip().lower() build_id = processed_crash.get('build', '').strip() release_channel = processed_crash.get('release_channel').strip() # Only run if we've got all the things we need if product and build_id and release_channel and product in self.SUPPORTED_PRODUCTS: # Convert the build_id to a str for lookups build_id = str(build_id) real_version = self._get_real_version(product, release_channel, build_id) if real_version: processed_crash['version'] = real_version return self.logger.info( 'betaversionrule: failed lookup %s %s %s %s', processed_crash.get('uuid'), product, release_channel, build_id ) # No real version, but this is an aurora or beta crash report, so we # tack on "b0" to make it match the channel processed_crash['version'] += 'b0' processor_meta.processor_notes.append( 'release channel is %s but no version data was found - added "b0" ' 'suffix to version number' % release_channel )
class BetaVersionRule(Rule): #: Hold at most this many items in cache; items are a key and a value #: both of which are short strings, so this doesn't take much memory CACHE_MAX_SIZE = 5000 #: Items in cache expire after 30 minutes by default SHORT_CACHE_TTL = 60 * 30 #: If we know it's good, cache it for 6 hours LONG_CACHE_TTL = 60 * 60 * 6 def __init__(self, config): super(BetaVersionRule, self).__init__(config) # NOTE(willkg): These config values come from Processor2015 instance. self.version_string_api = config.version_string_api self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL) def version(self): return '1.0' def _get_version_data(self, product, version, build_id): """Return the real version number of a specific product, version and build For example, beta builds of Firefox declare their version number as the major version (i.e. version 54.0b3 would say its version is 54.0). This database call returns the actual version number of said build (i.e. 54.0b3 for the previous example). :arg product: the product :arg version: the version as a string. e.g. "56.0" :arg build_id: the build_id as a string. :returns: ``None`` or the version string that should be used :raises requests.RequestException: raised if it has connection issues with the host specified in ``version_string_api`` """ if not (product and version and build_id): return None key = '%s:%s:%s' % (product, version, build_id) if key in self.cache: return self.cache[key] session = session_with_retries(self.version_string_api) resp = session.get(self.version_string_api, params={ 'product': product, 'version': version, 'build_id': build_id }) if resp.status_code == 200: hits = resp.json()['hits'] # Shimmy to add to ttl so as to distribute cache misses over time and reduce # HTTP requests from bunching up. shimmy = random.randint(1, 120) if hits: # If we got an answer we should keep it around for a while because it's # a real answer and it's not going to change so use the long ttl plus # a fudge factor. real_version = hits[0] self.cache.set(key, value=real_version, ttl=self.LONG_CACHE_TTL + shimmy) return real_version else: # We didn't get an answer which could mean that this is a weird build and there # is no answer or it could mean that ftpscraper hasn't picked up the relevant # build information or it could mean we're getting cached answers from the webapp. # Regardless, maybe in the future we get a better answer so we use the short # ttl plus a fudge factor. self.cache.set(key, value=None, ttl=self.SHORT_CACHE_TTL + shimmy) return None def _predicate(self, raw_crash, raw_dumps, processed_crash, proc_meta): # Beta and aurora versions send the wrong version in the crash report, # so we need to fix them. return processed_crash.get('release_channel', '').lower() in ('beta', 'aurora') def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta): try: # Sanitize the build id to avoid errors during the SQL query. try: build_id = int(processed_crash['build']) except ValueError: build_id = None real_version = self._get_version_data( processed_crash['product'], processed_crash['version'], build_id, ) if real_version: processed_crash['version'] = real_version else: # We don't have a real version to use, so we tack on "b0" to # make it better and match the channel. processed_crash['version'] += 'b0' processor_meta.processor_notes.append( 'release channel is %s but no version data was found ' '- added "b0" suffix to version number' % (processed_crash['release_channel'], )) except KeyError: return False except RequestException as exc: processed_crash['version'] += 'b0' processor_meta.processor_notes.append( 'could not connect to VersionString API - added "b0" suffix to version number' ) self.config.logger.exception('%s when connecting to %s', exc, self.version_string_api) return True
class BetaVersionRule(Rule): #: Hold at most this many items in cache; items are a key and a value #: both of which are short strings, so this doesn't take much memory CACHE_MAX_SIZE = 5000 #: Items in cache expire after 30 minutes by default SHORT_CACHE_TTL = 60 * 30 #: If we know it's good, cache it for 24 hours because it won't change LONG_CACHE_TTL = 60 * 60 * 24 def __init__(self, config): super(BetaVersionRule, self).__init__(config) # NOTE(willkg): These config values come from Processor2015 instance. self.buildhub_api = config.buildhub_api self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL) def version(self): return '1.0' def _get_version_data(self, product, build_id, channel): """Return the real version number of a specified product, build, channel For example, beta builds of Firefox declare their version number as the major version (i.e. version 54.0b3 would say its version is 54.0). This database call returns the actual version number of said build (i.e. 54.0b3 for the previous example). :arg product: the product :arg build_id: the build_id as a string :arg channel: the release channel :returns: ``None`` or the version string that should be used :raises requests.RequestException: raised if it has connection issues with the host specified in ``version_string_api`` """ # NOTE(willkg): AURORA LIVES! # # But seriously, if this is for Firefox/aurora and the build id is after # 20170601, then we ask Buildhub about devedition/aurora instead because # devedition is the aurora channel if (product, channel) == ('firefox', 'aurora') and build_id > '20170601': product = 'devedition' key = '%s:%s:%s' % (product, build_id, channel) if key in self.cache: return self.cache[key] session = session_with_retries(self.buildhub_api) query = { 'source.product': product, 'build.id': '"%s"' % build_id, 'target.channel': channel, '_limit': 1 } resp = session.get(self.buildhub_api, params=query) if resp.status_code == 200: hits = resp.json()['data'] # Shimmy to add to ttl so as to distribute cache misses over time and reduce # HTTP requests from bunching up. shimmy = random.randint(1, 120) if hits: # If we got an answer we should keep it around for a while because it's # a real answer and it's not going to change so use the long ttl plus # a fudge factor. real_version = hits[0]['target']['version'] self.cache.set(key, value=real_version, ttl=self.LONG_CACHE_TTL + shimmy) return real_version # We didn't get an answer which could mean that this is a weird # build and there is no answer or it could mean that Buildhub # doesn't know, yet. Maybe in the future we get a better answer # so we use the short ttl plus a fudge factor. self.cache.set(key, value=None, ttl=self.SHORT_CACHE_TTL + shimmy) return None def _predicate(self, raw_crash, raw_dumps, processed_crash, proc_meta): # Beta and aurora versions send the wrong version in the crash report, # so we need to fix them return processed_crash.get('release_channel', '').lower() in ('beta', 'aurora') def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta): product = processed_crash.get('product').strip() try: build_id = int(processed_crash.get('build').strip()) except ValueError: build_id = None release_channel = processed_crash.get('release_channel').strip() # If we're missing one of the magic ingredients, then there's nothing # to do if product and build_id and release_channel: try: real_version = self._get_version_data(product.lower(), str(build_id), release_channel) # If we got a real version, toss that in and we're done if real_version: processed_crash['version'] = real_version return True except RequestException as exc: processor_meta.processor_notes.append( 'could not connect to Buildhub') self.config.logger.exception('%s when connecting to %s', exc, self.version_string_api) # No real version, but this is an aurora or beta crash report, so we # tack on "b0" to make it match the channel processed_crash['version'] += 'b0' processor_meta.processor_notes.append( 'release channel is %s but no version data was found - added "b0" ' 'suffix to version number' % processed_crash['release_channel']) return True
def __init__(self, config): super(BetaVersionRule, self).__init__(config) # NOTE(willkg): These config values come from Processor2015 instance. self.buildhub_api = config.buildhub_api self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL)
class BetaVersionRule(Rule): #: Hold at most this many items in cache; items are a key and a value #: both of which are short strings, so this doesn't take much memory CACHE_MAX_SIZE = 5000 #: Items in cache expire after 30 minutes by default SHORT_CACHE_TTL = 60 * 30 #: If we know it's good, cache it for 24 hours because it won't change LONG_CACHE_TTL = 60 * 60 * 24 #: List of products to do lookups for SUPPORTED_PRODUCTS = ["firefox", "fennec", "fennecandroid"] def __init__(self, version_string_api): super().__init__() self.cache = ExpiringCache( max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL ) self.metrics = markus.get_metrics("processor.betaversionrule") # For looking up version strings self.version_string_api = version_string_api self.session = session_with_retries() def __repr__(self): return self.generate_repr(keys=["version_string_api"]) def _get_real_version(self, product, channel, build_id): """Return real version number from crashstats_productversion table :arg str product: the product :arg str channel: the release channel :arg int build_id: the build id as a string :returns: ``None`` or the version string that should be used """ # Fix the product so it matches the data in the table if (product, channel) == ("firefox", "aurora") and build_id > "20170601": product = "DevEdition" elif product == "firefox": product = "Firefox" elif product in ("fennec", "fennecandroid"): product = "Fennec" key = "%s:%s:%s" % (product, channel, build_id) if key in self.cache: self.metrics.incr("cache", tags=["result:hit"]) return self.cache[key] self.metrics.incr("cache", tags=["result:miss"]) resp = self.session.get( self.version_string_api, params={"product": product, "channel": channel, "build_id": build_id}, ) if resp.status_code != 200: versions = [] else: versions = resp.json()["hits"] if not versions: # We didn't get an answer which could mean that this is a weird # build and there is no answer or it could mean that Buildhub # doesn't know, yet. Maybe in the future we get a better answer so # we use the short ttl. self.metrics.incr("lookup", tags=["result:fail"]) self.cache.set(key, value=None, ttl=self.SHORT_CACHE_TTL) return None # If we got an answer we should keep it around for a while because it's # a real answer and it's not going to change so use the long ttl plus # a fudge factor. real_version = versions[0]["version_string"] self.metrics.incr("lookup", tags=["result:success"]) self.cache.set(key, value=real_version, ttl=self.LONG_CACHE_TTL) return real_version def predicate(self, raw_crash, raw_dumps, processed_crash, proc_meta): # Beta and aurora versions send the wrong version in the crash report, # so we need to fix them return processed_crash.get("release_channel", "").lower() in ("beta", "aurora") def action(self, raw_crash, raw_dumps, processed_crash, processor_meta): product = processed_crash.get("product", "").strip().lower() build_id = processed_crash.get("build", "").strip() release_channel = processed_crash.get("release_channel").strip() # Only run if we've got all the things we need if ( product and build_id and release_channel and product in self.SUPPORTED_PRODUCTS ): # Convert the build_id to a str for lookups build_id = str(build_id) real_version = self._get_real_version(product, release_channel, build_id) if real_version: processed_crash["version"] = real_version return self.logger.info( "betaversionrule: failed lookup %s %s %s %s", processed_crash.get("uuid"), product, release_channel, build_id, ) # No real version, but this is an aurora or beta crash report, so we # tack on "b0" to make it match the channel processed_crash["version"] += "b0" processor_meta.processor_notes.append( 'release channel is %s but no version data was found - added "b0" ' "suffix to version number" % release_channel )
class BetaVersionRule(Rule): #: Hold at most this many items in cache; items are a key and a value #: both of which are short strings, so this doesn't take much memory CACHE_MAX_SIZE = 5000 #: Items in cache expire after 30 minutes by default SHORT_CACHE_TTL = 60 * 30 #: If we know it's good, cache it for 24 hours because it won't change LONG_CACHE_TTL = 60 * 60 * 24 #: List of products to do lookups for SUPPORTED_PRODUCTS = ['firefox', 'fennec', 'fennecandroid'] def __init__(self, config): super(BetaVersionRule, self).__init__(config) self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL) self.metrics = markus.get_metrics('processor.betaversionrule') # NOTE(willkg): These config values come from Processor2015 instance and are # used for lookup in product_versions self.conn_context = config.database_class(config) def _get_real_version(self, product, channel, build_id): """Return real version number from crashstats_productversion table :arg str product: the product :arg str channel: the release channel :arg int build_id: the build id as a string :returns: ``None`` or the version string that should be used """ # Fix the product so it matches the data in the table if (product, channel) == ('firefox', 'aurora') and build_id > '20170601': product = 'DevEdition' elif product == 'firefox': product = 'Firefox' elif product in ('fennec', 'fennecandroid'): product = 'Fennec' key = '%s:%s:%s' % (product, channel, build_id) if key in self.cache: self.metrics.incr('cache', tags=['result:hit']) return self.cache[key] sql = """ SELECT version_string FROM crashstats_productversion WHERE product_name = %(product)s AND release_channel = %(channel)s AND build_id = %(build_id)s """ params = { 'product': product, 'channel': channel, 'build_id': build_id } with self.conn_context() as conn: versions = execute_query_fetchall(conn, sql, params) if versions: # Flatten version results from a list of tuples to a list of versions versions = [version[0] for version in versions] if versions: if 'b' in versions[0]: # If we're looking at betas which have a "b" in the versions, # then ignore "rc" versions because they didn't get released versions = [version for version in versions if 'rc' not in version] else: # If we're looking at non-betas, then only return "rc" # versions because this crash report is in the beta channel # and not the release channel versions = [version for version in versions if 'rc' in version] if not versions: # We didn't get an answer which could mean that this is a weird # build and there is no answer or it could mean that Buildhub # doesn't know, yet. Maybe in the future we get a better answer so # we use the short ttl. self.metrics.incr('lookup', tags=['result:fail']) self.cache.set(key, value=None, ttl=self.SHORT_CACHE_TTL) return None # If we got an answer we should keep it around for a while because it's # a real answer and it's not going to change so use the long ttl plus # a fudge factor. real_version = versions[0] self.metrics.incr('lookup', tags=['result:success']) self.cache.set(key, value=real_version, ttl=self.LONG_CACHE_TTL) return real_version def predicate(self, raw_crash, raw_dumps, processed_crash, proc_meta): # Beta and aurora versions send the wrong version in the crash report, # so we need to fix them return processed_crash.get('release_channel', '').lower() in ('beta', 'aurora') def action(self, raw_crash, raw_dumps, processed_crash, processor_meta): product = processed_crash.get('product', '').strip().lower() build_id = processed_crash.get('build', '').strip() release_channel = processed_crash.get('release_channel').strip() # Only run if we've got all the things we need if product and build_id and release_channel and product in self.SUPPORTED_PRODUCTS: # Convert the build_id to a str for lookups build_id = str(build_id) real_version = self._get_real_version(product, release_channel, build_id) if real_version: processed_crash['version'] = real_version return self.config.logger.info( 'betaversionrule: failed lookup %s %s %s %s', processed_crash.get('uuid'), product, release_channel, build_id ) # No real version, but this is an aurora or beta crash report, so we # tack on "b0" to make it match the channel processed_crash['version'] += 'b0' processor_meta.processor_notes.append( 'release channel is %s but no version data was found - added "b0" ' 'suffix to version number' % release_channel )
class BetaVersionRule(Rule): #: Hold at most this many items in cache; items are a key and a value #: both of which are short strings, so this doesn't take much memory CACHE_MAX_SIZE = 5000 #: Items in cache expire after 30 minutes by default SHORT_CACHE_TTL = 60 * 30 #: If we know it's good, cache it for 24 hours because it won't change LONG_CACHE_TTL = 60 * 60 * 24 #: List of products to do lookups for SUPPORTED_PRODUCTS = ['firefox', 'fennec', 'fennecandroid'] def __init__(self, config): super().__init__(config) self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL) self.metrics = markus.get_metrics('processor.betaversionrule') # For looking up version strings self.version_string_api = config.version_string_api self.session = session_with_retries() def _get_real_version(self, product, channel, build_id): """Return real version number from crashstats_productversion table :arg str product: the product :arg str channel: the release channel :arg int build_id: the build id as a string :returns: ``None`` or the version string that should be used """ # Fix the product so it matches the data in the table if (product, channel) == ('firefox', 'aurora') and build_id > '20170601': product = 'DevEdition' elif product == 'firefox': product = 'Firefox' elif product in ('fennec', 'fennecandroid'): product = 'Fennec' key = '%s:%s:%s' % (product, channel, build_id) if key in self.cache: self.metrics.incr('cache', tags=['result:hit']) return self.cache[key] self.metrics.incr('cache', tags=['result:miss']) resp = self.session.get(self.version_string_api, params={ 'product': product, 'channel': channel, 'build_id': build_id }) if resp.status_code != 200: versions = [] else: versions = resp.json()['hits'] if not versions: # We didn't get an answer which could mean that this is a weird # build and there is no answer or it could mean that Buildhub # doesn't know, yet. Maybe in the future we get a better answer so # we use the short ttl. self.metrics.incr('lookup', tags=['result:fail']) self.cache.set(key, value=None, ttl=self.SHORT_CACHE_TTL) return None # If we got an answer we should keep it around for a while because it's # a real answer and it's not going to change so use the long ttl plus # a fudge factor. real_version = versions[0]['version_string'] self.metrics.incr('lookup', tags=['result:success']) self.cache.set(key, value=real_version, ttl=self.LONG_CACHE_TTL) return real_version def predicate(self, raw_crash, raw_dumps, processed_crash, proc_meta): # Beta and aurora versions send the wrong version in the crash report, # so we need to fix them return processed_crash.get('release_channel', '').lower() in ('beta', 'aurora') def action(self, raw_crash, raw_dumps, processed_crash, processor_meta): product = processed_crash.get('product', '').strip().lower() build_id = processed_crash.get('build', '').strip() release_channel = processed_crash.get('release_channel').strip() # Only run if we've got all the things we need if product and build_id and release_channel and product in self.SUPPORTED_PRODUCTS: # Convert the build_id to a str for lookups build_id = str(build_id) real_version = self._get_real_version(product, release_channel, build_id) if real_version: processed_crash['version'] = real_version return self.logger.info('betaversionrule: failed lookup %s %s %s %s', processed_crash.get('uuid'), product, release_channel, build_id) # No real version, but this is an aurora or beta crash report, so we # tack on "b0" to make it match the channel processed_crash['version'] += 'b0' processor_meta.processor_notes.append( 'release channel is %s but no version data was found - added "b0" ' 'suffix to version number' % release_channel)