def test_expiration(self, mock_utc_now):
# Mock utc_now to return the current time so we can set the expiry for
# the key
now = utc_now()
mock_utc_now.return_value = now
cache = ExpiringCache(default_ttl=100)
cache['foo'] = 'bar'
assert len(cache) == 1
cache.set('long_foo', value='bar2', ttl=1000)
assert len(cache) == 2
# default ttl is 100, so 99 seconds into the future, we should get back
# both cached values
mock_utc_now.return_value = now + datetime.timedelta(seconds=99)
assert cache['foo'] == 'bar'
assert cache['long_foo'] == 'bar2'
assert len(cache) == 2
# ttl is 100, so 101 seconds into the future, we should get a KeyError
# for one cached key and the other should be fine
mock_utc_now.return_value = now + datetime.timedelta(seconds=101)
with pytest.raises(KeyError):
cache['foo']
assert cache['long_foo'] == 'bar2'
assert len(cache) == 1
class BetaVersionRule(Rule):
#: Hold at most this many items in cache; items are a key and a value
#: both of which are short strings, so this doesn't take much memory
CACHE_MAX_SIZE = 5000
#: Items in cache expire after 30 minutes by default
SHORT_CACHE_TTL = 60 * 30
#: If we know it's good, cache it for 24 hours because it won't change
LONG_CACHE_TTL = 60 * 60 * 24
#: List of products to do lookups for
SUPPORTED_PRODUCTS = ["firefox", "fennec", "fennecandroid"]
def __init__(self, version_string_api):
super().__init__()
self.cache = ExpiringCache(
max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL
)
self.metrics = markus.get_metrics("processor.betaversionrule")
# For looking up version strings
self.version_string_api = version_string_api
self.session = session_with_retries()
def __repr__(self):
return self.generate_repr(keys=["version_string_api"])
def _get_real_version(self, product, channel, build_id):
"""Return real version number from crashstats_productversion table
:arg str product: the product
:arg str channel: the release channel
:arg int build_id: the build id as a string
:returns: ``None`` or the version string that should be used
"""
# Fix the product so it matches the data in the table
if (product, channel) == ("firefox", "aurora") and build_id > "20170601":
product = "DevEdition"
elif product == "firefox":
product = "Firefox"
elif product in ("fennec", "fennecandroid"):
product = "Fennec"
key = "%s:%s:%s" % (product, channel, build_id)
if key in self.cache:
self.metrics.incr("cache", tags=["result:hit"])
return self.cache[key]
self.metrics.incr("cache", tags=["result:miss"])
resp = self.session.get(
self.version_string_api,
params={"product": product, "channel": channel, "build_id": build_id},
)
if resp.status_code != 200:
versions = []
else:
versions = resp.json()["hits"]
if not versions:
# We didn't get an answer which could mean that this is a weird
# build and there is no answer or it could mean that Buildhub
# doesn't know, yet. Maybe in the future we get a better answer so
# we use the short ttl.
self.metrics.incr("lookup", tags=["result:fail"])
self.cache.set(key, value=None, ttl=self.SHORT_CACHE_TTL)
return None
# If we got an answer we should keep it around for a while because it's
# a real answer and it's not going to change so use the long ttl plus
# a fudge factor.
real_version = versions[0]["version_string"]
self.metrics.incr("lookup", tags=["result:success"])
self.cache.set(key, value=real_version, ttl=self.LONG_CACHE_TTL)
return real_version
def predicate(self, raw_crash, raw_dumps, processed_crash, proc_meta):
# Beta and aurora versions send the wrong version in the crash report,
# so we need to fix them
return processed_crash.get("release_channel", "").lower() in ("beta", "aurora")
def action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
product = processed_crash.get("product", "").strip().lower()
build_id = processed_crash.get("build", "").strip()
release_channel = processed_crash.get("release_channel").strip()
# Only run if we've got all the things we need
if (
product
and build_id
and release_channel
and product in self.SUPPORTED_PRODUCTS
):
# Convert the build_id to a str for lookups
build_id = str(build_id)
real_version = self._get_real_version(product, release_channel, build_id)
if real_version:
processed_crash["version"] = real_version
return
self.logger.info(
"betaversionrule: failed lookup %s %s %s %s",
processed_crash.get("uuid"),
product,
release_channel,
build_id,
)
# No real version, but this is an aurora or beta crash report, so we
# tack on "b0" to make it match the channel
processed_crash["version"] += "b0"
processor_meta.processor_notes.append(
'release channel is %s but no version data was found - added "b0" '
"suffix to version number" % release_channel
)
class BetaVersionRule(Rule):
#: Hold at most this many items in cache; items are a key and a value
#: both of which are short strings, so this doesn't take much memory
CACHE_MAX_SIZE = 5000
#: Items in cache expire after 30 minutes by default
SHORT_CACHE_TTL = 60 * 30
#: If we know it's good, cache it for 24 hours because it won't change
LONG_CACHE_TTL = 60 * 60 * 24
#: List of products to do lookups for
SUPPORTED_PRODUCTS = ['firefox', 'fennec', 'fennecandroid']
def __init__(self, config):
super(BetaVersionRule, self).__init__(config)
self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL)
self.metrics = markus.get_metrics('processor.betaversionrule')
# NOTE(willkg): These config values come from Processor2015 instance and are
# used for lookup in product_versions
self.conn_context = config.database_class(config)
def _get_real_version(self, product, channel, build_id):
"""Return real version number from crashstats_productversion table
:arg str product: the product
:arg str channel: the release channel
:arg int build_id: the build id as a string
:returns: ``None`` or the version string that should be used
"""
# Fix the product so it matches the data in the table
if (product, channel) == ('firefox', 'aurora') and build_id > '20170601':
product = 'DevEdition'
elif product == 'firefox':
product = 'Firefox'
elif product in ('fennec', 'fennecandroid'):
product = 'Fennec'
key = '%s:%s:%s' % (product, channel, build_id)
if key in self.cache:
self.metrics.incr('cache', tags=['result:hit'])
return self.cache[key]
sql = """
SELECT version_string
FROM crashstats_productversion
WHERE product_name = %(product)s
AND release_channel = %(channel)s
AND build_id = %(build_id)s
"""
params = {
'product': product,
'channel': channel,
'build_id': build_id
}
with self.conn_context() as conn:
versions = execute_query_fetchall(conn, sql, params)
if versions:
# Flatten version results from a list of tuples to a list of versions
versions = [version[0] for version in versions]
if versions:
if 'b' in versions[0]:
# If we're looking at betas which have a "b" in the versions,
# then ignore "rc" versions because they didn't get released
versions = [version for version in versions if 'rc' not in version]
else:
# If we're looking at non-betas, then only return "rc"
# versions because this crash report is in the beta channel
# and not the release channel
versions = [version for version in versions if 'rc' in version]
if not versions:
# We didn't get an answer which could mean that this is a weird
# build and there is no answer or it could mean that Buildhub
# doesn't know, yet. Maybe in the future we get a better answer so
# we use the short ttl.
self.metrics.incr('lookup', tags=['result:fail'])
self.cache.set(key, value=None, ttl=self.SHORT_CACHE_TTL)
return None
# If we got an answer we should keep it around for a while because it's
# a real answer and it's not going to change so use the long ttl plus
# a fudge factor.
real_version = versions[0]
self.metrics.incr('lookup', tags=['result:success'])
self.cache.set(key, value=real_version, ttl=self.LONG_CACHE_TTL)
return real_version
def predicate(self, raw_crash, raw_dumps, processed_crash, proc_meta):
# Beta and aurora versions send the wrong version in the crash report,
# so we need to fix them
return processed_crash.get('release_channel', '').lower() in ('beta', 'aurora')
def action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
product = processed_crash.get('product', '').strip().lower()
build_id = processed_crash.get('build', '').strip()
release_channel = processed_crash.get('release_channel').strip()
# Only run if we've got all the things we need
if product and build_id and release_channel and product in self.SUPPORTED_PRODUCTS:
# Convert the build_id to a str for lookups
build_id = str(build_id)
real_version = self._get_real_version(product, release_channel, build_id)
if real_version:
processed_crash['version'] = real_version
return
self.config.logger.info(
'betaversionrule: failed lookup %s %s %s %s',
processed_crash.get('uuid'),
product,
release_channel,
build_id
)
# No real version, but this is an aurora or beta crash report, so we
# tack on "b0" to make it match the channel
processed_crash['version'] += 'b0'
processor_meta.processor_notes.append(
'release channel is %s but no version data was found - added "b0" '
'suffix to version number' % release_channel
)
class BetaVersionRule(Rule):
#: Hold at most this many items in cache; items are a key and a value
#: both of which are short strings, so this doesn't take much memory
CACHE_MAX_SIZE = 5000
#: Items in cache expire after 30 minutes by default
SHORT_CACHE_TTL = 60 * 30
#: If we know it's good, cache it for 24 hours because it won't change
LONG_CACHE_TTL = 60 * 60 * 24
#: List of products to do lookups for
SUPPORTED_PRODUCTS = ['firefox', 'fennec', 'fennecandroid']
def __init__(self, config):
super().__init__(config)
self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE,
default_ttl=self.SHORT_CACHE_TTL)
self.metrics = markus.get_metrics('processor.betaversionrule')
# For looking up version strings
self.version_string_api = config.version_string_api
self.session = session_with_retries()
def _get_real_version(self, product, channel, build_id):
"""Return real version number from crashstats_productversion table
:arg str product: the product
:arg str channel: the release channel
:arg int build_id: the build id as a string
:returns: ``None`` or the version string that should be used
"""
# Fix the product so it matches the data in the table
if (product, channel) == ('firefox',
'aurora') and build_id > '20170601':
product = 'DevEdition'
elif product == 'firefox':
product = 'Firefox'
elif product in ('fennec', 'fennecandroid'):
product = 'Fennec'
key = '%s:%s:%s' % (product, channel, build_id)
if key in self.cache:
self.metrics.incr('cache', tags=['result:hit'])
return self.cache[key]
self.metrics.incr('cache', tags=['result:miss'])
resp = self.session.get(self.version_string_api,
params={
'product': product,
'channel': channel,
'build_id': build_id
})
if resp.status_code != 200:
versions = []
else:
versions = resp.json()['hits']
if not versions:
# We didn't get an answer which could mean that this is a weird
# build and there is no answer or it could mean that Buildhub
# doesn't know, yet. Maybe in the future we get a better answer so
# we use the short ttl.
self.metrics.incr('lookup', tags=['result:fail'])
self.cache.set(key, value=None, ttl=self.SHORT_CACHE_TTL)
return None
# If we got an answer we should keep it around for a while because it's
# a real answer and it's not going to change so use the long ttl plus
# a fudge factor.
real_version = versions[0]['version_string']
self.metrics.incr('lookup', tags=['result:success'])
self.cache.set(key, value=real_version, ttl=self.LONG_CACHE_TTL)
return real_version
def predicate(self, raw_crash, raw_dumps, processed_crash, proc_meta):
# Beta and aurora versions send the wrong version in the crash report,
# so we need to fix them
return processed_crash.get('release_channel',
'').lower() in ('beta', 'aurora')
def action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
product = processed_crash.get('product', '').strip().lower()
build_id = processed_crash.get('build', '').strip()
release_channel = processed_crash.get('release_channel').strip()
# Only run if we've got all the things we need
if product and build_id and release_channel and product in self.SUPPORTED_PRODUCTS:
# Convert the build_id to a str for lookups
build_id = str(build_id)
real_version = self._get_real_version(product, release_channel,
build_id)
if real_version:
processed_crash['version'] = real_version
return
self.logger.info('betaversionrule: failed lookup %s %s %s %s',
processed_crash.get('uuid'), product,
release_channel, build_id)
# No real version, but this is an aurora or beta crash report, so we
# tack on "b0" to make it match the channel
processed_crash['version'] += 'b0'
processor_meta.processor_notes.append(
'release channel is %s but no version data was found - added "b0" '
'suffix to version number' % release_channel)
class BetaVersionRule(Rule):
#: Hold at most this many items in cache; items are a key and a value
#: both of which are short strings, so this doesn't take much memory
CACHE_MAX_SIZE = 5000
#: Items in cache expire after 30 minutes by default
SHORT_CACHE_TTL = 60 * 30
#: If we know it's good, cache it for 24 hours because it won't change
LONG_CACHE_TTL = 60 * 60 * 24
def __init__(self, config):
super(BetaVersionRule, self).__init__(config)
# NOTE(willkg): These config values come from Processor2015 instance.
self.buildhub_api = config.buildhub_api
self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE,
default_ttl=self.SHORT_CACHE_TTL)
def version(self):
return '1.0'
def _get_version_data(self, product, build_id, channel):
"""Return the real version number of a specified product, build, channel
For example, beta builds of Firefox declare their version number as the
major version (i.e. version 54.0b3 would say its version is 54.0). This
database call returns the actual version number of said build (i.e.
54.0b3 for the previous example).
:arg product: the product
:arg build_id: the build_id as a string
:arg channel: the release channel
:returns: ``None`` or the version string that should be used
:raises requests.RequestException: raised if it has connection issues with
the host specified in ``version_string_api``
"""
# NOTE(willkg): AURORA LIVES!
#
# But seriously, if this is for Firefox/aurora and the build id is after
# 20170601, then we ask Buildhub about devedition/aurora instead because
# devedition is the aurora channel
if (product, channel) == ('firefox',
'aurora') and build_id > '20170601':
product = 'devedition'
key = '%s:%s:%s' % (product, build_id, channel)
if key in self.cache:
return self.cache[key]
session = session_with_retries(self.buildhub_api)
query = {
'source.product': product,
'build.id': '"%s"' % build_id,
'target.channel': channel,
'_limit': 1
}
resp = session.get(self.buildhub_api, params=query)
if resp.status_code == 200:
hits = resp.json()['data']
# Shimmy to add to ttl so as to distribute cache misses over time and reduce
# HTTP requests from bunching up.
shimmy = random.randint(1, 120)
if hits:
# If we got an answer we should keep it around for a while because it's
# a real answer and it's not going to change so use the long ttl plus
# a fudge factor.
real_version = hits[0]['target']['version']
self.cache.set(key,
value=real_version,
ttl=self.LONG_CACHE_TTL + shimmy)
return real_version
# We didn't get an answer which could mean that this is a weird
# build and there is no answer or it could mean that Buildhub
# doesn't know, yet. Maybe in the future we get a better answer
# so we use the short ttl plus a fudge factor.
self.cache.set(key, value=None, ttl=self.SHORT_CACHE_TTL + shimmy)
return None
def _predicate(self, raw_crash, raw_dumps, processed_crash, proc_meta):
# Beta and aurora versions send the wrong version in the crash report,
# so we need to fix them
return processed_crash.get('release_channel',
'').lower() in ('beta', 'aurora')
def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
product = processed_crash.get('product').strip()
try:
build_id = int(processed_crash.get('build').strip())
except ValueError:
build_id = None
release_channel = processed_crash.get('release_channel').strip()
# If we're missing one of the magic ingredients, then there's nothing
# to do
if product and build_id and release_channel:
try:
real_version = self._get_version_data(product.lower(),
str(build_id),
release_channel)
# If we got a real version, toss that in and we're done
if real_version:
processed_crash['version'] = real_version
return True
except RequestException as exc:
processor_meta.processor_notes.append(
'could not connect to Buildhub')
self.config.logger.exception('%s when connecting to %s', exc,
self.version_string_api)
# No real version, but this is an aurora or beta crash report, so we
# tack on "b0" to make it match the channel
processed_crash['version'] += 'b0'
processor_meta.processor_notes.append(
'release channel is %s but no version data was found - added "b0" '
'suffix to version number' % processed_crash['release_channel'])
return True
class BetaVersionRule(Rule):
#: Hold at most this many items in cache; items are a key and a value
#: both of which are short strings, so this doesn't take much memory
CACHE_MAX_SIZE = 5000
#: Items in cache expire after 30 minutes by default
SHORT_CACHE_TTL = 60 * 30
#: If we know it's good, cache it for 6 hours
LONG_CACHE_TTL = 60 * 60 * 6
def __init__(self, config):
super(BetaVersionRule, self).__init__(config)
# NOTE(willkg): These config values come from Processor2015 instance.
self.version_string_api = config.version_string_api
self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE,
default_ttl=self.SHORT_CACHE_TTL)
def version(self):
return '1.0'
def _get_version_data(self, product, version, build_id):
"""Return the real version number of a specific product, version and build
For example, beta builds of Firefox declare their version number as the
major version (i.e. version 54.0b3 would say its version is 54.0). This
database call returns the actual version number of said build (i.e.
54.0b3 for the previous example).
:arg product: the product
:arg version: the version as a string. e.g. "56.0"
:arg build_id: the build_id as a string.
:returns: ``None`` or the version string that should be used
:raises requests.RequestException: raised if it has connection issues with
the host specified in ``version_string_api``
"""
if not (product and version and build_id):
return None
key = '%s:%s:%s' % (product, version, build_id)
if key in self.cache:
return self.cache[key]
session = session_with_retries(self.version_string_api)
resp = session.get(self.version_string_api,
params={
'product': product,
'version': version,
'build_id': build_id
})
if resp.status_code == 200:
hits = resp.json()['hits']
# Shimmy to add to ttl so as to distribute cache misses over time and reduce
# HTTP requests from bunching up.
shimmy = random.randint(1, 120)
if hits:
# If we got an answer we should keep it around for a while because it's
# a real answer and it's not going to change so use the long ttl plus
# a fudge factor.
real_version = hits[0]
self.cache.set(key,
value=real_version,
ttl=self.LONG_CACHE_TTL + shimmy)
return real_version
else:
# We didn't get an answer which could mean that this is a weird build and there
# is no answer or it could mean that ftpscraper hasn't picked up the relevant
# build information or it could mean we're getting cached answers from the webapp.
# Regardless, maybe in the future we get a better answer so we use the short
# ttl plus a fudge factor.
self.cache.set(key,
value=None,
ttl=self.SHORT_CACHE_TTL + shimmy)
return None
def _predicate(self, raw_crash, raw_dumps, processed_crash, proc_meta):
# Beta and aurora versions send the wrong version in the crash report,
# so we need to fix them.
return processed_crash.get('release_channel',
'').lower() in ('beta', 'aurora')
def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
try:
# Sanitize the build id to avoid errors during the SQL query.
try:
build_id = int(processed_crash['build'])
except ValueError:
build_id = None
real_version = self._get_version_data(
processed_crash['product'],
processed_crash['version'],
build_id,
)
if real_version:
processed_crash['version'] = real_version
else:
# We don't have a real version to use, so we tack on "b0" to
# make it better and match the channel.
processed_crash['version'] += 'b0'
processor_meta.processor_notes.append(
'release channel is %s but no version data was found '
'- added "b0" suffix to version number' %
(processed_crash['release_channel'], ))
except KeyError:
return False
except RequestException as exc:
processed_crash['version'] += 'b0'
processor_meta.processor_notes.append(
'could not connect to VersionString API - added "b0" suffix to version number'
)
self.config.logger.exception('%s when connecting to %s', exc,
self.version_string_api)
return True
class BetaVersionRule(Rule):
#: Hold at most this many items in cache; items are a key and a value
#: both of which are short strings, so this doesn't take much memory
CACHE_MAX_SIZE = 5000
#: Items in cache expire after 30 minutes by default
SHORT_CACHE_TTL = 60 * 30
#: If we know it's good, cache it for 24 hours because it won't change
LONG_CACHE_TTL = 60 * 60 * 24
#: List of products to do lookups for
SUPPORTED_PRODUCTS = ['firefox', 'fennec', 'fennecandroid']
def __init__(self, config):
super().__init__(config)
self.cache = ExpiringCache(max_size=self.CACHE_MAX_SIZE, default_ttl=self.SHORT_CACHE_TTL)
self.metrics = markus.get_metrics('processor.betaversionrule')
# For looking up version strings
self.version_string_api = config.version_string_api
self.session = session_with_retries()
def _get_real_version(self, product, channel, build_id):
"""Return real version number from crashstats_productversion table
:arg str product: the product
:arg str channel: the release channel
:arg int build_id: the build id as a string
:returns: ``None`` or the version string that should be used
"""
# Fix the product so it matches the data in the table
if (product, channel) == ('firefox', 'aurora') and build_id > '20170601':
product = 'DevEdition'
elif product == 'firefox':
product = 'Firefox'
elif product in ('fennec', 'fennecandroid'):
product = 'Fennec'
key = '%s:%s:%s' % (product, channel, build_id)
if key in self.cache:
self.metrics.incr('cache', tags=['result:hit'])
return self.cache[key]
self.metrics.incr('cache', tags=['result:miss'])
resp = self.session.get(self.version_string_api, params={
'product': product,
'channel': channel,
'build_id': build_id
})
if resp.status_code != 200:
versions = []
else:
versions = resp.json()['hits']
if not versions:
# We didn't get an answer which could mean that this is a weird
# build and there is no answer or it could mean that Buildhub
# doesn't know, yet. Maybe in the future we get a better answer so
# we use the short ttl.
self.metrics.incr('lookup', tags=['result:fail'])
self.cache.set(key, value=None, ttl=self.SHORT_CACHE_TTL)
return None
# If we got an answer we should keep it around for a while because it's
# a real answer and it's not going to change so use the long ttl plus
# a fudge factor.
real_version = versions[0]['version_string']
self.metrics.incr('lookup', tags=['result:success'])
self.cache.set(key, value=real_version, ttl=self.LONG_CACHE_TTL)
return real_version
def predicate(self, raw_crash, raw_dumps, processed_crash, proc_meta):
# Beta and aurora versions send the wrong version in the crash report,
# so we need to fix them
return processed_crash.get('release_channel', '').lower() in ('beta', 'aurora')
def action(self, raw_crash, raw_dumps, processed_crash, processor_meta):
product = processed_crash.get('product', '').strip().lower()
build_id = processed_crash.get('build', '').strip()
release_channel = processed_crash.get('release_channel').strip()
# Only run if we've got all the things we need
if product and build_id and release_channel and product in self.SUPPORTED_PRODUCTS:
# Convert the build_id to a str for lookups
build_id = str(build_id)
real_version = self._get_real_version(product, release_channel, build_id)
if real_version:
processed_crash['version'] = real_version
return
self.logger.info(
'betaversionrule: failed lookup %s %s %s %s',
processed_crash.get('uuid'),
product,
release_channel,
build_id
)
# No real version, but this is an aurora or beta crash report, so we
# tack on "b0" to make it match the channel
processed_crash['version'] += 'b0'
processor_meta.processor_notes.append(
'release channel is %s but no version data was found - added "b0" '
'suffix to version number' % release_channel
)