def test_key_incr(self, epoch_mock):
epoch_mock.return_value = 123456
cache_ns_key(self.namespace) # Sets ns to 123456
ns_key = cache_ns_key(self.namespace, increment=True)
expected = '123457:ns:%s' % self.namespace
assert ns_key == expected
assert cache_ns_key(self.namespace) == expected
def test_key_incr(self, epoch_mock):
epoch_mock.return_value = 123456
cache_ns_key(self.namespace) # Sets ns to 123456
ns_key = cache_ns_key(self.namespace, increment=True)
expected = '123457:ns:%s' % self.namespace
eq_(ns_key, expected)
eq_(cache_ns_key(self.namespace), expected)
def test_key_incr(self, epoch_mock):
epoch_mock.return_value = 1549383758398
cache_ns_key(self.namespace) # Sets ns to 1549383758398
ns_key = cache_ns_key(self.namespace, increment=True) # Increments it.
expected = '1549383758399:ns:%s' % self.namespace
assert ns_key == expected
assert cache_ns_key(self.namespace) == expected
def get_key(cls, key=None, invalidate=False):
namespace = "riscore"
if not key: # Assuming we're invalidating the namespace.
cache_ns_key(namespace, invalidate)
return
else:
# Using cache_ns_key so each cache val is invalidated together.
ns_key = cache_ns_key(namespace, invalidate)
return "%s:%s" % (ns_key, key)
def get_key(cls, key=None, invalidate=False):
namespace = 'riscore'
if not key: # Assuming we're invalidating the namespace.
cache_ns_key(namespace, invalidate)
return
else:
# Using cache_ns_key so each cache val is invalidated together.
ns_key = cache_ns_key(namespace, invalidate)
return '%s:%s' % (ns_key, key)
def update_incompatible_appversions(data, **kw):
"""Updates the incompatible_versions table for this version."""
log.info('Updating incompatible_versions for %s versions.' % len(data))
addon_ids = set()
for version_id in data:
# This is here to handle both post_save and post_delete hooks.
IncompatibleVersions.objects.filter(version=version_id).delete()
try:
version = Version.objects.get(pk=version_id)
except Version.DoesNotExist:
log.info('Version ID [%d] not found. Incompatible versions were '
'cleared.' % version_id)
return
addon_ids.add(version.addon_id)
try:
compat = CompatOverride.objects.get(addon=version.addon)
except CompatOverride.DoesNotExist:
log.info('Compat override for addon with version ID [%d] not '
'found. Incompatible versions were cleared.' % version_id)
return
app_ranges = []
ranges = compat.collapsed_ranges()
for range in ranges:
if range.min == '0' and range.max == '*':
# Wildcard range, add all app ranges
app_ranges.extend(range.apps)
else:
# Since we can't rely on add-on version numbers, get the min
# and max ID values and find versions whose ID is within those
# ranges, being careful with wildcards.
min_id = max_id = None
if range.min == '0':
versions = (Version.objects.filter(
addon=version.addon_id).order_by('id').values_list(
'id', flat=True)[:1])
if versions:
min_id = versions[0]
else:
try:
min_id = Version.objects.get(addon=version.addon_id,
version=range.min).id
except Version.DoesNotExist:
pass
if range.max == '*':
versions = (Version.objects.filter(
addon=version.addon_id).order_by('-id').values_list(
'id', flat=True)[:1])
if versions:
max_id = versions[0]
else:
try:
max_id = Version.objects.get(addon=version.addon_id,
version=range.max).id
except Version.DoesNotExist:
pass
if min_id and max_id:
if min_id <= version.id <= max_id:
app_ranges.extend(range.apps)
for app_range in app_ranges:
IncompatibleVersions.objects.create(version=version,
app=app_range.app.id,
min_app_version=app_range.min,
max_app_version=app_range.max)
log.info(
'Added incompatible version for version ID [%d]: '
'app:%d, %s -> %s' %
(version_id, app_range.app.id, app_range.min, app_range.max))
# Increment namespace cache of compat versions.
for addon_id in addon_ids:
cache_ns_key('d2c-versions:%s' % addon_id, increment=True)
def find_compatible_version(addon,
app_id,
app_version=None,
platform=None,
compat_mode='strict'):
"""Returns the newest compatible version (ordered by version id desc)
for the given addon."""
if not app_id:
return None
if platform:
# We include platform_id=1 always in the SQL so we skip it here.
platform = platform.lower()
if platform != 'all' and platform in amo.PLATFORM_DICT:
platform = amo.PLATFORM_DICT[platform].id
else:
platform = None
log.debug(u'Checking compatibility for add-on ID:%s, APP:%s, V:%s, '
u'OS:%s, Mode:%s' %
(addon.id, app_id, app_version, platform, compat_mode))
valid_file_statuses = ','.join(map(str, addon.valid_file_statuses))
data = {
'id': addon.id,
'app_id': app_id,
'platform': platform,
'valid_file_statuses': valid_file_statuses,
'channel': amo.RELEASE_CHANNEL_LISTED,
}
if app_version:
data.update(version_int=version_int(app_version))
else:
# We can't perform the search queries for strict or normal without
# an app version.
compat_mode = 'ignore'
ns_key = cache_ns_key('d2c-versions:%s' % addon.id)
cache_key = '%s:%s:%s:%s:%s' % (ns_key, app_id, app_version, platform,
compat_mode)
version_id = cache.get(cache_key)
if version_id is not None:
log.debug(u'Found compatible version in cache: %s => %s' %
(cache_key, version_id))
if version_id == 0:
return None
else:
try:
return Version.objects.get(pk=version_id)
except Version.DoesNotExist:
pass
raw_sql = [
"""
SELECT versions.*
FROM versions
INNER JOIN addons
ON addons.id = versions.addon_id AND addons.id = %(id)s
INNER JOIN applications_versions
ON applications_versions.version_id = versions.id
INNER JOIN appversions appmin
ON appmin.id = applications_versions.min
AND appmin.application_id = %(app_id)s
INNER JOIN appversions appmax
ON appmax.id = applications_versions.max
AND appmax.application_id = %(app_id)s
INNER JOIN files
ON files.version_id = versions.id AND
(files.platform_id = 1"""
]
if platform:
raw_sql.append(' OR files.platform_id = %(platform)s')
raw_sql.append(') WHERE files.status IN (%(valid_file_statuses)s) ')
raw_sql.append(' AND versions.channel = %(channel)s ')
if app_version:
raw_sql.append('AND appmin.version_int <= %(version_int)s ')
if compat_mode == 'ignore':
pass # No further SQL modification required.
elif compat_mode == 'normal':
raw_sql.append("""AND
CASE WHEN files.strict_compatibility = 1 OR
files.binary_components = 1
THEN appmax.version_int >= %(version_int)s ELSE 1 END
""")
# Filter out versions that don't have the minimum maxVersion
# requirement to qualify for default-to-compatible.
d2c_max = amo.D2C_MAX_VERSIONS.get(app_id)
if d2c_max:
data['d2c_max_version'] = version_int(d2c_max)
raw_sql.append("AND appmax.version_int >= %(d2c_max_version)s ")
# Filter out versions found in compat overrides
raw_sql.append("""AND
NOT versions.id IN (
SELECT version_id FROM incompatible_versions
WHERE app_id=%(app_id)s AND
(min_app_version='0' AND
max_app_version_int >= %(version_int)s) OR
(min_app_version_int <= %(version_int)s AND
max_app_version='*') OR
(min_app_version_int <= %(version_int)s AND
max_app_version_int >= %(version_int)s)) """)
else: # Not defined or 'strict'.
raw_sql.append('AND appmax.version_int >= %(version_int)s ')
raw_sql.append('ORDER BY versions.id DESC LIMIT 1;')
version = Version.objects.raw(''.join(raw_sql) % data)
if version:
version = version[0]
version_id = version.id
else:
version = None
version_id = 0
log.debug(u'Caching compat version %s => %s' % (cache_key, version_id))
cache.set(cache_key, version_id, None)
return version
def test_no_preexisting_key_incr(self, epoch_mock):
epoch_mock.return_value = 123456
eq_(cache_ns_key(self.namespace, increment=True),
'123456:ns:%s' % self.namespace)
def find_compatible_version(addon, app_id, app_version=None, platform=None,
compat_mode='strict'):
"""Returns the newest compatible version (ordered by version id desc)
for the given addon."""
if not app_id:
return None
if platform:
# We include platform_id=1 always in the SQL so we skip it here.
platform = platform.lower()
if platform != 'all' and platform in amo.PLATFORM_DICT:
platform = amo.PLATFORM_DICT[platform].id
else:
platform = None
log.debug(u'Checking compatibility for add-on ID:%s, APP:%s, V:%s, '
u'OS:%s, Mode:%s' % (addon.id, app_id, app_version, platform,
compat_mode))
valid_file_statuses = ','.join(map(str, addon.valid_file_statuses))
data = {
'id': addon.id,
'app_id': app_id,
'platform': platform,
'valid_file_statuses': valid_file_statuses,
'channel': amo.RELEASE_CHANNEL_LISTED,
}
if app_version:
data.update(version_int=version_int(app_version))
else:
# We can't perform the search queries for strict or normal without
# an app version.
compat_mode = 'ignore'
ns_key = cache_ns_key('d2c-versions:%s' % addon.id)
cache_key = '%s:%s:%s:%s:%s' % (ns_key, app_id, app_version, platform,
compat_mode)
version_id = cache.get(cache_key)
if version_id is not None:
log.debug(u'Found compatible version in cache: %s => %s' % (
cache_key, version_id))
if version_id == 0:
return None
else:
try:
return Version.objects.get(pk=version_id)
except Version.DoesNotExist:
pass
raw_sql = ["""
SELECT versions.*
FROM versions
INNER JOIN addons
ON addons.id = versions.addon_id AND addons.id = %(id)s
INNER JOIN applications_versions
ON applications_versions.version_id = versions.id
INNER JOIN appversions appmin
ON appmin.id = applications_versions.min
AND appmin.application_id = %(app_id)s
INNER JOIN appversions appmax
ON appmax.id = applications_versions.max
AND appmax.application_id = %(app_id)s
INNER JOIN files
ON files.version_id = versions.id AND
(files.platform_id = 1"""]
if platform:
raw_sql.append(' OR files.platform_id = %(platform)s')
raw_sql.append(') WHERE files.status IN (%(valid_file_statuses)s) ')
raw_sql.append(' AND versions.channel = %(channel)s ')
if app_version:
raw_sql.append('AND appmin.version_int <= %(version_int)s ')
if compat_mode == 'ignore':
pass # No further SQL modification required.
elif compat_mode == 'normal':
raw_sql.append("""AND
CASE WHEN files.strict_compatibility = 1 OR
files.binary_components = 1
THEN appmax.version_int >= %(version_int)s ELSE 1 END
""")
# Filter out versions that don't have the minimum maxVersion
# requirement to qualify for default-to-compatible.
d2c_max = amo.D2C_MAX_VERSIONS.get(app_id)
if d2c_max:
data['d2c_max_version'] = version_int(d2c_max)
raw_sql.append(
"AND appmax.version_int >= %(d2c_max_version)s ")
# Filter out versions found in compat overrides
raw_sql.append("""AND
NOT versions.id IN (
SELECT version_id FROM incompatible_versions
WHERE app_id=%(app_id)s AND
(min_app_version='0' AND
max_app_version_int >= %(version_int)s) OR
(min_app_version_int <= %(version_int)s AND
max_app_version='*') OR
(min_app_version_int <= %(version_int)s AND
max_app_version_int >= %(version_int)s)) """)
else: # Not defined or 'strict'.
raw_sql.append('AND appmax.version_int >= %(version_int)s ')
raw_sql.append('ORDER BY versions.id DESC LIMIT 1;')
version = Version.objects.raw(''.join(raw_sql) % data)
if version:
version = version[0]
version_id = version.id
else:
version = None
version_id = 0
log.debug(u'Caching compat version %s => %s' % (cache_key, version_id))
cache.set(cache_key, version_id, None)
return version
def update_incompatible_appversions(data, **kw):
"""Updates the incompatible_versions table for this version."""
log.info('Updating incompatible_versions for %s versions.' % len(data))
addon_ids = set()
for version_id in data:
# This is here to handle both post_save and post_delete hooks.
IncompatibleVersions.objects.filter(version=version_id).delete()
try:
version = Version.objects.get(pk=version_id)
except Version.DoesNotExist:
log.info('Version ID [%d] not found. Incompatible versions were '
'cleared.' % version_id)
return
addon_ids.add(version.addon_id)
try:
compat = CompatOverride.objects.get(addon=version.addon)
except CompatOverride.DoesNotExist:
log.info('Compat override for addon with version ID [%d] not '
'found. Incompatible versions were cleared.' % version_id)
return
app_ranges = []
ranges = compat.collapsed_ranges()
for range in ranges:
if range.min == '0' and range.max == '*':
# Wildcard range, add all app ranges
app_ranges.extend(range.apps)
else:
# Since we can't rely on add-on version numbers, get the min
# and max ID values and find versions whose ID is within those
# ranges, being careful with wildcards.
min_id = max_id = None
if range.min == '0':
versions = (Version.objects.filter(addon=version.addon_id)
.order_by('id')
.values_list('id', flat=True)[:1])
if versions:
min_id = versions[0]
else:
try:
min_id = Version.objects.get(addon=version.addon_id,
version=range.min).id
except Version.DoesNotExist:
pass
if range.max == '*':
versions = (Version.objects.filter(addon=version.addon_id)
.order_by('-id')
.values_list('id', flat=True)[:1])
if versions:
max_id = versions[0]
else:
try:
max_id = Version.objects.get(addon=version.addon_id,
version=range.max).id
except Version.DoesNotExist:
pass
if min_id and max_id:
if min_id <= version.id <= max_id:
app_ranges.extend(range.apps)
for app_range in app_ranges:
IncompatibleVersions.objects.create(version=version,
app=app_range.app.id,
min_app_version=app_range.min,
max_app_version=app_range.max)
log.info('Added incompatible version for version ID [%d]: '
'app:%d, %s -> %s' % (version_id, app_range.app.id,
app_range.min, app_range.max))
# Increment namespace cache of compat versions.
for addon_id in addon_ids:
cache_ns_key('d2c-versions:%s' % addon_id, increment=True)
def test_no_preexisting_key_incr(self, epoch_mock):
epoch_mock.return_value = 123456
assert cache_ns_key(self.namespace,
increment=True) == ('123456:ns:%s' %
self.namespace)
def test_no_preexisting_key_incr(self, epoch_mock):
epoch_mock.return_value = 1549383758398
assert cache_ns_key(self.namespace, increment=True) == (
'1549383758398:ns:%s' % self.namespace)
def test_no_preexisting_key(self, epoch_mock):
epoch_mock.return_value = 123456
eq_(cache_ns_key(self.namespace), '123456:ns:%s' % self.namespace)
def test_no_preexisting_key(self, epoch_mock):
epoch_mock.return_value = 1549383758398
assert cache_ns_key(self.namespace) == ('1549383758398:ns:%s' %
self.namespace)
def test_no_preexisting_key(self, epoch_mock):
epoch_mock.return_value = 123456
assert cache_ns_key(self.namespace) == '123456:ns:%s' % self.namespace