def update_user_ratings():
"""Update add-on author's ratings."""
cursor = connections[multidb.get_replica()].cursor()
# We build this query ahead of time because the cursor complains about data
# truncation if it does the parameters. Also, this query is surprisingly
# quick, <1sec for 6100 rows returned
q = """ SELECT
addons_users.user_id as user_id,
AVG(rating) as avg_rating
FROM reviews
INNER JOIN versions
INNER JOIN addons_users
INNER JOIN addons
ON reviews.version_id = versions.id
AND addons.id = versions.addon_id
AND addons_users.addon_id = addons.id
WHERE reviews.reply_to IS NULL
AND reviews.rating > 0
AND reviews.deleted = False
AND addons.status IN (%s)
AND addons.inactive = 0
GROUP BY addons_users.user_id
""" % (','.join(map(str, VALID_ADDON_STATUSES)))
cursor.execute(q)
d = cursor.fetchall()
cursor.close()
ts = [
update_user_ratings_task.subtask(args=[chunk])
for chunk in chunked(d, 1000)
]
group(ts).apply_async()
def get_blocks_from_guids(cls, guids):
"""Given a list of guids, return a list of Blocks - either existing
instances if the guid exists in a Block, or new instances otherwise.
"""
# load all the Addon instances together
using_db = get_replica()
addons = list(cls.get_addons_for_guids_qs(guids).using(using_db))
# And then any existing block instances
existing_blocks = {
block.guid: block
for block in cls.objects.using(using_db).filter(guid__in=guids)
}
for addon in addons:
# get the existing block object or create a new instance
block = existing_blocks.get(addon.guid, None)
if block:
# if it exists hook up the addon instance
block.addon = addon
else:
# otherwise create a new Block
block = Block(addon=addon)
existing_blocks[block.guid] = block
return list(existing_blocks.values())
def update_user_ratings():
"""Update add-on author's ratings."""
cursor = connections[multidb.get_replica()].cursor()
# We build this query ahead of time because the cursor complains about data
# truncation if it does the parameters. Also, this query is surprisingly
# quick, <1sec for 6100 rows returned
q = """ SELECT
addons_users.user_id as user_id,
AVG(rating) as avg_rating
FROM reviews
INNER JOIN versions
INNER JOIN addons_users
INNER JOIN addons
ON reviews.version_id = versions.id
AND addons.id = versions.addon_id
AND addons_users.addon_id = addons.id
WHERE reviews.reply_to IS NULL
AND reviews.rating > 0
AND addons.status IN (%s)
GROUP BY addons_users.user_id
""" % (",".join(map(str, VALID_ADDON_STATUSES)))
cursor.execute(q)
d = cursor.fetchall()
cursor.close()
ts = [update_user_ratings_task.subtask(args=[chunk])
for chunk in chunked(d, 1000)]
group(ts).apply_async()
def get_blocks_from_guids(cls, guids):
"""Given a list of guids, return a list of Blocks - either existing
instances if the guid exists in a Block, or new instances otherwise.
"""
# load all the Addon instances together
using_db = get_replica()
addons = list(
Addon.unfiltered.using(using_db).filter(
guid__in=guids).no_transforms())
# And then any existing block instances
existing_blocks = {
block.guid: block
for block in cls.objects.using(using_db).filter(guid__in=guids)
}
blocks = []
for addon in addons:
# get the existing block object or create a new instance
block = existing_blocks.get(addon.guid, None)
if block:
# if it exists hook up the addon instance
block.addon = addon
else:
# otherwise create a new Block
block = cls(addon=addon)
blocks.append(block)
blocks.extend(block for guid, block in existing_blocks.items()
if block not in blocks)
return blocks
def test_pinned_reads(self):
"""Test PinningReplicaRouter.db_for_read() when pinned and when
not."""
router = PinningReplicaRouter()
eq_(router.db_for_read(None), get_replica())
pin_this_thread()
eq_(router.db_for_read(None), DEFAULT_DB_ALIAS)
def test_pinned_reads(self):
"""Test PinningReplicaRouter.db_for_read() when pinned and when
not."""
router = PinningReplicaRouter()
eq_(router.db_for_read(None), get_replica())
pin_this_thread()
eq_(router.db_for_read(None), DEFAULT_DB_ALIAS)
def test_db_write_decorator(self):
def read_view(req):
eq_(router.db_for_read(None), get_replica())
return HttpResponse()
@db_write
def write_view(req):
eq_(router.db_for_read(None), DEFAULT_DB_ALIAS)
return HttpResponse()
router = PinningReplicaRouter()
eq_(router.db_for_read(None), get_replica())
write_view(HttpRequest())
read_view(HttpRequest())
def test_db_write_decorator(self):
def read_view(req):
eq_(router.db_for_read(None), get_replica())
return HttpResponse()
@db_write
def write_view(req):
eq_(router.db_for_read(None), DEFAULT_DB_ALIAS)
return HttpResponse()
router = PinningReplicaRouter()
eq_(router.db_for_read(None), get_replica())
write_view(HttpRequest())
read_view(HttpRequest())
def update_addon_average_daily_users():
"""Update add-ons ADU totals."""
if not waffle.switch_is_active('local-statistics-processing'):
return False
raise_if_reindex_in_progress('amo')
cursor = connections[multidb.get_replica()].cursor()
q = """SELECT addon_id, AVG(`count`)
FROM update_counts
WHERE `date` > DATE_SUB(CURDATE(), INTERVAL 13 DAY)
GROUP BY addon_id
ORDER BY addon_id"""
cursor.execute(q)
d = cursor.fetchall()
cursor.close()
ts = [_update_addon_average_daily_users.subtask(args=[chunk])
for chunk in chunked(d, 250)]
group(ts).apply_async()
def update_addon_average_daily_users():
"""Update add-ons ADU totals."""
if not waffle.switch_is_active('local-statistics-processing'):
return False
raise_if_reindex_in_progress('amo')
cursor = connections[multidb.get_replica()].cursor()
q = """SELECT addon_id, AVG(`count`)
FROM update_counts
WHERE `date` > DATE_SUB(CURDATE(), INTERVAL 13 DAY)
GROUP BY addon_id
ORDER BY addon_id"""
cursor.execute(q)
d = cursor.fetchall()
cursor.close()
ts = [
_update_addon_average_daily_users.subtask(args=[chunk])
for chunk in chunked(d, 250)
]
group(ts).apply_async()
def update_addon_average_daily_users():
"""Update add-ons ADU totals."""
if not waffle.switch_is_active('local-statistics-processing'):
return False
kwargs = {'id_field': 'pk'}
if waffle.switch_is_active('use-bigquery-for-addon-adu'):
# BigQuery does not have data for add-ons with type other than those in
# `ADDON_TYPES_WITH_STATS` so we use download counts instead.
# See: https://github.com/mozilla/addons-server/issues/14609
amo_counts = dict(
Addon.objects.exclude(type__in=amo.ADDON_TYPES_WITH_STATS).exclude(
guid__isnull=True).exclude(guid__exact='').annotate(
count=Coalesce(Sum('downloadcount__count'),
0)).values_list('guid', 'count')
# Just to make order predictable in tests, we order by id. This
# matches the GROUP BY being generated so it should be safe.
.order_by('id'))
counts = dict(get_addons_and_average_daily_users_from_bigquery())
counts.update(amo_counts)
counts = list(counts.items())
# BigQuery stores GUIDs, not AMO primary keys.
kwargs['id_field'] = 'guid'
else:
raise_if_reindex_in_progress('amo')
cursor = connections[multidb.get_replica()].cursor()
q = """SELECT addon_id, AVG(`count`)
FROM update_counts
WHERE `date` > DATE_SUB(CURDATE(), INTERVAL 13 DAY)
GROUP BY addon_id
ORDER BY addon_id"""
cursor.execute(q)
counts = cursor.fetchall()
cursor.close()
ts = [
_update_addon_average_daily_users.subtask(args=[chunk], kwargs=kwargs)
for chunk in chunked(counts, 250)
]
group(ts).apply_async()
def test_db_for_read(self):
eq_(ReplicaRouter().db_for_read(None), get_replica())
def import_block_from_blocklist(record):
legacy_id = record.get('id')
using_db = get_replica()
log.info('Processing block id: [%s]', legacy_id)
legacy_import, import_created = LegacyImport.objects.update_or_create(
legacy_id=legacy_id,
defaults={
'record': record,
'timestamp': record.get('last_modified')
},
)
if not import_created:
log.info('LegacyRS %s: updating existing LegacyImport object',
legacy_id)
existing_block_ids = list(
Block.objects.filter(legacy_id__in=(legacy_id,
f'*{legacy_id}')).values_list(
'id', flat=True))
guid = record.get('guid')
if not guid:
legacy_import.outcome = LegacyImport.OUTCOME_MISSINGGUID
legacy_import.save()
log.error('LegacyRS %s: GUID is falsey, skipping.', legacy_id)
return
version_range = (record.get('versionRange') or [{}])[0]
target_application = version_range.get('targetApplication') or [{}]
target_GUID = target_application[0].get('guid')
if target_GUID and target_GUID != amo.FIREFOX.guid:
legacy_import.outcome = LegacyImport.OUTCOME_NOTFIREFOX
legacy_import.save()
log.error(
'LegacyRS %s: targetApplication (%s) is not Firefox, skipping.',
legacy_id,
target_GUID,
)
return
block_kw = {
'min_version': version_range.get('minVersion', '0'),
'max_version': version_range.get('maxVersion', '*'),
'url': record.get('details', {}).get('bug') or '',
'reason': record.get('details', {}).get('why') or '',
'legacy_id': legacy_id,
'updated_by': get_task_user(),
}
modified_date = datetime.fromtimestamp(
record.get('last_modified', datetime_to_ts()) / 1000)
if guid.startswith('/'):
# need to escape the {} brackets or mysql chokes.
guid_regexp = bracket_open_regex.sub(r'\{', guid[1:-1])
guid_regexp = bracket_close_regex.sub(r'\}', guid_regexp)
# we're going to try to split the regex into a list for efficiency.
guids_list = split_regex_to_list(guid_regexp)
if guids_list:
log.info(
'LegacyRS %s: Broke down regex into list; '
'attempting to create Blocks for guids in %s',
legacy_id,
guids_list,
)
statsd.incr('blocklist.tasks.import_blocklist.record_guid',
count=len(guids_list))
addons_guids_qs = (Addon.unfiltered.using(using_db).filter(
guid__in=guids_list).values_list('guid', flat=True))
else:
log.info(
'LegacyRS %s: Unable to break down regex into list; '
'attempting to create Blocks for guids matching [%s]',
legacy_id,
guid_regexp,
)
# mysql doesn't support \d - only [:digit:]
guid_regexp = guid_regexp.replace(r'\d', '[[:digit:]]')
addons_guids_qs = (Addon.unfiltered.using(using_db).filter(
guid__regex=guid_regexp).values_list('guid', flat=True))
# We need to mark this id in a way so we know its from a
# regex guid - otherwise we might accidentally overwrite it.
block_kw['legacy_id'] = '*' + block_kw['legacy_id']
regex = True
else:
log.info('LegacyRS %s: Attempting to create a Block for guid [%s]',
legacy_id, guid)
statsd.incr('blocklist.tasks.import_blocklist.record_guid')
addons_guids_qs = (Addon.unfiltered.using(using_db).filter(
guid=guid).values_list('guid', flat=True))
regex = False
new_blocks = []
for guid in addons_guids_qs:
valid_files_qs = File.objects.filter(version__addon__guid=guid,
is_webextension=True)
if not valid_files_qs.exists():
log.info(
'LegacyRS %s: Skipped Block for [%s] because it has no '
'webextension files',
legacy_id,
guid,
)
statsd.incr('blocklist.tasks.import_blocklist.block_skipped')
continue
(block,
created) = Block.objects.update_or_create(guid=guid,
defaults=dict(guid=guid,
**block_kw))
block_activity_log_save(block, change=not created)
if created:
log.info('LegacyRS %s: Added Block for [%s]', legacy_id, guid)
statsd.incr('blocklist.tasks.import_blocklist.block_added')
block.update(modified=modified_date)
else:
log.info('LegacyRS %s: Updated Block for [%s]', legacy_id, guid)
statsd.incr('blocklist.tasks.import_blocklist.block_updated')
new_blocks.append(block)
if new_blocks:
legacy_import.outcome = (LegacyImport.OUTCOME_REGEXBLOCKS
if regex else LegacyImport.OUTCOME_BLOCK)
else:
legacy_import.outcome = LegacyImport.OUTCOME_NOMATCH
log.info('LegacyRS %s: No addon found', legacy_id)
if not import_created:
# now reconcile the blocks that were connected to the import last time
# but weren't changed this time - i.e. blocks we need to delete
delete_qs = Block.objects.filter(id__in=existing_block_ids).exclude(
id__in=(block.id for block in new_blocks))
for block in delete_qs:
block_activity_log_delete(block,
delete_user=block_kw['updated_by'])
block.delete()
statsd.incr('blocklist.tasks.import_blocklist.block_deleted')
legacy_import.save()
if import_created:
statsd.incr('blocklist.tasks.import_blocklist.new_record_processed')
else:
statsd.incr(
'blocklist.tasks.import_blocklist.modified_record_processed')
def test_allow_migrate(self):
router = ReplicaRouter()
assert router.allow_migrate(DEFAULT_DB_ALIAS, 'dummy')
assert not router.allow_migrate(get_replica(), 'dummy')
def read_view(req):
eq_(router.db_for_read(None), get_replica())
return HttpResponse()
def test_db_for_read(self):
eq_(ReplicaRouter().db_for_read(None), get_replica())
def test_allow_syncdb(self):
router = ReplicaRouter()
assert router.allow_syncdb(DEFAULT_DB_ALIAS, None)
assert not router.allow_syncdb(get_replica(), None)
def test_allow_migrate(self):
router = ReplicaRouter()
assert router.allow_migrate(DEFAULT_DB_ALIAS, 'dummy')
assert not router.allow_migrate(get_replica(), 'dummy')
def import_block_from_blocklist(record):
kinto_id = record.get('id')
using_db = get_replica()
log.debug('Processing block id: [%s]', kinto_id)
kinto_import = KintoImport(kinto_id=kinto_id, record=record)
guid = record.get('guid')
if not guid:
kinto_import.outcome = KintoImport.OUTCOME_MISSINGGUID
kinto_import.save()
log.error('Kinto %s: GUID is falsey, skipping.', kinto_id)
return
version_range = record.get('versionRange', [{}])[0]
target_application = version_range.get('targetApplication') or [{}]
target_GUID = target_application[0].get('guid')
if target_GUID and target_GUID != amo.FIREFOX.guid:
kinto_import.outcome = KintoImport.OUTCOME_NOTFIREFOX
kinto_import.save()
log.error('Kinto %s: targetApplication (%s) is not Firefox, skipping.',
kinto_id, target_GUID)
return
block_kw = {
'min_version': version_range.get('minVersion', '0'),
'max_version': version_range.get('maxVersion', '*'),
'url': record.get('details', {}).get('bug') or '',
'reason': record.get('details', {}).get('why') or '',
'kinto_id': kinto_id,
'include_in_legacy': True,
'updated_by': get_task_user(),
}
modified_date = datetime.fromtimestamp(
record.get('last_modified',
time.time() * 1000) / 1000)
if guid.startswith('/'):
# need to escape the {} brackets or mysql chokes.
guid_regexp = bracket_open_regex.sub(r'\{', guid[1:-1])
guid_regexp = bracket_close_regex.sub(r'\}', guid_regexp)
# we're going to try to split the regex into a list for efficiency.
guids_list = split_regex_to_list(guid_regexp)
if guids_list:
log.debug(
'Kinto %s: Broke down regex into list; '
'attempting to create Blocks for guids in %s', kinto_id,
guids_list)
addons_guids_qs = Addon.unfiltered.using(using_db).filter(
guid__in=guids_list).values_list('guid', flat=True)
else:
log.debug(
'Kinto %s: Unable to break down regex into list; '
'attempting to create Blocks for guids matching [%s]',
kinto_id, guid_regexp)
# mysql doesn't support \d - only [:digit:]
guid_regexp = guid_regexp.replace(r'\d', '[[:digit:]]')
addons_guids_qs = Addon.unfiltered.using(using_db).filter(
guid__regex=guid_regexp).values_list('guid', flat=True)
# We need to mark this id in a way so we know its from a
# regex guid - otherwise we might accidentally overwrite it.
block_kw['kinto_id'] = '*' + block_kw['kinto_id']
regex = True
else:
log.debug('Kinto %s: Attempting to create a Block for guid [%s]',
kinto_id, guid)
addons_guids_qs = Addon.unfiltered.using(using_db).filter(
guid=guid).values_list('guid', flat=True)
regex = False
new_blocks = []
for guid in addons_guids_qs:
valid_files_qs = File.objects.filter(version__addon__guid=guid,
is_webextension=True)
if not valid_files_qs.exists():
log.debug(
'Kinto %s: Skipped Block for [%s] because it has no '
'webextension files', kinto_id, guid)
continue
(block,
created) = Block.objects.update_or_create(guid=guid,
defaults=dict(guid=guid,
**block_kw))
block_activity_log_save(block, change=not created)
if created:
log.debug('Kinto %s: Added Block for [%s]', kinto_id, guid)
block.update(modified=modified_date)
else:
log.debug('Kinto %s: Updated Block for [%s]', kinto_id, guid)
new_blocks.append(block)
if new_blocks:
kinto_import.outcome = (KintoImport.OUTCOME_REGEXBLOCKS
if regex else KintoImport.OUTCOME_BLOCK)
else:
kinto_import.outcome = KintoImport.OUTCOME_NOMATCH
log.debug('Kinto %s: No addon found', kinto_id)
kinto_import.save()
def read_view(req):
eq_(router.db_for_read(None), get_replica())
return HttpResponse()
def test_allow_syncdb(self):
router = ReplicaRouter()
assert router.allow_syncdb(DEFAULT_DB_ALIAS, None)
assert not router.allow_syncdb(get_replica(), None)
