def update_user_ratings(): """Update add-on author's ratings.""" cursor = connections[multidb.get_replica()].cursor() # We build this query ahead of time because the cursor complains about data # truncation if it does the parameters. Also, this query is surprisingly # quick, <1sec for 6100 rows returned q = """ SELECT addons_users.user_id as user_id, AVG(rating) as avg_rating FROM reviews INNER JOIN versions INNER JOIN addons_users INNER JOIN addons ON reviews.version_id = versions.id AND addons.id = versions.addon_id AND addons_users.addon_id = addons.id WHERE reviews.reply_to IS NULL AND reviews.rating > 0 AND reviews.deleted = False AND addons.status IN (%s) AND addons.inactive = 0 GROUP BY addons_users.user_id """ % (','.join(map(str, VALID_ADDON_STATUSES))) cursor.execute(q) d = cursor.fetchall() cursor.close() ts = [ update_user_ratings_task.subtask(args=[chunk]) for chunk in chunked(d, 1000) ] group(ts).apply_async()
def get_blocks_from_guids(cls, guids): """Given a list of guids, return a list of Blocks - either existing instances if the guid exists in a Block, or new instances otherwise. """ # load all the Addon instances together using_db = get_replica() addons = list(cls.get_addons_for_guids_qs(guids).using(using_db)) # And then any existing block instances existing_blocks = { block.guid: block for block in cls.objects.using(using_db).filter(guid__in=guids) } for addon in addons: # get the existing block object or create a new instance block = existing_blocks.get(addon.guid, None) if block: # if it exists hook up the addon instance block.addon = addon else: # otherwise create a new Block block = Block(addon=addon) existing_blocks[block.guid] = block return list(existing_blocks.values())
def update_user_ratings(): """Update add-on author's ratings.""" cursor = connections[multidb.get_replica()].cursor() # We build this query ahead of time because the cursor complains about data # truncation if it does the parameters. Also, this query is surprisingly # quick, <1sec for 6100 rows returned q = """ SELECT addons_users.user_id as user_id, AVG(rating) as avg_rating FROM reviews INNER JOIN versions INNER JOIN addons_users INNER JOIN addons ON reviews.version_id = versions.id AND addons.id = versions.addon_id AND addons_users.addon_id = addons.id WHERE reviews.reply_to IS NULL AND reviews.rating > 0 AND addons.status IN (%s) GROUP BY addons_users.user_id """ % (",".join(map(str, VALID_ADDON_STATUSES))) cursor.execute(q) d = cursor.fetchall() cursor.close() ts = [update_user_ratings_task.subtask(args=[chunk]) for chunk in chunked(d, 1000)] group(ts).apply_async()
def get_blocks_from_guids(cls, guids): """Given a list of guids, return a list of Blocks - either existing instances if the guid exists in a Block, or new instances otherwise. """ # load all the Addon instances together using_db = get_replica() addons = list( Addon.unfiltered.using(using_db).filter( guid__in=guids).no_transforms()) # And then any existing block instances existing_blocks = { block.guid: block for block in cls.objects.using(using_db).filter(guid__in=guids) } blocks = [] for addon in addons: # get the existing block object or create a new instance block = existing_blocks.get(addon.guid, None) if block: # if it exists hook up the addon instance block.addon = addon else: # otherwise create a new Block block = cls(addon=addon) blocks.append(block) blocks.extend(block for guid, block in existing_blocks.items() if block not in blocks) return blocks
def test_pinned_reads(self): """Test PinningReplicaRouter.db_for_read() when pinned and when not.""" router = PinningReplicaRouter() eq_(router.db_for_read(None), get_replica()) pin_this_thread() eq_(router.db_for_read(None), DEFAULT_DB_ALIAS)
def test_db_write_decorator(self): def read_view(req): eq_(router.db_for_read(None), get_replica()) return HttpResponse() @db_write def write_view(req): eq_(router.db_for_read(None), DEFAULT_DB_ALIAS) return HttpResponse() router = PinningReplicaRouter() eq_(router.db_for_read(None), get_replica()) write_view(HttpRequest()) read_view(HttpRequest())
def update_addon_average_daily_users(): """Update add-ons ADU totals.""" if not waffle.switch_is_active('local-statistics-processing'): return False raise_if_reindex_in_progress('amo') cursor = connections[multidb.get_replica()].cursor() q = """SELECT addon_id, AVG(`count`) FROM update_counts WHERE `date` > DATE_SUB(CURDATE(), INTERVAL 13 DAY) GROUP BY addon_id ORDER BY addon_id""" cursor.execute(q) d = cursor.fetchall() cursor.close() ts = [_update_addon_average_daily_users.subtask(args=[chunk]) for chunk in chunked(d, 250)] group(ts).apply_async()
def update_addon_average_daily_users(): """Update add-ons ADU totals.""" if not waffle.switch_is_active('local-statistics-processing'): return False raise_if_reindex_in_progress('amo') cursor = connections[multidb.get_replica()].cursor() q = """SELECT addon_id, AVG(`count`) FROM update_counts WHERE `date` > DATE_SUB(CURDATE(), INTERVAL 13 DAY) GROUP BY addon_id ORDER BY addon_id""" cursor.execute(q) d = cursor.fetchall() cursor.close() ts = [ _update_addon_average_daily_users.subtask(args=[chunk]) for chunk in chunked(d, 250) ] group(ts).apply_async()
def update_addon_average_daily_users(): """Update add-ons ADU totals.""" if not waffle.switch_is_active('local-statistics-processing'): return False kwargs = {'id_field': 'pk'} if waffle.switch_is_active('use-bigquery-for-addon-adu'): # BigQuery does not have data for add-ons with type other than those in # `ADDON_TYPES_WITH_STATS` so we use download counts instead. # See: https://github.com/mozilla/addons-server/issues/14609 amo_counts = dict( Addon.objects.exclude(type__in=amo.ADDON_TYPES_WITH_STATS).exclude( guid__isnull=True).exclude(guid__exact='').annotate( count=Coalesce(Sum('downloadcount__count'), 0)).values_list('guid', 'count') # Just to make order predictable in tests, we order by id. This # matches the GROUP BY being generated so it should be safe. .order_by('id')) counts = dict(get_addons_and_average_daily_users_from_bigquery()) counts.update(amo_counts) counts = list(counts.items()) # BigQuery stores GUIDs, not AMO primary keys. kwargs['id_field'] = 'guid' else: raise_if_reindex_in_progress('amo') cursor = connections[multidb.get_replica()].cursor() q = """SELECT addon_id, AVG(`count`) FROM update_counts WHERE `date` > DATE_SUB(CURDATE(), INTERVAL 13 DAY) GROUP BY addon_id ORDER BY addon_id""" cursor.execute(q) counts = cursor.fetchall() cursor.close() ts = [ _update_addon_average_daily_users.subtask(args=[chunk], kwargs=kwargs) for chunk in chunked(counts, 250) ] group(ts).apply_async()
def test_db_for_read(self): eq_(ReplicaRouter().db_for_read(None), get_replica())
def import_block_from_blocklist(record): legacy_id = record.get('id') using_db = get_replica() log.info('Processing block id: [%s]', legacy_id) legacy_import, import_created = LegacyImport.objects.update_or_create( legacy_id=legacy_id, defaults={ 'record': record, 'timestamp': record.get('last_modified') }, ) if not import_created: log.info('LegacyRS %s: updating existing LegacyImport object', legacy_id) existing_block_ids = list( Block.objects.filter(legacy_id__in=(legacy_id, f'*{legacy_id}')).values_list( 'id', flat=True)) guid = record.get('guid') if not guid: legacy_import.outcome = LegacyImport.OUTCOME_MISSINGGUID legacy_import.save() log.error('LegacyRS %s: GUID is falsey, skipping.', legacy_id) return version_range = (record.get('versionRange') or [{}])[0] target_application = version_range.get('targetApplication') or [{}] target_GUID = target_application[0].get('guid') if target_GUID and target_GUID != amo.FIREFOX.guid: legacy_import.outcome = LegacyImport.OUTCOME_NOTFIREFOX legacy_import.save() log.error( 'LegacyRS %s: targetApplication (%s) is not Firefox, skipping.', legacy_id, target_GUID, ) return block_kw = { 'min_version': version_range.get('minVersion', '0'), 'max_version': version_range.get('maxVersion', '*'), 'url': record.get('details', {}).get('bug') or '', 'reason': record.get('details', {}).get('why') or '', 'legacy_id': legacy_id, 'updated_by': get_task_user(), } modified_date = datetime.fromtimestamp( record.get('last_modified', datetime_to_ts()) / 1000) if guid.startswith('/'): # need to escape the {} brackets or mysql chokes. guid_regexp = bracket_open_regex.sub(r'\{', guid[1:-1]) guid_regexp = bracket_close_regex.sub(r'\}', guid_regexp) # we're going to try to split the regex into a list for efficiency. guids_list = split_regex_to_list(guid_regexp) if guids_list: log.info( 'LegacyRS %s: Broke down regex into list; ' 'attempting to create Blocks for guids in %s', legacy_id, guids_list, ) statsd.incr('blocklist.tasks.import_blocklist.record_guid', count=len(guids_list)) addons_guids_qs = (Addon.unfiltered.using(using_db).filter( guid__in=guids_list).values_list('guid', flat=True)) else: log.info( 'LegacyRS %s: Unable to break down regex into list; ' 'attempting to create Blocks for guids matching [%s]', legacy_id, guid_regexp, ) # mysql doesn't support \d - only [:digit:] guid_regexp = guid_regexp.replace(r'\d', '[[:digit:]]') addons_guids_qs = (Addon.unfiltered.using(using_db).filter( guid__regex=guid_regexp).values_list('guid', flat=True)) # We need to mark this id in a way so we know its from a # regex guid - otherwise we might accidentally overwrite it. block_kw['legacy_id'] = '*' + block_kw['legacy_id'] regex = True else: log.info('LegacyRS %s: Attempting to create a Block for guid [%s]', legacy_id, guid) statsd.incr('blocklist.tasks.import_blocklist.record_guid') addons_guids_qs = (Addon.unfiltered.using(using_db).filter( guid=guid).values_list('guid', flat=True)) regex = False new_blocks = [] for guid in addons_guids_qs: valid_files_qs = File.objects.filter(version__addon__guid=guid, is_webextension=True) if not valid_files_qs.exists(): log.info( 'LegacyRS %s: Skipped Block for [%s] because it has no ' 'webextension files', legacy_id, guid, ) statsd.incr('blocklist.tasks.import_blocklist.block_skipped') continue (block, created) = Block.objects.update_or_create(guid=guid, defaults=dict(guid=guid, **block_kw)) block_activity_log_save(block, change=not created) if created: log.info('LegacyRS %s: Added Block for [%s]', legacy_id, guid) statsd.incr('blocklist.tasks.import_blocklist.block_added') block.update(modified=modified_date) else: log.info('LegacyRS %s: Updated Block for [%s]', legacy_id, guid) statsd.incr('blocklist.tasks.import_blocklist.block_updated') new_blocks.append(block) if new_blocks: legacy_import.outcome = (LegacyImport.OUTCOME_REGEXBLOCKS if regex else LegacyImport.OUTCOME_BLOCK) else: legacy_import.outcome = LegacyImport.OUTCOME_NOMATCH log.info('LegacyRS %s: No addon found', legacy_id) if not import_created: # now reconcile the blocks that were connected to the import last time # but weren't changed this time - i.e. blocks we need to delete delete_qs = Block.objects.filter(id__in=existing_block_ids).exclude( id__in=(block.id for block in new_blocks)) for block in delete_qs: block_activity_log_delete(block, delete_user=block_kw['updated_by']) block.delete() statsd.incr('blocklist.tasks.import_blocklist.block_deleted') legacy_import.save() if import_created: statsd.incr('blocklist.tasks.import_blocklist.new_record_processed') else: statsd.incr( 'blocklist.tasks.import_blocklist.modified_record_processed')
def test_allow_migrate(self): router = ReplicaRouter() assert router.allow_migrate(DEFAULT_DB_ALIAS, 'dummy') assert not router.allow_migrate(get_replica(), 'dummy')
def read_view(req): eq_(router.db_for_read(None), get_replica()) return HttpResponse()
def test_allow_syncdb(self): router = ReplicaRouter() assert router.allow_syncdb(DEFAULT_DB_ALIAS, None) assert not router.allow_syncdb(get_replica(), None)
def import_block_from_blocklist(record): kinto_id = record.get('id') using_db = get_replica() log.debug('Processing block id: [%s]', kinto_id) kinto_import = KintoImport(kinto_id=kinto_id, record=record) guid = record.get('guid') if not guid: kinto_import.outcome = KintoImport.OUTCOME_MISSINGGUID kinto_import.save() log.error('Kinto %s: GUID is falsey, skipping.', kinto_id) return version_range = record.get('versionRange', [{}])[0] target_application = version_range.get('targetApplication') or [{}] target_GUID = target_application[0].get('guid') if target_GUID and target_GUID != amo.FIREFOX.guid: kinto_import.outcome = KintoImport.OUTCOME_NOTFIREFOX kinto_import.save() log.error('Kinto %s: targetApplication (%s) is not Firefox, skipping.', kinto_id, target_GUID) return block_kw = { 'min_version': version_range.get('minVersion', '0'), 'max_version': version_range.get('maxVersion', '*'), 'url': record.get('details', {}).get('bug') or '', 'reason': record.get('details', {}).get('why') or '', 'kinto_id': kinto_id, 'include_in_legacy': True, 'updated_by': get_task_user(), } modified_date = datetime.fromtimestamp( record.get('last_modified', time.time() * 1000) / 1000) if guid.startswith('/'): # need to escape the {} brackets or mysql chokes. guid_regexp = bracket_open_regex.sub(r'\{', guid[1:-1]) guid_regexp = bracket_close_regex.sub(r'\}', guid_regexp) # we're going to try to split the regex into a list for efficiency. guids_list = split_regex_to_list(guid_regexp) if guids_list: log.debug( 'Kinto %s: Broke down regex into list; ' 'attempting to create Blocks for guids in %s', kinto_id, guids_list) addons_guids_qs = Addon.unfiltered.using(using_db).filter( guid__in=guids_list).values_list('guid', flat=True) else: log.debug( 'Kinto %s: Unable to break down regex into list; ' 'attempting to create Blocks for guids matching [%s]', kinto_id, guid_regexp) # mysql doesn't support \d - only [:digit:] guid_regexp = guid_regexp.replace(r'\d', '[[:digit:]]') addons_guids_qs = Addon.unfiltered.using(using_db).filter( guid__regex=guid_regexp).values_list('guid', flat=True) # We need to mark this id in a way so we know its from a # regex guid - otherwise we might accidentally overwrite it. block_kw['kinto_id'] = '*' + block_kw['kinto_id'] regex = True else: log.debug('Kinto %s: Attempting to create a Block for guid [%s]', kinto_id, guid) addons_guids_qs = Addon.unfiltered.using(using_db).filter( guid=guid).values_list('guid', flat=True) regex = False new_blocks = [] for guid in addons_guids_qs: valid_files_qs = File.objects.filter(version__addon__guid=guid, is_webextension=True) if not valid_files_qs.exists(): log.debug( 'Kinto %s: Skipped Block for [%s] because it has no ' 'webextension files', kinto_id, guid) continue (block, created) = Block.objects.update_or_create(guid=guid, defaults=dict(guid=guid, **block_kw)) block_activity_log_save(block, change=not created) if created: log.debug('Kinto %s: Added Block for [%s]', kinto_id, guid) block.update(modified=modified_date) else: log.debug('Kinto %s: Updated Block for [%s]', kinto_id, guid) new_blocks.append(block) if new_blocks: kinto_import.outcome = (KintoImport.OUTCOME_REGEXBLOCKS if regex else KintoImport.OUTCOME_BLOCK) else: kinto_import.outcome = KintoImport.OUTCOME_NOMATCH log.debug('Kinto %s: No addon found', kinto_id) kinto_import.save()