def index_all_profiles():
# Get an es object, delete index and re-create it
es = get_es(timeout=settings.ES_INDEXING_TIMEOUT)
mappings = {'mappings':
{UserProfile._meta.db_table: UserProfile.get_mapping()}}
def _recreate_index(index):
try:
es.delete_index_if_exists(index)
except pyes.exceptions.IndexMissingException:
pass
es.create_index(index, settings=mappings)
_recreate_index(settings.ES_INDEXES['default'])
_recreate_index(settings.ES_INDEXES['public'])
# mozillians index
ids = UserProfile.objects.complete().values_list('id', flat=True)
ts = [index_objects.subtask(args=[UserProfile, chunk, False])
for chunk in chunked(sorted(list(ids)), 150)]
# public index
ids = (UserProfile.objects.complete().public_indexable()
.privacy_level(PUBLIC).values_list('id', flat=True))
ts += [index_objects.subtask(args=[UserProfile, chunk, True])
for chunk in chunked(sorted(list(ids)), 150)]
TaskSet(ts).apply_async()
def index_all_profiles():
# Get an es object, delete index and re-create it
es = get_es(timeout=settings.ES_INDEXING_TIMEOUT)
mappings = {
'mappings': {
UserProfileMappingType.get_mapping_type_name():
UserProfileMappingType.get_mapping()
}
}
def _recreate_index(index):
es.indices.delete(index=index, ignore=[400, 404])
es.indices.create(index, body=mappings)
_recreate_index(settings.ES_INDEXES['default'])
_recreate_index(settings.ES_INDEXES['public'])
# mozillians index
ids = UserProfile.objects.complete().values_list('id', flat=True)
ts = [
index_objects.subtask(args=[UserProfileMappingType, chunk, 150, False])
for chunk in chunked(sorted(list(ids)), 150)
]
# public index
ts += [
index_objects.subtask(args=[UserProfileMappingType, chunk, 150, True])
for chunk in chunked(sorted(list(ids)), 150)
]
TaskSet(ts).apply_async()
def update_package_activity():
"""Recalculates package activity rating for all packages"""
ids = Package.objects.all().values_list('id', flat=True)
log.info("Updating package activity for %s packages" % len(ids))
with establish_connection() as conn:
for chunk in chunked(ids, 100):
tasks.calculate_activity_rating.apply_async(args=[chunk], connection=conn)
def index_all_profiles():
from elasticutils import tasks
ids = (UserProfile.objects.values_list('id', flat=True))
ts = [tasks.index_objects.subtask(args=[UserProfile, chunk])
for chunk in chunked(sorted(list(ids)), 150)]
TaskSet(ts).apply_async()
def index_all():
"""This reindexes all the known packages and libraries."""
ids = Package.objects.all().values_list('id', flat=True)
log.info("Indexing %s packages" % len(ids))
with establish_connection() as conn:
for chunk in chunked(ids, 100):
tasks.index_all.apply_async(args=[chunk], connection=conn)
def index_all_profiles():
# Get an es object, delete index and re-create it
index = settings.ES_INDEXES['default']
es = get_es(timeout=settings.ES_INDEXING_TIMEOUT)
try:
es.delete_index_if_exists(index)
except pyes.exceptions.IndexMissingException:
pass
mappings = {
'mappings': {
UserProfile._meta.db_table: UserProfile.get_mapping()
}
}
es.create_index(index, settings=mappings)
ids = (UserProfile.objects.exclude(full_name='').values_list('id',
flat=True))
ts = [
tasks.index_objects.subtask(args=[UserProfile, chunk])
for chunk in chunked(sorted(list(ids)), 150)
]
TaskSet(ts).apply_async()
def job():
from elasticutils import tasks
ids = (model.objects.values_list('id', flat=True))
ts = [tasks.index_objects.subtask(args=[chunk])
for chunk in chunked(sorted(list(ids)), chunk_size)]
TaskSet(ts).apply_async()
def reindex_videos():
from elasticutils import tasks
ids = Video.objects.values_list("id", flat=True)
ts = [tasks.index_objects.subtask(args=[Video, chunk]) for chunk in chunked(sorted(list(ids)), 150)]
TaskSet(ts).apply_async()
def index_all():
"""This reindexes all the known packages and libraries."""
ids = Package.objects.all().values_list('id', flat=True)
log.info("Indexing %s packages" % len(ids))
with establish_connection() as conn:
for chunk in chunked(ids, 100):
tasks.index_all.apply_async(args=[chunk], connection=conn)
def index_all_profiles():
ids = (UserProfile.objects.values_list('id', flat=True))
ts = [
tasks.index_objects.subtask(args=[UserProfile, chunk])
for chunk in chunked(sorted(list(ids)), 150)
]
TaskSet(ts).apply_async()
def update_package_activity():
"""Recalculates package activity rating for all packages"""
ids = Package.objects.all().values_list('id', flat=True)
log.info("Updating package activity for %s packages" % len(ids))
with establish_connection() as conn:
for chunk in chunked(ids, 100):
tasks.calculate_activity_rating.apply_async(args=[chunk],
connection=conn)
def job():
from elasticutils import tasks
ids = (model.objects.values_list('id', flat=True))
ts = [
tasks.index_objects.subtask(args=[chunk])
for chunk in chunked(sorted(list(ids)), chunk_size)
]
TaskSet(ts).apply_async()
def index_all():
"""
This reindexes all the Opinions in usage. This is not intended to be run
other than to initially seed Elastic Search.
"""
ids = (Opinion.objects
.filter(_type__in=[i.id for i in input.OPINION_USAGE])
.values_list('id', flat=True))
with establish_connection() as conn:
for chunk in chunked(ids, 1000):
tasks.add_to_index.apply_async(args=[chunk], connection=conn)
def index_all():
"""
This reindexes all the Opinions in usage. This is not intended to be run
other than to initially seed Elastic Search.
"""
ids = (Opinion.objects.filter(
_type__in=[i.id for i in input.OPINION_USAGE]).values_list('id',
flat=True))
with establish_connection() as conn:
for chunk in chunked(ids, 1000):
tasks.add_to_index.apply_async(args=[chunk], connection=conn)
def index_all_profiles():
# Get an es object, delete index and re-create it
es = get_es(timeout=settings.ES_INDEXING_TIMEOUT)
mappings = {'mappings': {UserProfileMappingType.get_mapping_type_name():
UserProfileMappingType.get_mapping()}}
def _recreate_index(index):
es.indices.delete(index=index, ignore=[400, 404])
es.indices.create(index, body=mappings)
_recreate_index(settings.ES_INDEXES['default'])
_recreate_index(settings.ES_INDEXES['public'])
# mozillians index
ids = UserProfile.objects.complete().values_list('id', flat=True)
ts = [index_objects.subtask(args=[UserProfileMappingType, chunk, 150, False])
for chunk in chunked(sorted(list(ids)), 150)]
# public index
ts += [index_objects.subtask(args=[UserProfileMappingType, chunk, 150, True])
for chunk in chunked(sorted(list(ids)), 150)]
TaskSet(ts).apply_async()
def index_all_profiles():
# Get an es object, delete index and re-create it
index = settings.ES_INDEXES['default']
es = get_es(timeout=settings.ES_INDEXING_TIMEOUT)
try:
es.delete_index_if_exists(index)
except pyes.exceptions.IndexMissingException:
pass
mappings = {'mappings':
{UserProfile._meta.db_table: UserProfile.get_mapping()}}
es.create_index(index, settings=mappings)
ids = (UserProfile.objects.values_list('id', flat=True))
ts = [tasks.index_objects.subtask(args=[UserProfile, chunk])
for chunk in chunked(sorted(list(ids)), 150)]
TaskSet(ts).apply_async()
