예제 #1
0
def index_all_profiles():
    # Get an es object, delete index and re-create it
    es = get_es(timeout=settings.ES_INDEXING_TIMEOUT)
    mappings = {'mappings':
                {UserProfile._meta.db_table: UserProfile.get_mapping()}}

    def _recreate_index(index):
        try:
            es.delete_index_if_exists(index)
        except pyes.exceptions.IndexMissingException:
            pass
        es.create_index(index, settings=mappings)
    _recreate_index(settings.ES_INDEXES['default'])
    _recreate_index(settings.ES_INDEXES['public'])

    # mozillians index
    ids = UserProfile.objects.complete().values_list('id', flat=True)
    ts = [index_objects.subtask(args=[UserProfile, chunk, False])
          for chunk in chunked(sorted(list(ids)), 150)]

    # public index
    ids = (UserProfile.objects.complete().public_indexable()
           .privacy_level(PUBLIC).values_list('id', flat=True))
    ts += [index_objects.subtask(args=[UserProfile, chunk, True])
           for chunk in chunked(sorted(list(ids)), 150)]

    TaskSet(ts).apply_async()
예제 #2
0
def index_all_profiles():
    # Get an es object, delete index and re-create it
    es = get_es(timeout=settings.ES_INDEXING_TIMEOUT)
    mappings = {
        'mappings': {
            UserProfileMappingType.get_mapping_type_name():
            UserProfileMappingType.get_mapping()
        }
    }

    def _recreate_index(index):
        es.indices.delete(index=index, ignore=[400, 404])
        es.indices.create(index, body=mappings)

    _recreate_index(settings.ES_INDEXES['default'])
    _recreate_index(settings.ES_INDEXES['public'])

    # mozillians index
    ids = UserProfile.objects.complete().values_list('id', flat=True)
    ts = [
        index_objects.subtask(args=[UserProfileMappingType, chunk, 150, False])
        for chunk in chunked(sorted(list(ids)), 150)
    ]

    # public index
    ts += [
        index_objects.subtask(args=[UserProfileMappingType, chunk, 150, True])
        for chunk in chunked(sorted(list(ids)), 150)
    ]

    TaskSet(ts).apply_async()
예제 #3
0
파일: cron.py 프로젝트: Jnull/FlightDeck
def update_package_activity():
    """Recalculates package activity rating for all packages"""
    ids = Package.objects.all().values_list('id', flat=True)
    log.info("Updating package activity for %s packages" % len(ids))
    with establish_connection() as conn:
        for chunk in chunked(ids, 100):
            tasks.calculate_activity_rating.apply_async(args=[chunk], connection=conn)
예제 #4
0
파일: cron.py 프로젝트: jdm/mozillians
def index_all_profiles():
    from elasticutils import tasks

    ids = (UserProfile.objects.values_list('id', flat=True))
    ts = [tasks.index_objects.subtask(args=[UserProfile, chunk])
          for chunk in chunked(sorted(list(ids)), 150)]
    TaskSet(ts).apply_async()
예제 #5
0
def index_all():
    """This reindexes all the known packages and libraries."""
    ids = Package.objects.all().values_list('id', flat=True)
    log.info("Indexing %s packages" % len(ids))
    with establish_connection() as conn:
        for chunk in chunked(ids, 100):
            tasks.index_all.apply_async(args=[chunk], connection=conn)
예제 #6
0
def index_all_profiles():
    # Get an es object, delete index and re-create it

    index = settings.ES_INDEXES['default']
    es = get_es(timeout=settings.ES_INDEXING_TIMEOUT)
    try:
        es.delete_index_if_exists(index)
    except pyes.exceptions.IndexMissingException:
        pass

    mappings = {
        'mappings': {
            UserProfile._meta.db_table: UserProfile.get_mapping()
        }
    }

    es.create_index(index, settings=mappings)

    ids = (UserProfile.objects.exclude(full_name='').values_list('id',
                                                                 flat=True))
    ts = [
        tasks.index_objects.subtask(args=[UserProfile, chunk])
        for chunk in chunked(sorted(list(ids)), 150)
    ]
    TaskSet(ts).apply_async()
예제 #7
0
파일: cron.py 프로젝트: aparo/elasticutils
    def job():
        from elasticutils import tasks

        ids = (model.objects.values_list('id', flat=True))
        ts = [tasks.index_objects.subtask(args=[chunk])
              for chunk in chunked(sorted(list(ids)), chunk_size)]
        TaskSet(ts).apply_async()
예제 #8
0
def reindex_videos():
    from elasticutils import tasks

    ids = Video.objects.values_list("id", flat=True)

    ts = [tasks.index_objects.subtask(args=[Video, chunk]) for chunk in chunked(sorted(list(ids)), 150)]
    TaskSet(ts).apply_async()
예제 #9
0
def index_all():
    """This reindexes all the known packages and libraries."""
    ids = Package.objects.all().values_list('id', flat=True)
    log.info("Indexing %s packages" % len(ids))
    with establish_connection() as conn:
        for chunk in chunked(ids, 100):
            tasks.index_all.apply_async(args=[chunk], connection=conn)
예제 #10
0
파일: cron.py 프로젝트: sumlaj/mozillians
def index_all_profiles():
    ids = (UserProfile.objects.values_list('id', flat=True))
    ts = [
        tasks.index_objects.subtask(args=[UserProfile, chunk])
        for chunk in chunked(sorted(list(ids)), 150)
    ]
    TaskSet(ts).apply_async()
예제 #11
0
def update_package_activity():
    """Recalculates package activity rating for all packages"""
    ids = Package.objects.all().values_list('id', flat=True)
    log.info("Updating package activity for %s packages" % len(ids))
    with establish_connection() as conn:
        for chunk in chunked(ids, 100):
            tasks.calculate_activity_rating.apply_async(args=[chunk],
                                                        connection=conn)
예제 #12
0
    def job():
        from elasticutils import tasks

        ids = (model.objects.values_list('id', flat=True))
        ts = [
            tasks.index_objects.subtask(args=[chunk])
            for chunk in chunked(sorted(list(ids)), chunk_size)
        ]
        TaskSet(ts).apply_async()
예제 #13
0
def index_all():
    """
    This reindexes all the Opinions in usage.  This is not intended to be run
    other than to initially seed Elastic Search.
    """
    ids = (Opinion.objects
           .filter(_type__in=[i.id for i in input.OPINION_USAGE])
           .values_list('id', flat=True))
    with establish_connection() as conn:
        for chunk in chunked(ids, 1000):
            tasks.add_to_index.apply_async(args=[chunk], connection=conn)
예제 #14
0
파일: cron.py 프로젝트: tofumatt/reporter
def index_all():
    """
    This reindexes all the Opinions in usage.  This is not intended to be run
    other than to initially seed Elastic Search.
    """
    ids = (Opinion.objects.filter(
        _type__in=[i.id for i in input.OPINION_USAGE]).values_list('id',
                                                                   flat=True))
    with establish_connection() as conn:
        for chunk in chunked(ids, 1000):
            tasks.add_to_index.apply_async(args=[chunk], connection=conn)
예제 #15
0
def index_all_profiles():
    # Get an es object, delete index and re-create it
    es = get_es(timeout=settings.ES_INDEXING_TIMEOUT)
    mappings = {'mappings': {UserProfileMappingType.get_mapping_type_name():
                             UserProfileMappingType.get_mapping()}}

    def _recreate_index(index):
        es.indices.delete(index=index, ignore=[400, 404])
        es.indices.create(index, body=mappings)

    _recreate_index(settings.ES_INDEXES['default'])
    _recreate_index(settings.ES_INDEXES['public'])

    # mozillians index
    ids = UserProfile.objects.complete().values_list('id', flat=True)
    ts = [index_objects.subtask(args=[UserProfileMappingType, chunk, 150, False])
          for chunk in chunked(sorted(list(ids)), 150)]

    # public index
    ts += [index_objects.subtask(args=[UserProfileMappingType, chunk, 150, True])
           for chunk in chunked(sorted(list(ids)), 150)]

    TaskSet(ts).apply_async()
예제 #16
0
def index_all_profiles():
    # Get an es object, delete index and re-create it

    index = settings.ES_INDEXES['default']
    es = get_es(timeout=settings.ES_INDEXING_TIMEOUT)
    try:
        es.delete_index_if_exists(index)
    except pyes.exceptions.IndexMissingException:
        pass

    mappings = {'mappings':
                {UserProfile._meta.db_table: UserProfile.get_mapping()}}

    es.create_index(index, settings=mappings)

    ids = (UserProfile.objects.values_list('id', flat=True))
    ts = [tasks.index_objects.subtask(args=[UserProfile, chunk])
          for chunk in chunked(sorted(list(ids)), 150)]
    TaskSet(ts).apply_async()