Example #1
0
def es_index(mock_elastic_index):
    # Create the index.
    index = Index()
    index_name = index.timestamped_index()
    index.create_index(index_name)
    index.update_aliases(index_name)
    # Update mapping
    proj = ProjectIndex()
    proj.put_mapping()
    page = PageIndex()
    page.put_mapping()
    sec = SectionIndex()
    sec.put_mapping()

    yield index
    index.delete_index(index_name=index_name)
Example #2
0
def project_search(request):
    query = request.GET.get('q', None)
    if query is None:
        return Response({'error': 'Need project and q'}, status=status.HTTP_400_BAD_REQUEST)

    log.debug("(API Project Search) %s" % (query))
    body = {
        "query": {
            "function_score": {
                "field_value_factor": {"field": "weight"},
                "query": {
                    "bool": {
                        "should": [
                            {"match": {"name": {"query": query, "boost": 10}}},
                            {"match": {"description": {"query": query}}},
                        ]
                    }
                }
            }
        },
        "fields": ["name", "slug", "description", "lang"]
    }
    results = ProjectIndex().search(body)

    return Response({'results': results})
def es_index(mock_elastic_index):
    # Create the index.
    index = Index()
    index_name = index.timestamped_index()
    index.create_index(index_name)
    index.update_aliases(index_name)
    # Update mapping
    proj = ProjectIndex()
    proj.put_mapping()
    page = PageIndex()
    page.put_mapping()
    sec = SectionIndex()
    sec.put_mapping()

    yield index
    index.delete_index(index_name=index_name)
    def handle(self, *args, **options):
        """Provision new ES instance"""
        index = Index()
        index_name = index.timestamped_index()

        log.info("Creating indexes..")
        index.create_index(index_name)
        index.update_aliases(index_name)

        log.info("Updating mappings..")
        proj = ProjectIndex()
        proj.put_mapping()
        page = PageIndex()
        page.put_mapping()
        sec = SectionIndex()
        sec.put_mapping()
        log.info("Done!")
Example #5
0
def index_search_request(version,
                         page_list,
                         commit,
                         project_scale,
                         page_scale,
                         section=True,
                         delete=True):
    """
    Update search indexes with build output JSON.

    In order to keep sub-projects all indexed on the same shard, indexes will be
    updated using the parent project's slug as the routing value.
    """
    # TODO refactor this function
    # pylint: disable=too-many-locals
    project = version.project

    log_msg = ' '.join([page['path'] for page in page_list])
    log.info("Updating search index: project=%s pages=[%s]", project.slug,
             log_msg)

    project_obj = ProjectIndex()
    project_obj.index_document(
        data={
            'id': project.pk,
            'name': project.name,
            'slug': project.slug,
            'description': project.description,
            'lang': project.language,
            'author': [user.username for user in project.users.all()],
            'url': project.get_absolute_url(),
            'tags': None,
            'weight': project_scale,
        })

    page_obj = PageIndex()
    section_obj = SectionIndex()
    index_list = []
    section_index_list = []
    routes = [project.slug]
    routes.extend([p.parent.slug for p in project.superprojects.all()])
    for page in page_list:
        log.debug("Indexing page: %s:%s", project.slug, page['path'])
        to_hash = '-'.join([project.slug, version.slug, page['path']])
        page_id = hashlib.md5(to_hash.encode('utf-8')).hexdigest()
        index_list.append({
            'id': page_id,
            'project': project.slug,
            'version': version.slug,
            'path': page['path'],
            'title': page['title'],
            'headers': page['headers'],
            'content': page['content'],
            'taxonomy': None,
            'commit': commit,
            'weight': page_scale + project_scale,
        })
        if section:
            for sect in page['sections']:
                id_to_hash = '-'.join(
                    [project.slug, version.slug, page['path'], sect['id']])
                section_index_list.append({
                    'id':
                    (hashlib.md5(id_to_hash.encode('utf-8')).hexdigest()),
                    'project':
                    project.slug,
                    'version':
                    version.slug,
                    'path':
                    page['path'],
                    'page_id':
                    sect['id'],
                    'title':
                    sect['title'],
                    'content':
                    sect['content'],
                    'weight':
                    page_scale,
                })
            for route in routes:
                section_obj.bulk_index(section_index_list,
                                       parent=page_id,
                                       routing=route)

    for route in routes:
        page_obj.bulk_index(index_list, parent=project.slug, routing=route)

    if delete:
        log.info("Deleting files not in commit: %s", commit)
        # TODO: AK Make sure this works
        delete_query = {
            "query": {
                "bool": {
                    "must": [
                        {
                            "term": {
                                "project": project.slug,
                            }
                        },
                        {
                            "term": {
                                "version": version.slug,
                            }
                        },
                    ],
                    "must_not": {
                        "term": {
                            "commit": commit
                        }
                    }
                }
            }
        }
        page_obj.delete_document(body=delete_query)
Example #6
0
def index_search_request(version, page_list, commit, project_scale, page_scale,
                         section=True, delete=True):
    """Update search indexes with build output JSON

    In order to keep sub-projects all indexed on the same shard, indexes will be
    updated using the parent project's slug as the routing value.
    """
    # TODO refactor this function
    # pylint: disable=too-many-locals
    project = version.project

    log_msg = ' '.join([page['path'] for page in page_list])
    log.info("Updating search index: project=%s pages=[%s]",
             project.slug, log_msg)

    project_obj = ProjectIndex()
    project_obj.index_document(data={
        'id': project.pk,
        'name': project.name,
        'slug': project.slug,
        'description': project.description,
        'lang': project.language,
        'author': [user.username for user in project.users.all()],
        'url': project.get_absolute_url(),
        'tags': None,
        'weight': project_scale,
    })

    page_obj = PageIndex()
    section_obj = SectionIndex()
    index_list = []
    section_index_list = []
    routes = [project.slug]
    routes.extend([p.parent.slug for p in project.superprojects.all()])
    for page in page_list:
        log.debug("Indexing page: %s:%s", project.slug, page['path'])
        to_hash = '-'.join([project.slug, version.slug, page['path']])
        page_id = hashlib.md5(to_hash.encode('utf-8')).hexdigest()
        index_list.append({
            'id': page_id,
            'project': project.slug,
            'version': version.slug,
            'path': page['path'],
            'title': page['title'],
            'headers': page['headers'],
            'content': page['content'],
            'taxonomy': None,
            'commit': commit,
            'weight': page_scale + project_scale,
        })
        if section:
            for sect in page['sections']:
                id_to_hash = '-'.join([project.slug, version.slug,
                                       page['path'], sect['id']])
                section_index_list.append({
                    'id': (hashlib.md5(id_to_hash.encode('utf-8')).hexdigest()),
                    'project': project.slug,
                    'version': version.slug,
                    'path': page['path'],
                    'page_id': sect['id'],
                    'title': sect['title'],
                    'content': sect['content'],
                    'weight': page_scale,
                })
            for route in routes:
                section_obj.bulk_index(section_index_list, parent=page_id,
                                       routing=route)

    for route in routes:
        page_obj.bulk_index(index_list, parent=project.slug, routing=route)

    if delete:
        log.info("Deleting files not in commit: %s", commit)
        # TODO: AK Make sure this works
        delete_query = {
            "query": {
                "bool": {
                    "must": [
                        {"term": {"project": project.slug, }},
                        {"term": {"version": version.slug, }},
                    ],
                    "must_not": {
                        "term": {
                            "commit": commit
                        }
                    }
                }
            }
        }
        page_obj.delete_document(body=delete_query)
Example #7
0
def index_search_request(version, page_list, commit, project_scale, page_scale,
                         section=True, delete=True):
    log_msg = ' '.join([page['path'] for page in page_list])
    log.info("(Server Search) Indexing Pages: %s [%s]" % (
        version.project.slug, log_msg))
    project = version.project
    page_obj = PageIndex()
    section_obj = SectionIndex()

    # tags = [tag.name for tag in project.tags.all()]

    project_obj = ProjectIndex()
    project_obj.index_document(data={
        'id': project.pk,
        'name': project.name,
        'slug': project.slug,
        'description': project.description,
        'lang': project.language,
        'author': [user.username for user in project.users.all()],
        'url': project.get_absolute_url(),
        'tags': None,
        'weight': project_scale,
    })

    index_list = []
    section_index_list = []
    for page in page_list:
        log.debug("(API Index) %s:%s" % (project.slug, page['path']))
        page_id = hashlib.md5('%s-%s-%s' % (project.slug, version.slug, page['path'])).hexdigest()
        index_list.append({
            'id': page_id,
            'project': project.slug,
            'version': version.slug,
            'path': page['path'],
            'title': page['title'],
            'headers': page['headers'],
            'content': page['content'],
            'taxonomy': None,
            'commit': commit,
            'weight': page_scale + project_scale,
        })
        if section:
            for section in page['sections']:
                section_index_list.append({
                    'id': hashlib.md5(
                        '%s-%s-%s-%s' % (project.slug, version.slug,
                                         page['path'], section['id'])
                    ).hexdigest(),
                    'project': project.slug,
                    'version': version.slug,
                    'path': page['path'],
                    'page_id': section['id'],
                    'title': section['title'],
                    'content': section['content'],
                    'weight': page_scale,
                })
            section_obj.bulk_index(section_index_list, parent=page_id, routing=project.slug)

    page_obj.bulk_index(index_list, parent=project.slug)

    if delete:
        log.info("(Server Search) Deleting files not in commit: %s" % commit)
        # TODO: AK Make sure this works
        delete_query = {
            "query": {
                "bool": {
                    "must": [
                        {"term": {"project": project.slug, }},
                        {"term": {"version": version.slug, }},
                    ],
                    "must_not": {
                        "term": {
                            "commit": commit
                        }
                    }
                }
            }
        }
        page_obj.delete_document(body=delete_query)
Example #8
0
def index_search_request(version,
                         page_list,
                         commit,
                         project_scale,
                         page_scale,
                         section=True,
                         delete=True):
    log_msg = ' '.join([page['path'] for page in page_list])
    log.info("(Server Search) Indexing Pages: %s [%s]" %
             (version.project.slug, log_msg))
    project = version.project
    page_obj = PageIndex()
    section_obj = SectionIndex()

    # tags = [tag.name for tag in project.tags.all()]

    project_obj = ProjectIndex()
    project_obj.index_document(
        data={
            'id': project.pk,
            'name': project.name,
            'slug': project.slug,
            'description': project.description,
            'lang': project.language,
            'author': [user.username for user in project.users.all()],
            'url': project.get_absolute_url(),
            'tags': None,
            'weight': project_scale,
        })

    index_list = []
    section_index_list = []
    for page in page_list:
        log.debug("(API Index) %s:%s" % (project.slug, page['path']))
        page_id = hashlib.md5(
            '%s-%s-%s' %
            (project.slug, version.slug, page['path'])).hexdigest()
        index_list.append({
            'id': page_id,
            'project': project.slug,
            'version': version.slug,
            'path': page['path'],
            'title': page['title'],
            'headers': page['headers'],
            'content': page['content'],
            'taxonomy': None,
            'commit': commit,
            'weight': page_scale + project_scale,
        })
        if section:
            for section in page['sections']:
                section_index_list.append({
                    'id':
                    hashlib.md5('%s-%s-%s-%s' %
                                (project.slug, version.slug, page['path'],
                                 section['id'])).hexdigest(),
                    'project':
                    project.slug,
                    'version':
                    version.slug,
                    'path':
                    page['path'],
                    'page_id':
                    section['id'],
                    'title':
                    section['title'],
                    'content':
                    section['content'],
                    'weight':
                    page_scale,
                })
            section_obj.bulk_index(section_index_list,
                                   parent=page_id,
                                   routing=project.slug)

    page_obj.bulk_index(index_list, parent=project.slug)

    if delete:
        log.info("(Server Search) Deleting files not in commit: %s" % commit)
        # TODO: AK Make sure this works
        delete_query = {
            "query": {
                "bool": {
                    "must": [
                        {
                            "term": {
                                "project": project.slug,
                            }
                        },
                        {
                            "term": {
                                "version": version.slug,
                            }
                        },
                    ],
                    "must_not": {
                        "term": {
                            "commit": commit
                        }
                    }
                }
            }
        }
        page_obj.delete_document(body=delete_query)
from readthedocs.search.indexes import Index, PageIndex, ProjectIndex, SectionIndex

# Create the index.
index = Index()
index_name = index.timestamped_index()
index.create_index(index_name)
index.update_aliases(index_name)
# Update mapping
proj = ProjectIndex()
proj.put_mapping()
page = PageIndex()
page.put_mapping()
sec = SectionIndex()
sec.put_mapping()