Exemplo n.º 1
0
def setup_mkt_indexes():
    """
    Define explicit ES mappings for models. If a field is not explicitly
    defined and a field is inserted, ES will dynamically guess the type and
    insert it, in a schemaless manner.
    """
    es = elasticutils.get_es()
    for model in [Contribution, InappPayment]:
        index = model._get_index()
        create_es_index_if_missing(index)

        mapping = {
            'properties': {
                'id': {'type': 'long'},
                'date': {'format': 'dateOptionalTime',
                         'type': 'date'},
                'count': {'type': 'long'},
                'revenue': {'type': 'double'},

                # Try to tell ES not to 'analyze' the field to querying with
                # hyphens and lowercase letters.
                'currency': {'type': 'string',
                             'index': 'not_analyzed'},
                'source': {'type': 'string',
                           'index': 'not_analyzed'},
                'inapp': {'type': 'string',
                          'index': 'not_analyzed'}
            }
        }

        es.put_mapping(model._meta.db_table, mapping,
                       model._get_index())
Exemplo n.º 2
0
def setup_indexes():
    es = elasticutils.get_es()
    for model in CollectionCount, DownloadCount, UpdateCount:
        index = model._get_index()
        create_es_index_if_missing(index)

        mapping = {
            'properties': {
                'id': {
                    'type': 'long'
                },
                'count': {
                    'type': 'long'
                },
                'data': {
                    'dynamic': 'true',
                    'properties': {
                        'v': {
                            'type': 'long'
                        },
                        'k': {
                            'type': 'string'
                        }
                    }
                },
                'date': {
                    'format': 'dateOptionalTime',
                    'type': 'date'
                }
            }
        }
        es.put_mapping(model._meta.db_table, mapping, model._get_index())
Exemplo n.º 3
0
def setup_mapping():
    """Set up the addons index mapping."""
    # Mapping describes how elasticsearch handles a document during indexing.
    # Most fields are detected and mapped automatically.
    appver = {'dynamic': False, 'properties': {'max': {'type': 'long'},
                                               'min': {'type': 'long'}}}
    mapping = {
        # Optional boosting during indexing.
        '_boost': {'name': '_boost', 'null_value': 1.0},
        'properties': {
            # Turn off analysis on name so we can sort by it.
            'name_sort': {'type': 'string', 'index': 'not_analyzed'},
            # Adding word-delimiter to split on camelcase and punctuation.
            'name': {'type': 'string',
                     'analyzer': 'standardPlusWordDelimiter'},
            'summary': {'type': 'string',
                        'analyzer': 'snowball'},
            'description': {'type': 'string',
                            'analyzer': 'snowball'},
            'tags': {'type': 'string',
                     'index': 'not_analyzed',
                     'index_name': 'tag'},
            'platforms': {'type': 'integer', 'index_name': 'platform'},
            'appversion': {'properties': dict((app.id, appver)
                                              for app in amo.APP_USAGE)},
        },
    }
    # Add room for language-specific indexes.
    for analyzer in amo.SEARCH_ANALYZER_MAP:
        mapping['properties']['name_' + analyzer] = {
            'type': 'string',
            'analyzer': analyzer,
        }
        mapping['properties']['summary_' + analyzer] = {
            'type': 'string',
            'analyzer': analyzer,
        }
        mapping['properties']['description_' + analyzer] = {
            'type': 'string',
            'analyzer': analyzer,
        }

    es = elasticutils.get_es()
    # Adjust the mapping for all models at once because fields are shared
    # across all doc types in an index. If we forget to adjust one of them
    # we'll get burned later on.
    for model in Addon, AppCompat, Collection, UserProfile:
        index = model._get_index()
        create_es_index_if_missing(index)
        try:
            es.put_mapping(model._meta.db_table, mapping, index)
        except pyes.ElasticSearchException, e:
            log.error(e)
Exemplo n.º 4
0
def setup_indexes():
    es = elasticutils.get_es()
    for model in CollectionCount, DownloadCount, UpdateCount:
        index = model._get_index()
        create_es_index_if_missing(index)

        mapping = {
            "properties": {
                "id": {"type": "long"},
                "count": {"type": "long"},
                "data": {"dynamic": "true", "properties": {"v": {"type": "long"}, "k": {"type": "string"}}},
                "date": {"format": "dateOptionalTime", "type": "date"},
            }
        }
        es.put_mapping(model._meta.db_table, mapping, model._get_index())
Exemplo n.º 5
0
def setup_indexes(index=None, aliased=True):
    es = amo.search.get_es()
    for model in CollectionCount, DownloadCount, UpdateCount:
        index = index or model._get_index()
        index = create_es_index_if_missing(index, aliased=aliased)

        mapping = {
            'properties': {
                'id': {
                    'type': 'long'
                },
                'count': {
                    'type': 'long'
                },
                'data': {
                    'dynamic': 'true',
                    'properties': {
                        'v': {
                            'type': 'long'
                        },
                        'k': {
                            'type': 'string'
                        }
                    }
                },
                'date': {
                    'format': 'dateOptionalTime',
                    'type': 'date'
                }
            }
        }
        es.put_mapping(model._meta.db_table, mapping, index)
Exemplo n.º 6
0
def setup_mkt_indexes(index=None, aliased=True):
    """
    Define explicit ES mappings for models. If a field is not explicitly
    defined and a field is inserted, ES will dynamically guess the type and
    insert it, in a schemaless manner.
    """
    es = elasticutils.get_es()
    for model in [Contribution, InappPayment]:
        index = index or model._get_index()
        index = create_es_index_if_missing(index, aliased=aliased)

        mapping = {
            "properties": {
                "id": {"type": "long"},
                "date": {"format": "dateOptionalTime", "type": "date"},
                "count": {"type": "long"},
                "revenue": {"type": "double"},
                # Try to tell ES not to 'analyze' the field to querying with
                # hyphens and lowercase letters.
                "currency": {"type": "string", "index": "not_analyzed"},
                "source": {"type": "string", "index": "not_analyzed"},
                "inapp": {"type": "string", "index": "not_analyzed"},
            }
        }

        es.put_mapping(model._meta.db_table, mapping, index)
Exemplo n.º 7
0
def setup_mapping(index=None, aliased=True):
    """Set up the addons index mapping."""
    # Mapping describes how elasticsearch handles a document during indexing.
    # Most fields are detected and mapped automatically.
    appver = {'dynamic': False, 'properties': {'max': {'type': 'long'},
                                               'min': {'type': 'long'}}}
    mapping = {
        # Optional boosting during indexing.
        '_boost': {'name': '_boost', 'null_value': 1.0},
        'properties': {
            # Turn off analysis on name so we can sort by it.
            'name_sort': {'type': 'string', 'index': 'not_analyzed'},
            # Adding word-delimiter to split on camelcase and punctuation.
            'name': {'type': 'string',
                     'analyzer': 'standardPlusWordDelimiter'},
            'summary': {'type': 'string',
                        'analyzer': 'snowball'},
            'description': {'type': 'string',
                            'analyzer': 'snowball'},
            'tags': {'type': 'string',
                     'index': 'not_analyzed',
                     'index_name': 'tag'},
            'platforms': {'type': 'integer', 'index_name': 'platform'},
            'appversion': {'properties': dict((app.id, appver)
                                              for app in amo.APP_USAGE)},
        },
    }
    # Add room for language-specific indexes.
    for analyzer in amo.SEARCH_ANALYZER_MAP:
        if (not settings.ES_USE_PLUGINS and
            analyzer in amo.SEARCH_ANALYZER_PLUGINS):
            log.info('While creating mapping, skipping the %s analyzer'
                     % analyzer)
            continue

        mapping['properties']['name_' + analyzer] = {
            'type': 'string',
            'analyzer': analyzer,
        }
        mapping['properties']['summary_' + analyzer] = {
            'type': 'string',
            'analyzer': analyzer,
        }
        mapping['properties']['description_' + analyzer] = {
            'type': 'string',
            'analyzer': analyzer,
        }

    es = amo.search.get_es()
    # Adjust the mapping for all models at once because fields are shared
    # across all doc types in an index. If we forget to adjust one of them
    # we'll get burned later on.
    for model in Addon, AppCompat, Collection, UserProfile:
        index = index or model._get_index()
        index = create_es_index_if_missing(index, aliased=aliased)
        try:
            es.put_mapping(model._meta.db_table, mapping, index)
        except pyes.ElasticSearchException, e:
            log.error(e)
Exemplo n.º 8
0
def elastic(request):
    INDEX = settings.ES_INDEXES['default']
    es = elasticutils.get_es()
    mappings = {
        'addons': reindex_addons,
        'apps': reindex_apps,
        'collections': reindex_collections,
        'compat': compatibility_report,
        'users': reindex_users,
        'stats_latest': index_latest_stats,
        'mkt_stats': index_mkt_stats,
        'mkt_stats_latest': index_latest_mkt_stats
    }
    if request.method == 'POST':
        if request.POST.get('recreate'):
            es.delete_index_if_exists(INDEX)
            # We must set up the mappings before we create the index again.
            setup_mapping()
            setup_indexes()
            if setup_mkt_indexes:
                setup_mkt_indexes()
            create_es_index_if_missing(INDEX)
            messages.info(request, 'Deleting %s index.' % INDEX)
        if request.POST.get('reindex') in mappings:
            name = request.POST['reindex']
            # Reindex.
            if mappings.get(name):
                mappings[name]()
            messages.info(request, 'Reindexing %s.' % name)
        return redirect('zadmin.elastic')

    indexes = set(settings.ES_INDEXES.values())
    es_mappings = es.get_mapping(None, indexes)
    ctx = {
        'index': INDEX,
        'nodes': es.cluster_nodes(),
        'health': es.cluster_health(),
        'state': es.cluster_state(),
        'mappings': [(index, es_mappings.get(index, {})) for index in indexes],
        'choices': mappings,
    }
    return jingo.render(request, 'zadmin/elastic.html', ctx)
Exemplo n.º 9
0
def elastic(request):
    INDEX = settings.ES_INDEXES["default"]
    es = elasticutils.get_es()
    mappings = {
        "addons": reindex_addons,
        "apps": reindex_apps,
        "collections": reindex_collections,
        "compat": compatibility_report,
        "users": reindex_users,
        "stats_latest": index_latest_stats,
        "mkt_stats": index_mkt_stats,
        "mkt_stats_latest": index_latest_mkt_stats,
    }
    if request.method == "POST":
        if request.POST.get("recreate"):
            es.delete_index_if_exists(INDEX)
            # We must set up the mappings before we create the index again.
            setup_mapping()
            setup_indexes()
            if setup_mkt_indexes:
                setup_mkt_indexes()
            create_es_index_if_missing(INDEX)
            messages.info(request, "Deleting %s index." % INDEX)
        if request.POST.get("reindex") in mappings:
            name = request.POST["reindex"]
            # Reindex.
            if mappings.get(name):
                mappings[name]()
            messages.info(request, "Reindexing %s." % name)
        return redirect("zadmin.elastic")

    indexes = set(settings.ES_INDEXES.values())
    es_mappings = es.get_mapping(None, indexes)
    ctx = {
        "index": INDEX,
        "nodes": es.cluster_nodes(),
        "health": es.cluster_health(),
        "state": es.cluster_state(),
        "mappings": [(index, es_mappings.get(index, {})) for index in indexes],
        "choices": mappings,
    }
    return jingo.render(request, "zadmin/elastic.html", ctx)
Exemplo n.º 10
0
def elastic(request):
    INDEX = settings.ES_INDEXES['default']
    es = elasticutils.get_es()
    mappings = {'addons': reindex_addons,
                'apps': reindex_apps,
                'collections': reindex_collections,
                'compat': compatibility_report,
                'users': reindex_users,
                'stats_latest': index_latest_stats,
                'mkt_stats': index_mkt_stats,
                'mkt_stats_latest': index_latest_mkt_stats}
    if request.method == 'POST':
        if request.POST.get('recreate'):
            es.delete_index_if_exists(INDEX)
            # We must set up the mappings before we create the index again.
            setup_mapping()
            setup_indexes()
            if setup_mkt_indexes:
                setup_mkt_indexes()
            create_es_index_if_missing(INDEX)
            messages.info(request, 'Deleting %s index.' % INDEX)
        if request.POST.get('reindex') in mappings:
            name = request.POST['reindex']
            # Reindex.
            if mappings.get(name):
                mappings[name]()
            messages.info(request, 'Reindexing %s.' % name)
        return redirect('zadmin.elastic')

    indexes = set(settings.ES_INDEXES.values())
    es_mappings = es.get_mapping(None, indexes)
    ctx = {
        'index': INDEX,
        'nodes': es.cluster_nodes(),
        'health': es.cluster_health(),
        'state': es.cluster_state(),
        'mappings': [(index, es_mappings.get(index, {})) for index in indexes],
        'choices': mappings,
    }
    return jingo.render(request, 'zadmin/elastic.html', ctx)
Exemplo n.º 11
0
def setup_indexes(index=None, aliased=True):
    es = amo.search.get_es()
    for model in CollectionCount, DownloadCount, UpdateCount:
        index = index or model._get_index()
        index = create_es_index_if_missing(index, aliased=aliased)

        mapping = {
            'properties': {
                'id': {'type': 'long'},
                'count': {'type': 'long'},
                'data': {'dynamic': 'true',
                         'properties': {
                            'v': {'type': 'long'},
                            'k': {'type': 'string'}
                        }
                },
                'date': {'format': 'dateOptionalTime',
                         'type': 'date'}
            }
        }
        es.put_mapping(model._meta.db_table, mapping, index)