def setup_mkt_indexes():
"""
Define explicit ES mappings for models. If a field is not explicitly
defined and a field is inserted, ES will dynamically guess the type and
insert it, in a schemaless manner.
"""
es = elasticutils.get_es()
for model in [Contribution, InappPayment]:
index = model._get_index()
create_es_index_if_missing(index)
mapping = {
'properties': {
'id': {'type': 'long'},
'date': {'format': 'dateOptionalTime',
'type': 'date'},
'count': {'type': 'long'},
'revenue': {'type': 'double'},
# Try to tell ES not to 'analyze' the field to querying with
# hyphens and lowercase letters.
'currency': {'type': 'string',
'index': 'not_analyzed'},
'source': {'type': 'string',
'index': 'not_analyzed'},
'inapp': {'type': 'string',
'index': 'not_analyzed'}
}
}
es.put_mapping(model._meta.db_table, mapping,
model._get_index())
def setup_indexes():
es = elasticutils.get_es()
for model in CollectionCount, DownloadCount, UpdateCount:
index = model._get_index()
create_es_index_if_missing(index)
mapping = {
'properties': {
'id': {
'type': 'long'
},
'count': {
'type': 'long'
},
'data': {
'dynamic': 'true',
'properties': {
'v': {
'type': 'long'
},
'k': {
'type': 'string'
}
}
},
'date': {
'format': 'dateOptionalTime',
'type': 'date'
}
}
}
es.put_mapping(model._meta.db_table, mapping, model._get_index())
def setup_mapping():
"""Set up the addons index mapping."""
# Mapping describes how elasticsearch handles a document during indexing.
# Most fields are detected and mapped automatically.
appver = {'dynamic': False, 'properties': {'max': {'type': 'long'},
'min': {'type': 'long'}}}
mapping = {
# Optional boosting during indexing.
'_boost': {'name': '_boost', 'null_value': 1.0},
'properties': {
# Turn off analysis on name so we can sort by it.
'name_sort': {'type': 'string', 'index': 'not_analyzed'},
# Adding word-delimiter to split on camelcase and punctuation.
'name': {'type': 'string',
'analyzer': 'standardPlusWordDelimiter'},
'summary': {'type': 'string',
'analyzer': 'snowball'},
'description': {'type': 'string',
'analyzer': 'snowball'},
'tags': {'type': 'string',
'index': 'not_analyzed',
'index_name': 'tag'},
'platforms': {'type': 'integer', 'index_name': 'platform'},
'appversion': {'properties': dict((app.id, appver)
for app in amo.APP_USAGE)},
},
}
# Add room for language-specific indexes.
for analyzer in amo.SEARCH_ANALYZER_MAP:
mapping['properties']['name_' + analyzer] = {
'type': 'string',
'analyzer': analyzer,
}
mapping['properties']['summary_' + analyzer] = {
'type': 'string',
'analyzer': analyzer,
}
mapping['properties']['description_' + analyzer] = {
'type': 'string',
'analyzer': analyzer,
}
es = elasticutils.get_es()
# Adjust the mapping for all models at once because fields are shared
# across all doc types in an index. If we forget to adjust one of them
# we'll get burned later on.
for model in Addon, AppCompat, Collection, UserProfile:
index = model._get_index()
create_es_index_if_missing(index)
try:
es.put_mapping(model._meta.db_table, mapping, index)
except pyes.ElasticSearchException, e:
log.error(e)
def setup_indexes():
es = elasticutils.get_es()
for model in CollectionCount, DownloadCount, UpdateCount:
index = model._get_index()
create_es_index_if_missing(index)
mapping = {
"properties": {
"id": {"type": "long"},
"count": {"type": "long"},
"data": {"dynamic": "true", "properties": {"v": {"type": "long"}, "k": {"type": "string"}}},
"date": {"format": "dateOptionalTime", "type": "date"},
}
}
es.put_mapping(model._meta.db_table, mapping, model._get_index())
def setup_indexes(index=None, aliased=True):
es = amo.search.get_es()
for model in CollectionCount, DownloadCount, UpdateCount:
index = index or model._get_index()
index = create_es_index_if_missing(index, aliased=aliased)
mapping = {
'properties': {
'id': {
'type': 'long'
},
'count': {
'type': 'long'
},
'data': {
'dynamic': 'true',
'properties': {
'v': {
'type': 'long'
},
'k': {
'type': 'string'
}
}
},
'date': {
'format': 'dateOptionalTime',
'type': 'date'
}
}
}
es.put_mapping(model._meta.db_table, mapping, index)
def setup_mkt_indexes(index=None, aliased=True):
"""
Define explicit ES mappings for models. If a field is not explicitly
defined and a field is inserted, ES will dynamically guess the type and
insert it, in a schemaless manner.
"""
es = elasticutils.get_es()
for model in [Contribution, InappPayment]:
index = index or model._get_index()
index = create_es_index_if_missing(index, aliased=aliased)
mapping = {
"properties": {
"id": {"type": "long"},
"date": {"format": "dateOptionalTime", "type": "date"},
"count": {"type": "long"},
"revenue": {"type": "double"},
# Try to tell ES not to 'analyze' the field to querying with
# hyphens and lowercase letters.
"currency": {"type": "string", "index": "not_analyzed"},
"source": {"type": "string", "index": "not_analyzed"},
"inapp": {"type": "string", "index": "not_analyzed"},
}
}
es.put_mapping(model._meta.db_table, mapping, index)
def setup_mapping(index=None, aliased=True):
"""Set up the addons index mapping."""
# Mapping describes how elasticsearch handles a document during indexing.
# Most fields are detected and mapped automatically.
appver = {'dynamic': False, 'properties': {'max': {'type': 'long'},
'min': {'type': 'long'}}}
mapping = {
# Optional boosting during indexing.
'_boost': {'name': '_boost', 'null_value': 1.0},
'properties': {
# Turn off analysis on name so we can sort by it.
'name_sort': {'type': 'string', 'index': 'not_analyzed'},
# Adding word-delimiter to split on camelcase and punctuation.
'name': {'type': 'string',
'analyzer': 'standardPlusWordDelimiter'},
'summary': {'type': 'string',
'analyzer': 'snowball'},
'description': {'type': 'string',
'analyzer': 'snowball'},
'tags': {'type': 'string',
'index': 'not_analyzed',
'index_name': 'tag'},
'platforms': {'type': 'integer', 'index_name': 'platform'},
'appversion': {'properties': dict((app.id, appver)
for app in amo.APP_USAGE)},
},
}
# Add room for language-specific indexes.
for analyzer in amo.SEARCH_ANALYZER_MAP:
if (not settings.ES_USE_PLUGINS and
analyzer in amo.SEARCH_ANALYZER_PLUGINS):
log.info('While creating mapping, skipping the %s analyzer'
% analyzer)
continue
mapping['properties']['name_' + analyzer] = {
'type': 'string',
'analyzer': analyzer,
}
mapping['properties']['summary_' + analyzer] = {
'type': 'string',
'analyzer': analyzer,
}
mapping['properties']['description_' + analyzer] = {
'type': 'string',
'analyzer': analyzer,
}
es = amo.search.get_es()
# Adjust the mapping for all models at once because fields are shared
# across all doc types in an index. If we forget to adjust one of them
# we'll get burned later on.
for model in Addon, AppCompat, Collection, UserProfile:
index = index or model._get_index()
index = create_es_index_if_missing(index, aliased=aliased)
try:
es.put_mapping(model._meta.db_table, mapping, index)
except pyes.ElasticSearchException, e:
log.error(e)
def elastic(request):
INDEX = settings.ES_INDEXES['default']
es = elasticutils.get_es()
mappings = {
'addons': reindex_addons,
'apps': reindex_apps,
'collections': reindex_collections,
'compat': compatibility_report,
'users': reindex_users,
'stats_latest': index_latest_stats,
'mkt_stats': index_mkt_stats,
'mkt_stats_latest': index_latest_mkt_stats
}
if request.method == 'POST':
if request.POST.get('recreate'):
es.delete_index_if_exists(INDEX)
# We must set up the mappings before we create the index again.
setup_mapping()
setup_indexes()
if setup_mkt_indexes:
setup_mkt_indexes()
create_es_index_if_missing(INDEX)
messages.info(request, 'Deleting %s index.' % INDEX)
if request.POST.get('reindex') in mappings:
name = request.POST['reindex']
# Reindex.
if mappings.get(name):
mappings[name]()
messages.info(request, 'Reindexing %s.' % name)
return redirect('zadmin.elastic')
indexes = set(settings.ES_INDEXES.values())
es_mappings = es.get_mapping(None, indexes)
ctx = {
'index': INDEX,
'nodes': es.cluster_nodes(),
'health': es.cluster_health(),
'state': es.cluster_state(),
'mappings': [(index, es_mappings.get(index, {})) for index in indexes],
'choices': mappings,
}
return jingo.render(request, 'zadmin/elastic.html', ctx)
def elastic(request):
INDEX = settings.ES_INDEXES["default"]
es = elasticutils.get_es()
mappings = {
"addons": reindex_addons,
"apps": reindex_apps,
"collections": reindex_collections,
"compat": compatibility_report,
"users": reindex_users,
"stats_latest": index_latest_stats,
"mkt_stats": index_mkt_stats,
"mkt_stats_latest": index_latest_mkt_stats,
}
if request.method == "POST":
if request.POST.get("recreate"):
es.delete_index_if_exists(INDEX)
# We must set up the mappings before we create the index again.
setup_mapping()
setup_indexes()
if setup_mkt_indexes:
setup_mkt_indexes()
create_es_index_if_missing(INDEX)
messages.info(request, "Deleting %s index." % INDEX)
if request.POST.get("reindex") in mappings:
name = request.POST["reindex"]
# Reindex.
if mappings.get(name):
mappings[name]()
messages.info(request, "Reindexing %s." % name)
return redirect("zadmin.elastic")
indexes = set(settings.ES_INDEXES.values())
es_mappings = es.get_mapping(None, indexes)
ctx = {
"index": INDEX,
"nodes": es.cluster_nodes(),
"health": es.cluster_health(),
"state": es.cluster_state(),
"mappings": [(index, es_mappings.get(index, {})) for index in indexes],
"choices": mappings,
}
return jingo.render(request, "zadmin/elastic.html", ctx)
def elastic(request):
INDEX = settings.ES_INDEXES['default']
es = elasticutils.get_es()
mappings = {'addons': reindex_addons,
'apps': reindex_apps,
'collections': reindex_collections,
'compat': compatibility_report,
'users': reindex_users,
'stats_latest': index_latest_stats,
'mkt_stats': index_mkt_stats,
'mkt_stats_latest': index_latest_mkt_stats}
if request.method == 'POST':
if request.POST.get('recreate'):
es.delete_index_if_exists(INDEX)
# We must set up the mappings before we create the index again.
setup_mapping()
setup_indexes()
if setup_mkt_indexes:
setup_mkt_indexes()
create_es_index_if_missing(INDEX)
messages.info(request, 'Deleting %s index.' % INDEX)
if request.POST.get('reindex') in mappings:
name = request.POST['reindex']
# Reindex.
if mappings.get(name):
mappings[name]()
messages.info(request, 'Reindexing %s.' % name)
return redirect('zadmin.elastic')
indexes = set(settings.ES_INDEXES.values())
es_mappings = es.get_mapping(None, indexes)
ctx = {
'index': INDEX,
'nodes': es.cluster_nodes(),
'health': es.cluster_health(),
'state': es.cluster_state(),
'mappings': [(index, es_mappings.get(index, {})) for index in indexes],
'choices': mappings,
}
return jingo.render(request, 'zadmin/elastic.html', ctx)
def setup_indexes(index=None, aliased=True):
es = amo.search.get_es()
for model in CollectionCount, DownloadCount, UpdateCount:
index = index or model._get_index()
index = create_es_index_if_missing(index, aliased=aliased)
mapping = {
'properties': {
'id': {'type': 'long'},
'count': {'type': 'long'},
'data': {'dynamic': 'true',
'properties': {
'v': {'type': 'long'},
'k': {'type': 'string'}
}
},
'date': {'format': 'dateOptionalTime',
'type': 'date'}
}
}
es.put_mapping(model._meta.db_table, mapping, index)