def switch_indices(backing_index, object_type):
"""
Switch the default index to point to the backing index, and delete the reindex alias
Args:
backing_index (str): The backing index of the reindex alias
object_type (str): The object type for the index (post, comment, etc)
"""
conn = get_conn()
actions = []
old_backing_indexes = []
default_alias = get_default_alias_name(object_type)
global_alias = get_default_alias_name(ALIAS_ALL_INDICES)
if conn.indices.exists_alias(name=default_alias):
# Should only be one backing index in normal circumstances
old_backing_indexes = list(
conn.indices.get_alias(name=default_alias).keys())
for index in old_backing_indexes:
actions.extend([
{
"remove": {
"index": index,
"alias": default_alias
}
},
{
"remove": {
"index": index,
"alias": global_alias
}
},
])
actions.extend([
{
"add": {
"index": backing_index,
"alias": default_alias
}
},
{
"add": {
"index": backing_index,
"alias": global_alias
}
},
])
conn.indices.update_aliases({"actions": actions})
refresh_index(backing_index)
for index in old_backing_indexes:
conn.indices.delete(index)
# Finally, remove the link to the reindexing alias
conn.indices.delete_alias(name=get_reindexing_alias_name(object_type),
index=backing_index)
def find_similar_resources(*, user, value_doc):
"""
Execute a "more like this" query to find learning resources that are similar to the one provided.
Args:
user (User): The user executing the search
value_doc (dict):
a document representing the data fields we want to search with
Returns:
dict: The Elasticsearch response dict
"""
index = get_default_alias_name(ALIAS_ALL_INDICES)
search = Search(index=index)
search = _apply_general_query_filters(search, user)
search = search.filter(Q("terms", object_type=LEARNING_RESOURCE_TYPES))
search = search.query(
MoreLikeThis(
like={
"doc": value_doc,
"fields": list(value_doc.keys())
},
fields=SIMILAR_RESOURCE_RELEVANT_FIELDS,
min_term_freq=settings.OPEN_RESOURCES_MIN_TERM_FREQ,
min_doc_freq=settings.OPEN_RESOURCES_MIN_DOC_FREQ,
))
response = search.execute()
return [
hit.to_dict() for hit in response.hits
if (hit["id"] != value_doc.get("id", None)
or hit["object_type"] != value_doc.get("object_type", None))
][0:settings.OPEN_DISCUSSIONS_SIMILAR_RESOURCES_COUNT]
def find_related_documents(*, user, post_id):
"""
Execute a "more like this" query to find posts that are related to a specific post
Args:
user (User): The user executing the search
post_id (str): The id of the post that you want to find related posts for
Returns:
dict: The Elasticsearch response dict
"""
index = get_default_alias_name(ALIAS_ALL_INDICES)
search = Search(index=index)
search = _apply_general_query_filters(search, user)
search = search.query(
MoreLikeThis(
like={
"_id": gen_post_id(post_id),
"_type": GLOBAL_DOC_TYPE
},
fields=RELATED_POST_RELEVANT_FIELDS,
min_term_freq=1,
min_doc_freq=1,
))
# Limit results to the number indicated in settings
search = search[0:settings.OPEN_DISCUSSIONS_RELATED_POST_COUNT]
return search.execute().to_dict()
def execute_learn_search(*, user, query):
"""
Execute a learning resources search based on the query
Args:
user (User): The user executing the search. Used to determine filters to enforce permissions.
query (dict): The Elasticsearch query constructed in the frontend
Returns:
dict: The Elasticsearch response dict
"""
index = get_default_alias_name(ALIAS_ALL_INDICES)
search = Search(index=index)
search.update_from_dict(query)
search = _apply_learning_query_filters(search, user)
return transform_results(search.execute().to_dict(), user)
def mocked_es(mocker, settings):
"""Mocked ES client objects/functions"""
index_name = "test"
settings.ELASTICSEARCH_INDEX = index_name
conn = mocker.Mock()
get_conn_patch = mocker.patch("search.indexing_api.get_conn",
autospec=True,
return_value=conn)
mocker.patch("search.connection.get_conn", autospec=True)
default_alias = get_default_alias_name(POST_TYPE)
reindex_alias = get_reindexing_alias_name(POST_TYPE)
yield SimpleNamespace(
get_conn=get_conn_patch,
conn=conn,
index_name=index_name,
default_alias=default_alias,
reindex_alias=reindex_alias,
active_aliases=[default_alias, reindex_alias],
)
def get_similar_topics(value_doc, num_topics, min_term_freq, min_doc_freq):
"""
Get a list of similar topics based on text values
Args:
value_doc (dict):
a document representing the data fields we want to search with
num_topics (int):
number of topics to return
min_term_freq (int):
minimum times a term needs to show up in input
min_doc_freq (int):
minimum times a term needs to show up in docs
Returns:
list of str:
list of topic values
"""
index = get_default_alias_name(ALIAS_ALL_INDICES)
search = Search(index=index)
search = search.filter(Q("terms", object_type=[COURSE_TYPE]))
search = search.query(
MoreLikeThis(
like=[{
"doc": value_doc,
"fields": list(value_doc.keys())
}],
fields=[
"course_id", "title", "short_description", "full_description"
],
min_term_freq=min_term_freq,
min_doc_freq=min_doc_freq,
))
search = search.source(includes="topics")
response = search.execute()
topics = [topic for hit in response.hits for topic in hit.topics]
counter = Counter(topics)
return list(dict(counter.most_common(num_topics)).keys())
def test_switch_indices(mocked_es, mocker, default_exists, object_type):
"""
switch_indices should atomically remove the old backing index
for the default alias and replace it with the new one
"""
refresh_mock = mocker.patch("search.indexing_api.refresh_index",
autospec=True)
conn_mock = mocked_es.conn
conn_mock.indices.exists_alias.return_value = default_exists
old_backing_index = "old_backing"
conn_mock.indices.get_alias.return_value.keys.return_value = [
old_backing_index
]
backing_index = "backing"
switch_indices(backing_index, object_type)
conn_mock.indices.delete_alias.assert_any_call(
name=get_reindexing_alias_name(object_type), index=backing_index)
default_alias = get_default_alias_name(object_type)
all_alias = get_default_alias_name(ALIAS_ALL_INDICES)
conn_mock.indices.exists_alias.assert_called_once_with(name=default_alias)
actions = []
if default_exists:
actions.extend([
{
"remove": {
"index": old_backing_index,
"alias": default_alias
}
},
{
"remove": {
"index": old_backing_index,
"alias": all_alias
}
},
])
actions.extend([
{
"add": {
"index": backing_index,
"alias": default_alias
}
},
{
"add": {
"index": backing_index,
"alias": all_alias
}
},
])
conn_mock.indices.update_aliases.assert_called_once_with(
{"actions": actions})
refresh_mock.assert_called_once_with(backing_index)
if default_exists:
conn_mock.indices.delete.assert_called_once_with(old_backing_index)
else:
assert conn_mock.indices.delete.called is False
conn_mock.indices.delete_alias.assert_called_once_with(
name=get_reindexing_alias_name(object_type), index=backing_index)