def refresh_all_default_indices():
"""
Refresh all default indexes
"""
for index_type in ALL_INDEX_TYPES:
alias = get_default_alias(index_type)
refresh_index(alias)
def document_needs_updating(enrollment):
"""
Get the document from elasticsearch and see if it matches what's in the database
Args:
enrollment (ProgramEnrollment): A program enrollment
Returns:
bool: True if the document needs to be updated via reindex
"""
index = get_default_alias(PRIVATE_ENROLLMENT_INDEX_TYPE)
conn = get_conn()
try:
document = conn.get(index=index, id=enrollment.id)
except NotFoundError:
return True
serialized_enrollment = serialize_program_enrolled_user(enrollment)
del serialized_enrollment['_id']
source = document['_source']
if serialized_enrollment != source:
# Convert OrderedDict to dict
reserialized_enrollment = json.loads(json.dumps(serialized_enrollment))
diff = make_patch(source, reserialized_enrollment).patch
serialized_diff = json.dumps(diff, indent=" ")
log.info("Difference found for enrollment %s: %s", enrollment, serialized_diff)
return True
return False
def refresh_all_default_indices():
"""
Refresh all default indexes
"""
for index_type in ALL_INDEX_TYPES:
alias = get_default_alias(index_type)
refresh_index(alias)
def _search_percolate_queries(program_enrollment):
"""
Find all PercolateQuery ids whose queries match a user document
Args:
program_enrollment (ProgramEnrollment): A ProgramEnrollment
Returns:
list of int: A list of PercolateQuery ids
"""
conn = get_conn()
percolate_index = get_default_alias(PERCOLATE_INDEX_TYPE)
doc = serialize_program_enrolled_user(program_enrollment)
if not doc:
return []
# We don't need this to search for percolator queries and
# it causes a dynamic mapping failure so we need to remove it
del doc['_id']
body = {"query": {"percolate": {"field": "query", "document": doc}}}
result = conn.search(percolate_index, GLOBAL_DOC_TYPE, body=body)
failures = result.get('_shards', {}).get('failures', [])
if len(failures) > 0:
raise PercolateException("Failed to percolate: {}".format(failures))
return [int(row['_id']) for row in result['hits']['hits']]
def document_needs_updating(enrollment):
"""
Get the document from elasticsearch and see if it matches what's in the database
Args:
enrollment (ProgramEnrollment): A program enrollment
Returns:
bool: True if the document needs to be updated via reindex
"""
index = get_default_alias(PRIVATE_ENROLLMENT_INDEX_TYPE)
conn = get_conn()
try:
document = conn.get(index=index,
doc_type=GLOBAL_DOC_TYPE,
id=enrollment.id)
except NotFoundError:
return True
serialized_enrollment = serialize_program_enrolled_user(enrollment)
del serialized_enrollment['_id']
source = document['_source']
if serialized_enrollment != source:
# Convert OrderedDict to dict
reserialized_enrollment = json.loads(json.dumps(serialized_enrollment))
diff = make_patch(source, reserialized_enrollment).patch
serialized_diff = json.dumps(diff, indent=" ")
log.info("Difference found for enrollment %s: %s", enrollment,
serialized_diff)
return True
return False
def test_create_search_obj_metadata(self, is_advance_search_capable, expected_index_type):
"""
Test that Search objects are created with proper metadata
"""
user = self.user if is_advance_search_capable else self.learner
search_param_dict = {'size': 50}
with patch('search.api.Search.update_from_dict', autospec=True) as mock_update_from_dict:
search_obj = create_search_obj(
user,
search_param_dict=search_param_dict,
)
expected_alias = get_default_alias(expected_index_type)
assert search_obj._index == [expected_alias] # pylint: disable=protected-access
assert mock_update_from_dict.call_count == 2
assert isinstance(mock_update_from_dict.call_args[0][0], Search)
assert mock_update_from_dict.call_args[0][1] == search_param_dict
def create_search_obj(user,
search_param_dict=None,
filter_on_email_optin=False):
"""
Creates a search object and prepares it with metadata and query parameters that
we want to apply for all ES requests
Args:
user (User): User object
search_param_dict (dict): A dict representing the body of an ES query
filter_on_email_optin (bool): If true, filter out profiles where email_optin != True
Returns:
Search: elasticsearch_dsl Search object
"""
staff_program_ids = get_advance_searchable_program_ids(user)
is_advance_search_capable = bool(staff_program_ids)
index_type = PRIVATE_ENROLLMENT_INDEX_TYPE if is_advance_search_capable else PUBLIC_ENROLLMENT_INDEX_TYPE
index = get_default_alias(index_type)
search_obj = Search(index=index)
# Update from search params first so our server-side filtering will overwrite it if necessary
if search_param_dict is not None:
search_obj.update_from_dict(search_param_dict)
if not is_advance_search_capable:
# Learners can't search for other learners with privacy set to private
search_obj = search_obj.filter(
~Q('term', **{'profile.account_privacy': Profile.PRIVATE}) # pylint: disable=invalid-unary-operand-type
)
# Limit results to one of the programs the user is staff on
search_obj = search_obj.filter(
create_program_limit_query(
user,
staff_program_ids,
filter_on_email_optin=filter_on_email_optin))
# Filter so that only filled_out profiles are seen
search_obj = search_obj.filter(Q('term', **{'profile.filled_out': True}))
# Force size to be the one we set on the server
update_dict = {'size': settings.ELASTICSEARCH_DEFAULT_PAGE_SIZE}
if search_param_dict is not None and search_param_dict.get(
'from') is not None:
update_dict['from'] = search_param_dict['from']
search_obj.update_from_dict(update_dict)
return search_obj
def create_search_obj(user, search_param_dict=None, filter_on_email_optin=False):
"""
Creates a search object and prepares it with metadata and query parameters that
we want to apply for all ES requests
Args:
user (User): User object
search_param_dict (dict): A dict representing the body of an ES query
filter_on_email_optin (bool): If true, filter out profiles where email_optin != True
Returns:
Search: elasticsearch_dsl Search object
"""
staff_program_ids = get_advance_searchable_program_ids(user)
is_advance_search_capable = bool(staff_program_ids)
index_type = PRIVATE_ENROLLMENT_INDEX_TYPE if is_advance_search_capable else PUBLIC_ENROLLMENT_INDEX_TYPE
index = get_default_alias(index_type)
search_obj = Search(index=index)
# Update from search params first so our server-side filtering will overwrite it if necessary
if search_param_dict is not None:
search_obj.update_from_dict(search_param_dict)
if not is_advance_search_capable:
# Learners can't search for other learners with privacy set to private
search_obj = search_obj.filter(
~Q('term', **{'profile.account_privacy': Profile.PRIVATE}) # pylint: disable=invalid-unary-operand-type
)
# Limit results to one of the programs the user is staff on
search_obj = search_obj.filter(create_program_limit_query(
user,
staff_program_ids,
filter_on_email_optin=filter_on_email_optin
))
# Filter so that only filled_out profiles are seen
search_obj = search_obj.filter(
Q('term', **{'profile.filled_out': True})
)
# Force size to be the one we set on the server
update_dict = {'size': settings.ELASTICSEARCH_DEFAULT_PAGE_SIZE}
if search_param_dict is not None and search_param_dict.get('from') is not None:
update_dict['from'] = search_param_dict['from']
search_obj.update_from_dict(update_dict)
return search_obj
def test_create_search_obj_metadata(self, is_advance_search_capable,
expected_index_type):
"""
Test that Search objects are created with proper metadata
"""
user = self.user if is_advance_search_capable else self.learner
search_param_dict = {'size': 50}
with patch('search.api.Search.update_from_dict',
autospec=True) as mock_update_from_dict:
search_obj = create_search_obj(
user,
search_param_dict=search_param_dict,
)
expected_alias = get_default_alias(expected_index_type)
assert search_obj._index == [expected_alias] # pylint: disable=protected-access
assert mock_update_from_dict.call_count == 2
assert isinstance(mock_update_from_dict.call_args[0][0], Search)
assert mock_update_from_dict.call_args[0][1] == search_param_dict
def test_get_aliases(self, is_reindex, index_type, expected_indices):
"""
We should choose the correct alias and doc type given the circumstances
"""
conn = get_conn(verify=False)
alias = make_alias_name(index_type, is_reindexing=False)
backing_index = make_backing_index_name()
# Skip the mapping because it's invalid for 2.x schema, and we don't need it here
clear_and_create_index(backing_index, index_type=index_type, skip_mapping=True)
conn.indices.put_alias(index=backing_index, name=alias)
if is_reindex:
conn.indices.put_alias(index=backing_index, name=make_alias_name(index_type, is_reindexing=True))
aliases = get_aliases(index_type)
assert aliases == list(expected_indices)
assert get_default_alias(index_type) == aliases[0]
def _search_percolate_queries(program_enrollment):
"""
Find all PercolateQuery ids whose queries match a user document
Args:
program_enrollment (ProgramEnrollment): A ProgramEnrollment
Returns:
list of int: A list of PercolateQuery ids
"""
conn = get_conn()
percolate_index = get_default_alias(PERCOLATE_INDEX_TYPE)
doc = serialize_program_enrolled_user(program_enrollment)
if not doc:
return []
# We don't need this to search for percolator queries and
# it causes a dynamic mapping failure so we need to remove it
del doc['_id']
result = conn.percolate(percolate_index, GLOBAL_DOC_TYPE, body={"doc": doc})
failures = result.get('_shards', {}).get('failures', [])
if len(failures) > 0:
raise PercolateException("Failed to percolate: {}".format(failures))
return [int(row['_id']) for row in result['matches']]
def test_get_aliases(self, is_reindex, index_type, expected_indices):
"""
We should choose the correct alias and doc type given the circumstances
"""
conn = get_conn(verify=False)
alias = make_alias_name(index_type, is_reindexing=False)
backing_index = make_backing_index_name()
# Skip the mapping because it's invalid for 2.x schema, and we don't need it here
clear_and_create_index(backing_index,
index_type=index_type,
skip_mapping=True)
conn.indices.put_alias(index=backing_index, name=alias)
if is_reindex:
conn.indices.put_alias(index=backing_index,
name=make_alias_name(index_type,
is_reindexing=True))
aliases = get_aliases(index_type)
assert aliases == list(expected_indices)
assert get_default_alias(index_type) == aliases[0]
def get_mappings(self, index_type):
"""Gets mapping data"""
alias = get_default_alias(index_type)
refresh_index(alias)
mapping = self.conn.indices.get_mapping(index=alias)
return list(mapping.values())[0]['mappings']
def get_mappings(self, index_type):
"""Gets mapping data"""
alias = get_default_alias(index_type)
refresh_index(alias)
mapping = self.conn.indices.get_mapping(index=alias)
return list(mapping.values())[0]['mappings']
def get_percolate_query(self, _id):
"""Get percolate query"""
index = get_default_alias(PERCOLATE_INDEX_TYPE)
return self.conn.get(id=_id, index=index)
def search(self, index_type):
"""Gets full index data from the _search endpoint"""
alias = get_default_alias(index_type)
refresh_index(alias)
return self.conn.search(index=alias)['hits']
def get_default_backing_index(self, index_type):
"""Get the default backing index"""
alias = get_default_alias(index_type)
return list(self.conn.indices.get_alias(name=alias).keys())[0]
def get_default_backing_index(self, index_type):
"""Get the default backing index"""
alias = get_default_alias(index_type)
return list(self.conn.indices.get_alias(name=alias).keys())[0]
def get_percolate_query(self, _id):
"""Get percolate query"""
index = get_default_alias(PERCOLATE_INDEX_TYPE)
return self.conn.get(id=_id, doc_type=GLOBAL_DOC_TYPE, index=index)
def search(self, index_type):
"""Gets full index data from the _search endpoint"""
alias = get_default_alias(index_type)
refresh_index(alias)
return self.conn.search(index=alias)['hits']