def assert_search(results, program_enrollments, *, index_type):
"""
Assert that search results match program-enrolled users
"""
assert results['total'] == len(
program_enrollments) * DOC_TYPES_PER_ENROLLMENT
sources_advanced = get_sources(results)
sorted_program_enrollments = sorted(
program_enrollments,
key=lambda program_enrollment: program_enrollment.id)
if index_type == PRIVATE_ENROLLMENT_INDEX_TYPE:
serialized = [
remove_es_keys(serialize_program_enrolled_user(program_enrollment))
for program_enrollment in sorted_program_enrollments
]
elif index_type == PUBLIC_ENROLLMENT_INDEX_TYPE:
serialized = [
remove_es_keys(
serialize_public_enrolled_user(
serialize_program_enrolled_user(program_enrollment)))
for program_enrollment in sorted_program_enrollments
]
else:
raise Exception("Unexpected index type")
assert serialized == sources_advanced
def document_needs_updating(enrollment):
"""
Get the document from elasticsearch and see if it matches what's in the database
Args:
enrollment (ProgramEnrollment): A program enrollment
Returns:
bool: True if the document needs to be updated via reindex
"""
index = get_default_alias(PRIVATE_ENROLLMENT_INDEX_TYPE)
conn = get_conn()
try:
document = conn.get(index=index,
doc_type=GLOBAL_DOC_TYPE,
id=enrollment.id)
except NotFoundError:
return True
serialized_enrollment = serialize_program_enrolled_user(enrollment)
del serialized_enrollment['_id']
source = document['_source']
if serialized_enrollment != source:
# Convert OrderedDict to dict
reserialized_enrollment = json.loads(json.dumps(serialized_enrollment))
diff = make_patch(source, reserialized_enrollment).patch
serialized_diff = json.dumps(diff, indent=" ")
log.info("Difference found for enrollment %s: %s", enrollment,
serialized_diff)
return True
return False
def _search_percolate_queries(program_enrollment):
"""
Find all PercolateQuery ids whose queries match a user document
Args:
program_enrollment (ProgramEnrollment): A ProgramEnrollment
Returns:
list of int: A list of PercolateQuery ids
"""
conn = get_conn()
percolate_index = get_default_alias(PERCOLATE_INDEX_TYPE)
doc = serialize_program_enrolled_user(program_enrollment)
if not doc:
return []
# We don't need this to search for percolator queries and
# it causes a dynamic mapping failure so we need to remove it
del doc['_id']
body = {"query": {"percolate": {"field": "query", "document": doc}}}
result = conn.search(percolate_index, GLOBAL_DOC_TYPE, body=body)
failures = result.get('_shards', {}).get('failures', [])
if len(failures) > 0:
raise PercolateException("Failed to percolate: {}".format(failures))
return [int(row['_id']) for row in result['hits']['hits']]
def document_needs_updating(enrollment):
"""
Get the document from elasticsearch and see if it matches what's in the database
Args:
enrollment (ProgramEnrollment): A program enrollment
Returns:
bool: True if the document needs to be updated via reindex
"""
index = get_default_alias(PRIVATE_ENROLLMENT_INDEX_TYPE)
conn = get_conn()
try:
document = conn.get(index=index, id=enrollment.id)
except NotFoundError:
return True
serialized_enrollment = serialize_program_enrolled_user(enrollment)
del serialized_enrollment['_id']
source = document['_source']
if serialized_enrollment != source:
# Convert OrderedDict to dict
reserialized_enrollment = json.loads(json.dumps(serialized_enrollment))
diff = make_patch(source, reserialized_enrollment).patch
serialized_diff = json.dumps(diff, indent=" ")
log.info("Difference found for enrollment %s: %s", enrollment, serialized_diff)
return True
return False
def test_program_enrolled_user_serializer(self): # pylint: disable=no-self-use
"""
Asserts the output of the serializer for program-enrolled users (ProgramEnrollments)
"""
with mute_signals(post_save):
profile = ProfileFactory.create()
EducationFactory.create(profile=profile)
EmploymentFactory.create(profile=profile)
program = ProgramFactory.create()
course = CourseFactory.create(program=program)
course_runs = [
CourseRunFactory.create(course=course) for _ in range(2)
]
for course_run in course_runs:
CachedCertificateFactory.create(user=profile.user,
course_run=course_run)
CachedEnrollmentFactory.create(user=profile.user,
course_run=course_run)
program_enrollment = ProgramEnrollment.objects.create(
user=profile.user, program=program)
assert serialize_program_enrolled_user(program_enrollment) == {
'_id': program_enrollment.id,
'id': program_enrollment.id,
'user_id': profile.user.id,
'email': profile.user.email,
'profile': ProfileSerializer().to_representation(profile),
'program': UserProgramSerializer.serialize(program_enrollment)
}
def test_public_enrolled_user_serializer(self):
"""
Asserts the output of the public serializer for program-enrolled users (ProgramEnrollments)
"""
profile = self.profile
program_enrollment = self.program_enrollment
serialized = serialize_program_enrolled_user(program_enrollment)
assert serialize_public_enrolled_user(serialized) == {
'_id': program_enrollment.id,
'id': program_enrollment.id,
'user_id': profile.user.id,
'profile': {
'first_name': profile.first_name,
'last_name': profile.last_name,
'full_name': profile.full_name,
'preferred_name': profile.preferred_name,
'romanized_first_name': profile.romanized_first_name,
'romanized_last_name': profile.romanized_last_name,
'image': '/media/{}'.format(profile.image),
'image_small': '/media/{}'.format(profile.image_small),
'image_medium': '/media/{}'.format(profile.image_medium),
'username': None, # bug in ProfileSerializer, issue #3166
'filled_out': profile.filled_out,
'account_privacy': profile.account_privacy,
'country': profile.country,
'state_or_territory': profile.state_or_territory,
'city': profile.city,
'birth_country': profile.birth_country,
'work_history': serialized['profile']['work_history'],
},
'program': {
'id':
program_enrollment.program.id,
'enrollments': [{
'course_title': enrollment['course_title'],
'semester': enrollment['semester']
} for enrollment in serialized['program']['enrollments']],
'courses': [{
'course_title': enrollment['course_title']
} for enrollment in serialized['program']['courses']],
'course_runs': [{
'semester': semester_enrolled['semester']
} for semester_enrolled in serialized['program']['course_runs']
],
'is_learner':
True,
'total_courses':
1,
}
}
def test_program_enrolled_user_serializer(self):
"""
Asserts the output of the serializer for program-enrolled users (ProgramEnrollments)
"""
profile = self.profile
program_enrollment = self.program_enrollment
assert serialize_program_enrolled_user(program_enrollment) == {
'_id': program_enrollment.id,
'id': program_enrollment.id,
'user_id': profile.user.id,
'email': profile.user.email,
'profile': filter_current_work(ProfileSerializer(profile).data),
'program': UserProgramSearchSerializer.serialize(program_enrollment)
}
def test_public_enrolled_user_serializer(self):
"""
Asserts the output of the public serializer for program-enrolled users (ProgramEnrollments)
"""
profile = self.profile
program_enrollment = self.program_enrollment
serialized = serialize_program_enrolled_user(program_enrollment)
assert serialize_public_enrolled_user(serialized) == {
'_id': program_enrollment.id,
'id': program_enrollment.id,
'user_id': profile.user.id,
'profile': {
'first_name': profile.first_name,
'last_name': profile.last_name,
'full_name': profile.full_name,
'preferred_name': profile.preferred_name,
'romanized_first_name': profile.romanized_first_name,
'romanized_last_name': profile.romanized_last_name,
'image': '/media/{}'.format(profile.image),
'image_small': '/media/{}'.format(profile.image_small),
'image_medium': '/media/{}'.format(profile.image_medium),
'username': None, # bug in ProfileSerializer, issue #3166
'filled_out': profile.filled_out,
'account_privacy': profile.account_privacy,
'country': profile.country,
'state_or_territory': profile.state_or_territory,
'city': profile.city,
'birth_country': profile.birth_country,
'work_history': serialized['profile']['work_history'],
},
'program': {
'id': program_enrollment.program.id,
'enrollments': [{
'course_title': enrollment['course_title'],
'semester': enrollment['semester']
} for enrollment in serialized['program']['enrollments']],
'courses': [{
'course_title': enrollment['course_title']
} for enrollment in serialized['program']['courses']],
'course_runs': [{
'semester': semester_enrolled['semester']
} for semester_enrolled in serialized['program']['course_runs']],
'is_learner': True,
'total_courses': 1,
}
}
def assert_search(results, program_enrollments, *, index_type):
"""
Assert that search results match program-enrolled users
"""
assert results['total'] == len(program_enrollments) * DOC_TYPES_PER_ENROLLMENT
sources_advanced = get_sources(results)
sorted_program_enrollments = sorted(program_enrollments, key=lambda program_enrollment: program_enrollment.id)
if index_type == PRIVATE_ENROLLMENT_INDEX_TYPE:
serialized = [
remove_es_keys(serialize_program_enrolled_user(program_enrollment))
for program_enrollment in sorted_program_enrollments
]
elif index_type == PUBLIC_ENROLLMENT_INDEX_TYPE:
serialized = [
remove_es_keys(serialize_public_enrolled_user(
serialize_program_enrolled_user(program_enrollment)
))
for program_enrollment in sorted_program_enrollments
]
else:
raise Exception("Unexpected index type")
assert serialized == sources_advanced
def test_program_enrolled_user_serializer(self):
"""
Asserts the output of the serializer for program-enrolled users (ProgramEnrollments)
"""
profile = self.profile
program_enrollment = self.program_enrollment
assert serialize_program_enrolled_user(program_enrollment) == {
'_id': program_enrollment.id,
'id': program_enrollment.id,
'user_id': profile.user.id,
'email': profile.user.email,
'profile': filter_current_work(ProfileSerializer(profile).data),
'program':
UserProgramSearchSerializer.serialize(program_enrollment)
}
def test_index_program_enrolled_users(self, mock_on_commit):
"""
Test that index_program_enrolled_users indexes an iterable of program-enrolled users
"""
num_enrollments = 10
chunk_size = 4
with mute_signals(post_save):
program_enrollments = [
ProgramEnrollmentFactory.create() for _ in range(num_enrollments)
]
for enrollment in program_enrollments:
ProfileFactory.create(user=enrollment.user)
private = [serialize_program_enrolled_user(enrollment) for enrollment in program_enrollments]
private_dicts = {serialized['id']: serialized for serialized in private}
public = [serialize_public_enrolled_user(serialized) for serialized in private]
public_dicts = {serialized['id']: serialized for serialized in public}
with patch(
'search.indexing_api._index_chunk', autospec=True, return_value=0
) as index_chunk, patch(
'search.indexing_api.serialize_program_enrolled_user', autospec=True,
side_effect=lambda x: private_dicts[x.id]
) as serialize_mock, patch(
'search.indexing_api.serialize_public_enrolled_user', autospec=True,
side_effect=lambda x: public_dicts[x['id']]
) as serialize_public_mock:
index_program_enrolled_users(program_enrollments, chunk_size=chunk_size)
assert index_chunk.call_count == 6 # 10 enrollments divided in chunks of 4, times the number of types (2)
public_index = make_alias_name(PUBLIC_ENROLLMENT_INDEX_TYPE, is_reindexing=False)
private_index = make_alias_name(PRIVATE_ENROLLMENT_INDEX_TYPE, is_reindexing=False)
for offset in range(0, num_enrollments, chunk_size):
# each enrollment should get yielded twice to account for each doctype
index_chunk.assert_any_call(
public[offset:offset+4], # ordered dicts FTW
index=public_index
)
index_chunk.assert_any_call(
private[offset:offset+4],
index=private_index
)
assert serialize_mock.call_count == len(program_enrollments)
assert serialize_public_mock.call_count == len(program_enrollments)
for enrollment in program_enrollments:
serialize_mock.assert_any_call(enrollment)
serialize_public_mock.assert_any_call(private_dicts[enrollment.id])
def assert_search(results, program_enrollments):
"""
Assert that search results match program-enrolled users
"""
assert results['total'] == len(program_enrollments)
sources = sorted([hit['_source'] for hit in results['hits']],
key=lambda hit: hit['id'])
sorted_program_enrollments = sorted(
program_enrollments,
key=lambda program_enrollment: program_enrollment.id)
serialized = [
dict_without_key(serialize_program_enrolled_user(program_enrollment),
"_id")
for program_enrollment in sorted_program_enrollments
]
assert serialized == sources
def _search_percolate_queries(program_enrollment):
"""
Find all PercolateQuery ids whose queries match a user document
Args:
program_enrollment (ProgramEnrollment): A ProgramEnrollment
Returns:
list of int: A list of PercolateQuery ids
"""
conn = get_conn()
percolate_index = get_default_alias(PERCOLATE_INDEX_TYPE)
doc = serialize_program_enrolled_user(program_enrollment)
if not doc:
return []
# We don't need this to search for percolator queries and
# it causes a dynamic mapping failure so we need to remove it
del doc['_id']
result = conn.percolate(percolate_index, GLOBAL_DOC_TYPE, body={"doc": doc})
failures = result.get('_shards', {}).get('failures', [])
if len(failures) > 0:
raise PercolateException("Failed to percolate: {}".format(failures))
return [int(row['_id']) for row in result['matches']]
def test_index_program_enrolled_users(self, mock_on_commit):
"""
Test that index_program_enrolled_users indexes an iterable of program-enrolled users
"""
num_enrollments = 10
chunk_size = 4
with mute_signals(post_save):
program_enrollments = [
ProgramEnrollmentFactory.create()
for _ in range(num_enrollments)
]
for enrollment in program_enrollments:
ProfileFactory.create(user=enrollment.user)
private = [
serialize_program_enrolled_user(enrollment)
for enrollment in program_enrollments
]
private_dicts = {
serialized['id']: serialized
for serialized in private
}
public = [
serialize_public_enrolled_user(serialized)
for serialized in private
]
public_dicts = {serialized['id']: serialized for serialized in public}
with patch('search.indexing_api._index_chunk',
autospec=True,
return_value=0) as index_chunk, patch(
'search.indexing_api.serialize_program_enrolled_user',
autospec=True,
side_effect=lambda x: private_dicts[x.id]
) as serialize_mock, patch(
'search.indexing_api.serialize_public_enrolled_user',
autospec=True,
side_effect=lambda x: public_dicts[x[
'id']]) as serialize_public_mock:
index_program_enrolled_users(program_enrollments,
chunk_size=chunk_size)
assert index_chunk.call_count == 6 # 10 enrollments divided in chunks of 4, times the number of types (2)
public_index = make_alias_name(PUBLIC_ENROLLMENT_INDEX_TYPE,
is_reindexing=False)
private_index = make_alias_name(PRIVATE_ENROLLMENT_INDEX_TYPE,
is_reindexing=False)
for offset in range(0, num_enrollments, chunk_size):
# each enrollment should get yielded twice to account for each doctype
index_chunk.assert_any_call(
public[offset:offset + 4], # ordered dicts FTW
index=public_index)
index_chunk.assert_any_call(private[offset:offset + 4],
index=private_index)
assert serialize_mock.call_count == len(program_enrollments)
assert serialize_public_mock.call_count == len(program_enrollments)
for enrollment in program_enrollments:
serialize_mock.assert_any_call(enrollment)
serialize_public_mock.assert_any_call(
private_dicts[enrollment.id])