def test_index_manager_regenerate_indices_from_broken_state(self, *args):
"""
`regenerate_indices` should succeed and give us a working ElasticSearch
when it runs and finds a broken state (eg. with an existing, incorrect
index with the name of an alias).
This can occur when ES restarts and an update signal is triggered before
Richie had a chance to bootstrap ES.
"""
# Create a course and trigger a signal to index it. This will create a
# broken "richie_test_courses" index
course = CourseFactory(should_publish=True)
apply_es_action_to_course(course.extended_object, "index", "en")
self.assertIsNotNone(ES_INDICES_CLIENT.get("richie_test_courses"))
# Call our `regenerate_indices command`
creation_datetime = datetime(2010, 1, 1, tzinfo=timezone.utc)
creation_string = creation_datetime.strftime("%Y-%m-%d-%Hh%Mm%S.%fs")
with mock.patch.object(timezone, "now",
return_value=creation_datetime):
regenerate_indices(None)
# No error was thrown, the courses index (like all others) was bootstrapped
self.assertIsNotNone(
ES_INDICES_CLIENT.get(f"richie_test_courses_{creation_string}"))
# The expected alias is associated with the index
self.assertEqual(
list(ES_INDICES_CLIENT.get_alias("richie_test_courses").keys())[0],
f"richie_test_courses_{creation_string}",
)
def test_index_manager_regenerate_indices(self, *args):
"""
Make sure indices are created, aliases updated and old, no longer useful indices
are pruned when the `regenerate_elasticsearch` function is called.
"""
# Create an unrelated index with an alias to make sure it is unaffected by our operations
ES_INDICES_CLIENT.create(index="unrelated_index")
ES_INDICES_CLIENT.put_alias(index="unrelated_index",
name="unrelated_index_alias")
self.assertIsNotNone(
ES_INDICES_CLIENT.get("unrelated_index")["unrelated_index"])
self.assertEqual(
list(ES_INDICES_CLIENT.get_alias("unrelated_index_alias").keys())
[0],
"unrelated_index",
)
# Create all our indices from scratch
# Use a mocked timezone.now to check the names of our indices as they include a datetime
creation1_datetime = datetime(2010, 1, 1, tzinfo=timezone.utc)
creation1_string = creation1_datetime.strftime("%Y-%m-%d-%Hh%Mm%S.%fs")
with mock.patch.object(timezone,
"now",
return_value=creation1_datetime):
regenerate_indices(None)
expected_indices = [
"richie_test_categories",
"richie_test_courses",
"richie_test_organizations",
"richie_test_persons",
]
# All indices were created and properly aliased
for alias_name in expected_indices:
new_index_name = f"{alias_name}_{creation1_string}"
# The index is created
self.assertIsNotNone(
ES_INDICES_CLIENT.get(new_index_name)[new_index_name])
# The expected alias is associated with the index
self.assertEqual(
list(ES_INDICES_CLIENT.get_alias(alias_name).keys())[0],
new_index_name)
# Now regenerate the indices, replacing the ones we just created
creation2_datetime = datetime(2011, 2, 2, tzinfo=timezone.utc)
creation2_string = creation2_datetime.strftime("%Y-%m-%d-%Hh%Mm%S.%fs")
with mock.patch.object(timezone,
"now",
return_value=creation2_datetime):
regenerate_indices(None)
# All indices were replaced and aliases updated
for alias_name in expected_indices:
# The index is created
new_index_name = f"{alias_name}_{creation2_string}"
self.assertIsNotNone(
ES_INDICES_CLIENT.get(new_index_name)[new_index_name])
# The expected alias is associated with the new index
self.assertEqual(
list(ES_INDICES_CLIENT.get_alias(alias_name).keys())[0],
new_index_name)
# The previous version of the index is still around
creation1_index_name = f"{alias_name}_{creation1_string}"
self.assertIsNotNone(
ES_INDICES_CLIENT.get(creation1_index_name)
[creation1_index_name])
# But not aliased any more
self.assertEqual(
ES_INDICES_CLIENT.get(creation1_index_name)
[creation1_index_name]["aliases"],
{},
)
# Regenerate indices again to make sure versions n-2 of indices are
# deleted (not just unaliased)
creation3_datetime = datetime(2012, 3, 3, tzinfo=timezone.utc)
creation3_string = creation3_datetime.strftime("%Y-%m-%d-%Hh%Mm%S.%fs")
with mock.patch.object(timezone,
"now",
return_value=creation3_datetime):
regenerate_indices(None)
# All indices were replaced and had their aliases changed
for index_name in expected_indices:
new_index_name = f"{index_name}_{creation3_string}"
# The index is created
self.assertIsNotNone(
ES_INDICES_CLIENT.get(new_index_name)[new_index_name])
# The expected alias is associated with the new index
self.assertEqual(
list(ES_INDICES_CLIENT.get_alias(index_name).keys())[0],
new_index_name)
# The previous version of the index is still around
creation2_index_name = f"{alias_name}_{creation2_string}"
self.assertIsNotNone(
ES_INDICES_CLIENT.get(creation2_index_name)
[creation2_index_name])
# But not aliased any more
self.assertEqual(
ES_INDICES_CLIENT.get(creation2_index_name)
[creation2_index_name]["aliases"],
{},
)
# Version n-2 of the index does not exist any more
with self.assertRaises(NotFoundError):
ES_INDICES_CLIENT.get(f"{index_name}_{creation1_string}")
# Make sure our unrelated index was unaffected through regenerations
self.assertIsNotNone(
ES_INDICES_CLIENT.get("unrelated_index")["unrelated_index"])
self.assertEqual(
list(ES_INDICES_CLIENT.get_alias("unrelated_index_alias").keys())
[0],
"unrelated_index",
)
def test_index_manager_perform_create_index(self):
"""
Perform all side-effects through the ES client and return the index name (incl. timestamp)
"""
# Create an indexable from scratch that mimicks the expected shape of the dynamic
# import in es_index
class IndexableClass:
"""Indexable stub"""
index_name = "richie_courses"
mapping = {
"properties": {
"code": {
"type": "keyword"
},
"name": {
"type": "text"
}
}
}
# pylint: disable=no-self-use
def get_es_documents(self, index, action="index"):
"""Stub method"""
for i in range(0, 10):
yield {
"_id": i,
"_index": index,
"_op_type": action,
"code": f"course-{i:d}",
"name": f"Course Number {i:d}",
}
indexable = IndexableClass()
# Set a fake time to check the name of the index
now = datetime(2016, 5, 4, 3, 12, 33, 123456, tzinfo=pytz.utc)
# Make sure our index is empty before we call the function
self.assertEqual(ES_INDICES_CLIENT.get_alias("*"), {})
mock_logger = mock.Mock(spec=["info"])
with mock.patch.object(timezone, "now", return_value=now):
new_index = perform_create_index(indexable, mock_logger)
ES_INDICES_CLIENT.refresh()
self.assertEqual(new_index,
"richie_courses_2016-05-04-03h12m33.123456s")
self.assertEqual(ES_CLIENT.count()["count"], 10)
self.assertEqual(
ES_INDICES_CLIENT.get_mapping(),
{
"richie_courses_2016-05-04-03h12m33.123456s": {
"mappings": {
"properties": {
"code": {
"type": "keyword"
},
"name": {
"type": "text"
},
}
}
}
},
)
mock_logger.info.assert_called()