def test_reindex(self):
src = 'sl-old'
dst = 'sl-new'
single = ['OS::Neutron::Net']
mult = ['OS::Neutron::Net', 'OS::Nova::Server', 'OS::Glance::Image']
reindex_name = 'searchlight.elasticsearch.plugins.utils.helper_reindex'
expected_single = {
'query': {
'bool': {
'filter': {
'terms': {
'_type': single
}
}
}
},
'version': 'true'
}
expected_mult = {
'query': {
'bool': {
'filter': {
'terms': {
'_type': mult
}
}
}
},
'version': 'true'
}
# Set up the ES mock.
mock_engine = mock.Mock()
with mock.patch('searchlight.elasticsearch.get_api') as mock_api:
with mock.patch(reindex_name) as mock_reindex:
# Plug in the ES mock.
mock_api.return_value = mock_engine
# Test #1: Reindex a single index.
plugin_utils.reindex(src, dst, single)
mock_api.assert_called_with()
mock_reindex.assert_called_with(client=mock_engine,
source_index=src,
target_index=dst,
query=expected_single)
# Test #2: Reindex multiple indexes.
plugin_utils.reindex(src, dst, mult)
mock_reindex.assert_called_with(client=mock_engine,
source_index=src,
target_index=dst,
query=expected_mult)
def test_reindex_with_plugin_and_es(self):
"""Verify the reindexing functionality using both the plugin reindex
and the elasticsearch reindex methods for the reindexing. We want to
verify: the number of documents during reindex, the number of aliases
during the reindex, the number of documents after the reindex and the
aliases after the reindex.
"""
alias_listener = self.role_plugin.alias_name_listener
alias_search = self.role_plugin.alias_name_search
resource_group = self.role_plugin.resource_group_name
non_role_doc_type = self.non_role_plugin.document_type
# Create a set of documents in ElasticSearch.
self.create_es_documents(alias_listener)
self.verify_initial_state()
# Create and prepare a new index.
new_index = es_utils.create_new_index(resource_group)
self.role_plugin.prepare_index(index_name=new_index)
self.non_role_plugin.prepare_index(index_name=new_index)
# Set up the aliases.
es_utils.setup_alias(new_index, alias_search, alias_listener)
es_alias = self._get_elasticsearch_aliases([])
self.assertEqual(2, len(es_alias))
# Reindex. For role, use the plugin. For non-role use ElasticSearch.
self.role_plugin.index_initial_data()
reindex = [non_role_doc_type]
es_utils.reindex(src_index=alias_listener,
dst_index=new_index,
type_list=reindex)
self._flush_elasticsearch(alias_listener)
self.verify_reindex_state(new_index)
# Update aliases.
old_index = es_utils.alias_search_update(alias_search, new_index)
es_utils.delete_index(old_index)
self._flush_elasticsearch(alias_listener)
self.verify_new_alias_state(new_index=new_index,
alias_search=alias_search,
alias_listener=alias_listener)
def test_reindex_with_plugin_and_es(self):
"""Verify the reindexing functionality using both the plugin reindex
and the elasticsearch reindex methods for the reindexing. We want to
verify: the number of documents during reindex, the number of aliases
during the reindex, the number of documents after the reindex and the
aliases after the reindex.
"""
alias_listener = self.role_plugin.alias_name_listener
alias_search = self.role_plugin.alias_name_search
resource_group = self.role_plugin.resource_group_name
non_role_doc_type = self.non_role_plugin.document_type
# Create a set of documents in ElasticSearch.
self.create_es_documents(alias_listener)
self.verify_initial_state()
# Create and prepare a new index.
new_index = es_utils.create_new_index(resource_group)
self.role_plugin.prepare_index(index_name=new_index)
self.non_role_plugin.prepare_index(index_name=new_index)
# Set up the aliases.
es_utils.setup_alias(new_index, alias_search, alias_listener)
es_alias = self._get_elasticsearch_aliases([])
self.assertEqual(2, len(es_alias))
# Reindex. For role, use the plugin. For non-role use ElasticSearch.
self.role_plugin.initial_indexing()
reindex = [non_role_doc_type]
es_utils.reindex(src_index=alias_listener, dst_index=new_index,
type_list=reindex)
self._flush_elasticsearch(alias_listener)
self.verify_reindex_state(new_index)
# Update aliases.
old_index = es_utils.alias_search_update(alias_search, new_index)
es_utils.delete_index(old_index)
self._flush_elasticsearch(alias_listener)
self.verify_new_alias_state(new_index=new_index,
alias_search=alias_search,
alias_listener=alias_listener)
def test_reindex(self):
src = 'sl-old'
dst = 'sl-new'
single = ['OS::Neutron::Net']
mult = ['OS::Neutron::Net', 'OS::Nova::Server', 'OS::Glance::Image']
reindex_name = 'searchlight.elasticsearch.plugins.utils.helper_reindex'
expected_single = {'query': {
'filtered': {
'filter': {
'terms': {
'_type': single}
}
}}, 'version': 'true'}
expected_mult = {'query': {
'filtered': {
'filter': {
'terms': {
'_type': mult}
}
}}, 'version': 'true'}
# Set up the ES mock.
mock_engine = mock.Mock()
with mock.patch('searchlight.elasticsearch.get_api') as mock_api:
with mock.patch(reindex_name) as mock_reindex:
# Plug in the ES mock.
mock_api.return_value = mock_engine
# Test #1: Reindex a single index.
plugin_utils.reindex(src, dst, single)
mock_api.assert_called_with()
mock_reindex.assert_called_with(client=mock_engine,
source_index=src,
target_index=dst,
query=expected_single)
# Test #2: Reindex multiple indexes.
plugin_utils.reindex(src, dst, mult)
mock_reindex.assert_called_with(client=mock_engine,
source_index=src,
target_index=dst,
query=expected_mult)
def _es_reindex_worker(self, es_reindex, resource_groups, index_names):
"""Helper to re-index using the ES reindex helper, allowing all ES
re-indexes to occur simultaneously. We may need to cleanup. See
sig_handler() for more info.
"""
for group in six.iterkeys(index_names):
# Grab the correct tuple as a list, convert list to a
# single tuple, extract second member (the search
# alias) of tuple.
plugins_reindex = [
doc_type for doc_type, plugin in es_reindex.items()
if plugin.resource_group_name == group
]
alias_search = \
[a for a in resource_groups if a[0] == group][0][1]
LOG.info(
"ES Reindex start from %(src)s to %(dst)s "
"for types %(types)s" % {
'src': alias_search,
'dst': index_names[group],
'types': ', '.join(plugins_reindex)
})
dst_index = index_names[group]
try:
es_utils.reindex(src_index=alias_search,
dst_index=dst_index,
type_list=plugins_reindex)
es_utils.refresh_index(dst_index)
LOG.info(
"ES Reindex end from %(src)s to %(dst)s "
"for types %(types)s" % {
'src': alias_search,
'dst': index_names[group],
'types': ', '.join(plugins_reindex)
})
except Exception as e:
LOG.exception("Failed to setup index extension "
"%(ex)s: %(e)s" % {
'ex': dst_index,
'e': e
})
raise
def sync(self, group=None, _type=None, force=False):
# Verify all indices and types have registered plugins.
# index and _type are lists because of nargs='*'
group = group.split(',') if group else []
_type = _type.split(',') if _type else []
group_set = set(group)
type_set = set(_type)
"""
The caller can specify a sync based on either the Document Type or the
Resource Group. With the Zero Downtime functionality, we are using
aliases to index into ElasticSearch. We now have multiple Document
Types sharing a single alias. If any member of a Resource Group (an
ES alias) is re-syncing *all* members of that Resoruce Group needs
to re-sync.
The final list of plugins to use for re-syncing *must* come only from
the Resource Group specifications. The "type" list is used only to make
the "group" list complete. We need a two pass algorithm for this.
First pass: Analyze the plugins according to the "type" list. This
turns a type in the "type" list to a group in the "group" list.
Second pass: Analyze the plugins according to the "group" list. Create
the plugin list that will be used for re-syncing.
Note: We cannot call any plugin's sync() during these two passes. The
sync needs to be a separate step. The API states that if any invalid
plugin was specified by the caller, the entire operation fails.
"""
# First Pass: Document Types.
if _type:
for res_type, ext in six.iteritems(utils.get_search_plugins()):
plugin_obj = ext.obj
type_set.discard(plugin_obj.get_document_type())
if plugin_obj.get_document_type() in _type:
group.append(plugin_obj.resource_group_name)
# Second Pass: Resource Groups (including those from types).
# This pass is a little tricky. If "group" is empty, it implies every
# resource gets re-synced. The command group_set.discard() is a no-op
# when "group" is empty.
resource_groups = []
plugin_objs = {}
plugins_list = []
for res_type, ext in six.iteritems(utils.get_search_plugins()):
plugin_obj = ext.obj
group_set.discard(plugin_obj.resource_group_name)
if (not group) or (plugin_obj.resource_group_name in group):
plugins_list.append((res_type, ext))
plugin_objs[plugin_obj.resource_group_name] = plugin_obj
if not (plugin_obj.resource_group_name,
plugin_obj.alias_name_search,
plugin_obj.alias_name_listener) in resource_groups:
resource_groups.append((plugin_obj.resource_group_name,
plugin_obj.alias_name_search,
plugin_obj.alias_name_listener))
if group_set or type_set:
print("Some index names or types do not have plugins "
"registered. Index names: %s. Types: %s" %
(",".join(group_set) or "<None>", ",".join(type_set)
or "<None>"))
print("Aborting.")
sys.exit(1)
if not force:
# For display purpose, we want to iterate on only parthenogenetic
# plugins that are not the children of another plugin. If there
# are children plugins they will be displayed when we call
# get_index_display_name(). Therefore any child plugins in the
# display list, will be listed twice.
display_plugins = []
for res, ext in plugins_list:
if not ext.obj.parent_plugin:
display_plugins.append((res, ext))
def format_selection(selection):
resource_type, ext = selection
return ' ' + ext.obj.get_index_display_name()
# Grab the first element in the first (and only) tuple.
group = resource_groups[0][0]
print("\nAll resource types within Resource Group \"%(group)s\""
" must be re-indexed" % {'group': group})
print("\nResource types (and aliases) matching selection:\n%s\n" %
'\n'.join(map(format_selection, sorted(display_plugins))))
ans = six.moves.input(
"Indexing will NOT delete existing data or mapping(s). It "
"will reindex all resources. \nUse '--force' to suppress "
"this message.\nOK to continue? [y/n]: ")
if ans.lower() != 'y':
print("Aborting.")
sys.exit(0)
# Start the re-indexing process
# Step #1: Create new indexes for each Resource Group Type.
# The index needs to be fully functional before it gets
# added to any aliases. This inclues all settings and
# mappings. Only then can we add it to the aliases. We first
# need to create all indexes. This is done by resource group.
# We cache and turn off new indexes' refresh intervals,
# this will improve the the performance of data re-syncing.
# After data get re-synced, set the refresh interval back.
# Once all indexes are created, we need to initialize the
# indexes. This is done by document type.
# NB: The aliases remain unchanged for this step.
index_names = {}
refresh_intervals = {}
try:
for group, search, listen in resource_groups:
index_name = es_utils.create_new_index(group)
index_names[group] = index_name
refresh_intervals[index_name] = \
es_utils.get_index_refresh_interval(index_name)
# Disable refresh interval by setting its value to -1
es_utils.set_index_refresh_interval(index_name, -1)
for resource_type, ext in plugins_list:
plugin_obj = ext.obj
group_name = plugin_obj.resource_group_name
plugin_obj.prepare_index(index_name=index_names[group_name])
except Exception:
LOG.error(
_LE("Error creating index or mapping, aborting "
"without indexing"))
es_utils.alias_error_cleanup(index_names)
raise
# Step #2: Modify new index to play well with multiple indices.
# There is a "feature" of Elasticsearch where some types of
# queries do not work across multiple indices if there are no
# mappings for the specified document types. This is an issue we
# run into with our RBAC functionality. We need to modify the new
# index to work for these cases. We will grab all document types
# from the plugins and add a mapping for them as needed to the newly
# created indices.
doc_type_info = []
for res_type, ext in six.iteritems(utils.get_search_plugins()):
doc_type_info.append(
(ext.obj.get_document_type(), ext.obj.parent_plugin_type))
for index in list(index_names.values()):
es_utils.add_extra_mappings(index_name=index,
doc_type_info=doc_type_info)
# Step #3: Set up the aliases for all Resource Type Group.
# These actions need to happen outside of the plugins. Now that
# the indexes are created and fully functional we can associate
# them with the aliases.
# NB: The indexes remain unchanged for this step.
for group, search, listen in resource_groups:
try:
es_utils.setup_alias(index_names[group], search, listen)
except Exception as e:
LOG.error(
_LE("Failed to setup alias for resource group "
"%(g)s: %(e)s") % {
'g': group,
'e': e
})
es_utils.alias_error_cleanup(index_names)
raise
# Step #4: Re-index all resource types in this Resource Type Group.
# As an optimization, if any types are explicitly requested, we
# will index them from their service APIs. The rest will be
# indexed from an existing ES index, if one exists.
# NB: The "search" and "listener" aliases remain unchanged for this
# step.
es_reindex = []
plugins_to_index = copy.copy(plugins_list)
if _type:
for resource_type, ext in plugins_list:
doc_type = ext.obj.get_document_type()
if doc_type not in _type:
es_reindex.append(doc_type)
plugins_to_index.remove((resource_type, ext))
# Call plugin API as needed.
if plugins_to_index:
for res, ext in plugins_to_index:
plugin_obj = ext.obj
gname = plugin_obj.resource_group_name
try:
plugin_obj.initial_indexing(index_name=index_names[gname])
es_utils.refresh_index(index_names[gname])
except exceptions.EndpointNotFound:
LOG.warning(
_LW("Service is not available for plugin: "
"%(ext)s") % {"ext": ext.name})
except Exception as e:
LOG.error(
_LE("Failed to setup index extension "
"%(ex)s: %(e)s") % {
'ex': ext.name,
'e': e
})
es_utils.alias_error_cleanup(index_names)
raise
# Call ElasticSearch for the rest, if needed.
if es_reindex:
for group in six.iterkeys(index_names):
# Grab the correct tuple as a list, convert list to a single
# tuple, extract second member (the search alias) of tuple.
alias_search = \
[a for a in resource_groups if a[0] == group][0][1]
try:
es_utils.reindex(src_index=alias_search,
dst_index=index_names[group],
type_list=es_reindex)
es_utils.refresh_index(index_names[group])
except Exception as e:
LOG.error(
_LE("Failed to setup index extension "
"%(ex)s: %(e)s") % {
'ex': ext.name,
'e': e
})
es_utils.alias_error_cleanup(index_names)
raise
# Step #5: Update the "search" alias.
# All re-indexing has occurred. The index/alias is the same for
# all resource types within this Resource Group. These actions need
# to happen outside of the plugins. Also restore refresh interval
# for indexes, this will make data in the indexes become searchable.
# NB: The "listener" alias remains unchanged for this step.
for index_name, interval in refresh_intervals.items():
es_utils.set_index_refresh_interval(index_name, interval)
old_index = {}
for group, search, listen in resource_groups:
old_index[group] = \
es_utils.alias_search_update(search, index_names[group])
# Step #6: Update the "listener" alias.
# The "search" alias has been updated. This involves both removing
# the old index from the alias as well as deleting the old index.
# These actions need to happen outside of the plugins.
# NB: The "search" alias remains unchanged for this step.
for group, search, listen in resource_groups:
try:
# If any exception raises, ignore and continue to delete
# any other old indexes.
es_utils.delete_index(old_index[group])
except Exception as e:
LOG.error(encodeutils.exception_to_unicode(e))