def reindex(session, es, request):
"""Reindex all annotations into a new index, and update the alias."""
if get_aliased_index(es) is None:
raise RuntimeError('cannot reindex if current index is not aliased')
settings = request.find_service(name='settings')
new_index = configure_index(es)
try:
settings.put(SETTING_NEW_INDEX, new_index)
request.tm.commit()
indexer = BatchIndexer(session,
es,
request,
target_index=new_index,
op_type='create')
errored = indexer.index()
if errored:
log.debug('failed to index {} annotations, retrying...'.format(
len(errored)))
errored = indexer.index(errored)
if errored:
log.warn('failed to index {} annotations: {!r}'.format(
len(errored), errored))
update_aliased_index(es, new_index)
finally:
settings.delete(SETTING_NEW_INDEX)
request.tm.commit()
def test_raises_if_called_for_concrete_index(self, client):
"""Raise if called for a concrete index."""
client.conn.indices.get_alias.side_effect = (
elasticsearch.exceptions.NotFoundError('test', 'test desc'))
with pytest.raises(RuntimeError):
update_aliased_index(client, 'new-target')
def test_updates_index_atomically(self, client):
"""Update the alias atomically."""
client.conn.indices.get_alias.return_value = {
"old-target": {
"aliases": {
"foo": {}
}
}
}
update_aliased_index(client, "new-target")
client.conn.indices.update_aliases.assert_called_once_with(
body={
"actions": [
{
"add": {
"index": "new-target",
"alias": "foo"
}
},
{
"remove": {
"index": "old-target",
"alias": "foo"
}
},
]
})
def test_updates_index_atomically(self, client):
"""Update the alias atomically."""
client.conn.indices.get_alias.return_value = {
'old-target': {
'aliases': {
'foo': {}
}
},
}
update_aliased_index(client, 'new-target')
client.conn.indices.update_aliases.assert_called_once_with(
body={
'actions': [
{
'add': {
'index': 'new-target',
'alias': 'foo'
}
},
{
'remove': {
'index': 'old-target',
'alias': 'foo'
}
},
],
})
def test_raises_if_called_for_concrete_index(self, client):
"""Raise if called for a concrete index."""
client.conn.indices.get_alias.side_effect = (
elasticsearch1.exceptions.NotFoundError('test', 'test desc'))
with pytest.raises(RuntimeError):
update_aliased_index(client, 'new-target')
def test_raises_if_called_for_concrete_index(self, mock_es_client):
"""Raise if called for a concrete index."""
mock_es_client.conn.indices.get_alias.side_effect = (
elasticsearch.exceptions.NotFoundError("test", "test desc")
)
with pytest.raises(RuntimeError):
update_aliased_index(mock_es_client, "new-target")
def test_updates_index_atomically(self, client):
"""Update the alias atomically."""
client.conn.indices.get_alias.return_value = {
'old-target': {'aliases': {'foo': {}}},
}
update_aliased_index(client, 'new-target')
client.conn.indices.update_aliases.assert_called_once_with(body={
'actions': [
{'add': {'index': 'new-target', 'alias': 'foo'}},
{'remove': {'index': 'old-target', 'alias': 'foo'}},
],
})
def test_updates_index_atomically(self, client):
"""Update the alias atomically."""
client.conn.indices.get_alias.return_value = {
"old-target": {"aliases": {"foo": {}}}
}
update_aliased_index(client, "new-target")
client.conn.indices.update_aliases.assert_called_once_with(
body={
"actions": [
{"add": {"index": "new-target", "alias": "foo"}},
{"remove": {"index": "old-target", "alias": "foo"}},
]
}
)
def reindex(session, es, request):
"""Reindex all annotations into a new index, and update the alias."""
current_index = get_aliased_index(es)
if current_index is None:
raise RuntimeError('cannot reindex if current index is not aliased')
settings = request.find_service(name='settings')
# Preload userids of shadowbanned users.
nipsa_svc = request.find_service(name='nipsa')
nipsa_svc.fetch_all_flagged_userids()
new_index = configure_index(es)
log.info('configured new index {}'.format(new_index))
setting_name = 'reindex.new_index'
if es.version < (2, ):
setting_name = 'reindex.new_index'
try:
settings.put(setting_name, new_index)
request.tm.commit()
log.info('reindexing annotations into new index {}'.format(new_index))
indexer = BatchIndexer(session,
es,
request,
target_index=new_index,
op_type='create')
errored = indexer.index()
if errored:
log.debug('failed to index {} annotations, retrying...'.format(
len(errored)))
errored = indexer.index(errored)
if errored:
log.warn('failed to index {} annotations: {!r}'.format(
len(errored), errored))
log.info('making new index {} current'.format(new_index))
update_aliased_index(es, new_index)
log.info('removing previous index {}'.format(current_index))
delete_index(es, current_index)
finally:
settings.delete(setting_name)
request.tm.commit()
def reindex(session, es, request):
"""Reindex all annotations into a new index, and update the alias."""
current_index = get_aliased_index(es)
if current_index is None:
raise RuntimeError("cannot reindex if current index is not aliased")
settings = request.find_service(name="settings")
# Preload userids of shadowbanned users.
nipsa_svc = request.find_service(name="nipsa")
nipsa_svc.fetch_all_flagged_userids()
new_index = configure_index(es)
log.info("configured new index {}".format(new_index))
setting_name = "reindex.new_index"
try:
settings.put(setting_name, new_index)
request.tm.commit()
log.info("reindexing annotations into new index {}".format(new_index))
indexer = BatchIndexer(
session, es, request, target_index=new_index, op_type="create"
)
errored = indexer.index()
if errored:
log.debug(
"failed to index {} annotations, retrying...".format(len(errored))
)
errored = indexer.index(errored)
if errored:
log.warning(
"failed to index {} annotations: {!r}".format(len(errored), errored)
)
log.info("making new index {} current".format(new_index))
update_aliased_index(es, new_index)
log.info("removing previous index {}".format(current_index))
delete_index(es, current_index)
finally:
settings.delete(setting_name)
request.tm.commit()
def reindex(session, es, request):
"""Reindex all annotations into a new index, and update the alias."""
current_index = get_aliased_index(es)
if current_index is None:
raise RuntimeError("cannot reindex if current index is not aliased")
settings = request.find_service(name="settings")
# Preload userids of shadowbanned users.
nipsa_svc = request.find_service(name="nipsa")
nipsa_svc.fetch_all_flagged_userids()
new_index = configure_index(es)
log.info("configured new index %s", new_index)
setting_name = "reindex.new_index"
try:
settings.put(setting_name, new_index)
request.tm.commit()
log.info("reindexing annotations into new index %s", new_index)
indexer = BatchIndexer(
session, es, request, target_index=new_index, op_type="create"
)
errored = indexer.index()
if errored:
log.debug("failed to index %d annotations, retrying...", len(errored))
errored = indexer.index(errored)
if errored:
log.warning("failed to index %d annotations: %r", len(errored), errored)
log.info("making new index %s current", new_index)
update_aliased_index(es, new_index)
log.info("removing previous index %s", current_index)
delete_index(es, current_index)
finally:
settings.delete(setting_name)
request.tm.commit()
