def test_sets_correct_mappings_and_settings(self, client):
configure_index(client)
client.conn.indices.create.assert_called_once_with(
mock.ANY,
body={
'mappings': {'annotation': ANNOTATION_MAPPING},
'settings': {'analysis': ANALYSIS_SETTINGS},
})
def test_sets_correct_mappings_and_settings(self, client):
configure_index(client)
client.conn.indices.create.assert_called_once_with(
mock.ANY,
body={
'mappings': {'annotation': ANNOTATION_MAPPING},
'settings': {'analysis': ANALYSIS_SETTINGS},
})
def test_sets_correct_mappings_and_settings(self, mock_es_client):
configure_index(mock_es_client)
mock_es_client.conn.indices.create.assert_called_once_with(
Any(),
body={
"mappings": {"annotation": ANNOTATION_MAPPING},
"settings": {"analysis": ANALYSIS_SETTINGS},
},
)
def test_sets_correct_mappings_and_settings_for_es7(self, mock_es_client):
mock_es_client.mapping_type = "_doc"
mock_es_client.server_version = Version("7.10.0")
configure_index(mock_es_client)
mock_es_client.conn.indices.create.assert_called_once_with(
Any(),
body={
"mappings": ANNOTATION_MAPPING,
"settings": {"analysis": ANALYSIS_SETTINGS},
},
)
def reindex(session, es, request):
"""Reindex all annotations into a new index, and update the alias."""
if get_aliased_index(es) is None:
raise RuntimeError('cannot reindex if current index is not aliased')
settings = request.find_service(name='settings')
new_index = configure_index(es)
try:
settings.put(SETTING_NEW_INDEX, new_index)
request.tm.commit()
indexer = BatchIndexer(session,
es,
request,
target_index=new_index,
op_type='create')
errored = indexer.index()
if errored:
log.debug('failed to index {} annotations, retrying...'.format(
len(errored)))
errored = indexer.index(errored)
if errored:
log.warn('failed to index {} annotations: {!r}'.format(
len(errored), errored))
update_aliased_index(es, new_index)
finally:
settings.delete(SETTING_NEW_INDEX)
request.tm.commit()
def reindex(session, es, request):
"""Reindex all annotations into a new index, and update the alias."""
current_index = get_aliased_index(es)
if current_index is None:
raise RuntimeError('cannot reindex if current index is not aliased')
settings = request.find_service(name='settings')
# Preload userids of shadowbanned users.
nipsa_svc = request.find_service(name='nipsa')
nipsa_svc.fetch_all_flagged_userids()
new_index = configure_index(es)
log.info('configured new index {}'.format(new_index))
setting_name = 'reindex.new_index'
if es.version < (2, ):
setting_name = 'reindex.new_index'
try:
settings.put(setting_name, new_index)
request.tm.commit()
log.info('reindexing annotations into new index {}'.format(new_index))
indexer = BatchIndexer(session,
es,
request,
target_index=new_index,
op_type='create')
errored = indexer.index()
if errored:
log.debug('failed to index {} annotations, retrying...'.format(
len(errored)))
errored = indexer.index(errored)
if errored:
log.warn('failed to index {} annotations: {!r}'.format(
len(errored), errored))
log.info('making new index {} current'.format(new_index))
update_aliased_index(es, new_index)
log.info('removing previous index {}'.format(current_index))
delete_index(es, current_index)
finally:
settings.delete(setting_name)
request.tm.commit()
def reindex(session, es, request):
"""Reindex all annotations into a new index, and update the alias."""
current_index = get_aliased_index(es)
if current_index is None:
raise RuntimeError("cannot reindex if current index is not aliased")
settings = request.find_service(name="settings")
# Preload userids of shadowbanned users.
nipsa_svc = request.find_service(name="nipsa")
nipsa_svc.fetch_all_flagged_userids()
new_index = configure_index(es)
log.info("configured new index {}".format(new_index))
setting_name = "reindex.new_index"
try:
settings.put(setting_name, new_index)
request.tm.commit()
log.info("reindexing annotations into new index {}".format(new_index))
indexer = BatchIndexer(
session, es, request, target_index=new_index, op_type="create"
)
errored = indexer.index()
if errored:
log.debug(
"failed to index {} annotations, retrying...".format(len(errored))
)
errored = indexer.index(errored)
if errored:
log.warning(
"failed to index {} annotations: {!r}".format(len(errored), errored)
)
log.info("making new index {} current".format(new_index))
update_aliased_index(es, new_index)
log.info("removing previous index {}".format(current_index))
delete_index(es, current_index)
finally:
settings.delete(setting_name)
request.tm.commit()
def reindex(session, es, request):
"""Reindex all annotations into a new index, and update the alias."""
current_index = get_aliased_index(es)
if current_index is None:
raise RuntimeError("cannot reindex if current index is not aliased")
settings = request.find_service(name="settings")
# Preload userids of shadowbanned users.
nipsa_svc = request.find_service(name="nipsa")
nipsa_svc.fetch_all_flagged_userids()
new_index = configure_index(es)
log.info("configured new index %s", new_index)
setting_name = "reindex.new_index"
try:
settings.put(setting_name, new_index)
request.tm.commit()
log.info("reindexing annotations into new index %s", new_index)
indexer = BatchIndexer(
session, es, request, target_index=new_index, op_type="create"
)
errored = indexer.index()
if errored:
log.debug("failed to index %d annotations, retrying...", len(errored))
errored = indexer.index(errored)
if errored:
log.warning("failed to index %d annotations: %r", len(errored), errored)
log.info("making new index %s current", new_index)
update_aliased_index(es, new_index)
log.info("removing previous index %s", current_index)
delete_index(es, current_index)
finally:
settings.delete(setting_name)
request.tm.commit()
def test_returns_index_name(self, client, matchers):
name = configure_index(client)
assert name == matchers.Regex('foo-[0-9a-f]{8}')
def test_creates_randomly_named_index(self, client, matchers):
configure_index(client)
client.conn.indices.create.assert_called_once_with(
matchers.Regex('foo-[0-9a-f]{8}'),
body=mock.ANY)
def test_returns_index_name(self, client):
name = configure_index(client)
assert name == Any.string.matching("foo-[0-9a-f]{8}")
def test_creates_randomly_named_index(self, client):
configure_index(client)
client.conn.indices.create.assert_called_once_with(
Any.string.matching("foo-[0-9a-f]{8}"), body=Any())
def test_returns_index_name(self, client, matchers):
name = configure_index(client)
assert name == matchers.Regex('foo-[0-9a-f]{8}')
def test_creates_randomly_named_index(self, client, matchers):
configure_index(client)
client.conn.indices.create.assert_called_once_with(
matchers.Regex('foo-[0-9a-f]{8}'),
body=mock.ANY)
def test_returns_index_name(self, mock_es_client):
name = configure_index(mock_es_client)
assert name == Any.string.matching(mock_es_client.index + "-[0-9a-f]{8}")