def reindex(src, dest):
from elasticsearch import helpers
from scrapi.processing.elasticsearch import DatabaseManager
dm = DatabaseManager()
dm.setup()
helpers.reindex(dm.es, src, dest)
dm.es.indices.delete(src)
def reindex(self):
elastic_client = Elasticsearch([{
"host": self.__host,
"port": self.__port
}])
index_client = IndicesClient(elastic_client)
# Create new index with necessory fields mapping
# , master_timeout=10, timeout=10
index_client.create(index=self.__target_index, body=self.__body)
# reindexind data from source index to target index
helpers.reindex(client=elastic_client,
source_index=self.__source_index,
target_index=self.__target_index)
# creating alias for target index
alias = {'actions': []}
# remove_action = {"remove": {"index": self.__source_index, "alias": self.__alias}}
add_action = {
"add": {
"index": self.__target_index,
"alias": self.__alias
}
}
# alias['actions'].append(remove_action)
alias['actions'].append(add_action)
# deleteing the source index
index_client.delete(index=self.__source_index)
index_client.update_aliases(body=alias)
def run(self, index_name=None):
# if no index name is passed then use the configured one
index_name = index_name if index_name else superdesk.app.config['ELASTICSEARCH_INDEX']
print('Starting index rebuilding for index: {}'.format(index_name))
if index_name not in self._get_available_indexes():
raise Exception("Index {} is not configured".format(index_name))
try:
es = get_es(superdesk.app.config['ELASTICSEARCH_URL'])
clone_name = index_name + '-' + get_random_string()
print('Creating index: ', clone_name)
superdesk.app.data.elastic.create_index(clone_name, superdesk.app.config['ELASTICSEARCH_SETTINGS'])
real_name = superdesk.app.data.elastic.get_index_by_alias(clone_name)
print('Putting mapping for index: ', clone_name)
superdesk.app.data.elastic.put_mapping(superdesk.app, clone_name)
print('Starting index rebuilding.')
reindex(es, index_name, clone_name)
print('Finished index rebuilding.')
print('Deleting index: ', index_name)
get_indices(es).delete(index_name)
print('Creating alias: ', index_name)
get_indices(es).put_alias(index=real_name, name=index_name)
print('Alias created.')
print('Deleting clone name alias')
get_indices(es).delete_alias(name=clone_name, index=real_name)
print('Deleted clone name alias')
except elasticsearch.exceptions.NotFoundError as nfe:
print(nfe)
print('Index {0} rebuilt successfully.'.format(index_name))
def test_children_are_reindexed_correctly(self):
helpers.reindex(self.client, 'test-index', 'real-index')
q = self.client.get(index='real-index', doc_type='question', id=42)
self.assertEquals(
{
'_id': '42',
'_index': 'real-index',
'_source': {},
'_type': 'question',
'_version': 1,
'found': True
}, q)
q = self.client.get(index='test-index',
doc_type='answer',
id=47,
parent=42)
if '_routing' in q:
self.assertEquals(q.pop('_routing'), '42')
self.assertEquals(
{
'_id': '47',
'_index': 'test-index',
'_source': {
'some': 'data'
},
'_type': 'answer',
'_version': 1,
'_parent': '42',
'found': True
}, q)
def test_reindex_accepts_a_query(self):
helpers.reindex(self.client,
"test_index",
"prod_index",
query={
"query": {
"filtered": {
"filter": {
"term": {
"_type": "answers"
}
}
}
}
})
self.client.indices.refresh()
self.assertTrue(self.client.indices.exists("prod_index"))
self.assertFalse(
self.client.indices.exists_type(index='prod_index',
doc_type='questions'))
self.assertEquals(
50,
self.client.count(index='prod_index', doc_type='answers')['count'])
self.assertEquals({
"answer": 42,
"correct": True
},
self.client.get(index="prod_index",
doc_type="answers",
id=42)['_source'])
def reindex(self, idx, newindex, alias_name=None, remap_alias=None, **kwargs): # are we an alias or an actual index? index = idx; alias = None alias_exists = False if self._esc.exists_alias(idx): alias = idx index = self.get_index_for_alias(idx) alias_exists = True if alias_name: alias = alias_name # does the new index exist? if not self._esc.exists( newindex ): # if new doesn't exist then create the mapping # as a copy of the old one. The idea being that the mapping # was changed index_mapping = self.get_index_map(index=idx) # using "idx" intentionally because models will be defined as alias self._esc.create( index=newindex, body=index_mapping.get(idx)) # have to use the index name as the key to the dict even though only one is returned. .create() only takes the mapping # map our documents helpers.reindex(self._es, index, newindex, **kwargs) if remap_alias or alias_name: if alias_exists and alias != alias_name: self._esc.delete_alias(alias) self._esc.put_alias(name=alias,index=newindex)
def reindex_func(src_idx, dst_idx):
es = Elasticsearch(self.es_clienthost)
try:
helpers.reindex(es, src_idx, dst_idx)
except:
print('Reindex failed for src_idx ' + src_idx +
' into dst_idx' + dst_idx)
def test_all_documents_get_moved(self):
bulk = []
for x in range(100):
bulk.append({
"index": {
"_index": "test_index",
"_type": "answers" if x % 2 == 0 else "questions",
"_id": x
}
})
bulk.append({"answer": x, "correct": x == 42})
self.client.bulk(bulk, refresh=True)
helpers.reindex(self.client, "test_index", "prod_index")
self.client.indices.refresh()
self.assertTrue(self.client.indices.exists("prod_index"))
self.assertEquals(
50,
self.client.count(index='prod_index',
doc_type='questions')['count'])
self.assertEquals(
50,
self.client.count(index='prod_index', doc_type='answers')['count'])
self.assertEquals({
"answer": 42,
"correct": True
},
self.client.get(index="prod_index",
doc_type="answers",
id=42)['_source'])
def test_reindex_passes_kwargs_to_scan_and_bulk(self):
helpers.reindex(self.client, "test_index", "prod_index", scan_kwargs={'q': 'type:answers'}, bulk_kwargs={'refresh': True})
self.assertTrue(self.client.indices.exists("prod_index"))
self.assertEquals(50, self.client.count(index='prod_index', q='type:answers')['count'])
self.assertEquals({"answer": 42, "correct": True, "type": "answers"}, self.client.get(index="prod_index", doc_type="post", id=42)['_source'])
def test_children_are_reindexed_correctly(self):
helpers.reindex(self.client, 'test-index', 'real-index')
q = self.client.get(
index='real-index',
doc_type='post',
id=42
)
self.assertEquals(
{
'_id': '42',
'_index': 'real-index',
'_source': {'question_answer': 'question'},
'_type': 'post',
'_version': 1,
'found': True
}, q
)
q = self.client.get(
index='test-index',
doc_type='post',
id=47,
routing=42
)
self.assertEquals(
{
'_routing': '42',
'_id': '47',
'_index': 'test-index',
'_source': {'some': 'data', 'question_answer': {'name': 'answer', 'parent': 42}},
'_type': 'post',
'_version': 1,
'found': True
}, q
)
def backup_index(es_source, es_target, source_index, target_index):
"""
Warning
------------------------
The existing target index is deleted.
"""
# delete old
if es_target.indices.exists(target_index):
es_target.indices.delete(target_index)
# index setup
mappings = es_source.indices.get_mapping(
index=source_index)[source_index]['mappings']
settings = es_source.indices.get_settings(
index=source_index)[source_index]['settings']
setup_index_template(es_target, target_index, settings, mappings)
if not _mappings_checking(es_source, es_target, source_index,
target_index):
raise Exception('mappings is not copied correctly')
if not _settings_checking(es_source, es_target, source_index,
target_index):
raise Exception('settings is not copied correctly')
# data
es_source.indices.refresh(source_index)
reindex(client=es_source,
target_client=es_target,
source_index=source_index,
target_index=target_index,
scroll=_SCROLL_TIME)
def reindex(args):
"""Reindex the annotations into a new Elasticsearch index."""
request = bootstrap(args)
# Configure the new index
search_config.configure_index(request.es, args.target)
# Reindex the annotations
es_helpers.reindex(client=request.es.conn,
source_index=request.es.index,
target_index=args.target)
if args.update_alias:
request.es.conn.indices.update_aliases(
body={
'actions': [
# Remove all existing aliases
{
"remove": {
"index": "*",
"alias": request.es.index
}
},
# Alias current index name to new target
{
"add": {
"index": args.target,
"alias": request.es.index
}
},
]
})
def run(self, index_name=None):
# if no index name is passed then use the configured one
index_name = index_name if index_name else superdesk.app.config['ELASTICSEARCH_INDEX']
print('Starting index rebuilding for index: {}'.format(index_name))
if index_name not in self._get_available_indexes():
raise Exception("Index {} is not configured".format(index_name))
try:
es = get_es(superdesk.app.config['ELASTICSEARCH_URL'])
clone_name = index_name + '-' + get_random_string()
print('Creating index: ', clone_name)
superdesk.app.data.elastic.create_index(clone_name, superdesk.app.config['ELASTICSEARCH_SETTINGS'])
real_name = superdesk.app.data.elastic.get_index_by_alias(clone_name)
print('Putting mapping for index: ', clone_name)
superdesk.app.data.elastic.put_mapping(superdesk.app, clone_name)
print('Starting index rebuilding.')
reindex(es, index_name, clone_name)
print('Finished index rebuilding.')
print('Deleting index: ', index_name)
get_indices(es).delete(index_name)
print('Creating alias: ', index_name)
get_indices(es).put_alias(index=real_name, name=index_name)
print('Alias created.')
print('Deleting clone name alias')
get_indices(es).delete_alias(name=clone_name, index=real_name)
print('Deleted clone name alias')
except elasticsearch.exceptions.NotFoundError as nfe:
print(nfe)
print('Index {0} rebuilt successfully.'.format(index_name))
def test_reindex_accepts_a_query(self):
helpers.reindex(
self.client,
"test_index",
"prod_index",
query={
"query": {
"bool": {
"filter": {
"term": {
"type": "answers"
}
}
}
}
},
)
self.client.indices.refresh()
self.assertTrue(self.client.indices.exists("prod_index"))
self.assertEqual(
50,
self.client.count(index="prod_index", q="type:answers")["count"])
self.assertEqual(
{
"answer": 42,
"correct": True,
"type": "answers"
},
self.client.get(index="prod_index", id=42)["_source"],
)
def test_children_are_reindexed_correctly(self):
helpers.reindex(self.client, 'test-index', 'real-index')
q = self.client.get(index='real-index', doc_type='post', id=42)
self.assertEquals(
{
'_id': '42',
'_index': 'real-index',
'_source': {
'question_answer': 'question'
},
'_type': 'post',
'_version': 1,
'found': True
}, q)
q = self.client.get(index='test-index',
doc_type='post',
id=47,
routing=42)
self.assertEquals(
{
'_routing': '42',
'_id': '47',
'_index': 'test-index',
'_source': {
'some': 'data',
'question_answer': {
'name': 'answer',
'parent': 42
}
},
'_type': 'post',
'_version': 1,
'found': True
}, q)
def clone_index(use_helper, from_index, to_index):
"""Clone an index"""
from elasticsearch_dsl import Search
from elasticsearch.helpers import reindex
click.clear()
if not es.client.indices.exists(index=to_index):
click.secho('%s not existing!'.format(to_index), fg='red')
return 1
cnt = Search(using=es.client, index=to_index).count()
message = 'Index %s already exists (%d records). Overwrite?' % (to_index,
cnt)
click.confirm(message, abort=True)
if use_helper:
reindex(client=es.client,
source_index=from_index,
target_index=to_index)
else:
es.client.reindex(body=dict(source=dict(index=from_index),
dest=dict(index=to_index)),
wait_for_completion=False)
def test_children_are_reindexed_correctly(self):
helpers.reindex(self.client, "test-index", "real-index")
q = self.client.get(index="real-index", id=42)
self.assertEqual(
{
"_id": "42",
"_index": "real-index",
"_primary_term": 1,
"_seq_no": 0,
"_source": {"question_answer": "question"},
"_version": 1,
"found": True,
},
q,
)
q = self.client.get(index="test-index", id=47, routing=42)
self.assertEqual(
{
"_routing": "42",
"_id": "47",
"_index": "test-index",
"_primary_term": 1,
"_seq_no": 1,
"_source": {
"some": "data",
"question_answer": {"name": "answer", "parent": 42},
},
"_version": 1,
"found": True,
},
q,
)
def test_children_are_reindexed_correctly(self):
helpers.reindex(self.client, "test-index", "real-index")
q = self.client.get(index="real-index", id=42)
self.assertEquals(
{
"_id": "42",
"_index": "real-index",
"_primary_term": 1,
"_seq_no": 0,
"_source": {"question_answer": "question"},
"_type": "_doc",
"_version": 1,
"found": True,
},
q,
)
q = self.client.get(index="test-index", id=47, routing=42)
self.assertEquals(
{
"_routing": "42",
"_id": "47",
"_index": "test-index",
"_primary_term": 1,
"_seq_no": 1,
"_source": {
"some": "data",
"question_answer": {"name": "answer", "parent": 42},
},
"_type": "_doc",
"_version": 1,
"found": True,
},
q,
)
def test_reindex_passes_kwargs_to_scan_and_bulk(self):
helpers.reindex(self.client, "test_index", "prod_index", scan_kwargs={'doc_type': 'answers'}, bulk_kwargs={'refresh': True})
self.assertTrue(self.client.indices.exists("prod_index"))
self.assertFalse(self.client.indices.exists_type(index='prod_index', doc_type='questions'))
self.assertEquals(50, self.client.count(index='prod_index', doc_type='answers')['count'])
self.assertEquals({"answer": 42, "correct": True}, self.client.get(index="prod_index", doc_type="answers", id=42)['_source'])
def reindex_data(data):
try:
request_data = json.loads(data)
es = Elasticsearch()
helpers.reindex(client=es, source_index=request_data.get("source"), target_index=request_data.get("target"))
return {"data":"sucess"}
except elasticsearch.ElasticsearchException,e:
return {"new":dir(e),"mes":e.message, "error":e.errors}
def test_reindex_accepts_a_query(self):
helpers.reindex(self.client, "test_index", "prod_index", query={"query": {"bool": {"filter": {"term": {"type": "answers"}}}}})
self.client.indices.refresh()
self.assertTrue(self.client.indices.exists("prod_index"))
self.assertEquals(50, self.client.count(index='prod_index', q='type:answers')['count'])
self.assertEquals({"answer": 42, "correct": True, "type": "answers"}, self.client.get(index="prod_index", doc_type="post", id=42)['_source'])
def history(self):
Logger.debug("{},begin history!".format(datetime.datetime.now()))
helpers.reindex(self.es,
self.active_index,
self.history_index,
query=self.query_body,
chunk_size=1000)
Logger.debug("{},history finished!".format(datetime.datetime.now()))
def reindex_data(data):
try:
request_data = json.loads(data)
es = Elasticsearch()
helpers.reindex(client=es, source_index=request_data.get("source"), target_index=request_data.get("target"))
return {"data":"sucess"}
except elasticsearch.ElasticsearchException,e:
return {"new":dir(e),"mes":e.message, "error":e.errors}
def test_all_documents_get_moved(self):
helpers.reindex(self.client, "test_index", "prod_index")
self.client.indices.refresh()
self.assertTrue(self.client.indices.exists("prod_index"))
self.assertEquals(50, self.client.count(index='prod_index', doc_type='questions')['count'])
self.assertEquals(50, self.client.count(index='prod_index', doc_type='answers')['count'])
self.assertEquals({"answer": 42, "correct": True}, self.client.get(index="prod_index", doc_type="answers", id=42)['_source'])
def reindex(src, dest):
''' Reindexes documents from index {src} to index {dest}'''
from elasticsearch import helpers
from scrapi.processing.elasticsearch import DatabaseManager
dm = DatabaseManager()
dm.setup()
helpers.reindex(dm.es, src, dest)
dm.es.indices.delete(src)
def execute(self):
"""
Reindex elastic index using existing documents
"""
client = elasticsearch.Elasticsearch(
hosts=settings.ELASTIC_SEARCH_HOSTS)
log.debug('Backup index %s to %s ', self.index, self.target)
helpers.reindex(client, self.index, self.target)
def reindex():
body = {"query": {"match_all": {}}} # get all document in index
helpers.reindex(es_src,
src_index_name,
des_index_name,
target_client=es_des,
query=body)
print('Success reindexing')
def test_all_documents_get_moved(self):
helpers.reindex(self.client, "test_index", "prod_index")
self.client.indices.refresh()
self.assertTrue(self.client.indices.exists("prod_index"))
self.assertEquals(50, self.client.count(index='prod_index', q='type:questions')['count'])
self.assertEquals(50, self.client.count(index='prod_index', q='type:answers')['count'])
self.assertEquals({"answer": 42, "correct": True, "type": "answers"}, self.client.get(index="prod_index", doc_type="post", id=42)['_source'])
def reindex(src, dest):
''' Reindexes documents from index {src} to index {dest}'''
from elasticsearch import helpers
from scrapi.processing.elasticsearch import DatabaseManager
dm = DatabaseManager()
dm.setup()
helpers.reindex(dm.es, src, dest)
dm.es.indices.delete(src)
def reindex(self, source_index, target_index=None):
if not target_index:
target_index = source_index
try:
helpers.reindex(es, source_index, target_index)
except Exception as e:
self.logger.error(
f'ERROR WHILE REINDEXING SOURCE: {source_index} TARGET: {target_index}'
+ traceback.format_exc())
raise GeneralException(message=str(e))
def execute(self):
"""
Reindex elastic index using existing documents
"""
client = elasticsearch.Elasticsearch(
hosts=settings.ELASTIC_SEARCH_HOSTS,
# sniff_on_start=True,
retry_on_timeout=True)
log.debug('Backup index %s to %s ', self.index, self.target)
helpers.reindex(client, self.index, self.target)
def copy_es_index(index_source, index_destination):
es = Elasticsearch()
# Delete and create target index
es.indices.delete(index=index_destination, ignore=[404])
maybe_create_index(index_destination)
reindex(es,
source_index=index_source,
target_index=index_destination,
chunk_size=500)
def run():
# reindex the documents from the old index onto the new index
helpers.reindex(elasticsearch, old_name, new_name)
# setup the alias for the new index
self.indices.put_alias(name=self.alias_name, index=new_name)
# delete the old alias
self.indices.delete_alias(name=self.alias_name, index=old_name)
# delete all the documents in the old index (keep the mapping in case one
# needs to role back to an older schema
self.delete_by_query(index=old_name,
body={"query": {"match_all": {}}})
def clone_index(self, new_indexname, index_conf=None):
'''Clone current index
All entries of the current index will be copied into the newly
created one named `new_indexname`
:param index_conf: Configuration to be used in the new index creation.
This param will be passed directly to :py:func:`DB.create_index`
'''
log.debug("Cloning index '{}' into '{}'".format(self.index_name, new_indexname))
self.create_index(indexname=new_indexname, index_conf=index_conf)
reindex(self.es, self.index_name, new_indexname)
def run():
# reindex the documents from the old index onto the new index
helpers.reindex(elasticsearch, old_name, new_name)
# setup the alias for the new index
self.indices.put_alias(name=self.alias_name, index=new_name)
# delete the old alias
self.indices.delete_alias(name=self.alias_name, index=old_name)
# delete all the documents in the old index (keep the mapping in case one
# needs to role back to an older schema
self.delete_by_query(index=old_name,
body={"query": {
"match_all": {}
}})
def migrate():
print("Starting migration...")
elastic = connect_to_elasticsearch()
index_name = create_new_index(elastic)
if elastic.indices.exists(ELASTICSEARCH_ALIAS):
print("Reindexing data from previous index...")
reindex(elastic, ELASTICSEARCH_ALIAS, index_name)
switch_alias_to_index(elastic, ELASTICSEARCH_ALIAS, index_name)
print("Done.")
def migrate():
print("Starting migration...")
elastic = connect_to_elasticsearch()
index_name = create_new_index(elastic)
if elastic.indices.exists(ELASTICSEARCH_ALIAS):
print("Reindexing data from previous index...")
reindex(elastic, ELASTICSEARCH_ALIAS, index_name)
switch_alias_to_index(elastic, ELASTICSEARCH_ALIAS, index_name)
print("Done.")
def reindex():
# To reindex a specified index and appends the new index with "-reindex" if the --new_index_name options has not been specified
args = parse_args()
src_index_name = args.reindex
if args.new_index_name is not None:
des_index_name = args.new_index_name
else:
des_index_name = src_index_name + "-reindex"
helpers.reindex(es(), src_index_name, des_index_name)
print(src_index_name + " has been reindexed to " + des_index_name)
def doreindex(srcidxlist,dstidx):
print "reindexing to " + dstidx
client = Elasticsearch([{'host':'localhost','port':9200}], sniff_on_start=True, sniff_on_connection_fail=True)
for row in srcidxlist:
print "Reindexing index " + row + " to " + dstidx + " please stand by and watch for errors"
try:
reindex(client,row,dstidx,scroll='5m')
except BulkIndexError:
print "Problems with one or more documents from the source index above due to incompatable data types. They will not be included in the destination index"
sys.exc_clear()
print "Confirm the documents were all reindexed\nrun curl -GET http://localhost:9200/_cat/indices?v | grep " + dstidx
return
def clone_index(self, new_indexname, index_conf=None):
'''Clone current index
All entries of the current index will be copied into the newly
created one named `new_indexname`
:param index_conf: Configuration to be used in the new index creation.
This param will be passed directly to :py:func:`DB.create_index`
'''
log.debug("Cloning index '{}' into '{}'".format(
self.index_name, new_indexname))
self.create_index(indexname=new_indexname, index_conf=index_conf)
reindex(self.es, self.index_name, new_indexname)
def test_all_documents_get_moved(self):
bulk = []
for x in range(100):
bulk.append({"index": {"_index": "test_index", "_type": "answers" if x % 2 == 0 else "questions", "_id": x}})
bulk.append({"answer": x, "correct": x == 42})
self.client.bulk(bulk, refresh=True)
helpers.reindex(self.client, "test_index", "prod_index")
self.client.indices.refresh()
self.assertTrue(self.client.indices.exists("prod_index"))
self.assertTrue(self.client.indices.exists_type("prod_index", "answers"))
self.assertTrue(self.client.indices.exists_type("prod_index", "questions"))
self.assertEquals(100, self.client.count(index='prod_index')['count'])
def migrate_indexes(aggregate_indexes=None, forensic_indexes=None):
"""
Updates index mappings
Args:
aggregate_indexes (list): A list of aggregate index names
forensic_indexes (list): A list of forensic index names
"""
version = 2
if aggregate_indexes is None:
aggregate_indexes = []
if forensic_indexes is None:
forensic_indexes = []
for aggregate_index_name in aggregate_indexes:
if not Index(aggregate_index_name).exists():
continue
aggregate_index = Index(aggregate_index_name)
doc = "doc"
fo_field = "published_policy.fo"
fo = "fo"
fo_mapping = aggregate_index.get_field_mapping(fields=[fo_field])
fo_mapping = fo_mapping[list(fo_mapping.keys())[0]]["mappings"]
if doc not in fo_mapping:
continue
fo_mapping = fo_mapping[doc][fo_field]["mapping"][fo]
fo_type = fo_mapping["type"]
if fo_type == "long":
new_index_name = "{0}-v{1}".format(aggregate_index_name, version)
body = {
"properties": {
"published_policy.fo": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
Index(new_index_name).create()
Index(new_index_name).put_mapping(doc_type=doc, body=body)
reindex(connections.get_connection(), aggregate_index_name,
new_index_name)
Index(aggregate_index_name).delete()
for forensic_index in forensic_indexes:
pass
def main():
args = parse_args()
should_apply = args.apply
print(should_apply)
es = Elasticsearch([{'host': host}])
print_count("Source [before]", es.count(index=source))
print_count("Target [before]", es.count(index=target))
if (args.apply):
reindex(es, source, target, chunk_size=5000, scroll='30m')
print_count("Source [after]", es.count(index=source))
print_count("Target [after]", es.count(index=target))
def migrate_index(conn, index_name, current_version=INDEX_VERSION):
new_index = create_index(
conn,
version=current_version,
minor_version=0,
setup_read_alias=False,
)
reindex(conn, source_index=index_name, target_index=new_index._name)
swap_aliases(
conn,
old_index_name=index_name,
new_index_name=new_index._name,
)
delete_index(conn, index_name)
return new_index
def run_request(self, index=None):
if not index:
self.show_index_list_panel(self.run)
return
if not sublime.ok_cancel_dialog("Are you sure you want to reindex?",
ok_title='Reindex'):
return
sublime.status_message("Reindex: start ... please waite.")
options = dict(client=self.client,
source_index=index,
target_index=index,
target_client=self.client,
chunk_size=self.settings.chunk_size,
scroll=self.settings.scroll_size,
scan_kwargs=dict(),
bulk_kwargs=dict(index=index,
stats_only=False,
expand_action_callback=expand_action))
success, errors = reindex(**options)
if errors:
return dict(command=self.command_name,
index=index,
status="ERROR",
errors=errors)
return dict(command=self.command_name,
index=index,
status="SUCCESS",
docs=success)
def reindex():
"""
To reindex a specified index and appends the new index with "-reindex"
if the --new_index_name options has not been specified
"""
args = parse_args()
src_index_name = args.reindex
if args.new_index_name is not None:
des_index_name = args.new_index_name
else:
des_index_name = src_index_name + "-reindex"
helpers.reindex(esSrc(), src_index_name, des_index_name, None, esDest())
print(src_index_name + " has been reindexed to " + des_index_name)
def test_all_documents_get_moved(self):
helpers.reindex(self.client, "test_index", "prod_index")
self.client.indices.refresh()
self.assertTrue(self.client.indices.exists("prod_index"))
self.assertEqual(
50, self.client.count(index="prod_index", q="type:questions")["count"]
)
self.assertEqual(
50, self.client.count(index="prod_index", q="type:answers")["count"]
)
self.assertEqual(
{"answer": 42, "correct": True, "type": "answers"},
self.client.get(index="prod_index", id=42)["_source"],
)
def test_all_documents_get_moved(self):
helpers.reindex(self.client, "test_index", "prod_index")
self.client.indices.refresh()
self.assertTrue(self.client.indices.exists("prod_index"))
self.assertEquals(
50, self.client.count(index="prod_index", q="type:questions")["count"]
)
self.assertEquals(
50, self.client.count(index="prod_index", q="type:answers")["count"]
)
self.assertEquals(
{"answer": 42, "correct": True, "type": "answers"},
self.client.get(index="prod_index", id=42)["_source"],
)
def reindex():
# create index
if not es.indices.exists("tweets_index"):
res_index = es.indices.create(index="tweets_index", ignore=400, body=settings)
print("index created: ", res_index)
res = helpers.reindex(es,"tests_index","tweets_index")
print(res)
def test_reindex_passes_kwargs_to_scan_and_bulk(self):
helpers.reindex(
self.client,
"test_index",
"prod_index",
scan_kwargs={"q": "type:answers"},
bulk_kwargs={"refresh": True},
)
self.assertTrue(self.client.indices.exists("prod_index"))
self.assertEquals(
50, self.client.count(index="prod_index", q="type:answers")["count"]
)
self.assertEquals(
{"answer": 42, "correct": True, "type": "answers"},
self.client.get(index="prod_index", id=42)["_source"],
)
def cp_index(src_client=None, src_index=None, target_client=None, target_index=None, chunk_size=1000):
""" Reindexes from src to dst es """
check_not_empty(src_client)
check_not_empty(src_index)
target_client = target_client or src_client
target_index = target_index or src_index
ok = cp_metadata(src_client, src_index, target_client, target_index)
if ok:
print "Copy documents..."
reindex(
client=src_client,
source_index=src_index,
target_client=target_client,
target_index=target_index,
chunk_size=chunk_size,
query={"query": {"match_all": {}}}
)
print "Data copied!"
def reindex(self, old_index, new_index):
"""Reindex documents using the current mappings."""
conn = self.conn
if not conn.indices.exists(old_index):
raise ValueError("Index {0} does not exist!".format(old_index))
if conn.indices.exists(new_index):
self._print("Index {0} already exists. "
"The mapping will not be changed.".format(new_index))
else:
# Create the new index with (presumably) new mapping config
conn.indices.create(new_index, body=self.get_index_config())
# Do the actual reindexing.
self._print("Reindexing {0} to {1}...".format(old_index, new_index))
helpers.reindex(conn, old_index, new_index)
self._print("Reindexing done.")
def reindex(args):
"""Reindex the annotations into a new ElasticSearch index."""
request = bootstrap(args)
# Configure the new index
search_config.configure_index(request.es, args.target)
# Reindex the annotations
es_helpers.reindex(client=request.es.conn,
source_index=request.es.index,
target_index=args.target)
if args.update_alias:
request.es.conn.indices.update_aliases(body={'actions': [
# Remove all existing aliases
{"remove": {"index": "*", "alias": request.es.index}},
# Alias current index name to new target
{"add": {"index": args.target, "alias": request.es.index}},
]})
def test_children_are_reindexed_correctly(self):
helpers.reindex(self.client, 'test-index', 'real-index')
q = self.client.get(
index='real-index',
doc_type='question',
id=42,
fields=['_source']
)
if 'fields' in q:
q.update(q.pop('fields'))
self.assertEquals(
{
'_id': '42',
'_index': 'real-index',
'_source': {},
'_type': 'question',
'_version': 1,
'found': True
}, q
)
q = self.client.get(
index='test-index',
doc_type='answer',
id=47,
parent=42,
fields=['_source', '_parent']
)
if 'fields' in q:
q.update(q.pop('fields'))
if '_routing' in q:
self.assertEquals(q.pop('_routing'), '42')
self.assertEquals(
{
'_id': '47',
'_index': 'test-index',
'_source': {'some': 'data'},
'_type': 'answer',
'_version': 1,
'_parent': '42',
'found': True
}, q
)
def set_up_index(idx):
alias = es.indices.get_aliases(index=idx)
if not alias or not alias.keys() or idx in alias.keys():
# Deal with empty indices or the first migration
index = '{}_v1'.format(idx)
search.create_index(index=index)
logger.info("Reindexing {0} to {1}_v1".format(idx, idx))
helpers.reindex(es, idx, index)
logger.info("Deleting {} index".format(idx))
es.indices.delete(index=idx)
es.indices.put_alias(idx, index)
else:
# Increment version
version = int(alias.keys()[0].split('_v')[1]) + 1
logger.info("Incrementing index version to {}".format(version))
index = '{0}_v{1}'.format(idx, version)
search.create_index(index=index)
logger.info("{} index created".format(index))
return index
def _setup_alias(self, mappings=None, settings=None):
"""since we only want a single index to be aliased at a time,
this method will check for the alias and delete the current alias if
it is different from the index_name.
"""
index = self.get_index_name()
self.elasticsearch.indices.create(
index="", body={"settings": settings, "mappings": mappings})
if self.alias_exists:
existing_index = self.index_name(self.current_revision_number)
new_index = self.index_name(self.current_revision_number + 1)
# reindex from the existing aliased index onto the new index
reindex(self.elasticsearch, existing_index, new_index,
chunk_size=self._chunk_size)
self.elasticsearch.indices.delete_alias(index=existing_index,
name=self.alias_name)
self.elasticsearch.indices.put_alias(
index=self.index_name, name=self.alias_name)
def update(self, doc_type):
alias_name = 'frisc_{}'.format(doc_type)
index_name = '{}_v1'.format(alias_name),
try:
if not self.esc.exists_alias(alias_name):
self.create(doc_type)
return
version_number = 0
old_index_name = ''
old_indexes = self.esc.get_alias(name=alias_name)
for index in old_indexes.keys():
match = re.search('^({})_v(\d+)$'.format(alias_name), index)
if match:
version = int(match.group(2))
if version > version_number:
version_number = version
old_index_name = match.group(0)
version_number += 1
index_name = '{}_v{}'.format(alias_name, version_number)
if self.esc.exists(index_name):
# raise soemthing
raise
self.__create__(index_name, type=doc_type)
reindex(self.es, old_index_name, index_name)
self.esc.update_aliases(
body={'actions': [
{'remove': {'alias': alias_name, 'index': old_index_name}},
{'add': {'alias': alias_name, 'index': index_name}}
]}
)
except es_exceptions.TransportError:
print("unable to connect to Elasticsearch")
def run(self):
index_name = superdesk.app.config['ELASTICSEARCH_INDEX']
print('Starting index rebuilding for index: ', index_name)
try:
es = get_es(superdesk.app.config['ELASTICSEARCH_URL'])
clone_name = index_name + '-' + get_random_string()
print('Creating index: ', clone_name)
superdesk.app.data.elastic.create_index(clone_name, superdesk.app.config['ELASTICSEARCH_SETTINGS'])
print('Putting mapping for index: ', clone_name)
superdesk.app.data.elastic.put_mapping(superdesk.app, clone_name)
print('Starting index rebuilding.')
reindex(es, index_name, clone_name)
print('Finished index rebuilding.')
print('Deleting index: ', index_name)
get_indices(es).delete(index_name)
print('Creating alias: ', index_name)
get_indices(es).put_alias(index=clone_name, name=index_name)
print('Alias created.')
except elasticsearch.exceptions.NotFoundError as nfe:
print(nfe)
print('Index {0} rebuilt successfully.'.format(index_name))
def main(settings):
props = ('_index', '_parent', '_percolate', '_routing', '_timestamp',
'_ttl', '_type', '_version', '_version_type', '_id', '_retry_on_conflict')
if settings.backuper.props_to_exclude is not None:
props = tuple(p for p in props if p not in settings.backuper.props_to_exclude)
es = Elasticsearch(settings.hosts)
if settings.verbosity:
sys.stdout.write('copying from {} to {}, documents {}\n'.format(settings.backuper.from_index, settings.backuper.index, settings.backuper.doc_type))
success, failed = helpers.reindex(es, settings.backuper.from_index, settings.backuper.index, scan_kwargs={'doc_type': settings.backuper.doc_type}, bulk_kwargs = {'expand_action_callback': make_expand(props)})
if settings.verbosity:
sys.stdout.write('Documents copied: {}. Documents Failed {}\n'.format(success, failed))