def bulk(self, docs, index="", doc_type="", op_type='index'):
'''
bulk sample:
{"_op_type":"index", _index" : "test", "_type" : "type1", "_id" : "1" , "_source":{"field1":"value1", "field2":"value2"}}
{ "_op_type":"delete" , "_index" : "test", "_type" : "type1", "_id" : "2" }
'''
index_ = self.index if index == "" else index
doc_type_ = self.doc_type if doc_type == "" else doc_type
allow_op = ['index', 'delete']
if op_type not in allow_op:
raise exceptions.RequestError(
400,
'{"msg":"op_type is not allowed, you can use index or delete"}'
)
actions = []
for doc in docs:
action = {}
action["_index"] = index_
action["_type"] = doc_type_
action["_id"] = doc["_id"]
if op_type == 'index':
del doc["_id"]
action["_source"] = doc
action["_op_type"] = op_type
actions.append(action)
return helpers.parallel_bulk(self.es, actions)
def test_multiple_alias_exception_detection(self):
alias_exc = es_exceptions.RequestError(
400, "Something " + helper.ALIAS_EXCEPTION_STRING, {})
self.assertTrue(helper._is_multiple_alias_exception(alias_exc))
other_exc = Exception("Blah blah")
self.assertFalse(helper._is_multiple_alias_exception(other_exc))
def test_multiple_alias_exception_elasticsearch2(self):
# The es-2 format is different
alias_exc = es_exceptions.RequestError(
400,
'illegal_argument_exception',
{"error": {
"root_cause": [{
"type": "illegal_argument_exception",
"reason": "Something " + helper.ALIAS_EXCEPTION_STRING
}],
"type": "illegal_argument_exception",
"reason": "Something " + helper.ALIAS_EXCEPTION_STRING
}})
self.assertTrue(helper._is_multiple_alias_exception(alias_exc))
def create_db():
"""Create the Elasticsearch index for Annotations and Documents."""
# Check for required plugin(s)
_ensure_es_plugins(es.conn)
models = [Annotation, Document]
mappings = {}
analysis = {}
# Collect the mappings and analysis settings
for model in models:
mappings.update(model.get_mapping())
for section, items in model.get_analysis().items():
existing_items = analysis.setdefault(section, {})
for name in items:
if name in existing_items:
fmt = "Duplicate definition of 'index.analysis.{}.{}'."
msg = fmt.format(section, name)
raise RuntimeError(msg)
existing_items.update(items)
# Create the index
try:
# Pylint issue #258: https://bitbucket.org/logilab/pylint/issue/258
#
# pylint: disable=unexpected-keyword-arg
response = es.conn.indices.create(es.index,
ignore=400,
body={
'mappings': mappings,
'settings': {
'analysis': analysis
},
})
except elasticsearch_exceptions.ConnectionError as e:
msg = ('Can not access Elasticsearch at {0}! '
'Check to ensure it is running.').format(es.host)
raise elasticsearch_exceptions.ConnectionError('N/A', msg, e)
# Bad request (400) is ignored above, to prevent warnings in the log, but
# the failure could be for reasons other than that the index exists. If so,
# raise the error here.
if 'error' in response and 'IndexAlreadyExists' not in response['error']:
raise elasticsearch_exceptions.RequestError(400, response['error'])
# Update analysis settings
settings = es.conn.indices.get_settings(index=es.index)
existing = settings[es.index]['settings']['index'].get('analysis', {})
if existing != analysis:
try:
es.conn.indices.close(index=es.index)
es.conn.indices.put_settings(index=es.index,
body={'analysis': analysis})
finally:
es.conn.indices.open(index=es.index)
# Update mappings
try:
for doc_type, body in mappings.items():
es.conn.indices.put_mapping(index=es.index,
doc_type=doc_type,
body=body)
except elasticsearch_exceptions.RequestError as e:
if e.error.startswith('MergeMappingException'):
date = time.strftime('%Y-%m-%d')
message = ("Elasticsearch index mapping is incorrect! Please "
"reindex it. For example, run: "
"./bin/hypothesis reindex {0} {1} {1}-{2}".format(
'yourconfig.ini', es.index, date))
log.critical(message)
raise RuntimeError(message)
raise