def bulk(self, docs, index="", doc_type="", op_type='index'): ''' bulk sample: {"_op_type":"index", _index" : "test", "_type" : "type1", "_id" : "1" , "_source":{"field1":"value1", "field2":"value2"}} { "_op_type":"delete" , "_index" : "test", "_type" : "type1", "_id" : "2" } ''' index_ = self.index if index == "" else index doc_type_ = self.doc_type if doc_type == "" else doc_type allow_op = ['index', 'delete'] if op_type not in allow_op: raise exceptions.RequestError( 400, '{"msg":"op_type is not allowed, you can use index or delete"}' ) actions = [] for doc in docs: action = {} action["_index"] = index_ action["_type"] = doc_type_ action["_id"] = doc["_id"] if op_type == 'index': del doc["_id"] action["_source"] = doc action["_op_type"] = op_type actions.append(action) return helpers.parallel_bulk(self.es, actions)
def test_multiple_alias_exception_detection(self): alias_exc = es_exceptions.RequestError( 400, "Something " + helper.ALIAS_EXCEPTION_STRING, {}) self.assertTrue(helper._is_multiple_alias_exception(alias_exc)) other_exc = Exception("Blah blah") self.assertFalse(helper._is_multiple_alias_exception(other_exc))
def test_multiple_alias_exception_elasticsearch2(self): # The es-2 format is different alias_exc = es_exceptions.RequestError( 400, 'illegal_argument_exception', {"error": { "root_cause": [{ "type": "illegal_argument_exception", "reason": "Something " + helper.ALIAS_EXCEPTION_STRING }], "type": "illegal_argument_exception", "reason": "Something " + helper.ALIAS_EXCEPTION_STRING }}) self.assertTrue(helper._is_multiple_alias_exception(alias_exc))
def create_db(): """Create the Elasticsearch index for Annotations and Documents.""" # Check for required plugin(s) _ensure_es_plugins(es.conn) models = [Annotation, Document] mappings = {} analysis = {} # Collect the mappings and analysis settings for model in models: mappings.update(model.get_mapping()) for section, items in model.get_analysis().items(): existing_items = analysis.setdefault(section, {}) for name in items: if name in existing_items: fmt = "Duplicate definition of 'index.analysis.{}.{}'." msg = fmt.format(section, name) raise RuntimeError(msg) existing_items.update(items) # Create the index try: # Pylint issue #258: https://bitbucket.org/logilab/pylint/issue/258 # # pylint: disable=unexpected-keyword-arg response = es.conn.indices.create(es.index, ignore=400, body={ 'mappings': mappings, 'settings': { 'analysis': analysis }, }) except elasticsearch_exceptions.ConnectionError as e: msg = ('Can not access Elasticsearch at {0}! ' 'Check to ensure it is running.').format(es.host) raise elasticsearch_exceptions.ConnectionError('N/A', msg, e) # Bad request (400) is ignored above, to prevent warnings in the log, but # the failure could be for reasons other than that the index exists. If so, # raise the error here. if 'error' in response and 'IndexAlreadyExists' not in response['error']: raise elasticsearch_exceptions.RequestError(400, response['error']) # Update analysis settings settings = es.conn.indices.get_settings(index=es.index) existing = settings[es.index]['settings']['index'].get('analysis', {}) if existing != analysis: try: es.conn.indices.close(index=es.index) es.conn.indices.put_settings(index=es.index, body={'analysis': analysis}) finally: es.conn.indices.open(index=es.index) # Update mappings try: for doc_type, body in mappings.items(): es.conn.indices.put_mapping(index=es.index, doc_type=doc_type, body=body) except elasticsearch_exceptions.RequestError as e: if e.error.startswith('MergeMappingException'): date = time.strftime('%Y-%m-%d') message = ("Elasticsearch index mapping is incorrect! Please " "reindex it. For example, run: " "./bin/hypothesis reindex {0} {1} {1}-{2}".format( 'yourconfig.ini', es.index, date)) log.critical(message) raise RuntimeError(message) raise