def test_getitem_missing(self): provider = ESTermIndexWeightingProvider(self.es, self.index, initial_weights=[('ba', 1), ('knark', 2), ('knirk', 3), ('foo', 3)]) IndicesClient(self.es).refresh(self.index) self.assertRaises(KeyError, lambda: provider['notfound']) self.assertRaises(KeyError, lambda: provider['ba', 'notfound']) provider = ESTermIndexWeightingProvider(self.es, self.index, initial_weights=[('ba', 1), ('knark', 2), ('knirk', 3), ('foo', 3)], missing='ignore') IndicesClient(self.es).refresh(self.index) self.assertIsNone(provider['notfound']) self.assertEqual([('ba', 1)], list(provider['ba', 'notfound']))
def __init__(self, type='elastic', url=None, base=None, name=None, server_base=None, content=None, default=None): if not (url and name): raise Exception('both url and name-parameters required') self.url = url self.base = base self.server_base = server_base self.index_name = name self.elastic = Elasticsearch(url) self.indices = IndicesClient(self.elastic) self.index_content = content is not None self.content = content self.default = default or '' self.index_map = {} if not self.indices.exists(self.index_name): try: self.indices.create(self.index_name, ignore=400, body=mapping) self.index_existed = False except RequestError as e: if e.error == 'resource_already_exists_exception': self.indices = IndicesClient(self.elastic) self.index_existed = True else: raise e else: self.index_existed = True for c in (content or {}).get('parts', {}).values(): prefix = content.get('index_prefix', '') index = prefix + c['index_name'] self.index_map[c['type']] = index if not self.indices.exists(index): try: self.indices.create(index, ignore=400, body=mapping) except RequestError as e: if e.error == 'resource_already_exists_exception': self.indices = IndicesClient(self.elastic) else: raise e
def __init__(self, conf, queue): self.conf = conf host = self.conf.get("host", "es") port = self.conf.get("port", 9200) self.log = logging.getLogger("pulsar.indexer") logging.getLogger("elasticsearch").setLevel(logging.INFO) self.log.debug("port: %r" % port) self.es = Elasticsearch([{"host": host, "port": port}]) self.cluster_client = ClusterClient(self.es) health = self.cluster_client.health() if not health or health.get("number_of_nodes") < 1: raise Exception("No Elasticsearch nodes found: %r" % health) # Put our template self.indices_client = IndicesClient(self.es) self.index_prefix = self.conf.get("index_prefix", self.INDEX_PREFIX) self.indices_client.put_template( name=self.index_prefix, body=open("conf/es-template.json").read()) self.log.info("Put template to ES for pulsar indexes") self.last_event_time = time() self.index_prefix = self.index_prefix + "-" self.index_name = self.get_index_name() self.queue = queue self.counter = 0 self.stats_checkpoint = time() self.stats_every = 10000 try: # This will block as it reads from the queue self.bulk(self.es, self.iterator(), stats_only=True) except Exception as e: self.log.exception("Error with bulk", exc_info=e)
def test_tokenize(self): test_case = { "text": "2018-06-08T00:00:00Z INFO GET /v1/bundles/7ef8966b-45ef-4e0a-a51b-44a865372050.2018-06-08T230333.785338Z?param1=1¶m2=2 {\"key\": \"value\"}" } index_name = self.es_client._format_today_index_name(self.index_prefix) index_client = IndicesClient(TestESClient.es) with self.new_index(index_name): response = index_client.analyze(index=index_name, body=test_case) tokens = [t['token'] for t in response['tokens']] self.assertEqual(set(tokens), { '7ef8966b-45ef-4e0a-a51b-44a865372050', '2018-06-08T230333.785338Z', ':', 'INFO', '1', '2', 'v1', 'bundles', 'key', 'GET', 'param2', 'param1', '2018-06-08T00:00:00Z', 'value' }) self.assertEqual(len(tokens), 14)
def __init__(self, driver, connection_name, connection_config): super(ElasticsearchConnection, self).__init__(driver, connection_name, connection_config) self.uri = self.connection_config.get('uri').split(',') self.cnx_opts = {} use_ssl = self.connection_config.get('use_ssl', True) if isinstance(use_ssl, str): if use_ssl.lower() == 'false': use_ssl = False else: use_ssl = True self.cnx_opts['use_ssl'] = use_ssl if use_ssl: verify_certs = self.connection_config.get('verify_certs', True) if isinstance(verify_certs, str): if verify_certs.lower() == 'false': verify_certs = False else: verify_certs = True self.cnx_opts['verify_certs'] = verify_certs self.cnx_opts['ca_certs'] = self.connection_config.get( 'ca_certs', None) self.cnx_opts['client_cert'] = self.connection_config.get( 'client_cert', None) self.cnx_opts['client_key'] = self.connection_config.get( 'client_key', None) self.es = Elasticsearch(self.uri, **self.cnx_opts) try: self.log.debug("Elasticsearch info: %s" % self.es.info()) except Exception as e: self.log.warn("An error occured on estabilishing " "connection to Elasticsearch: %s" % e) self.ic = IndicesClient(self.es)
def create_index_conf(): indices_client = IndicesClient(models.client) index_name = 'conf' doc_type = index_name if indices_client.exists(index_name): indices_client.delete(index=index_name) indices_client.create(index=index_name)
def test_index_manager_regenerate_indices_from_broken_state(self, *args): """ `regenerate_indices` should succeed and give us a working ElasticSearch when it runs and finds a broken state (eg. with an existing, incorrect index with the name of an alias). This can occur when ES restarts and an update signal is triggered before Richie had a chance to bootstrap ES. """ # The indices client will be used to test the actual indices in ElasticSearch indices_client = IndicesClient(client=ES_CLIENT) # Create a course and trigger a signal to index it. This will create a # broken "richie_test_courses" index course = CourseFactory(should_publish=True) update_course(course.extended_object, "en") self.assertIsNotNone(indices_client.get("richie_test_courses")) # Call our `regenerate_indices command` creation_datetime = datetime(2010, 1, 1, tzinfo=timezone.utc) creation_string = creation_datetime.strftime("%Y-%m-%d-%Hh%Mm%S.%fs") with mock.patch.object(timezone, "now", return_value=creation_datetime): regenerate_indices(None) # No error was thrown, the courses index (like all others) was bootstrapped self.assertIsNotNone( indices_client.get(f"richie_test_courses_{creation_string}") ) # The expected alias is associated with the index self.assertEqual( list(indices_client.get_alias("richie_test_courses").keys())[0], f"richie_test_courses_{creation_string}", )
def __init__(self, host, port, db_config): self.es = Elasticsearch([{"host": host, "port": port}]) if not self.es.ping(): error( "Cannot connect to elasticsearch cluster for users. Check database configuration in user_db_config.json." ) exit(0) index = db_config["index"] self.index = index["name"] user_type = db_config["user_type"] self.user_type_name = user_type["name"] mappings = dict() if "mapping" in user_type: mappings[self.user_type_name] = user_type["mapping"] body = dict() if "settings" in index: body["settings"] = index["settings"] if mappings: body["mappings"] = mappings try: self.indices_client = IndicesClient(self.es) if not self.indices_client.exists(self.index): self.indices_client.create(index=self.index, body=body) except TransportError: error( "Error while creating elasticsearch cluster for users. Check type mappings in user_db_config.json." ) print(traceback.format_exc()) exit(0)
def get_es_indices(): es_idx = IndicesClient(es) indicies = es_idx.get('appcompat-*') result = [] for index_name,v in indicies.iteritems(): result.append((index_name,index_name,v['settings']['index']['creation_date'])) return result
def handle(self, *args, **options): es = Elasticsearch(hosts=[{'host': 'localhost', 'port': 9200}]) fop=open('spider/management/commands/'+str(argv[2]), 'r') inds = IndicesClient(es) mapping={ "mappings": { "product_type": { "properties": { "code": { "type" : "string" },"name": {"type" : "string"},"img": {"type" : "string"},"url": {"type" : "string"},"price_reg": {"type" : "float"},"price_discount": {"type" : "float"}}}}} if not inds.exists(index='gearbest_index'): inds.create(index='gearbest_index',body=mapping) print 'gearbest_index created' for jsonline in fop: jobj=loads(jsonline) del jobj["_type"] es.index(index="gearbest_index",doc_type='product_type', body=jobj, id=jobj['code']) disc=0 reg=0 if len(jobj['price_discount'])>0: disc = float(jobj['price_discount'][0]) if len(jobj['price_reg'])>0: reg = float(jobj['price_reg'][0]) #insert="INSERT into 'price_gb' ('price','price_disc','code','date') values ("+str(reg)+", "+str(disc)+", '"+str(jobj['code'])+"', '"+str(datetime.today())+"')" #cursor = connection.cursor() #cursor.execute(insert) add_price=Price_gb(price=reg,price_disc=disc,code=str(jobj['code']),date=datetime.date.today()) add_price.save() print 'code='+str(jobj['code'])
def _elastic(self, doc_id=None, doc={}, option='create'): """ option: init: 初始化文档结构。(当config.ini中的init为True时才会执行。) create: 若文档已存在,则不执行任何操作。 若文档不存在,则直接创建。 update: 若文档已存在,则直接更新。 若文档不存在,则不执行任何操作。 delete: 若文档已存在,则直接删除。若文档不存在,则不执行任何操作。 """ esclient = Elasticsearch([self.elastic]) status = 'Success !' if 'create' == option: try: esclient.create( index=self.elastic['index'], doc_type=self.elastic['type'], id=doc_id, body=doc, ) except ConflictError: status = 'Fail(existsd) !' elif 'update' == option: try: esclient.update( index=self.elastic['index'], doc_type=self.elastic['type'], id=doc_id, body={'doc': doc}, ) except NotFoundError: status = 'Fail(not existsd) !' elif 'delete' == option: try: esclient.delete( index=self.elastic['index'], doc_type=self.elastic['type'], id=doc_id, ) except NotFoundError: status = 'Fail(not existsd) !' elif 'init' == option: try: IndicesClient(esclient).create( index=self.elastic['index'], body=doc, ) except RequestError: status = 'Fail(existsd) !' self.logger.record('Sync@%s < %s-%s-%s > %s' % ( option, self.elastic['index'], self.elastic['type'], doc_id, status, ))
def _create_index(self): es_index = IndicesClient(self._es) if es_index.exists(self._store_index): logging.info('Index ' + self._store_index + ' already exists. Skipping index creation.') return None es_mapping = { "mappings": { 'last_runtime': { 'properties': { 'plugin_name': { 'index': 'not_analyzed', 'type': 'string' }, 'rule_name': { 'index': 'not_analyzed', 'type': 'string' }, 'plugin_sid': { 'index': 'not_analyzed', 'type': 'long' }, '@timestamp': { 'format': 'dateOptionalTime||epoch_millis', 'type': 'date' } } } } } self._es.indices.create(self._store_index, body=es_mapping) time.sleep(1)
def prepare_index(self, courses): """ Not a test. This method is doing the heavy lifting for the tests in this class: - prepare the Elasticsearch index, - execute the query. """ self.create_filter_pages() # Index these 4 courses in Elasticsearch indices_client = IndicesClient(client=ES_CLIENT) # Delete any existing indices so we get a clean slate indices_client.delete(index="_all") # Create an index we'll use to test the ES features indices_client.create(index="test_courses") indices_client.close(index="test_courses") indices_client.put_settings(body=ANALYSIS_SETTINGS, index="test_courses") indices_client.open(index="test_courses") # Use the default courses mapping from the Indexer indices_client.put_mapping(body=CoursesIndexer.mapping, doc_type="course", index="test_courses") # Add the sorting script ES_CLIENT.put_script(id="state", body=CoursesIndexer.scripts["state"]) # Actually insert our courses in the index actions = [{ "_id": course["id"], "_index": "test_courses", "_op_type": "create", "_type": "course", **course, } for course in courses] bulk(actions=actions, chunk_size=500, client=ES_CLIENT) indices_client.refresh()
def indices(self): """ Returns an elasticsearch.client.IndicesClient instance """ if not self.connected: self.connect() return IndicesClient(self.client)
def create_index_survey(): indices_client = IndicesClient(models.client) index_name = models.SurveyMap._meta.es_index_name if indices_client.exists(index_name): indices_client.delete(index=index_name) indices_client.create(index=index_name) #put_settings(models.ScentemotionMap) # add qstfld fields es_mapping = models.SurveyMap._meta.es_mapping for qst, mapping in survey.qst2fld.items(): fields = mapping[0] field_type = mapping[1] if field_type == 'nested_qst_ans': for field in fields: if field not in es_mapping['properties']: es_mapping['properties'][field] = {} es_mapping['properties'][field]['type'] = 'nested' es_mapping['properties'][field]['properties'] = {} es_mapping['properties'][field]['properties']['question'] = {'type' : 'text', 'fields' : {'keyword' : {'type' : 'keyword', 'ignore_above' : 256}}} es_mapping['properties'][field]['properties']['answer'] = {'type' : 'text', 'fields' : {'keyword' : {'type' : 'keyword', 'ignore_above' : 256}}} #'type' : 'nested', #'properties' : { # 'question' : {'type' : 'text', 'fields' : {'keyword' : {'type' : 'keyword', 'ignore_above' : 256}}}, # 'answer' : {'type' : 'text', 'fields' : {'keyword' : {'type' : 'keyword', 'ignore_above' : 256}}}, # } #}, indices_client.put_mapping( doc_type=models.SurveyMap._meta.es_type_name, #body=models.SurveyMap._meta.es_mapping, body=es_mapping, index=index_name )
def setUp(self): """ Make sure tests get clean indexes to run """ self.indices_client = IndicesClient(client=settings.ES_CLIENT) # Delete any existing indexes so we get a clean slate self.indices_client.delete(index="_all") # Create an index we'll use to test the ES features self.indices_client.create(index="test_index") # Add a mapping for a test document type. It needs to include different fields for the # various features we'll be running tests on self.indices_client.put_mapping( body={ "properties": { "datetime_field": { "type": "date" }, "keyword_field": { "type": "keyword" }, "text_field": { "type": "text" }, } }, doc_type="test_doc", index="test_index", )
def handle(self, *args, **options): client = IndicesClient(client=settings.ES_CLIENT) self.recreate_index( client, **{ 'index_name': Sofa._meta.es_index_name, 'doc_type': Sofa._meta.es_type_name, 'body': Sofa._meta.es_mapping }) self.push_db_to_index(client, SofaSerializerES, Sofa.objects.all().iterator()) self.recreate_index( client, **{ 'index_name': Bed._meta.es_index_name, 'doc_type': Bed._meta.es_type_name, 'body': Bed._meta.es_mapping }) self.push_db_to_index(client, BedSerializerES, Bed.objects.all().iterator()) self.recreate_index( client, **{ 'index_name': Dining._meta.es_index_name, 'doc_type': Dining._meta.es_type_name, 'body': Dining._meta.es_mapping }) self.push_db_to_index(client, DiningSerializerES, Dining.objects.all().iterator())
def main(): # Define the globals global index_names global STARTED_TIMESTAMP global es global es_indices try: # Initiate the elasticsearch session using ES low-level client. # By default nodes are randomized before passed into the pool and round-robin strategy is used for load balancing. es = Elasticsearch(ES_HOSTS, timeout=30) es_indices = IndicesClient(es) except: print("Could not connect to elasticsearch!") sys.exit(1) print("Creating indices.. \n"), indices = generate_indices() print("Done!\n") print("GET Settings \n"), print json.dumps(es_indices.get_settings(index="_all"), sort_keys=True, indent=4, separators=(',', ': ')) print("Done!\n") # We will Clean up the indices by default # Default: True if CLEANUP: print("Cleaning up created indices.. "), cleanup_indices() print("Done!\n")
def perform_create_index(indexable, logger): """ Create a new index in ElasticSearch from an indexable instance """ indices_client = IndicesClient(client=settings.ES_CLIENT) # Create a new index name, suffixing its name with a timestamp new_index = "{:s}_{:s}".format( indexable.index_name, timezone.now().strftime("%Y-%m-%d-%Hh%Mm%S.%fs")) # Create the new index logger.info( 'Creating a new Elasticsearch index "{:s}"...'.format(new_index)) indices_client.create(index=new_index) indices_client.put_mapping(body=indexable.mapping, doc_type=indexable.document_type, index=new_index) # Populate the new index with data provided from our indexable class bulk( actions=indexable.get_data_for_es(new_index, "create"), chunk_size=settings.ES_CHUNK_SIZE, client=settings.ES_CLIENT, stats_only=True, ) # Return the name of the index we just created in ElasticSearch return new_index
def setUp(self): """ Make sure all indexes are deleted before each new test is run. """ super().setUp() self.indices_client = IndicesClient(client=settings.ES_CLIENT) self.indices_client.delete(index="_all")
def import_examples_into_es(examples: list): index_name = config.index_name type_name = config.type_name buck_size = config.buck_size es = Elasticsearch(config.es_url) es_index = IndicesClient(es) if es_index.exists(index=index_name): es_index.delete(index=index_name) # 创建索引 with open(config.es_index_json) as f: mappings = json.load(f) res = es.indices.create(index=index_name, body=mappings) # 数据批量导入es for i in range(len(examples)): examples[i] = { "_index": index_name, "_type": type_name, "_id": examples[i]["ntc_id"], "_source": examples[i] } for i in tqdm(range(ceil(len(examples) / buck_size)), desc="Import into ES"): bulk(es, actions=examples[i * buck_size: min((i + 1) * buck_size, len(examples))])
def setUp(self): """ Instantiate our ES client and make sure all indexes are deleted before each test """ super().setUp() self.indices_client = IndicesClient(client=ES_CLIENT) self.indices_client.delete(index="_all")
def perform_create_index(indexable, logger): """ Create a new index in ElasticSearch from an indexable instance """ indices_client = IndicesClient(client=ES_CLIENT) # Create a new index name, suffixing its name with a timestamp new_index = "{:s}_{:s}".format( indexable.index_name, timezone.now().strftime("%Y-%m-%d-%Hh%Mm%S.%fs")) # Create the new index logger.info( 'Creating a new Elasticsearch index "{:s}"...'.format(new_index)) indices_client.create(index=new_index) # The index needs to be closed before we set an analyzer indices_client.close(index=new_index) indices_client.put_settings(body=ANALYSIS_SETTINGS, index=new_index) indices_client.open(index=new_index) indices_client.put_mapping(body=indexable.mapping, doc_type=indexable.document_type, index=new_index) # Populate the new index with data provided from our indexable class richie_bulk(indexable.get_es_documents(new_index)) # Return the name of the index we just created in ElasticSearch return new_index
def reindex(self): elastic_client = Elasticsearch([{ "host": self.__host, "port": self.__port }]) index_client = IndicesClient(elastic_client) # Create new index with necessory fields mapping # , master_timeout=10, timeout=10 index_client.create(index=self.__target_index, body=self.__body) # reindexind data from source index to target index helpers.reindex(client=elastic_client, source_index=self.__source_index, target_index=self.__target_index) # creating alias for target index alias = {'actions': []} # remove_action = {"remove": {"index": self.__source_index, "alias": self.__alias}} add_action = { "add": { "index": self.__target_index, "alias": self.__alias } } # alias['actions'].append(remove_action) alias['actions'].append(add_action) # deleteing the source index index_client.delete(index=self.__source_index) index_client.update_aliases(body=alias)
def put_settings(obj): indices_client = IndicesClient(models.client) index_name = obj._meta.es_index_name indices_client.close(index=index_name) kwargs = { "analysis": { "analyzer": { "default": { "tokenizer": "standard", "filter": ["synonym"] }, "keepwords": { "tokenizer": "standard", "filter": ["keepwords"] }, }, "filter": { "synonym": { "type": "synonym", "synonyms_path": "synonym.txt" }, "keepwords": { "type": "keep", "keep_words_path": "keepwords.txt" }, } } } indices_client.put_settings(index=index_name, body=kwargs) indices_client.open(index=index_name)
def setUp(self): """ Starts a new connector for every test """ try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() self.connector = Connector( address='%s:%s' % (mongo_host, self.primary_p), oplog_checkpoint='config.txt', target_url=elastic_pair, ns_set=['test.test'], u_key='_id', auth_key=None, doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py', auto_commit_interval=0) # Clean out test databases try: self.elastic_doc._remove() except OperationFailed: try: # Create test.test index if necessary client = Elasticsearch(hosts=[elastic_pair]) idx_client = IndicesClient(client) idx_client.create(index='test.test') except es_exceptions.TransportError: pass self.conn.test.test.drop() self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) == 0)
def __init__(self, host, port, db_config): self.es = Elasticsearch([{"host": host, "port": port}]) try: if self.es.ping(): es_logger = logging.getLogger('elasticsearch') es_logger.setLevel(logging.CRITICAL) self.indices_client = IndicesClient(self.es) index_definitions = db_config["index_definitions"] self.settings = db_config["settings"] self.data_point_definition = index_definitions["data_point"] self.create_index_from_definition(self.data_point_definition, self.settings) self.data_point_type_name = self.data_point_definition["name"] self.data_point_index = self.data_point_definition[ "index_name"] self.definitions.append(self.data_point_definition) self.experiment_definition = index_definitions["experiment"] self.create_index_from_definition(self.experiment_definition, self.settings) self.experiment_type_name = self.experiment_definition["name"] self.experiment_index = self.experiment_definition[ "index_name"] self.definitions.append(self.experiment_definition) self.target_system_definition = index_definitions[ "target_system"] self.create_index_from_definition( self.target_system_definition, self.settings) self.target_system_type_name = self.target_system_definition[ "name"] self.target_system_index = self.target_system_definition[ "index_name"] self.definitions.append(self.target_system_definition) self.analysis_definition = index_definitions["analysis"] self.create_index_from_definition(self.analysis_definition, self.settings) self.analysis_type_name = self.analysis_definition["name"] self.analysis_index = self.analysis_definition["index_name"] self.definitions.append(self.analysis_definition) self.stage_definition = index_definitions["stage"] self.create_index_from_definition(self.stage_definition, self.settings) self.stage_type_name = self.stage_definition["name"] self.stage_index = self.stage_definition["index_name"] self.definitions.append(self.stage_definition) else: raise ConnectionError("Host/port values are not valid") except TransportError as err1: error( "TransportError while creating elasticsearch instance for experiments. Check type mappings in experiment_db_config.json." ) raise err1
def execute_query(self, querystring=""): """ Not a test. This method is doing the heavy lifting for the tests in this class: create and fill the index with our courses so we can run our queries and check our facet counts. It also executes the query and returns the result from the API. """ # Create the subject category page. This is necessary to link the subjects we # defined above with the "subjects" filter # As it is the only page we create, we expect it to have the path "0001" CategoryFactory(page_reverse_id="subjects", should_publish=True) # Index these 4 courses in Elasticsearch indices_client = IndicesClient(client=ES_CLIENT) # Delete any existing indices so we get a clean slate indices_client.delete(index="_all") # Create an index we'll use to test the ES features indices_client.create(index="test_courses") indices_client.close(index="test_courses") indices_client.put_settings(body=ANALYSIS_SETTINGS, index="test_courses") indices_client.open(index="test_courses") # Use the default courses mapping from the Indexer indices_client.put_mapping( body=CoursesIndexer.mapping, doc_type="course", index="test_courses" ) # Add the sorting script ES_CLIENT.put_script(id="state", body=CoursesIndexer.scripts["state"]) # Actually insert our courses in the index actions = [ { "_id": course["id"], "_index": "test_courses", "_op_type": "create", "_type": "course", "absolute_url": {"en": "url"}, "cover_image": {"en": "image"}, "title": {"en": "title"}, **course, "course_runs": [ { "languages": course_run["languages"], "start": arrow.utcnow().datetime, "end": arrow.utcnow().datetime, "enrollment_start": arrow.utcnow().datetime, "enrollment_end": arrow.utcnow().datetime, } for course_run in course["course_runs"] ], } for course in COURSES ] bulk(actions=actions, chunk_size=500, client=ES_CLIENT) indices_client.refresh() response = self.client.get(f"/api/v1.0/courses/?{querystring:s}") self.assertEqual(response.status_code, 200) return json.loads(response.content)
def execute_query(self, courses, querystring="", **extra): """ Not a test. Prepare the ElasticSearch index and execute the query in it. """ indices_client = IndicesClient(client=ES_CLIENT) # Delete any existing indices so we get a clean slate indices_client.delete(index="_all") # Create an index we'll use to test the ES features indices_client.create(index=COURSES_INDEX) # The index needs to be closed before we set an analyzer indices_client.close(index=COURSES_INDEX) indices_client.put_settings(body=ANALYSIS_SETTINGS, index=COURSES_INDEX) indices_client.open(index=COURSES_INDEX) # Use the default courses mapping from the Indexer indices_client.put_mapping(body=CoursesIndexer.mapping, doc_type="course", index=COURSES_INDEX) # Add the sorting script ES_CLIENT.put_script(id="score", body=CoursesIndexer.scripts["score"]) ES_CLIENT.put_script(id="state_field", body=CoursesIndexer.scripts["state_field"]) # Actually insert our courses in the index actions = [{ "_id": course["id"], "_index": COURSES_INDEX, "_op_type": "create", "_type": "course", "absolute_url": { "en": "en/url", "fr": "fr/url" }, "categories": ["1", "2", "3"], "cover_image": { "en": "en/image", "fr": "fr/image" }, "is_meta": False, "logo": { "en": "/en/some/img.png", "fr": "/fr/some/img.png" }, "nb_children": 0, "organizations": ["11", "12", "13"], **course, } for course in courses] bulk(actions=actions, chunk_size=500, client=ES_CLIENT) indices_client.refresh() results = self.client.get( f"/api/v1.0/courses/autocomplete/?{querystring:s}", **extra) self.assertEqual(results.status_code, 200) return json.loads(results.content)
def recreate_index(self, index_name, index_mapping): indices_client = IndicesClient(client=ES_CLIENT) if indices_client.exists(index_name): indices_client.delete(index=index_name) indices_client.create(index=index_name) indices_client.put_mapping(doc_type='page', index=index_name, body=index_mapping)