def write(self): logger.info("Started indexing valueset %s", self.valueset) lookup_es.delete_valueset(self.valueset['valueset']) lookup_es.index(lookup_es.VALUESET_INDEX_NAME, self.valueset) lookup_es.index_by_bulk(self.entities) logger.info("Finished indexing valueset %s", self.valueset)
def write(self): logger.info("Started indexing valueset %s", self.valueset) lookup_es.delete_valueset(self.valueset['valueset']) lookup_es.index(lookup_es.VALUESET_INDEX_NAME, self.valueset) entity_set = set() for entity in self.entities: if entity["entity"] in entity_set: continue lookup_es.index(lookup_es.ENTITY_INDEX_NAME, entity) entity_set.add(entity["entity"]) entity_set.clear() logger.info("Finished indexing valueset %s", self.valueset)
def write(self): logger.info("Started indexing valueset %s", self.valueset) lookup_es.delete_valueset(self.valueset['valueset']) lookup_es.index(lookup_es.VALUESET_INDEX_NAME, self.valueset) data = [] count = 0 for entity in self.entities: data.append(entity) if len(data) % 100000 == 0 or count == len(self.entities) - 1: logger.info("indexing bulk of %d", len(data)) lookup_es.index_by_bulk(data) data.clear() count += 1 logger.info("Finished indexing valueset %s", self.valueset)