def test_no_indexes_setup(self, patch): elasticSearchFunctions.setup('elasticsearch:9200', enabled=[]) elasticSearchFunctions.setup('elasticsearch:9200', enabled=['unknown']) patch.assert_not_called()
def test_default_setup(self, patch): elasticSearchFunctions.setup('elasticsearch:9200') patch.assert_called_with( ANY, ['aips', 'aipfiles', 'transfers', 'transferfiles'])
def test_only_transfers_setup(self, patch): elasticSearchFunctions.setup('elasticsearch:9200', enabled=['transfers']) patch.assert_called_with(ANY, ['transfers', 'transferfiles'])
def setUp(self): elasticSearchFunctions.setup('elasticsearch:9200') self.client = elasticSearchFunctions.get_client() self.aip_uuid = 'b34521a3-1c63-43dd-b901-584416f36c91' self.file_uuid = '268421a7-a986-4fa0-95c1-54176e508210'
def test_only_transfers_setup(self, patch): elasticSearchFunctions.setup("elasticsearch:9200", enabled=["transfers"]) patch.assert_called_with(ANY, ["transfers", "transferfiles"])
def test_default_setup(self, patch): elasticSearchFunctions.setup("elasticsearch:9200") patch.assert_called_with( ANY, ["aips", "aipfiles", "transfers", "transferfiles"] )
def setUp(self): hosts = os.environ.get('ELASTICSEARCH_SERVER', '127.0.0.1:9200') elasticSearchFunctions.setup(hosts) self.client = elasticSearchFunctions.get_client() self.aip_uuid = 'a1ee611a-a4f5-4ba9-b7ce-b92695746514'
def handle(self, *args, **options): # Check search enabled configuration if len(settings.SEARCH_ENABLED) == 0: self.error( "The Elasticsearch indexes are not enabled. Please, make sure " "to set the *_SEARCH_ENABLED environment variables to `true` " "to enable the AIPs and Transfers indexes, to `aips` " "to only enable the AIPs indexes or to `transfers` " "to only enable the Transfers indexes.") sys.exit(1) # Setup new cluster connection. Do not pass SEARCH_ENABLED # setting to avoid the creation of the indexes on setup, # and use the timeout passed to the command. elasticSearchFunctions.setup(settings.ELASTICSEARCH_SERVER, options["timeout"], []) es_client = elasticSearchFunctions.get_client() # Get enabled indexes based on setting indexes = [] if "aips" in settings.SEARCH_ENABLED: indexes.extend(["aips", "aipfiles"]) if "transfers" in settings.SEARCH_ENABLED: indexes.extend(["transfers", "transferfiles"]) # Delete all indexes and create enabled ones in new cluster self.info("Creating new indexes.") try: es_client.indices.delete(",".join(elasticSearchFunctions.INDEXES), ignore=404) elasticSearchFunctions.create_indexes_if_needed(es_client, indexes) except Exception as e: self.error( "The Elasticsearch indexes could not be recreated in {}. " "Error: {}".format(settings.ELASTICSEARCH_SERVER, e)) sys.exit(1) # Default body for reindex requests body = { "source": { "remote": { "host": options["host"], "socket_timeout": "{}s".format(options["timeout"]), "connect_timeout": "{}s".format(options["timeout"]), }, "index": "", "type": "", "size": options["size"], }, "dest": { "index": "", "type": elasticSearchFunctions.DOC_TYPE }, } # Add basic auth if options["username"] != "": body["source"]["remote"]["username"] = options["username"] if options["password"] != "": body["source"]["remote"]["password"] = options["password"] # Indexes and types to reindex indexes_relations = [ { "dest_index": "aips", "source_index": "aips", "source_type": "aip" }, { "dest_index": "aipfiles", "source_index": "aips", "source_type": "aipfile", }, { "dest_index": "transfers", "source_index": "transfers", "source_type": "transfer", }, { "dest_index": "transferfiles", "source_index": "transfers", "source_type": "transferfile", }, ] # Reindex documents from remote cluster fails = 0 for indexes_relation in indexes_relations: # Skip not enabled indexes if indexes_relation["dest_index"] not in indexes: continue # Update request body body["dest"]["index"] = indexes_relation["dest_index"] body["source"]["index"] = indexes_relation["source_index"] body["source"]["type"] = indexes_relation["source_type"] # Reindex request self.info("Reindexing %s:" % indexes_relation["dest_index"]) try: response = es_client.reindex(body=body) except elasticsearch.TransportError as exc: fails += 1 self.error("Error: {}. Details:\n{}".format(exc, exc.info)) except Exception as exc: fails += 1 self.error("Error: {}".format(exc)) else: self.info("Response:\n%s" % json.dumps(response, indent=4)) if fails > 0: self.error("%s reindex request(s) failed!" % fails) else: self.success("All reindex requests ended successfully!")