def test_es_award_loader_class(award_data_fixture, elasticsearch_award_index, baby_sleeps): elasticsearch_client = instantiate_elasticsearch_client() loader = Rapidloader(config, elasticsearch_client) assert loader.__class__.__name__ == "Rapidloader" loader.run_load_steps() assert elasticsearch_client.indices.exists(config["index_name"]) elasticsearch_client.indices.delete(index=config["index_name"], ignore_unavailable=False)
def test_es_award_loader_class(award_data_fixture, elasticsearch_award_index, baby_sleeps, monkeypatch): monkeypatch.setattr("usaspending_api.etl.es_etl_helpers.execute_sql_statement", mock_execute_sql) elasticsearch_client = instantiate_elasticsearch_client() loader = Rapidloader(config, elasticsearch_client) assert loader.__class__.__name__ == "Rapidloader" loader.run_load_steps() assert elasticsearch_client.indices.exists(config["index_name"]) elasticsearch_client.indices.delete(index=config["index_name"], ignore_unavailable=False)
def handle(self, *args, **options): elasticsearch_client = instantiate_elasticsearch_client() config = process_cli_parameters(options, elasticsearch_client) start = perf_counter() printf({"msg": f"Starting script\n{'=' * 56}"}) start_msg = "target index: {index_name} | FY(s): {fiscal_years} | Starting from: {starting_date}" printf({"msg": start_msg.format(**config)}) if config["load_type"] == "transactions": ensure_view_exists(settings.ES_TRANSACTIONS_ETL_VIEW_NAME) elif config["load_type"] == "awards": ensure_view_exists(settings.ES_AWARDS_ETL_VIEW_NAME) loader = Rapidloader(config, elasticsearch_client) loader.run_load_steps() loader.complete_process() printf({ "msg": "---------------------------------------------------------------" }) printf({"msg": f"Script completed in {perf_counter() - start:.2f}s"}) printf({ "msg": "---------------------------------------------------------------" })