def test_make_main_table_v3(self): with temporary_directory() as workdir: # Copy all test tables into the temporary directory publish_global_tables(SRC / "test" / "data", workdir) # Create the main table main_table_path = workdir / "main.csv" merge_output_tables(workdir, main_table_path, use_table_names=V3_TABLE_LIST) self._test_make_main_table_helper(main_table_path, OUTPUT_COLUMN_ADAPTER)
def test_make_main_table(self): with temporary_directory() as workdir: # Copy all test tables into the temporary directory copy_tables(SRC / "test" / "data", workdir) # Create the main table main_table_path = workdir / "main.csv" merge_output_tables(workdir, main_table_path) self._test_make_main_table_helper(main_table_path, {})
def publish_main_table() -> Response: with temporary_directory() as workdir: input_folder = workdir / "input" output_folder = workdir / "output" input_folder.mkdir(parents=True, exist_ok=True) output_folder.mkdir(parents=True, exist_ok=True) # Download the already published tables allowlist_filenames = [f"{table}.csv" for table in get_table_names()] download_folder(GCS_BUCKET_PROD, "v2", input_folder, lambda x: str(x) in allowlist_filenames) # Create the joint main table for all records main_table_path = output_folder / "main.csv" merge_output_tables(input_folder, main_table_path) logger.log_info("Main table created") # Upload the results to the prod bucket upload_folder(GCS_BUCKET_PROD, "v2", output_folder) return Response("OK", status=200)