def test_make_main_table_v3(self):
with temporary_directory() as workdir:
# Copy all test tables into the temporary directory
publish_global_tables(SRC / "test" / "data", workdir)
# Create the main table
main_table_path = workdir / "main.csv"
merge_output_tables(workdir, main_table_path, use_table_names=V3_TABLE_LIST)
self._test_make_main_table_helper(main_table_path, OUTPUT_COLUMN_ADAPTER)
def test_make_main_table(self):
with temporary_directory() as workdir:
# Copy all test tables into the temporary directory
copy_tables(SRC / "test" / "data", workdir)
# Create the main table
main_table_path = workdir / "main.csv"
merge_output_tables(workdir, main_table_path)
self._test_make_main_table_helper(main_table_path, {})
def publish_main_table() -> Response:
with temporary_directory() as workdir:
input_folder = workdir / "input"
output_folder = workdir / "output"
input_folder.mkdir(parents=True, exist_ok=True)
output_folder.mkdir(parents=True, exist_ok=True)
# Download the already published tables
allowlist_filenames = [f"{table}.csv" for table in get_table_names()]
download_folder(GCS_BUCKET_PROD, "v2", input_folder,
lambda x: str(x) in allowlist_filenames)
# Create the joint main table for all records
main_table_path = output_folder / "main.csv"
merge_output_tables(input_folder, main_table_path)
logger.log_info("Main table created")
# Upload the results to the prod bucket
upload_folder(GCS_BUCKET_PROD, "v2", output_folder)
return Response("OK", status=200)