def publish_global_tables_(prod_folder: str = "v2") -> Response:
prod_folder = _get_request_param("prod_folder", prod_folder)
with temporary_directory() as workdir:
tables_folder = workdir / "tables"
public_folder = workdir / "public"
tables_folder.mkdir(parents=True, exist_ok=True)
public_folder.mkdir(parents=True, exist_ok=True)
# Download all the combined tables into our local storage
download_folder(GCS_BUCKET_TEST, "tables", tables_folder)
# Publish the tables containing all location keys
table_names, column_adapter = None, None
if prod_folder == "v2":
table_names, column_adapter = V2_TABLE_LIST, {}
if prod_folder == "v3":
table_names, column_adapter = V3_TABLE_LIST, OUTPUT_COLUMN_ADAPTER
assert table_names is not None and column_adapter is not None
publish_global_tables(tables_folder,
public_folder,
use_table_names=table_names,
column_adapter=column_adapter)
# Upload the results to the prod bucket
upload_folder(GCS_BUCKET_PROD, prod_folder, public_folder)
return Response("OK", status=200)
def test_make_main_table_v3(self):
with temporary_directory() as workdir:
# Copy all test tables into the temporary directory
publish_global_tables(SRC / "test" / "data", workdir)
# Create the main table
main_table_path = workdir / "main.csv"
merge_output_tables(workdir, main_table_path, use_table_names=V3_TABLE_LIST)
self._test_make_main_table_helper(main_table_path, OUTPUT_COLUMN_ADAPTER)
def publish_v3_global_tables() -> Response:
with temporary_directory() as workdir:
tables_folder = workdir / "tables"
public_folder = workdir / "public"
tables_folder.mkdir(parents=True, exist_ok=True)
public_folder.mkdir(parents=True, exist_ok=True)
# Download all the combined tables into our local storage
download_folder(GCS_BUCKET_TEST, "tables", tables_folder)
# Publish the tables containing all location keys
publish_global_tables(tables_folder, public_folder)
logger.log_info("Global tables created")
# Upload the results to the prod bucket
upload_folder(GCS_BUCKET_PROD, "v3", public_folder)
return Response("OK", status=200)
def test_convert_to_json(self):
with temporary_directory() as workdir:
# Copy all test tables into the temporary directory
publish_global_tables(SRC / "test" / "data", workdir)
# Copy test tables again but under a subpath
subpath = workdir / "latest"
subpath.mkdir()
publish_global_tables(workdir, subpath)
# Convert all the tables to JSON under a new path
jsonpath = workdir / "json"
jsonpath.mkdir()
convert_tables_to_json(workdir, jsonpath)
# The JSON files should maintain the same relative path
for csv_file in workdir.glob("**/*.csv"):
self.assertTrue((workdir / "json" / f"{csv_file.stem}.json").exists())
self.assertTrue((workdir / "json" / "latest" / f"{csv_file.stem}.json").exists())
def test_import_tables_into_sqlite(self):
with temporary_directory() as workdir:
intermediate = workdir / "intermediate"
intermediate.mkdir(parents=True, exist_ok=True)
# Copy all test tables into the temporary directory
publish_global_tables(
SRC / "test" / "data", intermediate, use_table_names=V3_TABLE_LIST
)
# Create the SQLite file and open it
sqlite_output = workdir / "database.sqlite"
table_paths = list(intermediate.glob("*.csv"))
import_tables_into_sqlite(table_paths, sqlite_output)
with create_sqlite_database(sqlite_output) as conn:
# Verify that each table contains all the data
for table in table_paths:
temp_path = workdir / f"{table.stem}.csv"
table_export_csv(conn, _safe_table_name(table.stem), temp_path)
_compare_tables_equal(self, table, temp_path)