def publish_global_tables_(prod_folder: str = "v2") -> Response: prod_folder = _get_request_param("prod_folder", prod_folder) with temporary_directory() as workdir: tables_folder = workdir / "tables" public_folder = workdir / "public" tables_folder.mkdir(parents=True, exist_ok=True) public_folder.mkdir(parents=True, exist_ok=True) # Download all the combined tables into our local storage download_folder(GCS_BUCKET_TEST, "tables", tables_folder) # Publish the tables containing all location keys table_names, column_adapter = None, None if prod_folder == "v2": table_names, column_adapter = V2_TABLE_LIST, {} if prod_folder == "v3": table_names, column_adapter = V3_TABLE_LIST, OUTPUT_COLUMN_ADAPTER assert table_names is not None and column_adapter is not None publish_global_tables(tables_folder, public_folder, use_table_names=table_names, column_adapter=column_adapter) # Upload the results to the prod bucket upload_folder(GCS_BUCKET_PROD, prod_folder, public_folder) return Response("OK", status=200)
def test_make_main_table_v3(self): with temporary_directory() as workdir: # Copy all test tables into the temporary directory publish_global_tables(SRC / "test" / "data", workdir) # Create the main table main_table_path = workdir / "main.csv" merge_output_tables(workdir, main_table_path, use_table_names=V3_TABLE_LIST) self._test_make_main_table_helper(main_table_path, OUTPUT_COLUMN_ADAPTER)
def publish_v3_global_tables() -> Response: with temporary_directory() as workdir: tables_folder = workdir / "tables" public_folder = workdir / "public" tables_folder.mkdir(parents=True, exist_ok=True) public_folder.mkdir(parents=True, exist_ok=True) # Download all the combined tables into our local storage download_folder(GCS_BUCKET_TEST, "tables", tables_folder) # Publish the tables containing all location keys publish_global_tables(tables_folder, public_folder) logger.log_info("Global tables created") # Upload the results to the prod bucket upload_folder(GCS_BUCKET_PROD, "v3", public_folder) return Response("OK", status=200)
def test_convert_to_json(self): with temporary_directory() as workdir: # Copy all test tables into the temporary directory publish_global_tables(SRC / "test" / "data", workdir) # Copy test tables again but under a subpath subpath = workdir / "latest" subpath.mkdir() publish_global_tables(workdir, subpath) # Convert all the tables to JSON under a new path jsonpath = workdir / "json" jsonpath.mkdir() convert_tables_to_json(workdir, jsonpath) # The JSON files should maintain the same relative path for csv_file in workdir.glob("**/*.csv"): self.assertTrue((workdir / "json" / f"{csv_file.stem}.json").exists()) self.assertTrue((workdir / "json" / "latest" / f"{csv_file.stem}.json").exists())
def test_import_tables_into_sqlite(self): with temporary_directory() as workdir: intermediate = workdir / "intermediate" intermediate.mkdir(parents=True, exist_ok=True) # Copy all test tables into the temporary directory publish_global_tables( SRC / "test" / "data", intermediate, use_table_names=V3_TABLE_LIST ) # Create the SQLite file and open it sqlite_output = workdir / "database.sqlite" table_paths = list(intermediate.glob("*.csv")) import_tables_into_sqlite(table_paths, sqlite_output) with create_sqlite_database(sqlite_output) as conn: # Verify that each table contains all the data for table in table_paths: temp_path = workdir / f"{table.stem}.csv" table_export_csv(conn, _safe_table_name(table.stem), temp_path) _compare_tables_equal(self, table, temp_path)