def main(): """Publish UDFs.""" args = parser.parse_args() client = bigquery.Client(args.project_id) if args.dependency_dir: push_dependencies_to_gcs(args.gcs_bucket, args.gcs_path, args.dependency_dir, args.project_id) raw_udfs = read_udf_dirs(args.udf_dir) published_udfs = [] for raw_udf in raw_udfs: # get all dependencies for UDF and publish as persistent UDF udfs_to_publish = accumulate_dependencies([], raw_udfs, raw_udf) udfs_to_publish.append(raw_udf) for dep in udfs_to_publish: if dep not in published_udfs: publish_udf( raw_udfs[dep], client, args.project_id, args.gcs_bucket, args.gcs_path, raw_udfs.keys(), ) published_udfs.append(dep)
def main(): """Validate SQL examples.""" args = parser.parse_args() # parse UDFs parsed_udfs = read_udf_dirs(*args.project_dirs) for project_dir in args.project_dirs: if os.path.isdir(project_dir): for root, dirs, files in os.walk(project_dir): if os.path.basename(root) == EXAMPLE_DIR: for file in files: dry_run_sql = sql_for_dry_run(os.path.join(root, file), parsed_udfs, project_dir) # store sql in temporary file for dry_run tmp_dir = Path(tempfile.mkdtemp()) / Path(root) tmp_dir.mkdir(parents=True, exist_ok=True) tmp_example_file = tmp_dir / file tmp_example_file.write_text(dry_run_sql) dry_run_sql_file(str(tmp_example_file))
QueryParameter = Union[bigquery.ArrayQueryParameter, bigquery.ScalarQueryParameter, bigquery.StructQueryParameter, ] table_extensions = { "ndjson": bigquery.SourceFormat.NEWLINE_DELIMITED_JSON, "csv": bigquery.SourceFormat.CSV, "backup_info": bigquery.SourceFormat.DATASTORE_BACKUP, "export_metadata": bigquery.SourceFormat.DATASTORE_BACKUP, "avro": bigquery.SourceFormat.AVRO, "parquet": bigquery.SourceFormat.PARQUET, "orc": bigquery.SourceFormat.ORC, } raw_udfs = parse_udf.read_udf_dirs() @dataclass class Table: """Define info needed to create a table for a generated test.""" name: str source_format: str # a tuple means read via `load(*source_path)` and format as source_format # a string means source_path is already in source_format source_path: Union[str, Tuple[str, str]] # post_init fields schema: Optional[List[bigquery.SchemaField]] = None def __post_init__(self):
def udfs(): """Get all udfs and assertions.""" return read_udf_dirs("tests/assert", "udf", "udf_js")