def generate_dbt_models(destination_type: DestinationType, test_resource_name: str, test_root_dir: str):
"""
This is the normalization step generating dbt models files from the destination_catalog.json taken as input.
"""
catalog_processor = CatalogProcessor(os.path.join(test_root_dir, "models", "generated"), destination_type)
catalog_processor.process(
os.path.join("resources", test_resource_name, "data_input", "catalog.json"), "_airbyte_data", dbt_test_utils.target_schema
)
def process_catalog(self) -> None:
destination_type = DestinationType.from_string(self.config["integration_type"])
schema = self.config["schema"]
output = self.config["output_path"]
json_col = self.config["json_column"]
processor = CatalogProcessor(output_directory=output, destination_type=destination_type)
for catalog_file in self.config["catalog"]:
print(f"Processing {catalog_file}...")
processor.process(catalog_file=catalog_file, json_column_name=json_col, default_schema=schema)
def generate_dbt_models(destination_type: DestinationType, test_root_dir: str,
column_count: int):
"""
This is the normalization step generating dbt models files from the destination_catalog.json taken as input.
"""
output_directory = os.path.join(test_root_dir, "models", "generated")
shutil.rmtree(output_directory, ignore_errors=True)
catalog_processor = CatalogProcessor(output_directory, destination_type)
catalog_config = {
"streams": [{
"stream": {
"name":
dbt_test_utils.generate_random_string(
f"stream_with_{column_count}_columns"),
"json_schema": {
"type": ["null", "object"],
"properties": {},
},
"supported_sync_modes": ["incremental"],
"source_defined_cursor":
True,
"default_cursor_field": [],
},
"sync_mode": "incremental",
"cursor_field": [],
"destination_sync_mode": "overwrite",
}]
}
if column_count == 1:
catalog_config["streams"][0]["stream"]["json_schema"]["properties"][
"_airbyte_id"] = {
"type": "integer"
}
else:
for column in [
dbt_test_utils.random_string(5) for _ in range(column_count)
]:
catalog_config["streams"][0]["stream"]["json_schema"][
"properties"][column] = {
"type": "string"
}
catalog = os.path.join(test_root_dir, "catalog.json")
with open(catalog, "w") as fh:
fh.write(json.dumps(catalog_config))
catalog_processor.process(catalog, "_airbyte_data",
dbt_test_utils.target_schema)
