def test_resolve_names(destination_type: DestinationType, catalog_file: str): """ For a given catalog.json and destination, multiple cases can occur where naming becomes tricky. (especially since some destination like postgres have a very low limit to identifiers length of 64 characters) In case of nested objects/arrays in a stream, names can drag on to very long names. Tests are built here using resources files as follow: - `<name of source or test types>_catalog.json`: input catalog.json, typically as what source would provide. For example Hubspot, Stripe and Facebook catalog.json contains some level of nesting. (here, nested_catalog.json is an extracted smaller sample of stream/properties from the facebook catalog) - `<name of source or test types>_expected_names.json`: list of expected table names For the expected json files, it is possible to specialize the file to a certain destination. So if for example, the resources folder contains these two expected files: - edge_cases_catalog_expected_names.json - edge_cases_catalog_expected_postgres_names.json Then the test will be using the first edge_cases_catalog_expected_names.json except for Postgres destination where the expected table names will come from edge_cases_catalog_expected_postgres_names.json The content of the expected_*.json files are the serialization of the stream_processor.tables_registry.registry """ integration_type = destination_type.value tables_registry = TableNameRegistry(destination_type) catalog = read_json(f"resources/{catalog_file}.json") # process top level stream_processors = CatalogProcessor.build_stream_processor( catalog=catalog, json_column_name="'json_column_name_test'", default_schema="schema_test", name_transformer=DestinationNameTransformer(destination_type), destination_type=destination_type, tables_registry=tables_registry, ) for stream_processor in stream_processors: # Check properties if not stream_processor.properties: raise EOFError("Invalid Catalog: Unexpected empty properties in catalog") stream_processor.collect_table_names() for conflict in tables_registry.resolve_names(): print( f"WARN: Resolving conflict: {conflict.schema}.{conflict.table_name_conflict} " f"from '{'.'.join(conflict.json_path)}' into {conflict.table_name_resolved}" ) apply_function = identity if DestinationType.SNOWFLAKE.value == destination_type.value: apply_function = str.upper elif DestinationType.REDSHIFT.value == destination_type.value: apply_function = str.lower if os.path.exists(f"resources/{catalog_file}_expected_{integration_type.lower()}_names.json"): expected_names = read_json(f"resources/{catalog_file}_expected_{integration_type.lower()}_names.json", apply_function) else: expected_names = read_json(f"resources/{catalog_file}_expected_names.json", apply_function) assert tables_registry.to_dict(apply_function) == expected_names
def test_primary_key( primary_key: List[List[str]], column_type: str, expecting_exception: bool, expected_primary_keys: List[str], expected_final_primary_key_string: str, ): stream_processor = StreamProcessor.create( stream_name="test_primary_key", destination_type=DestinationType.POSTGRES, raw_schema="raw_schema", default_schema="default_schema", schema="schema_name", source_sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append_dedup, cursor_field=[], primary_key=primary_key, json_column_name="json_column_name", properties={ key: { "type": column_type } for key in expected_primary_keys }, tables_registry=TableNameRegistry(DestinationType.POSTGRES), from_table="", ) try: assert (", ".join( stream_processor.get_primary_key_partition( column_names=stream_processor.extract_column_names())) == expected_final_primary_key_string) except ValueError as e: if not expecting_exception: raise e
def process(self, catalog_file: str, json_column_name: str, default_schema: str): """ This method first parse and build models to handle top-level streams. In a second loop will go over the substreams that were nested in a breadth-first traversal manner. @param catalog_file input AirbyteCatalog file in JSON Schema describing the structure of the raw data @param json_column_name is the column name containing the JSON Blob with the raw data @param default_schema is the final schema where to output the final transformed data to """ tables_registry: TableNameRegistry = TableNameRegistry( self.destination_type) schema_to_source_tables: Dict[str, Set[str]] = {} catalog = read_json(catalog_file) # print(json.dumps(catalog, separators=(",", ":"))) substreams = [] stream_processors = self.build_stream_processor( catalog=catalog, json_column_name=json_column_name, default_schema=default_schema, name_transformer=self.name_transformer, destination_type=self.destination_type, tables_registry=tables_registry, ) for stream_processor in stream_processors: stream_processor.collect_table_names() for conflict in tables_registry.resolve_names(): print( f"WARN: Resolving conflict: {conflict.schema}.{conflict.table_name_conflict} " f"from '{'.'.join(conflict.json_path)}' into {conflict.table_name_resolved}" ) for stream_processor in stream_processors: # MySQL table names need to be manually truncated, because it does not do it automatically truncate = self.destination_type == DestinationType.MYSQL raw_table_name = self.name_transformer.normalize_table_name( f"_airbyte_raw_{stream_processor.stream_name}", truncate=truncate) add_table_to_sources(schema_to_source_tables, stream_processor.schema, raw_table_name) nested_processors = stream_processor.process() if nested_processors and len(nested_processors) > 0: substreams += nested_processors for file in stream_processor.sql_outputs: output_sql_file(os.path.join(self.output_directory, file), stream_processor.sql_outputs[file]) self.write_yaml_sources_file(schema_to_source_tables) self.process_substreams(substreams, tables_registry)
def test_get_simple_table_name(json_path: List[str], expected_postgres: str, expected_bigquery: str): """ Checks how to generate a simple and easy to understand name from a json path """ postgres_registry = TableNameRegistry(DestinationType.POSTGRES) actual_postgres_name = postgres_registry.get_simple_table_name(json_path) assert actual_postgres_name == expected_postgres assert len(actual_postgres_name) <= 43 # explicitly check for our max postgres length in case tests are changed in the future bigquery_registry = TableNameRegistry(DestinationType.BIGQUERY) actual_bigquery_name = bigquery_registry.get_simple_table_name(json_path) assert actual_bigquery_name == expected_bigquery
def test_cursor_field(cursor_field: List[str], expecting_exception: bool, expected_cursor_field: str): stream_processor = StreamProcessor.create( stream_name="test_cursor_field", destination_type=DestinationType.POSTGRES, raw_schema="raw_schema", schema="schema_name", source_sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append_dedup, cursor_field=cursor_field, primary_key=[], json_column_name="json_column_name", properties=dict(), tables_registry=TableNameRegistry(DestinationType.POSTGRES), from_table="", ) try: assert (stream_processor.get_cursor_field(column_names={ expected_cursor_field: (expected_cursor_field, "random") }) == expected_cursor_field) except ValueError as e: if not expecting_exception: raise e