def test_transform(self): input = { "host": "airbyte.io", "port": 5432, "username": "******", "password": "******", "database": "my_db", "schema": "public", } expected = self.get_base_config() expected["normalize"]["outputs"]["prod"] = { "type": "postgres", "dbname": "my_db", "host": "airbyte.io", "pass": "******", "port": 5432, "schema": "public", "threads": 32, "user": "******", } actual = TransformConfig().transform(DestinationType.postgres, input) assert expected == actual assert extract_schema( actual["normalize"]["outputs"]["prod"]) == "public"
def test_transform_bigquery(self): input = { "project_id": "my_project_id", "dataset_id": "my_dataset_id", "credentials_json": '{ "type": "service_account" }' } actual_output = TransformConfig().transform_bigquery(input) expected_output = { "type": "bigquery", "method": "service-account", "project": "my_project_id", "dataset": "my_dataset_id", "keyfile": "/tmp/bq_keyfile.json", "retries": 1, "threads": 32, } with open("/tmp/bq_keyfile.json", "r") as file: actual_keyfile = json.loads(file.read()) expected_keyfile = {"type": "service_account"} if os.path.exists("/tmp/bq_keyfile.json"): os.remove("/tmp/bq_keyfile.json") assert expected_output == actual_output assert expected_keyfile == actual_keyfile assert extract_schema(actual_output) == "my_dataset_id"
def test_transform_mssql(self): input = { "type": "sqlserver", "host": "airbyte.io", "port": 1433, "database": "my_db", "schema": "my_db", "username": "******", "password": "******", } actual = TransformConfig().transform_mysql(input) expected = { "type": "sqlserver", "server": "airbyte.io", "port": 1433, "schema": "my_db", "database": "my_db", "username": "******", "password": "******", } assert expected == actual # DBT schema is equivalent to MySQL database assert extract_schema(actual) == "my_db"
def test_transform_snowflake(self): input = { "host": "http://123abc.us-east-7.aws.snowflakecomputing.com", "role": "AIRBYTE_ROLE", "warehouse": "AIRBYTE_WAREHOUSE", "database": "AIRBYTE_DATABASE", "schema": "AIRBYTE_SCHEMA", "username": "******", "password": "******", } actual = TransformConfig().transform_snowflake(input) expected = { "account": "123abc.us-east-7.aws", "client_session_keep_alive": False, "database": "AIRBYTE_DATABASE", "password": "******", "query_tag": "normalization", "role": "AIRBYTE_ROLE", "schema": "AIRBYTE_SCHEMA", "threads": 32, "type": "snowflake", "user": "******", "warehouse": "AIRBYTE_WAREHOUSE", } assert expected == actual assert extract_schema(actual) == "AIRBYTE_SCHEMA"
def test_transform_postgres_ssh(self): input = { "host": "airbyte.io", "port": 5432, "username": "******", "password": "******", "database": "my_db", "schema": "public", "tunnel_method": { "tunnel_host": "1.2.3.4", "tunnel_method": "SSH_PASSWORD_AUTH", "tunnel_port": 22, "tunnel_user": "******", "tunnel_user_password": "******", }, } port = TransformConfig.pick_a_port() actual = TransformConfig().transform_postgres(input) expected = { "type": "postgres", "dbname": "my_db", "host": "localhost", "pass": "******", "port": port, "schema": "public", "threads": 8, "user": "******", } assert expected == actual assert extract_schema(actual) == "public"
def test_transform_bigquery(self): input = { "project_id": "my_project_id", "dataset_id": "my_dataset_id", "credentials_json": '{ "type": "service_account-json" }', "transformation_priority": "interactive", "dataset_location": "EU", } actual_output = TransformConfig().transform_bigquery(input) expected_output = { "type": "bigquery", "method": "service-account-json", "project": "my_project_id", "dataset": "my_dataset_id", "priority": "interactive", "keyfile_json": {"type": "service_account-json"}, "location": "EU", "retries": 3, "threads": 8, } actual_keyfile = actual_output["keyfile_json"] expected_keyfile = {"type": "service_account-json"} assert expected_output == actual_output assert expected_keyfile == actual_keyfile assert extract_schema(actual_output) == "my_dataset_id"
def test_transform_clickhouse(self): input = {"host": "airbyte.io", "port": 9440, "database": "default", "username": "******", "password": "******", "ssl": True} actual = TransformConfig().transform_clickhouse(input) expected = { "type": "clickhouse", "host": "airbyte.io", "port": 9440, "schema": "default", "user": "******", "password": "******", "secure": True, } assert expected == actual assert extract_schema(actual) == "default"
def test_transform_bigquery_with_embedded_project_id(self): input = {"project_id": "my_project_id", "dataset_id": "my_project_id:my_dataset_id"} actual_output = TransformConfig().transform_bigquery(input) expected_output = { "type": "bigquery", "method": "oauth", "project": "my_project_id", "dataset": "my_dataset_id", "priority": "interactive", "retries": 3, "threads": 8, } assert expected_output == actual_output assert extract_schema(actual_output) == "my_dataset_id"
def test_transform_snowflake_oauth(self): input = { "host": "http://123abc.us-east-7.aws.snowflakecomputing.com", "role": "AIRBYTE_ROLE", "warehouse": "AIRBYTE_WAREHOUSE", "database": "AIRBYTE_DATABASE", "schema": "AIRBYTE_SCHEMA", "username": "******", "credentials": { "auth_type": "OAuth2.0", "client_id": "AIRBYTE_CLIENT_ID", "access_token": "AIRBYTE_ACCESS_TOKEN", "client_secret": "AIRBYTE_CLIENT_SECRET", "refresh_token": "AIRBYTE_REFRESH_TOKEN", }, } actual = TransformConfig().transform_snowflake(input) expected = { "account": "123abc.us-east-7.aws", "client_session_keep_alive": False, "database": "AIRBYTE_DATABASE", "query_tag": "normalization", "role": "AIRBYTE_ROLE", "schema": "AIRBYTE_SCHEMA", "threads": 5, "retry_all": True, "retry_on_database_errors": True, "connect_retries": 3, "connect_timeout": 15, "type": "snowflake", "user": "******", "warehouse": "AIRBYTE_WAREHOUSE", "authenticator": "oauth", "oauth_client_id": "AIRBYTE_CLIENT_ID", "oauth_client_secret": "AIRBYTE_CLIENT_SECRET", "token": "AIRBYTE_REFRESH_TOKEN", } assert expected == actual assert extract_schema(actual) == "AIRBYTE_SCHEMA"
def test_transform_postgres(self): input = { "host": "airbyte.io", "port": 5432, "username": "******", "password": "******", "database": "my_db", "schema": "public", } actual = TransformConfig().transform_postgres(input) expected = { "type": "postgres", "dbname": "my_db", "host": "airbyte.io", "pass": "******", "port": 5432, "schema": "public", "threads": 32, "user": "******", } assert expected == actual assert extract_schema(actual) == "public"