def test_transform(self):
input = {
"host": "airbyte.io",
"port": 5432,
"username": "******",
"password": "******",
"database": "my_db",
"schema": "public",
}
expected = self.get_base_config()
expected["normalize"]["outputs"]["prod"] = {
"type": "postgres",
"dbname": "my_db",
"host": "airbyte.io",
"pass": "******",
"port": 5432,
"schema": "public",
"threads": 32,
"user": "******",
}
actual = TransformConfig().transform(DestinationType.postgres, input)
assert expected == actual
assert extract_schema(
actual["normalize"]["outputs"]["prod"]) == "public"
def test_transform_bigquery(self):
input = {
"project_id": "my_project_id",
"dataset_id": "my_dataset_id",
"credentials_json": '{ "type": "service_account" }'
}
actual_output = TransformConfig().transform_bigquery(input)
expected_output = {
"type": "bigquery",
"method": "service-account",
"project": "my_project_id",
"dataset": "my_dataset_id",
"keyfile": "/tmp/bq_keyfile.json",
"retries": 1,
"threads": 32,
}
with open("/tmp/bq_keyfile.json", "r") as file:
actual_keyfile = json.loads(file.read())
expected_keyfile = {"type": "service_account"}
if os.path.exists("/tmp/bq_keyfile.json"):
os.remove("/tmp/bq_keyfile.json")
assert expected_output == actual_output
assert expected_keyfile == actual_keyfile
assert extract_schema(actual_output) == "my_dataset_id"
def test_transform_mssql(self):
input = {
"type": "sqlserver",
"host": "airbyte.io",
"port": 1433,
"database": "my_db",
"schema": "my_db",
"username": "******",
"password": "******",
}
actual = TransformConfig().transform_mysql(input)
expected = {
"type": "sqlserver",
"server": "airbyte.io",
"port": 1433,
"schema": "my_db",
"database": "my_db",
"username": "******",
"password": "******",
}
assert expected == actual
# DBT schema is equivalent to MySQL database
assert extract_schema(actual) == "my_db"
def test_transform_snowflake(self):
input = {
"host": "http://123abc.us-east-7.aws.snowflakecomputing.com",
"role": "AIRBYTE_ROLE",
"warehouse": "AIRBYTE_WAREHOUSE",
"database": "AIRBYTE_DATABASE",
"schema": "AIRBYTE_SCHEMA",
"username": "******",
"password": "******",
}
actual = TransformConfig().transform_snowflake(input)
expected = {
"account": "123abc.us-east-7.aws",
"client_session_keep_alive": False,
"database": "AIRBYTE_DATABASE",
"password": "******",
"query_tag": "normalization",
"role": "AIRBYTE_ROLE",
"schema": "AIRBYTE_SCHEMA",
"threads": 32,
"type": "snowflake",
"user": "******",
"warehouse": "AIRBYTE_WAREHOUSE",
}
assert expected == actual
assert extract_schema(actual) == "AIRBYTE_SCHEMA"
def test_transform_postgres_ssh(self):
input = {
"host": "airbyte.io",
"port": 5432,
"username": "******",
"password": "******",
"database": "my_db",
"schema": "public",
"tunnel_method": {
"tunnel_host": "1.2.3.4",
"tunnel_method": "SSH_PASSWORD_AUTH",
"tunnel_port": 22,
"tunnel_user": "******",
"tunnel_user_password": "******",
},
}
port = TransformConfig.pick_a_port()
actual = TransformConfig().transform_postgres(input)
expected = {
"type": "postgres",
"dbname": "my_db",
"host": "localhost",
"pass": "******",
"port": port,
"schema": "public",
"threads": 8,
"user": "******",
}
assert expected == actual
assert extract_schema(actual) == "public"
def test_transform_bigquery(self):
input = {
"project_id": "my_project_id",
"dataset_id": "my_dataset_id",
"credentials_json": '{ "type": "service_account-json" }',
"transformation_priority": "interactive",
"dataset_location": "EU",
}
actual_output = TransformConfig().transform_bigquery(input)
expected_output = {
"type": "bigquery",
"method": "service-account-json",
"project": "my_project_id",
"dataset": "my_dataset_id",
"priority": "interactive",
"keyfile_json": {"type": "service_account-json"},
"location": "EU",
"retries": 3,
"threads": 8,
}
actual_keyfile = actual_output["keyfile_json"]
expected_keyfile = {"type": "service_account-json"}
assert expected_output == actual_output
assert expected_keyfile == actual_keyfile
assert extract_schema(actual_output) == "my_dataset_id"
def test_transform_clickhouse(self):
input = {"host": "airbyte.io", "port": 9440, "database": "default", "username": "******", "password": "******", "ssl": True}
actual = TransformConfig().transform_clickhouse(input)
expected = {
"type": "clickhouse",
"host": "airbyte.io",
"port": 9440,
"schema": "default",
"user": "******",
"password": "******",
"secure": True,
}
assert expected == actual
assert extract_schema(actual) == "default"
def test_transform_bigquery_with_embedded_project_id(self):
input = {"project_id": "my_project_id", "dataset_id": "my_project_id:my_dataset_id"}
actual_output = TransformConfig().transform_bigquery(input)
expected_output = {
"type": "bigquery",
"method": "oauth",
"project": "my_project_id",
"dataset": "my_dataset_id",
"priority": "interactive",
"retries": 3,
"threads": 8,
}
assert expected_output == actual_output
assert extract_schema(actual_output) == "my_dataset_id"
def test_transform_snowflake_oauth(self):
input = {
"host": "http://123abc.us-east-7.aws.snowflakecomputing.com",
"role": "AIRBYTE_ROLE",
"warehouse": "AIRBYTE_WAREHOUSE",
"database": "AIRBYTE_DATABASE",
"schema": "AIRBYTE_SCHEMA",
"username": "******",
"credentials": {
"auth_type": "OAuth2.0",
"client_id": "AIRBYTE_CLIENT_ID",
"access_token": "AIRBYTE_ACCESS_TOKEN",
"client_secret": "AIRBYTE_CLIENT_SECRET",
"refresh_token": "AIRBYTE_REFRESH_TOKEN",
},
}
actual = TransformConfig().transform_snowflake(input)
expected = {
"account": "123abc.us-east-7.aws",
"client_session_keep_alive": False,
"database": "AIRBYTE_DATABASE",
"query_tag": "normalization",
"role": "AIRBYTE_ROLE",
"schema": "AIRBYTE_SCHEMA",
"threads": 5,
"retry_all": True,
"retry_on_database_errors": True,
"connect_retries": 3,
"connect_timeout": 15,
"type": "snowflake",
"user": "******",
"warehouse": "AIRBYTE_WAREHOUSE",
"authenticator": "oauth",
"oauth_client_id": "AIRBYTE_CLIENT_ID",
"oauth_client_secret": "AIRBYTE_CLIENT_SECRET",
"token": "AIRBYTE_REFRESH_TOKEN",
}
assert expected == actual
assert extract_schema(actual) == "AIRBYTE_SCHEMA"
def test_transform_postgres(self):
input = {
"host": "airbyte.io",
"port": 5432,
"username": "******",
"password": "******",
"database": "my_db",
"schema": "public",
}
actual = TransformConfig().transform_postgres(input)
expected = {
"type": "postgres",
"dbname": "my_db",
"host": "airbyte.io",
"pass": "******",
"port": 5432,
"schema": "public",
"threads": 32,
"user": "******",
}
assert expected == actual
assert extract_schema(actual) == "public"
