def __init__(self, target: str) -> None:
super().__init__()
self.target: str = target
self.dialect = self.create_dialect(self.target)
self.warehouse_yml = WarehouseYml(dialect=self.dialect, name='test_warehouse')
self.warehouse: Optional[Warehouse] = Warehouse(self.warehouse_yml)
self.database: Optional[str] = None
self.create_database()
def __init__(self,
warehouse_yml_dict: dict,
warehouse_yml_path: str = 'warehouse-dict'):
super().__init__(description=warehouse_yml_path)
if isinstance(warehouse_yml_dict, dict):
self._push_context(object=warehouse_yml_dict,
name=self.description)
self.warehouse_yml = WarehouseYml()
self.warehouse_yml.name = self.get_str_required(KEY_NAME)
EnvVars.load_env_vars(self.warehouse_yml.name)
connection_dict = self.get_dict_required(KEY_CONNECTION)
if connection_dict:
self._push_context(object=connection_dict, name=KEY_CONNECTION)
self.warehouse_yml.dialect = Dialect.create(self)
if self.warehouse_yml.dialect:
self.warehouse_yml.dialect.validate_connection()
self._pop_context()
soda_account_dict = self.get_dict_optional(KEY_SODA_ACCOUNT)
if soda_account_dict:
self._push_context(object=soda_account_dict,
name=KEY_SODA_ACCOUNT)
self.warehouse_yml.soda_host = self.get_str_optional(
SODA_KEY_HOST, 'cloud.soda.io')
self.warehouse_yml.soda_port = self.get_int_optional(
SODA_KEY_PORT, 443)
self.warehouse_yml.soda_protocol = self.get_str_optional(
SODA_KEY_PROTOCOL, 'https')
self.warehouse_yml.soda_api_key_id = self.get_str_required_env(
SODA_KEY_API_KEY_ID)
self.warehouse_yml.soda_api_key_secret = self.get_str_required_env(
SODA_KEY_API_KEY_SECRET)
self._pop_context()
ingest_dict = self.get_dict_optional(KEY_INGEST)
if ingest_dict:
self._push_context(object=ingest_dict, name=KEY_INGEST)
self.warehouse_yml.dbt_cloud_api_token = self.get_str_optional(
DBT_CLOUD_KEY_API_TOKEN)
self._pop_context()
self.check_invalid_keys(VALID_WAREHOUSE_KEYS)
else:
self.error('No warehouse configuration provided')
def setup_get_warehouse_fixture(self):
"""self.target may be initialized by a test suite"""
if self.target is None:
self.target = os.getenv('SODA_TEST_TARGET', TARGET_POSTGRES)
warehouse_fixture = SqlTestCase.warehouse_fixtures_by_target.get(
self.target)
if warehouse_fixture is None:
logging.debug(f'Creating warehouse {self.target}')
warehouse_fixture = WarehouseFixture.create(self.target)
dialect = self.create_dialect(self.target)
warehouse_yml = WarehouseYml(dialect=dialect)
warehouse_fixture.warehouse = Warehouse(warehouse_yml)
warehouse_fixture.create_database()
SqlTestCase.warehouse_fixtures_by_target[
self.target] = warehouse_fixture
return warehouse_fixture
'ID': {
KEY_METRICS: ['distinct', 'uniqueness'],
'tests': {
'nomissing': 'missing_percentage < 3.0',
'noinvalid': 'invalid_count == 0'
}
}
}
}
scan_configuration_parser = ScanYmlParser(scan_configuration_dict,
'demodata-scan')
scan_configuration_parser.assert_no_warnings_or_errors()
dialect = SqlTestCase.create_dialect('postgres')
warehouse_yml = WarehouseYml(dialect=dialect)
warehouse = Warehouse(warehouse_yml)
row = warehouse.sql_fetchone('SELECT MIN(date), MAX(date) FROM demodata')
min_date = row[0]
max_date = row[1]
scan_results = []
date = min_date
while date != max_date:
timeslice = datetime(year=date.year, month=date.month,
day=date.day).isoformat()
variables = {'date': date.strftime("%Y-%m-%d")}
scan = Scan(warehouse=warehouse,
scan_yml=scan_configuration_parser.scan_yml,
'ID': {
KEY_METRICS: ['distinct', 'uniqueness'],
'tests': {
'nomissing': 'missing_percentage < 3.0',
'noinvalid': 'invalid_count == 0'
}
}
}
}
scan_configuration_parser = ScanYmlParser(scan_configuration_dict,
'demodata-scan')
scan_configuration_parser.assert_no_warnings_or_errors()
dialect = SqlTestCase.create_dialect('postgres')
warehouse_configuration = WarehouseYml(dialect=dialect)
warehouse = Warehouse(warehouse_configuration)
row = warehouse.sql_fetchone('SELECT MIN(date), MAX(date) FROM demodata')
min_date = row[0]
max_date = row[1]
scan_results = []
date = min_date
while date != max_date:
timeslice = datetime(year=date.year, month=date.month,
day=date.day).isoformat()
variables = {'date': date.strftime("%Y-%m-%d")}
scan = Scan(warehouse=warehouse,
scan_yml=scan_configuration_parser.scan_yml,