def __init__(self, target: str) -> None: super().__init__() self.target: str = target self.dialect = self.create_dialect(self.target) self.warehouse_yml = WarehouseYml(dialect=self.dialect, name='test_warehouse') self.warehouse: Optional[Warehouse] = Warehouse(self.warehouse_yml) self.database: Optional[str] = None self.create_database()
def __init__(self, warehouse_yml_dict: dict, warehouse_yml_path: str = 'warehouse-dict'): super().__init__(description=warehouse_yml_path) if isinstance(warehouse_yml_dict, dict): self._push_context(object=warehouse_yml_dict, name=self.description) self.warehouse_yml = WarehouseYml() self.warehouse_yml.name = self.get_str_required(KEY_NAME) EnvVars.load_env_vars(self.warehouse_yml.name) connection_dict = self.get_dict_required(KEY_CONNECTION) if connection_dict: self._push_context(object=connection_dict, name=KEY_CONNECTION) self.warehouse_yml.dialect = Dialect.create(self) if self.warehouse_yml.dialect: self.warehouse_yml.dialect.validate_connection() self._pop_context() soda_account_dict = self.get_dict_optional(KEY_SODA_ACCOUNT) if soda_account_dict: self._push_context(object=soda_account_dict, name=KEY_SODA_ACCOUNT) self.warehouse_yml.soda_host = self.get_str_optional( SODA_KEY_HOST, 'cloud.soda.io') self.warehouse_yml.soda_port = self.get_int_optional( SODA_KEY_PORT, 443) self.warehouse_yml.soda_protocol = self.get_str_optional( SODA_KEY_PROTOCOL, 'https') self.warehouse_yml.soda_api_key_id = self.get_str_required_env( SODA_KEY_API_KEY_ID) self.warehouse_yml.soda_api_key_secret = self.get_str_required_env( SODA_KEY_API_KEY_SECRET) self._pop_context() ingest_dict = self.get_dict_optional(KEY_INGEST) if ingest_dict: self._push_context(object=ingest_dict, name=KEY_INGEST) self.warehouse_yml.dbt_cloud_api_token = self.get_str_optional( DBT_CLOUD_KEY_API_TOKEN) self._pop_context() self.check_invalid_keys(VALID_WAREHOUSE_KEYS) else: self.error('No warehouse configuration provided')
def setup_get_warehouse_fixture(self): """self.target may be initialized by a test suite""" if self.target is None: self.target = os.getenv('SODA_TEST_TARGET', TARGET_POSTGRES) warehouse_fixture = SqlTestCase.warehouse_fixtures_by_target.get( self.target) if warehouse_fixture is None: logging.debug(f'Creating warehouse {self.target}') warehouse_fixture = WarehouseFixture.create(self.target) dialect = self.create_dialect(self.target) warehouse_yml = WarehouseYml(dialect=dialect) warehouse_fixture.warehouse = Warehouse(warehouse_yml) warehouse_fixture.create_database() SqlTestCase.warehouse_fixtures_by_target[ self.target] = warehouse_fixture return warehouse_fixture
'ID': { KEY_METRICS: ['distinct', 'uniqueness'], 'tests': { 'nomissing': 'missing_percentage < 3.0', 'noinvalid': 'invalid_count == 0' } } } } scan_configuration_parser = ScanYmlParser(scan_configuration_dict, 'demodata-scan') scan_configuration_parser.assert_no_warnings_or_errors() dialect = SqlTestCase.create_dialect('postgres') warehouse_yml = WarehouseYml(dialect=dialect) warehouse = Warehouse(warehouse_yml) row = warehouse.sql_fetchone('SELECT MIN(date), MAX(date) FROM demodata') min_date = row[0] max_date = row[1] scan_results = [] date = min_date while date != max_date: timeslice = datetime(year=date.year, month=date.month, day=date.day).isoformat() variables = {'date': date.strftime("%Y-%m-%d")} scan = Scan(warehouse=warehouse, scan_yml=scan_configuration_parser.scan_yml,
'ID': { KEY_METRICS: ['distinct', 'uniqueness'], 'tests': { 'nomissing': 'missing_percentage < 3.0', 'noinvalid': 'invalid_count == 0' } } } } scan_configuration_parser = ScanYmlParser(scan_configuration_dict, 'demodata-scan') scan_configuration_parser.assert_no_warnings_or_errors() dialect = SqlTestCase.create_dialect('postgres') warehouse_configuration = WarehouseYml(dialect=dialect) warehouse = Warehouse(warehouse_configuration) row = warehouse.sql_fetchone('SELECT MIN(date), MAX(date) FROM demodata') min_date = row[0] max_date = row[1] scan_results = [] date = min_date while date != max_date: timeslice = datetime(year=date.year, month=date.month, day=date.day).isoformat() variables = {'date': date.strftime("%Y-%m-%d")} scan = Scan(warehouse=warehouse, scan_yml=scan_configuration_parser.scan_yml,