def build(self): self._build_warehouse_yml() self._build_scan_yml() for parser in self.parsers: parser.assert_no_warnings_or_errors() if not self.scan_yml or not self.warehouse_yml: return from sodasql.scan.warehouse import Warehouse warehouse = Warehouse(self.warehouse_yml) self._create_soda_server_client() return Scan(warehouse=warehouse, scan_yml=self.scan_yml, variables=self.variables, soda_server_client=self.soda_server_client, time=self.time)
def build(self, offline: bool = False): self._build_warehouse_yml() self._build_scan_yml() for parser in self.parsers: parser.assert_no_warnings_or_errors() if not self.scan_yml or not self.warehouse_yml: return from sodasql.scan.warehouse import Warehouse warehouse = Warehouse(self.warehouse_yml) if not offline: self._create_soda_server_client() return Scan(warehouse=warehouse, scan_yml=self.scan_yml, variables=self.variables, soda_server_client=self.soda_server_client, time=self.time, scan_results_file=self.scan_results_json_path, failed_rows_processor=self.failed_rows_processor)
dialect = SqlTestCase.create_dialect('postgres') warehouse_yml = WarehouseYml(dialect=dialect) warehouse = Warehouse(warehouse_yml) row = warehouse.sql_fetchone('SELECT MIN(date), MAX(date) FROM demodata') min_date = row[0] max_date = row[1] scan_results = [] date = min_date while date != max_date: timeslice = datetime(year=date.year, month=date.month, day=date.day).isoformat() variables = {'date': date.strftime("%Y-%m-%d")} scan = Scan(warehouse=warehouse, scan_yml=scan_configuration_parser.scan_yml, variables=variables, time=timeslice) scan_results.append(scan.execute()) date = date + timedelta(days=1) print() print('Summary:') for scan_result in scan_results: print(f'Scan results:') print(f' Measurements: {len(scan_result.measurements)}') print( f' Test results: {len(scan_result.test_results)} of which {scan_result.get_test_failures_count()} failed' )
def create_scan(self, *args, **kwargs): # Purpose of this method is to enable dialects to override and customize the scan implementation from sodasql.scan.scan import Scan return Scan(*args, **kwargs)