def build(self):
self._build_warehouse_yml()
self._build_scan_yml()
for parser in self.parsers:
parser.assert_no_warnings_or_errors()
if not self.scan_yml or not self.warehouse_yml:
return
from sodasql.scan.warehouse import Warehouse
warehouse = Warehouse(self.warehouse_yml)
self._create_soda_server_client()
return Scan(warehouse=warehouse,
scan_yml=self.scan_yml,
variables=self.variables,
soda_server_client=self.soda_server_client,
time=self.time)
def build(self, offline: bool = False):
self._build_warehouse_yml()
self._build_scan_yml()
for parser in self.parsers:
parser.assert_no_warnings_or_errors()
if not self.scan_yml or not self.warehouse_yml:
return
from sodasql.scan.warehouse import Warehouse
warehouse = Warehouse(self.warehouse_yml)
if not offline:
self._create_soda_server_client()
return Scan(warehouse=warehouse,
scan_yml=self.scan_yml,
variables=self.variables,
soda_server_client=self.soda_server_client,
time=self.time,
scan_results_file=self.scan_results_json_path,
failed_rows_processor=self.failed_rows_processor)
dialect = SqlTestCase.create_dialect('postgres')
warehouse_yml = WarehouseYml(dialect=dialect)
warehouse = Warehouse(warehouse_yml)
row = warehouse.sql_fetchone('SELECT MIN(date), MAX(date) FROM demodata')
min_date = row[0]
max_date = row[1]
scan_results = []
date = min_date
while date != max_date:
timeslice = datetime(year=date.year, month=date.month,
day=date.day).isoformat()
variables = {'date': date.strftime("%Y-%m-%d")}
scan = Scan(warehouse=warehouse,
scan_yml=scan_configuration_parser.scan_yml,
variables=variables,
time=timeslice)
scan_results.append(scan.execute())
date = date + timedelta(days=1)
print()
print('Summary:')
for scan_result in scan_results:
print(f'Scan results:')
print(f' Measurements: {len(scan_result.measurements)}')
print(
f' Test results: {len(scan_result.test_results)} of which {scan_result.get_test_failures_count()} failed'
)
def create_scan(self, *args, **kwargs):
# Purpose of this method is to enable dialects to override and customize the scan implementation
from sodasql.scan.scan import Scan
return Scan(*args, **kwargs)