def create_fields(monkeypatch): monkeypatch.chdir(Path('e2e') / 'scenarios' / 'pano-field-cleanup') (Paths.fields_dir(Path('test_dataset')) / 'orphan_test_field.field.yaml').write_text(TEST_ORPHANED_FIELD) (Paths.fields_dir(Path('test_dataset')) / 'calculated_test_field.field.yaml').write_text(TEST_CALCULATED_FIELD) yield
def test_push_pull_e2e(monkeypatch): monkeypatch.chdir(Path('e2e') / 'scenarios' / 'pano-push-pull') dataset_dir = Path('test_dataset') dataset_file: Path = dataset_dir / PresetFileName.DATASET_YAML.value model_file: Path = dataset_dir / f'test_model{FileExtension.MODEL_YAML.value}' # Create company scoped field company_fields_dir = Paths.fields_dir(Path.cwd()) company_fields_dir.mkdir(exist_ok=True) company_field_file: Path = company_fields_dir / f'company_test_field{FileExtension.FIELD_YAML.value}' company_field_file.write_text(TEST_COMPANY_FIELD) # Create dataset and model to push dataset_dir.mkdir(exist_ok=True) dataset_file.write_text(TEST_DATASET) model_file.write_text(TEST_MODEL) # Create dataset scoped field dataset_fields_dir = Paths.fields_dir(dataset_dir) dataset_fields_dir.mkdir(exist_ok=True) dataset_field_file: Path = dataset_fields_dir / f'dataset_test_field{FileExtension.FIELD_YAML.value}' dataset_field_file.write_text(TEST_DATASET_FIELD) # Push dataset and model runner = CliRunner() result = runner.invoke(cli, ['push', '-y']) # Check push was successful assert result.exit_code == 0 # Delete local files so they can be re-created with pull dataset_file.unlink() model_file.unlink() company_field_file.unlink() dataset_field_file.unlink() # Pull dataset and model result = runner.invoke(cli, ['pull', '-y']) # Check pull was successful assert dataset_file.exists() assert model_file.exists() assert dataset_field_file.exists() assert company_field_file.exists() # Delete local dataset and model files dataset_file.unlink() model_file.unlink() company_field_file.unlink() dataset_field_file.unlink() # Push deleted dataset and model result = runner.invoke(cli, ['push', '-y']) # Check push was successful assert result.exit_code == 0
def delete_field(self, field: PanoField): """Delete field from local filesystem.""" assert field.file_name is not None if field.package is not None: # dataset-scope field path = Paths.fields_dir(self.cwd / field.package) / field.file_name else: # company-scope field path = Paths.fields_dir(self.cwd) / field.file_name logger.debug(f'About to delete field {field.id}') delete_file(path)
def test_validate_local_state_missing_field_file(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) dataset_dir = tmp_path / 'test_dataset' dataset_dir.mkdir() with (dataset_dir / PresetFileName.DATASET_YAML.value).open('w') as f: f.write(yaml.dump(VALID_DATASET)) with (dataset_dir / 'model1.model.yaml').open('w') as f: f.write( yaml.dump({ **VALID_MODEL_MINIMAL, 'fields': [{ 'data_reference': '"COLUMN1"', 'field_map': ['field_slug', 'field_slug_2'] }], })) field_dir = Paths.fields_dir(dataset_dir) field_dir.mkdir() with (field_dir / 'field_slug.field.yaml').open('w') as f: f.write(yaml.dump(VALID_FIELD_MINIMAL)) errors = validate_local_state() assert errors == [ MissingFieldFileError( field_slug='field_slug_2', dataset_slug='test_dataset', data_reference='"COLUMN1"', identifier=False, model_name='model1', ) ]
def test_validate_local_state_duplicate_dataset_scoped_field( tmp_path, monkeypatch, invalid_field): monkeypatch.chdir(tmp_path) dataset_dir = tmp_path / 'test_dataset' dataset_dir.mkdir() with (dataset_dir / PresetFileName.DATASET_YAML.value).open('w') as f: f.write(yaml.dump(VALID_DATASET)) with (dataset_dir / 'test_model.model.yaml').open('w') as f: f.write( yaml.dump({ **VALID_MODEL_MINIMAL, 'fields': [{ 'field_map': ['field_slug'], 'data_reference': '"FIELD_SLUG"' }] })) field_dir = Paths.fields_dir(dataset_dir) field_dir.mkdir() with (field_dir / 'first_field.field.yaml').open('w') as f: f.write(yaml.dump(VALID_FIELD_MINIMAL)) with (field_dir / 'duplicate.field.yaml').open('w') as f: f.write(yaml.dump(VALID_FIELD_MINIMAL)) errors = validate_local_state() assert len(errors) == 1
def test_reader_get_packages(tmp_path: Path): # scanned directory scanned_dir = tmp_path / SystemDirectory.SCANNED.value scanned_dir.mkdir() (scanned_dir / PresetFileName.DATASET_YAML.value).touch() # dataset with one model ds1_dir = tmp_path / 'dataset1' ds1_dir.mkdir() ds1_file = ds1_dir / PresetFileName.DATASET_YAML.value ds1_file.touch() model_file = ds1_dir / f'test_model{FileExtension.MODEL_YAML.value}' model_file.touch() ds1_fields_dir = Paths.fields_dir(ds1_dir) ds1_fields_dir.mkdir() field_file = ds1_fields_dir / f'test_field{FileExtension.FIELD_YAML.value}' field_file.touch() # empty dataset ds2_dir = tmp_path / 'dataset2' ds2_dir.mkdir() packages = list(FileReader(cwd=tmp_path).get_packages()) expected = [ FilePackage(name='dataset1', data_source_file=ds1_file, model_files=[model_file], field_files=[field_file]) ] assert packages == expected
def test_validate_local_state_valid(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) global_fields_dir = Paths.fields_dir(tmp_path) global_fields_dir.mkdir() dataset_dir = tmp_path / 'test_dataset' dataset_dir.mkdir() dataset_fields_dir = Paths.fields_dir(dataset_dir) dataset_fields_dir.mkdir() with (dataset_dir / PresetFileName.DATASET_YAML.value).open('w') as f: f.write(yaml.dump(VALID_DATASET)) model1 = {**VALID_MODEL_MINIMAL, 'model_name': 'sf.db.schema.table1'} model2 = { **VALID_MODEL_MINIMAL, 'model_name': 'sf.db.schema.table2', 'fields': [{ 'field_map': ['field_slug'], 'data_reference': '"FIELD_SLUG"' }], } with (dataset_dir / 'test_model-1.model.yaml').open('w') as f: f.write(yaml.dump(model1)) with (dataset_dir / 'test_model-2.model.yaml').open('w') as f: f.write(yaml.dump(model2)) with (global_fields_dir / 'company_field.field.yaml').open('w') as f: f.write(yaml.dump(VALID_FIELD_FULL)) with (dataset_fields_dir / 'first_field.field.yaml').open('w') as f: f.write(yaml.dump(VALID_FIELD_MINIMAL)) errors = validate_local_state() assert len(errors) == 0 state = get_state() assert len(state.models) == 2 assert len(state.data_sources) == 1 assert len(state.fields) == 2
def write_field(self, field: PanoField, *, package: Optional[str] = None, file_name: Optional[str] = None): """Write model to local filesystem.""" if file_name is None: file_name = f'{field.slug}{FileExtension.FIELD_YAML.value}' package = package if package is not None else field.data_source if package is not None: # dataset-scope field path = Paths.fields_dir(self.cwd / package) / file_name else: # company-scope field path = Paths.fields_dir(self.cwd) / file_name logger.debug(f'About to write field {field.id}') write_yaml(path, field.to_dict())
def test_validate_local_state_orphan_field_files(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) dataset_dir = tmp_path / 'test_dataset' dataset_dir.mkdir() with (dataset_dir / PresetFileName.DATASET_YAML.value).open('w') as f: f.write(yaml.dump(VALID_DATASET)) with (dataset_dir / 'test_model.model.yaml').open('w') as f: f.write( yaml.dump({ **VALID_MODEL_FULL, 'fields': [{ 'field_map': ['field_slug'], 'data_reference': '"FIELD_SLUG"' }], })) Paths.fields_dir(dataset_dir).mkdir() with (Paths.fields_dir(dataset_dir) / 'test_field.field.yaml').open('w') as f: f.write(yaml.dump(VALID_FIELD_MINIMAL)) with (Paths.fields_dir(dataset_dir) / 'calculated_field.field.yaml').open('w') as f: f.write( yaml.dump({ **VALID_FIELD_MINIMAL, 'slug': 'calculated_slug', 'calculation': '2+2' })) with (Paths.fields_dir(dataset_dir) / 'orphan_field.field.yaml').open('w') as f: f.write(yaml.dump({**VALID_FIELD_MINIMAL, 'slug': 'orphan_slug'})) errors = validate_local_state() assert errors == [ OrphanFieldFileError(field_slug='orphan_slug', dataset_slug='test_dataset') ]
def test_field_cleanup_e2e(_, create_fields): runner = CliRunner() result = runner.invoke(cli, ['field', 'cleanup', '-y']) fields_dir = Paths.fields_dir(Path('test_dataset')) assert result.exit_code == 0 assert {f.name for f in fields_dir.iterdir()} == { 'dataset_test_field.field.yaml', 'calculated_test_field.field.yaml', } assert {f.name for f in Paths.company_fields_dir().iterdir()} == {'company_test_field.field.yaml'}
def test_field_scaffold_e2e(_, clear_fields): runner = CliRunner() result = runner.invoke(cli, ['field', 'scaffold', '-y']) fields_dir = Paths.fields_dir(Path('test_dataset')) Paths.fields_dir(Path('test_dataset')) assert result.exit_code == 0 assert {f.name for f in fields_dir.iterdir()} == {'dataset_test_field.field.yaml'} assert ( (fields_dir / 'dataset_test_field.field.yaml').read_text() == """aggregation: type: group_by api_version: v1 data_type: text display_name: dataset_test_field field_type: dimension group: CLI slug: dataset_test_field """ )
def test_file_package_read_fields(tmp_path): field_file = Paths.fields_dir( tmp_path) / f'test_field{FileExtension.FIELD_YAML.value}' os.makedirs(os.path.dirname(field_file), exist_ok=True) with field_file.open('w') as f: f.write('slug: field_slug') package = FilePackage(name='dataset1', data_source_file=Mock(), model_files=[], field_files=[field_file]) assert list(package.read_fields()) == [({ 'slug': 'field_slug' }, field_file)]
def test_validate_local_state_invalid_dataset_scoped_field( tmp_path, monkeypatch, invalid_field): monkeypatch.chdir(tmp_path) dataset_dir = tmp_path / 'test_dataset' dataset_dir.mkdir() with (dataset_dir / PresetFileName.DATASET_YAML.value).open('w') as f: f.write(yaml.dump(VALID_DATASET)) field_dir = Paths.fields_dir(dataset_dir) field_dir.mkdir() with (field_dir / 'first_field.field.yaml').open('w') as f: f.write(yaml.dump(invalid_field)) errors = validate_local_state() assert len(errors) == 1
def clear_fields(monkeypatch): monkeypatch.chdir(Path('e2e') / 'scenarios' / 'pano-field-scaffold') # delete field files for f in Paths.fields_dir(Path('test_dataset')).iterdir(): f.unlink() yield