def create_fields(monkeypatch):
monkeypatch.chdir(Path('e2e') / 'scenarios' / 'pano-field-cleanup')
(Paths.fields_dir(Path('test_dataset')) /
'orphan_test_field.field.yaml').write_text(TEST_ORPHANED_FIELD)
(Paths.fields_dir(Path('test_dataset')) /
'calculated_test_field.field.yaml').write_text(TEST_CALCULATED_FIELD)
yield
def test_push_pull_e2e(monkeypatch):
monkeypatch.chdir(Path('e2e') / 'scenarios' / 'pano-push-pull')
dataset_dir = Path('test_dataset')
dataset_file: Path = dataset_dir / PresetFileName.DATASET_YAML.value
model_file: Path = dataset_dir / f'test_model{FileExtension.MODEL_YAML.value}'
# Create company scoped field
company_fields_dir = Paths.fields_dir(Path.cwd())
company_fields_dir.mkdir(exist_ok=True)
company_field_file: Path = company_fields_dir / f'company_test_field{FileExtension.FIELD_YAML.value}'
company_field_file.write_text(TEST_COMPANY_FIELD)
# Create dataset and model to push
dataset_dir.mkdir(exist_ok=True)
dataset_file.write_text(TEST_DATASET)
model_file.write_text(TEST_MODEL)
# Create dataset scoped field
dataset_fields_dir = Paths.fields_dir(dataset_dir)
dataset_fields_dir.mkdir(exist_ok=True)
dataset_field_file: Path = dataset_fields_dir / f'dataset_test_field{FileExtension.FIELD_YAML.value}'
dataset_field_file.write_text(TEST_DATASET_FIELD)
# Push dataset and model
runner = CliRunner()
result = runner.invoke(cli, ['push', '-y'])
# Check push was successful
assert result.exit_code == 0
# Delete local files so they can be re-created with pull
dataset_file.unlink()
model_file.unlink()
company_field_file.unlink()
dataset_field_file.unlink()
# Pull dataset and model
result = runner.invoke(cli, ['pull', '-y'])
# Check pull was successful
assert dataset_file.exists()
assert model_file.exists()
assert dataset_field_file.exists()
assert company_field_file.exists()
# Delete local dataset and model files
dataset_file.unlink()
model_file.unlink()
company_field_file.unlink()
dataset_field_file.unlink()
# Push deleted dataset and model
result = runner.invoke(cli, ['push', '-y'])
# Check push was successful
assert result.exit_code == 0
def delete_field(self, field: PanoField):
"""Delete field from local filesystem."""
assert field.file_name is not None
if field.package is not None:
# dataset-scope field
path = Paths.fields_dir(self.cwd / field.package) / field.file_name
else:
# company-scope field
path = Paths.fields_dir(self.cwd) / field.file_name
logger.debug(f'About to delete field {field.id}')
delete_file(path)
def test_validate_local_state_missing_field_file(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
dataset_dir = tmp_path / 'test_dataset'
dataset_dir.mkdir()
with (dataset_dir / PresetFileName.DATASET_YAML.value).open('w') as f:
f.write(yaml.dump(VALID_DATASET))
with (dataset_dir / 'model1.model.yaml').open('w') as f:
f.write(
yaml.dump({
**VALID_MODEL_MINIMAL,
'fields': [{
'data_reference': '"COLUMN1"',
'field_map': ['field_slug', 'field_slug_2']
}],
}))
field_dir = Paths.fields_dir(dataset_dir)
field_dir.mkdir()
with (field_dir / 'field_slug.field.yaml').open('w') as f:
f.write(yaml.dump(VALID_FIELD_MINIMAL))
errors = validate_local_state()
assert errors == [
MissingFieldFileError(
field_slug='field_slug_2',
dataset_slug='test_dataset',
data_reference='"COLUMN1"',
identifier=False,
model_name='model1',
)
]
def test_validate_local_state_duplicate_dataset_scoped_field(
tmp_path, monkeypatch, invalid_field):
monkeypatch.chdir(tmp_path)
dataset_dir = tmp_path / 'test_dataset'
dataset_dir.mkdir()
with (dataset_dir / PresetFileName.DATASET_YAML.value).open('w') as f:
f.write(yaml.dump(VALID_DATASET))
with (dataset_dir / 'test_model.model.yaml').open('w') as f:
f.write(
yaml.dump({
**VALID_MODEL_MINIMAL, 'fields': [{
'field_map': ['field_slug'],
'data_reference':
'"FIELD_SLUG"'
}]
}))
field_dir = Paths.fields_dir(dataset_dir)
field_dir.mkdir()
with (field_dir / 'first_field.field.yaml').open('w') as f:
f.write(yaml.dump(VALID_FIELD_MINIMAL))
with (field_dir / 'duplicate.field.yaml').open('w') as f:
f.write(yaml.dump(VALID_FIELD_MINIMAL))
errors = validate_local_state()
assert len(errors) == 1
def test_reader_get_packages(tmp_path: Path):
# scanned directory
scanned_dir = tmp_path / SystemDirectory.SCANNED.value
scanned_dir.mkdir()
(scanned_dir / PresetFileName.DATASET_YAML.value).touch()
# dataset with one model
ds1_dir = tmp_path / 'dataset1'
ds1_dir.mkdir()
ds1_file = ds1_dir / PresetFileName.DATASET_YAML.value
ds1_file.touch()
model_file = ds1_dir / f'test_model{FileExtension.MODEL_YAML.value}'
model_file.touch()
ds1_fields_dir = Paths.fields_dir(ds1_dir)
ds1_fields_dir.mkdir()
field_file = ds1_fields_dir / f'test_field{FileExtension.FIELD_YAML.value}'
field_file.touch()
# empty dataset
ds2_dir = tmp_path / 'dataset2'
ds2_dir.mkdir()
packages = list(FileReader(cwd=tmp_path).get_packages())
expected = [
FilePackage(name='dataset1',
data_source_file=ds1_file,
model_files=[model_file],
field_files=[field_file])
]
assert packages == expected
def test_validate_local_state_valid(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
global_fields_dir = Paths.fields_dir(tmp_path)
global_fields_dir.mkdir()
dataset_dir = tmp_path / 'test_dataset'
dataset_dir.mkdir()
dataset_fields_dir = Paths.fields_dir(dataset_dir)
dataset_fields_dir.mkdir()
with (dataset_dir / PresetFileName.DATASET_YAML.value).open('w') as f:
f.write(yaml.dump(VALID_DATASET))
model1 = {**VALID_MODEL_MINIMAL, 'model_name': 'sf.db.schema.table1'}
model2 = {
**VALID_MODEL_MINIMAL,
'model_name':
'sf.db.schema.table2',
'fields': [{
'field_map': ['field_slug'],
'data_reference': '"FIELD_SLUG"'
}],
}
with (dataset_dir / 'test_model-1.model.yaml').open('w') as f:
f.write(yaml.dump(model1))
with (dataset_dir / 'test_model-2.model.yaml').open('w') as f:
f.write(yaml.dump(model2))
with (global_fields_dir / 'company_field.field.yaml').open('w') as f:
f.write(yaml.dump(VALID_FIELD_FULL))
with (dataset_fields_dir / 'first_field.field.yaml').open('w') as f:
f.write(yaml.dump(VALID_FIELD_MINIMAL))
errors = validate_local_state()
assert len(errors) == 0
state = get_state()
assert len(state.models) == 2
assert len(state.data_sources) == 1
assert len(state.fields) == 2
def write_field(self,
field: PanoField,
*,
package: Optional[str] = None,
file_name: Optional[str] = None):
"""Write model to local filesystem."""
if file_name is None:
file_name = f'{field.slug}{FileExtension.FIELD_YAML.value}'
package = package if package is not None else field.data_source
if package is not None:
# dataset-scope field
path = Paths.fields_dir(self.cwd / package) / file_name
else:
# company-scope field
path = Paths.fields_dir(self.cwd) / file_name
logger.debug(f'About to write field {field.id}')
write_yaml(path, field.to_dict())
def test_validate_local_state_orphan_field_files(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
dataset_dir = tmp_path / 'test_dataset'
dataset_dir.mkdir()
with (dataset_dir / PresetFileName.DATASET_YAML.value).open('w') as f:
f.write(yaml.dump(VALID_DATASET))
with (dataset_dir / 'test_model.model.yaml').open('w') as f:
f.write(
yaml.dump({
**VALID_MODEL_FULL,
'fields': [{
'field_map': ['field_slug'],
'data_reference': '"FIELD_SLUG"'
}],
}))
Paths.fields_dir(dataset_dir).mkdir()
with (Paths.fields_dir(dataset_dir) /
'test_field.field.yaml').open('w') as f:
f.write(yaml.dump(VALID_FIELD_MINIMAL))
with (Paths.fields_dir(dataset_dir) /
'calculated_field.field.yaml').open('w') as f:
f.write(
yaml.dump({
**VALID_FIELD_MINIMAL, 'slug': 'calculated_slug',
'calculation': '2+2'
}))
with (Paths.fields_dir(dataset_dir) /
'orphan_field.field.yaml').open('w') as f:
f.write(yaml.dump({**VALID_FIELD_MINIMAL, 'slug': 'orphan_slug'}))
errors = validate_local_state()
assert errors == [
OrphanFieldFileError(field_slug='orphan_slug',
dataset_slug='test_dataset')
]
def test_field_cleanup_e2e(_, create_fields):
runner = CliRunner()
result = runner.invoke(cli, ['field', 'cleanup', '-y'])
fields_dir = Paths.fields_dir(Path('test_dataset'))
assert result.exit_code == 0
assert {f.name for f in fields_dir.iterdir()} == {
'dataset_test_field.field.yaml',
'calculated_test_field.field.yaml',
}
assert {f.name for f in Paths.company_fields_dir().iterdir()} == {'company_test_field.field.yaml'}
def test_field_scaffold_e2e(_, clear_fields):
runner = CliRunner()
result = runner.invoke(cli, ['field', 'scaffold', '-y'])
fields_dir = Paths.fields_dir(Path('test_dataset'))
Paths.fields_dir(Path('test_dataset'))
assert result.exit_code == 0
assert {f.name for f in fields_dir.iterdir()} == {'dataset_test_field.field.yaml'}
assert (
(fields_dir / 'dataset_test_field.field.yaml').read_text()
== """aggregation:
type: group_by
api_version: v1
data_type: text
display_name: dataset_test_field
field_type: dimension
group: CLI
slug: dataset_test_field
"""
)
def test_file_package_read_fields(tmp_path):
field_file = Paths.fields_dir(
tmp_path) / f'test_field{FileExtension.FIELD_YAML.value}'
os.makedirs(os.path.dirname(field_file), exist_ok=True)
with field_file.open('w') as f:
f.write('slug: field_slug')
package = FilePackage(name='dataset1',
data_source_file=Mock(),
model_files=[],
field_files=[field_file])
assert list(package.read_fields()) == [({
'slug': 'field_slug'
}, field_file)]
def test_validate_local_state_invalid_dataset_scoped_field(
tmp_path, monkeypatch, invalid_field):
monkeypatch.chdir(tmp_path)
dataset_dir = tmp_path / 'test_dataset'
dataset_dir.mkdir()
with (dataset_dir / PresetFileName.DATASET_YAML.value).open('w') as f:
f.write(yaml.dump(VALID_DATASET))
field_dir = Paths.fields_dir(dataset_dir)
field_dir.mkdir()
with (field_dir / 'first_field.field.yaml').open('w') as f:
f.write(yaml.dump(invalid_field))
errors = validate_local_state()
assert len(errors) == 1
def clear_fields(monkeypatch):
monkeypatch.chdir(Path('e2e') / 'scenarios' / 'pano-field-scaffold')
# delete field files
for f in Paths.fields_dir(Path('test_dataset')).iterdir():
f.unlink()
yield