def test_invalid_name(self):
dct = {
'name': 'log',
'version': '1.0',
'profile': 'test',
'project-root': '/usr/src/app',
}
with self.assertRaises(ValidationError):
Project.from_dict(dct)
def test_minimal(self):
dct = {
'name': 'test',
'version': '1.0',
'profile': 'test',
'project-root': '/usr/src/app',
}
project = Project(
name='test',
version='1.0',
profile='test',
project_root='/usr/src/app',
)
self.assert_from_dict(project, dct)
def validate(self):
try:
ProjectContract.from_dict(self.to_project_config())
except ValidationError as e:
raise DbtProjectError(validator_error_message(e)) from e
def from_project_config(cls, project_dict, packages_dict=None):
"""Create a project from its project and package configuration, as read
by yaml.safe_load().
:param project_dict dict: The dictionary as read from disk
:param packages_dict Optional[dict]: If it exists, the packages file as
read from disk.
:raises DbtProjectError: If the project is missing or invalid, or if
the packages file exists and is invalid.
:returns Project: The project, with defaults populated.
"""
try:
project_dict = cls._preprocess(project_dict)
except RecursionException:
raise DbtProjectError(
'Cycle detected: Project input has a reference to itself',
project=project_dict
)
# just for validation.
try:
ProjectContract.from_dict(project_dict)
except ValidationError as e:
raise DbtProjectError(validator_error_message(e)) from e
# name/version are required in the Project definition, so we can assume
# they are present
name = project_dict['name']
version = project_dict['version']
# this is added at project_dict parse time and should always be here
# once we see it.
project_root = project_dict['project-root']
# this is only optional in the sense that if it's not present, it needs
# to have been a cli argument.
profile_name = project_dict.get('profile')
# these are optional
source_paths = project_dict.get('source-paths', ['models'])
macro_paths = project_dict.get('macro-paths', ['macros'])
data_paths = project_dict.get('data-paths', ['data'])
test_paths = project_dict.get('test-paths', ['test'])
analysis_paths = project_dict.get('analysis-paths', [])
docs_paths = project_dict.get('docs-paths', source_paths[:])
target_path = project_dict.get('target-path', 'target')
snapshot_paths = project_dict.get('snapshot-paths', ['snapshots'])
# should this also include the modules path by default?
clean_targets = project_dict.get('clean-targets', [target_path])
log_path = project_dict.get('log-path', 'logs')
modules_path = project_dict.get('modules-path', 'dbt_modules')
# in the default case we'll populate this once we know the adapter type
quoting = project_dict.get('quoting', {})
models = project_dict.get('models', {})
on_run_start = project_dict.get('on-run-start', [])
on_run_end = project_dict.get('on-run-end', [])
seeds = project_dict.get('seeds', {})
snapshots = project_dict.get('snapshots', {})
dbt_raw_version = project_dict.get('require-dbt-version', '>=0.0.0')
query_comment = project_dict.get('query-comment', NoValue())
try:
dbt_version = _parse_versions(dbt_raw_version)
except SemverException as e:
raise DbtProjectError(str(e)) from e
try:
packages = package_config_from_data(packages_dict)
except ValidationError as e:
raise DbtProjectError(validator_error_message(e)) from e
project = cls(
project_name=name,
version=version,
project_root=project_root,
profile_name=profile_name,
source_paths=source_paths,
macro_paths=macro_paths,
data_paths=data_paths,
test_paths=test_paths,
analysis_paths=analysis_paths,
docs_paths=docs_paths,
target_path=target_path,
snapshot_paths=snapshot_paths,
clean_targets=clean_targets,
log_path=log_path,
modules_path=modules_path,
quoting=quoting,
models=models,
on_run_start=on_run_start,
on_run_end=on_run_end,
seeds=seeds,
snapshots=snapshots,
dbt_version=dbt_version,
packages=packages,
query_comment=query_comment,
)
# sanity check - this means an internal issue
project.validate()
return project
def create_project(self, rendered: RenderComponents) -> 'Project':
unrendered = RenderComponents(
project_dict=self.project_dict,
packages_dict=self.packages_dict,
selectors_dict=self.selectors_dict,
)
dbt_version = _get_required_version(
rendered.project_dict,
verify_version=self.verify_version,
)
try:
ProjectContract.validate(rendered.project_dict)
cfg = ProjectContract.from_dict(rendered.project_dict)
except ValidationError as e:
raise DbtProjectError(validator_error_message(e)) from e
# name/version are required in the Project definition, so we can assume
# they are present
name = cfg.name
version = cfg.version
# this is added at project_dict parse time and should always be here
# once we see it.
if cfg.project_root is None:
raise DbtProjectError('cfg must have a project root!')
else:
project_root = cfg.project_root
# this is only optional in the sense that if it's not present, it needs
# to have been a cli argument.
profile_name = cfg.profile
# these are all the defaults
source_paths: List[str] = value_or(cfg.source_paths, ['models'])
macro_paths: List[str] = value_or(cfg.macro_paths, ['macros'])
data_paths: List[str] = value_or(cfg.data_paths, ['data'])
test_paths: List[str] = value_or(cfg.test_paths, ['test'])
analysis_paths: List[str] = value_or(cfg.analysis_paths, [])
snapshot_paths: List[str] = value_or(cfg.snapshot_paths, ['snapshots'])
all_source_paths: List[str] = _all_source_paths(
source_paths, data_paths, snapshot_paths, analysis_paths,
macro_paths)
docs_paths: List[str] = value_or(cfg.docs_paths, all_source_paths)
asset_paths: List[str] = value_or(cfg.asset_paths, [])
target_path: str = value_or(cfg.target_path, 'target')
clean_targets: List[str] = value_or(cfg.clean_targets, [target_path])
log_path: str = value_or(cfg.log_path, 'logs')
modules_path: str = value_or(cfg.modules_path, 'dbt_modules')
# in the default case we'll populate this once we know the adapter type
# It would be nice to just pass along a Quoting here, but that would
# break many things
quoting: Dict[str, Any] = {}
if cfg.quoting is not None:
quoting = cfg.quoting.to_dict(omit_none=True)
models: Dict[str, Any]
seeds: Dict[str, Any]
snapshots: Dict[str, Any]
sources: Dict[str, Any]
vars_value: VarProvider
models = cfg.models
seeds = cfg.seeds
snapshots = cfg.snapshots
sources = cfg.sources
if cfg.vars is None:
vars_dict: Dict[str, Any] = {}
else:
vars_dict = cfg.vars
vars_value = VarProvider(vars_dict)
on_run_start: List[str] = value_or(cfg.on_run_start, [])
on_run_end: List[str] = value_or(cfg.on_run_end, [])
query_comment = _query_comment_from_cfg(cfg.query_comment)
packages = package_config_from_data(rendered.packages_dict)
selectors = selector_config_from_data(rendered.selectors_dict)
manifest_selectors: Dict[str, Any] = {}
if rendered.selectors_dict and rendered.selectors_dict['selectors']:
# this is a dict with a single key 'selectors' pointing to a list
# of dicts.
manifest_selectors = SelectorDict.parse_from_selectors_list(
rendered.selectors_dict['selectors'])
project = Project(
project_name=name,
version=version,
project_root=project_root,
profile_name=profile_name,
source_paths=source_paths,
macro_paths=macro_paths,
data_paths=data_paths,
test_paths=test_paths,
analysis_paths=analysis_paths,
docs_paths=docs_paths,
asset_paths=asset_paths,
target_path=target_path,
snapshot_paths=snapshot_paths,
clean_targets=clean_targets,
log_path=log_path,
modules_path=modules_path,
quoting=quoting,
models=models,
on_run_start=on_run_start,
on_run_end=on_run_end,
seeds=seeds,
snapshots=snapshots,
dbt_version=dbt_version,
packages=packages,
manifest_selectors=manifest_selectors,
selectors=selectors,
query_comment=query_comment,
sources=sources,
vars=vars_value,
config_version=cfg.config_version,
unrendered=unrendered,
)
# sanity check - this means an internal issue
project.validate()
return project