def package_config_from_data(packages_data: Dict[str, Any]):
if not packages_data:
packages_data = {'packages': []}
try:
PackageConfig.validate(packages_data)
packages = PackageConfig.from_dict(packages_data)
except ValidationError as e:
raise DbtProjectError(
MALFORMED_PACKAGE_ERROR.format(error=str(e.message))) from e
return packages
def test_from_args(self):
with temp_cd(self.project_dir):
config = dbt.config.RuntimeConfig.from_args(self.args)
self.assertEqual(config.project_name, 'my_test_project')
self.assertEqual(config.version, '0.0.1')
self.assertEqual(config.profile_name, 'default')
# on osx, for example, these are not necessarily equal due to /private
self.assertTrue(os.path.samefile(config.project_root,
self.project_dir))
self.assertEqual(config.source_paths, ['models'])
self.assertEqual(config.macro_paths, ['macros'])
self.assertEqual(config.data_paths, ['data'])
self.assertEqual(config.test_paths, ['test'])
self.assertEqual(config.analysis_paths, [])
self.assertEqual(config.docs_paths, ['models'])
self.assertEqual(config.target_path, 'target')
self.assertEqual(config.clean_targets, ['target'])
self.assertEqual(config.log_path, 'logs')
self.assertEqual(config.modules_path, 'dbt_modules')
self.assertEqual(config.quoting, {'database': True, 'identifier': True, 'schema': True})
self.assertEqual(config.models, {})
self.assertEqual(config.on_run_start, [])
self.assertEqual(config.on_run_end, [])
self.assertEqual(config.archive, [])
self.assertEqual(config.seeds, {})
self.assertEqual(config.packages, PackageConfig(packages=[]))
def test_defaults(self):
project = dbt.config.Project.from_project_config(
self.default_project_data
)
self.assertEqual(project.project_name, 'my_test_project')
self.assertEqual(project.version, '0.0.1')
self.assertEqual(project.profile_name, 'default')
self.assertEqual(project.project_root, '/invalid-root-path')
self.assertEqual(project.source_paths, ['models'])
self.assertEqual(project.macro_paths, ['macros'])
self.assertEqual(project.data_paths, ['data'])
self.assertEqual(project.test_paths, ['test'])
self.assertEqual(project.analysis_paths, [])
self.assertEqual(project.docs_paths, ['models'])
self.assertEqual(project.target_path, 'target')
self.assertEqual(project.clean_targets, ['target'])
self.assertEqual(project.log_path, 'logs')
self.assertEqual(project.modules_path, 'dbt_modules')
self.assertEqual(project.quoting, {})
self.assertEqual(project.models, {})
self.assertEqual(project.on_run_start, [])
self.assertEqual(project.on_run_end, [])
self.assertEqual(project.archive, [])
self.assertEqual(project.seeds, {})
self.assertEqual(project.packages, PackageConfig(packages=[]))
# just make sure str() doesn't crash anything, that's always
# embarrassing
str(project)
def package_config_from_data(packages_data):
if packages_data is None:
packages_data = {'packages': []}
try:
packages = PackageConfig(**packages_data)
except ValidationException as e:
raise DbtProjectError('Invalid package config: {}'.format(str(e)))
return packages
def package_config_from_data(packages_data):
if packages_data is None:
packages_data = {'packages': []}
try:
packages = PackageConfig.from_dict(packages_data)
except ValidationError as e:
raise DbtProjectError(
'Invalid package config: {}'.format(validator_error_message(e))
) from e
return packages
def test_dependency_resolution(self):
package_config = PackageConfig.from_dict({
'packages': [
{
'package': 'fishtown-analytics-test/a',
'version': '>0.1.2'
},
{
'package': 'fishtown-analytics-test/b',
'version': '0.2.1'
},
],
})
resolved = resolve_packages(package_config.packages, None)
self.assertEqual(len(resolved), 2)
self.assertEqual(resolved[0].name, 'fishtown-analytics-test/a')
self.assertEqual(resolved[0].version, '0.1.3')
self.assertEqual(resolved[1].name, 'fishtown-analytics-test/b')
self.assertEqual(resolved[1].version, '0.2.1')
def test_all_overrides(self):
self.default_project_data.update({
'source-paths': ['other-models'],
'macro-paths': ['other-macros'],
'data-paths': ['other-data'],
'test-paths': ['other-test'],
'analysis-paths': ['analysis'],
'docs-paths': ['docs'],
'target-path': 'other-target',
'clean-targets': ['another-target'],
'log-path': 'other-logs',
'modules-path': 'other-dbt_modules',
'quoting': {'identifier': False},
'models': {
'pre-hook': ['{{ logging.log_model_start_event() }}'],
'post-hook': ['{{ logging.log_model_end_event() }}'],
'my_test_project': {
'first': {
'enabled': False,
'sub': {
'enabled': True,
}
},
'second': {
'materialized': 'table',
},
},
'third_party': {
'third': {
'materialized': 'view',
},
},
},
'on-run-start': [
'{{ logging.log_run_start_event() }}',
],
'on-run-end': [
'{{ logging.log_run_end_event() }}',
],
'archive': [
{
'source_schema': 'my_schema',
'target_schema': 'my_other_schema',
'tables': [
{
'source_table': 'my_table',
'target_Table': 'my_table_archived',
'updated_at': 'updated_at_field',
'unique_key': 'table_id',
},
],
},
],
'seeds': {
'my_test_project': {
'enabled': True,
'schema': 'seed_data',
'post-hook': 'grant select on {{ this }} to bi_user',
},
},
})
packages = {
'packages': [
{
'local': 'foo',
},
{
'git': '[email protected]:fishtown-analytics/dbt-utils.git',
'revision': 'test-rev'
},
],
}
project = dbt.config.Project.from_project_config(
self.default_project_data, packages
)
self.assertEqual(project.project_name, 'my_test_project')
self.assertEqual(project.version, '0.0.1')
self.assertEqual(project.profile_name, 'default')
self.assertEqual(project.project_root, '/invalid-root-path')
self.assertEqual(project.source_paths, ['other-models'])
self.assertEqual(project.macro_paths, ['other-macros'])
self.assertEqual(project.data_paths, ['other-data'])
self.assertEqual(project.test_paths, ['other-test'])
self.assertEqual(project.analysis_paths, ['analysis'])
self.assertEqual(project.docs_paths, ['docs'])
self.assertEqual(project.target_path, 'other-target')
self.assertEqual(project.clean_targets, ['another-target'])
self.assertEqual(project.log_path, 'other-logs')
self.assertEqual(project.modules_path, 'other-dbt_modules')
self.assertEqual(project.quoting, {'identifier': False})
self.assertEqual(project.models, {
'pre-hook': ['{{ logging.log_model_start_event() }}'],
'post-hook': ['{{ logging.log_model_end_event() }}'],
'my_test_project': {
'first': {
'enabled': False,
'sub': {
'enabled': True,
}
},
'second': {
'materialized': 'table',
},
},
'third_party': {
'third': {
'materialized': 'view',
},
},
})
self.assertEqual(project.on_run_start, ['{{ logging.log_run_start_event() }}'])
self.assertEqual(project.on_run_end, ['{{ logging.log_run_end_event() }}'])
self.assertEqual(project.archive, [{
'source_schema': 'my_schema',
'target_schema': 'my_other_schema',
'tables': [
{
'source_table': 'my_table',
'target_Table': 'my_table_archived',
'updated_at': 'updated_at_field',
'unique_key': 'table_id',
},
],
}])
self.assertEqual(project.seeds, {
'my_test_project': {
'enabled': True,
'schema': 'seed_data',
'post-hook': 'grant select on {{ this }} to bi_user',
},
})
self.assertEqual(project.packages, PackageConfig(packages=[
{
'local': 'foo',
},
{
'git': '[email protected]:fishtown-analytics/dbt-utils.git',
'revision': 'test-rev'
},
]))
str(project)
json.dumps(project.to_project_config())