def incorporate(self, **kwargs):
"""
Given a list of kwargs, incorporate these arguments
into a new copy of this instance, and return the new
instance after validating.
"""
return type(self)(**deep_merge(self._contents, kwargs))
def _merge(self, *configs):
merged_config = {}
for config in configs:
intermediary_merged = deep_merge(merged_config.copy(),
config.copy())
merged_config.update(intermediary_merged)
return merged_config
def incorporate(self, **kwargs):
"""
Given a list of kwargs, incorporate these arguments
into a new copy of this instance, and return the new
instance after validating.
"""
existing = copy.deepcopy(dict(self))
updates = copy.deepcopy(kwargs)
return type(self)(**deep_merge(existing, updates))
def _merge(self, *configs):
merged_config = {}
for config in configs:
intermediary_merged = deep_merge(
merged_config.copy(), config.copy()
)
merged_config.update(intermediary_merged)
return merged_config
def __init__(self, **kwargs):
"""
Create and validate an instance. Note that if you override this, you
will want to do so by modifying kwargs and only then calling
super(NewClass, self).__init__(**kwargs).
"""
super(APIObject, self).__init__()
# note: deep_merge does a deep copy on its arguments.
self._contents = deep_merge(self.DEFAULTS, kwargs)
self.validate()
def __init__(self, *args, **kwargs):
"""
Create and validate an instance. Note that it's
not a good idea to override this.
"""
defaults = copy.deepcopy(self.DEFAULTS)
settings = copy.deepcopy(kwargs)
d = deep_merge(defaults, settings)
super(APIObject, self).__init__(*args, **d)
self.__dict__ = self
self.validate()
def create_from_source(cls: Type[Self], source: ParsedSourceDefinition,
**kwargs: Any) -> Self:
quote_policy = deep_merge(
cls.get_default_quote_policy().to_dict(),
source.quoting.to_dict(),
kwargs.get('quote_policy', {}),
)
return cls.create(database=source.database,
schema=source.schema,
identifier=source.identifier,
quote_policy=quote_policy,
**kwargs)
class CredentialsValidator(APIObject):
SCHEMA = deep_merge(
validator.SCHEMA, {
'properties': {
'type': {
'type': 'string'
},
'threads': {
'type': 'integer'
},
},
'required': (validator.SCHEMA.get('required', []) +
['type', 'threads']),
})
def merge(self, *configs: Dict[str, Any]) -> Dict[str, Any]:
merged_config: Dict[str, Any] = {}
for config in configs:
# Do not attempt to deep merge clobber fields
config = config.copy()
clobber = {
key: config.pop(key)
for key in list(config.keys()) if key in self.ClobberFields
}
intermediary_merged = deep_merge(merged_config, config)
intermediary_merged.update(clobber)
merged_config.update(intermediary_merged)
return merged_config
def _parse_template_docs(self, template, docfile):
for key, item in template.module.__dict__.items():
if type(item) != jinja2.runtime.Macro:
continue
if not key.startswith(DOCS_PREFIX):
continue
name = key.replace(DOCS_PREFIX, '')
unique_id = self.generate_unique_id(name)
merged = deep_merge(
docfile.to_dict(), {
'name': name,
'unique_id': unique_id,
'block_contents': item().strip(),
})
yield ParsedDocumentation.from_dict(merged)
return False
@property
def failed(self):
return None
RUN_MODEL_RESULT_CONTRACT = deep_merge(
PARTIAL_RESULT_CONTRACT,
{
'description': 'The result of a single node being run',
'properties': {
'skip': {
'type': 'boolean',
'description': 'True if this node was skipped',
},
# This is assigned by dbt.ui.printer.print_test_result_line, if a test
# has no error and a non-zero status
'fail': {
'type': ['boolean', 'null'],
'description': 'On tests, true if the test failed',
},
},
'required': ['skip', 'fail']
})
class RunModelResult(NodeSerializable):
SCHEMA = RUN_MODEL_RESULT_CONTRACT
def __init__(self,
node,
'required': ['packages'],
}
# the metadata from the registry has extra things that we don't care about.
REGISTRY_PACKAGE_METADATA_CONTRACT = deep_merge(
PACKAGE_FILE_CONTRACT,
{
'additionalProperties': True,
'properties': {
'name': {
'type': 'string',
},
'downloads': {
'type': 'object',
'additionalProperties': True,
'properties': {
'tarball': {
'type': 'string',
},
},
'required': ['tarball']
},
},
'required': PACKAGE_FILE_CONTRACT['required'][:] + ['downloads']
}
)
class PackageConfig(APIObject):
SCHEMA = PACKAGE_FILE_CONTRACT
'required': ['packages'],
}
# the metadata from the registry has extra things that we don't care about.
REGISTRY_PACKAGE_METADATA_CONTRACT = deep_merge(
PACKAGE_FILE_CONTRACT,
{
'additionalProperties': True,
'properties': {
'name': {
'type': 'string',
},
'downloads': {
'type': 'object',
'additionalProperties': True,
'properties': {
'tarball': {
'type': 'string',
},
},
'required': ['tarball']
},
},
'required': PACKAGE_FILE_CONTRACT['required'][:] + ['downloads']
}
)
class PackageConfig(APIObject):
SCHEMA = PACKAGE_FILE_CONTRACT
COMPILED_NODE_CONTRACT = deep_merge(
PARSED_NODE_CONTRACT,
{
# TODO: when we add 'extra_ctes' back in, flip this back to False
'additionalProperties':
True,
'properties': {
'compiled': {
'description':
('This is true after the node has been compiled, but ctes '
'have not necessarily been injected into the node.'),
'type':
'boolean'
},
'compiled_sql': {
'type': ['string', 'null'],
},
'extra_ctes_injected': {
'description':
('This is true after extra ctes have been injected into '
'the compiled node.'),
'type':
'boolean',
},
# TODO: add this back in, and add back to 'required' list
# 'extra_ctes': {
# 'type': 'array',
# 'items': {
# 'type': 'string',
# }
# },
'injected_sql': {
'type': ['string', 'null'],
},
},
'required':
PARSED_NODE_CONTRACT['required'] +
['compiled', 'compiled_sql', 'extra_ctes_injected', 'injected_sql']
})
PARSED_NODE_CONTRACT = deep_merge(
UNPARSED_NODE_CONTRACT,
{
'properties': {
'unique_id': {
'type': 'string',
'minLength': 1,
},
'fqn': {
'type': 'array',
'items': {
'type': 'string',
}
},
'schema': {
'type': 'string',
'description': (
'The actual database string that this will build into.'
)
},
'alias': {
'type': 'string',
'description': (
'The name of the relation that this will build into'
)
},
'refs': {
'type': 'array',
'items': {
'type': 'array',
'description': (
'The list of arguments passed to a single ref call.'
),
},
'description': (
'The list of call arguments, one list of arguments per '
'call.'
)
},
'depends_on': {
'type': 'object',
'additionalProperties': False,
'properties': {
'nodes': {
'type': 'array',
'items': {
'type': 'string',
'minLength': 1,
'description': (
'A node unique ID that this depends on.'
)
}
},
'macros': {
'type': 'array',
'items': {
'type': 'string',
'minLength': 1,
'description': (
'A macro unique ID that this depends on.'
)
}
},
},
'description': (
'A list of unique IDs for nodes and macros that this '
'node depends upon.'
),
'required': ['nodes', 'macros'],
},
# TODO: move this into a class property.
'empty': {
'type': 'boolean',
'description': 'True if the SQL is empty',
},
'config': CONFIG_CONTRACT,
'tags': {
'type': 'array',
'items': {
'type': 'string',
}
},
'description': {
'type': 'string',
'description': 'A user-supplied description of the model',
},
'columns': {
'type': 'object',
'properties': {
'.*': COLUMN_INFO_CONTRACT,
}
},
'patch_path': {
'type': 'string',
'description': (
'The path to the patch source if the node was patched'
),
},
'docrefs': {
'type': 'array',
'items': DOCREF_CONTRACT,
},
'build_path': {
'type': 'string',
'description': (
'In seeds, the path to the source file used during build.'
),
},
'column_name': {
'type': 'string',
'description': (
'In tests parsed from a v2 schema, the column the test is '
'associated with (if there is one)'
)
},
},
'required': UNPARSED_NODE_CONTRACT['required'] + [
'unique_id', 'fqn', 'schema', 'refs', 'depends_on', 'empty',
'config', 'tags', 'alias', 'columns', 'description'
]
}
)
def skipped(self):
return False
@property
def failed(self):
return None
RUN_MODEL_RESULT_CONTRACT = deep_merge(PARTIAL_RESULT_CONTRACT, {
'description': 'The result of a single node being run',
'properties': {
'skip': {
'type': 'boolean',
'description': 'True if this node was skipped',
},
# This is assigned by dbt.ui.printer.print_test_result_line, if a test
# has no error and a non-zero status
'fail': {
'type': ['boolean', 'null'],
'description': 'On tests, true if the test failed',
},
},
'required': ['skip', 'fail']
})
class RunModelResult(NodeSerializable):
SCHEMA = RUN_MODEL_RESULT_CONTRACT
def __init__(self, node, error=None, skip=False, status=None, failed=None,
thread_id=None, timing=None, execution_time=0):
if timing is None:
@property
def failed(self):
return None
RUN_MODEL_RESULT_CONTRACT = deep_merge(
PARTIAL_RESULT_CONTRACT, {
'description': 'The result of a single node being run',
'properties': {
'skip': {
'type': 'boolean',
'description': 'True if this node was skipped',
},
'warn': {
'type': ['boolean', 'null'],
'description': 'True if this node succeeded with a warning',
},
'fail': {
'type': ['boolean', 'null'],
'description': 'On tests, true if the test failed',
},
},
'required': ['skip', 'fail', 'warn']
})
class RunModelResult(NodeSerializable):
SCHEMA = RUN_MODEL_RESULT_CONTRACT
def __init__(self,
COMPILED_NODE_CONTRACT = deep_merge(
PARSED_NODE_CONTRACT,
{
# TODO: when we add 'extra_ctes' back in, flip this back to False
'additionalProperties':
True,
'properties': {
'compiled': {
'description':
('This is true after the node has been compiled, but ctes '
'have not necessarily been injected into the node.'),
'type':
'boolean'
},
'compiled_sql': {
'type': ['string', 'null'],
},
'extra_ctes_injected': {
'description':
('This is true after extra ctes have been injected into '
'the compiled node.'),
'type':
'boolean',
},
'extra_ctes': {
'type': 'array',
'description': 'The injected CTEs for a model',
'items': INJECTED_CTE_CONTRACT,
},
'injected_sql': {
'type': ['string', 'null'],
'description': 'The SQL after CTEs have been injected',
},
'wrapped_sql': {
'type': ['string', 'null'],
'description':
('The SQL after it has been wrapped (for tests, '
'operations, and analysis)'),
},
},
'required':
PARSED_NODE_CONTRACT['required'] + [
'compiled', 'compiled_sql', 'extra_ctes_injected', 'injected_sql',
'extra_ctes'
]
})
PARSED_NODE_CONTRACT = deep_merge(
UNPARSED_NODE_CONTRACT,
{
'properties': {
'unique_id': {
'type': 'string',
'minLength': 1,
},
'fqn': {
'type': 'array',
'items': {
'type': 'string',
}
},
'schema': {
'type':
'string',
'description':
('The actual database string that this will build into.')
},
'alias': {
'type':
'string',
'description':
('The name of the relation that this will build into')
},
'refs': {
'type':
'array',
'items': {
'type':
'array',
'description':
('The list of arguments passed to a single ref call.'),
},
'description':
('The list of call arguments, one list of arguments per '
'call.')
},
'depends_on': {
'type':
'object',
'additionalProperties':
False,
'properties': {
'nodes': {
'type': 'array',
'items': {
'type':
'string',
'minLength':
1,
'description':
('A node unique ID that this depends on.')
}
},
'macros': {
'type': 'array',
'items': {
'type':
'string',
'minLength':
1,
'description':
('A macro unique ID that this depends on.')
}
},
},
'description':
('A list of unique IDs for nodes and macros that this '
'node depends upon.'),
'required': ['nodes', 'macros'],
},
# TODO: move this into a class property.
'empty': {
'type': 'boolean',
'description': 'True if the SQL is empty',
},
'config': CONFIG_CONTRACT,
'tags': {
'type': 'array',
'items': {
'type': 'string',
}
},
},
'required':
UNPARSED_NODE_CONTRACT['required'] + [
'unique_id',
'fqn',
'schema',
'refs',
'depends_on',
'empty',
'config',
'tags',
'alias',
]
})
def incorporate(self, **kwargs):
value = self.to_dict()
value = deep_merge(value, kwargs)
return self.from_dict(value)
UNPARSED_NODE_CONTRACT = deep_merge(
UNPARSED_BASE_CONTRACT,
{
'properties': {
'name': {
'type': 'string',
'description': (
'Name of this node. For models, this is used as the '
'identifier in the database.'),
'minLength': 1,
},
'resource_type': {
'enum': [
NodeType.Model,
NodeType.Test,
NodeType.Analysis,
# Note: Hooks fail if you remove this, even though it's
# also allowed in ParsedMacro, which seems wrong.
# Maybe need to move hook operations into macros?
NodeType.Operation,
NodeType.Seed,
# we need this if parse_node is going to handle archives.
NodeType.Archive,
]
}
},
'required': UNPARSED_BASE_CONTRACT['required'] + [
'resource_type', 'name']
}
)
PARSED_NODE_CONTRACT = deep_merge(
UNPARSED_NODE_CONTRACT,
HAS_UNIQUE_ID_CONTRACT,
HAS_FQN_CONTRACT,
CAN_REF_CONTRACT,
HAS_DOCREFS_CONTRACT,
HAS_DESCRIPTION_CONTRACT,
HAS_CONFIG_CONTRACT,
COLUMN_TEST_CONTRACT,
HAS_RELATION_METADATA_CONTRACT,
{
'properties': {
'alias': {
'type':
'string',
'description':
('The name of the relation that this will build into')
},
# TODO: move this into a class property.
'empty': {
'type': 'boolean',
'description': 'True if the SQL is empty',
},
'tags': {
'type': 'array',
'items': {
'type': 'string',
}
},
# this is really nodes-only
'patch_path': {
'type':
'string',
'description':
('The path to the patch source if the node was patched'),
},
'build_path': {
'type':
'string',
'description':
('In seeds, the path to the source file used during build.'),
},
},
'required': ['empty', 'tags', 'alias'],
})
'raw_sql': {
'type': 'string',
'description': (
'For nodes defined in SQL files, this is just the contents '
'of that file. For schema tests, archives, etc. this is '
'generated by dbt.'),
},
'index': {
'type': 'integer',
}
},
'required': ['raw_sql']
}
UNPARSED_MACRO_CONTRACT = deep_merge(
UNPARSED_BASE_CONTRACT,
UNPARSED_HAS_SQL_CONTRACT
)
UNPARSED_NODE_CONTRACT = deep_merge(
UNPARSED_BASE_CONTRACT,
UNPARSED_HAS_SQL_CONTRACT,
{
'properties': {
'name': {
'type': 'string',
'description': (
'Name of this node. For models, this is used as the '
'identifier in the database.'),
'minLength': 1,
},
'resource_type': {
'raw_sql': {
'type':
'string',
'description':
('For nodes defined in SQL files, this is just the contents '
'of that file. For schema tests, archives, etc. this is '
'generated by dbt.'),
},
'index': {
'type': 'integer',
}
},
'required': ['raw_sql']
}
UNPARSED_MACRO_CONTRACT = deep_merge(UNPARSED_BASE_CONTRACT,
UNPARSED_HAS_SQL_CONTRACT)
UNPARSED_NODE_CONTRACT = deep_merge(
UNPARSED_BASE_CONTRACT,
UNPARSED_HAS_SQL_CONTRACT,
{
'properties': {
'name': {
'type':
'string',
'description':
('Name of this node. For models, this is used as the '
'identifier in the database.'),
'minLength':
1,
},
required.extend(arg['required'])
return required
CONFIG_CONTRACT = deep_merge(
PROJECT_CONTRACT,
PACKAGE_FILE_CONTRACT,
PROFILE_INFO_CONTRACT,
{
'properties': {
'cli_vars': {
'type': 'object',
'additionalProperties': True,
},
# override quoting: both 'identifier' and 'schema' must be
# populated
'quoting': {
'required': ['identifier', 'schema'],
},
},
'required': _merge_requirements(
['cli_vars'],
PROJECT_CONTRACT,
PACKAGE_FILE_CONTRACT,
PROFILE_INFO_CONTRACT
),
},
)
class Configuration(APIObject):
SCHEMA = CONFIG_CONTRACT
