def incorporate(self, **kwargs): """ Given a list of kwargs, incorporate these arguments into a new copy of this instance, and return the new instance after validating. """ return type(self)(**deep_merge(self._contents, kwargs))
def _merge(self, *configs): merged_config = {} for config in configs: intermediary_merged = deep_merge(merged_config.copy(), config.copy()) merged_config.update(intermediary_merged) return merged_config
def incorporate(self, **kwargs): """ Given a list of kwargs, incorporate these arguments into a new copy of this instance, and return the new instance after validating. """ existing = copy.deepcopy(dict(self)) updates = copy.deepcopy(kwargs) return type(self)(**deep_merge(existing, updates))
def _merge(self, *configs): merged_config = {} for config in configs: intermediary_merged = deep_merge( merged_config.copy(), config.copy() ) merged_config.update(intermediary_merged) return merged_config
def __init__(self, **kwargs): """ Create and validate an instance. Note that if you override this, you will want to do so by modifying kwargs and only then calling super(NewClass, self).__init__(**kwargs). """ super(APIObject, self).__init__() # note: deep_merge does a deep copy on its arguments. self._contents = deep_merge(self.DEFAULTS, kwargs) self.validate()
def __init__(self, *args, **kwargs): """ Create and validate an instance. Note that it's not a good idea to override this. """ defaults = copy.deepcopy(self.DEFAULTS) settings = copy.deepcopy(kwargs) d = deep_merge(defaults, settings) super(APIObject, self).__init__(*args, **d) self.__dict__ = self self.validate()
def create_from_source(cls: Type[Self], source: ParsedSourceDefinition, **kwargs: Any) -> Self: quote_policy = deep_merge( cls.get_default_quote_policy().to_dict(), source.quoting.to_dict(), kwargs.get('quote_policy', {}), ) return cls.create(database=source.database, schema=source.schema, identifier=source.identifier, quote_policy=quote_policy, **kwargs)
class CredentialsValidator(APIObject): SCHEMA = deep_merge( validator.SCHEMA, { 'properties': { 'type': { 'type': 'string' }, 'threads': { 'type': 'integer' }, }, 'required': (validator.SCHEMA.get('required', []) + ['type', 'threads']), })
def merge(self, *configs: Dict[str, Any]) -> Dict[str, Any]: merged_config: Dict[str, Any] = {} for config in configs: # Do not attempt to deep merge clobber fields config = config.copy() clobber = { key: config.pop(key) for key in list(config.keys()) if key in self.ClobberFields } intermediary_merged = deep_merge(merged_config, config) intermediary_merged.update(clobber) merged_config.update(intermediary_merged) return merged_config
def _parse_template_docs(self, template, docfile): for key, item in template.module.__dict__.items(): if type(item) != jinja2.runtime.Macro: continue if not key.startswith(DOCS_PREFIX): continue name = key.replace(DOCS_PREFIX, '') unique_id = self.generate_unique_id(name) merged = deep_merge( docfile.to_dict(), { 'name': name, 'unique_id': unique_id, 'block_contents': item().strip(), }) yield ParsedDocumentation.from_dict(merged)
return False @property def failed(self): return None RUN_MODEL_RESULT_CONTRACT = deep_merge( PARTIAL_RESULT_CONTRACT, { 'description': 'The result of a single node being run', 'properties': { 'skip': { 'type': 'boolean', 'description': 'True if this node was skipped', }, # This is assigned by dbt.ui.printer.print_test_result_line, if a test # has no error and a non-zero status 'fail': { 'type': ['boolean', 'null'], 'description': 'On tests, true if the test failed', }, }, 'required': ['skip', 'fail'] }) class RunModelResult(NodeSerializable): SCHEMA = RUN_MODEL_RESULT_CONTRACT def __init__(self, node,
'required': ['packages'], } # the metadata from the registry has extra things that we don't care about. REGISTRY_PACKAGE_METADATA_CONTRACT = deep_merge( PACKAGE_FILE_CONTRACT, { 'additionalProperties': True, 'properties': { 'name': { 'type': 'string', }, 'downloads': { 'type': 'object', 'additionalProperties': True, 'properties': { 'tarball': { 'type': 'string', }, }, 'required': ['tarball'] }, }, 'required': PACKAGE_FILE_CONTRACT['required'][:] + ['downloads'] } ) class PackageConfig(APIObject): SCHEMA = PACKAGE_FILE_CONTRACT
COMPILED_NODE_CONTRACT = deep_merge( PARSED_NODE_CONTRACT, { # TODO: when we add 'extra_ctes' back in, flip this back to False 'additionalProperties': True, 'properties': { 'compiled': { 'description': ('This is true after the node has been compiled, but ctes ' 'have not necessarily been injected into the node.'), 'type': 'boolean' }, 'compiled_sql': { 'type': ['string', 'null'], }, 'extra_ctes_injected': { 'description': ('This is true after extra ctes have been injected into ' 'the compiled node.'), 'type': 'boolean', }, # TODO: add this back in, and add back to 'required' list # 'extra_ctes': { # 'type': 'array', # 'items': { # 'type': 'string', # } # }, 'injected_sql': { 'type': ['string', 'null'], }, }, 'required': PARSED_NODE_CONTRACT['required'] + ['compiled', 'compiled_sql', 'extra_ctes_injected', 'injected_sql'] })
PARSED_NODE_CONTRACT = deep_merge( UNPARSED_NODE_CONTRACT, { 'properties': { 'unique_id': { 'type': 'string', 'minLength': 1, }, 'fqn': { 'type': 'array', 'items': { 'type': 'string', } }, 'schema': { 'type': 'string', 'description': ( 'The actual database string that this will build into.' ) }, 'alias': { 'type': 'string', 'description': ( 'The name of the relation that this will build into' ) }, 'refs': { 'type': 'array', 'items': { 'type': 'array', 'description': ( 'The list of arguments passed to a single ref call.' ), }, 'description': ( 'The list of call arguments, one list of arguments per ' 'call.' ) }, 'depends_on': { 'type': 'object', 'additionalProperties': False, 'properties': { 'nodes': { 'type': 'array', 'items': { 'type': 'string', 'minLength': 1, 'description': ( 'A node unique ID that this depends on.' ) } }, 'macros': { 'type': 'array', 'items': { 'type': 'string', 'minLength': 1, 'description': ( 'A macro unique ID that this depends on.' ) } }, }, 'description': ( 'A list of unique IDs for nodes and macros that this ' 'node depends upon.' ), 'required': ['nodes', 'macros'], }, # TODO: move this into a class property. 'empty': { 'type': 'boolean', 'description': 'True if the SQL is empty', }, 'config': CONFIG_CONTRACT, 'tags': { 'type': 'array', 'items': { 'type': 'string', } }, 'description': { 'type': 'string', 'description': 'A user-supplied description of the model', }, 'columns': { 'type': 'object', 'properties': { '.*': COLUMN_INFO_CONTRACT, } }, 'patch_path': { 'type': 'string', 'description': ( 'The path to the patch source if the node was patched' ), }, 'docrefs': { 'type': 'array', 'items': DOCREF_CONTRACT, }, 'build_path': { 'type': 'string', 'description': ( 'In seeds, the path to the source file used during build.' ), }, 'column_name': { 'type': 'string', 'description': ( 'In tests parsed from a v2 schema, the column the test is ' 'associated with (if there is one)' ) }, }, 'required': UNPARSED_NODE_CONTRACT['required'] + [ 'unique_id', 'fqn', 'schema', 'refs', 'depends_on', 'empty', 'config', 'tags', 'alias', 'columns', 'description' ] } )
def skipped(self): return False @property def failed(self): return None RUN_MODEL_RESULT_CONTRACT = deep_merge(PARTIAL_RESULT_CONTRACT, { 'description': 'The result of a single node being run', 'properties': { 'skip': { 'type': 'boolean', 'description': 'True if this node was skipped', }, # This is assigned by dbt.ui.printer.print_test_result_line, if a test # has no error and a non-zero status 'fail': { 'type': ['boolean', 'null'], 'description': 'On tests, true if the test failed', }, }, 'required': ['skip', 'fail'] }) class RunModelResult(NodeSerializable): SCHEMA = RUN_MODEL_RESULT_CONTRACT def __init__(self, node, error=None, skip=False, status=None, failed=None, thread_id=None, timing=None, execution_time=0): if timing is None:
@property def failed(self): return None RUN_MODEL_RESULT_CONTRACT = deep_merge( PARTIAL_RESULT_CONTRACT, { 'description': 'The result of a single node being run', 'properties': { 'skip': { 'type': 'boolean', 'description': 'True if this node was skipped', }, 'warn': { 'type': ['boolean', 'null'], 'description': 'True if this node succeeded with a warning', }, 'fail': { 'type': ['boolean', 'null'], 'description': 'On tests, true if the test failed', }, }, 'required': ['skip', 'fail', 'warn'] }) class RunModelResult(NodeSerializable): SCHEMA = RUN_MODEL_RESULT_CONTRACT def __init__(self,
COMPILED_NODE_CONTRACT = deep_merge( PARSED_NODE_CONTRACT, { # TODO: when we add 'extra_ctes' back in, flip this back to False 'additionalProperties': True, 'properties': { 'compiled': { 'description': ('This is true after the node has been compiled, but ctes ' 'have not necessarily been injected into the node.'), 'type': 'boolean' }, 'compiled_sql': { 'type': ['string', 'null'], }, 'extra_ctes_injected': { 'description': ('This is true after extra ctes have been injected into ' 'the compiled node.'), 'type': 'boolean', }, 'extra_ctes': { 'type': 'array', 'description': 'The injected CTEs for a model', 'items': INJECTED_CTE_CONTRACT, }, 'injected_sql': { 'type': ['string', 'null'], 'description': 'The SQL after CTEs have been injected', }, 'wrapped_sql': { 'type': ['string', 'null'], 'description': ('The SQL after it has been wrapped (for tests, ' 'operations, and analysis)'), }, }, 'required': PARSED_NODE_CONTRACT['required'] + [ 'compiled', 'compiled_sql', 'extra_ctes_injected', 'injected_sql', 'extra_ctes' ] })
PARSED_NODE_CONTRACT = deep_merge( UNPARSED_NODE_CONTRACT, { 'properties': { 'unique_id': { 'type': 'string', 'minLength': 1, }, 'fqn': { 'type': 'array', 'items': { 'type': 'string', } }, 'schema': { 'type': 'string', 'description': ('The actual database string that this will build into.') }, 'alias': { 'type': 'string', 'description': ('The name of the relation that this will build into') }, 'refs': { 'type': 'array', 'items': { 'type': 'array', 'description': ('The list of arguments passed to a single ref call.'), }, 'description': ('The list of call arguments, one list of arguments per ' 'call.') }, 'depends_on': { 'type': 'object', 'additionalProperties': False, 'properties': { 'nodes': { 'type': 'array', 'items': { 'type': 'string', 'minLength': 1, 'description': ('A node unique ID that this depends on.') } }, 'macros': { 'type': 'array', 'items': { 'type': 'string', 'minLength': 1, 'description': ('A macro unique ID that this depends on.') } }, }, 'description': ('A list of unique IDs for nodes and macros that this ' 'node depends upon.'), 'required': ['nodes', 'macros'], }, # TODO: move this into a class property. 'empty': { 'type': 'boolean', 'description': 'True if the SQL is empty', }, 'config': CONFIG_CONTRACT, 'tags': { 'type': 'array', 'items': { 'type': 'string', } }, }, 'required': UNPARSED_NODE_CONTRACT['required'] + [ 'unique_id', 'fqn', 'schema', 'refs', 'depends_on', 'empty', 'config', 'tags', 'alias', ] })
def incorporate(self, **kwargs): value = self.to_dict() value = deep_merge(value, kwargs) return self.from_dict(value)
UNPARSED_NODE_CONTRACT = deep_merge( UNPARSED_BASE_CONTRACT, { 'properties': { 'name': { 'type': 'string', 'description': ( 'Name of this node. For models, this is used as the ' 'identifier in the database.'), 'minLength': 1, }, 'resource_type': { 'enum': [ NodeType.Model, NodeType.Test, NodeType.Analysis, # Note: Hooks fail if you remove this, even though it's # also allowed in ParsedMacro, which seems wrong. # Maybe need to move hook operations into macros? NodeType.Operation, NodeType.Seed, # we need this if parse_node is going to handle archives. NodeType.Archive, ] } }, 'required': UNPARSED_BASE_CONTRACT['required'] + [ 'resource_type', 'name'] } )
PARSED_NODE_CONTRACT = deep_merge( UNPARSED_NODE_CONTRACT, HAS_UNIQUE_ID_CONTRACT, HAS_FQN_CONTRACT, CAN_REF_CONTRACT, HAS_DOCREFS_CONTRACT, HAS_DESCRIPTION_CONTRACT, HAS_CONFIG_CONTRACT, COLUMN_TEST_CONTRACT, HAS_RELATION_METADATA_CONTRACT, { 'properties': { 'alias': { 'type': 'string', 'description': ('The name of the relation that this will build into') }, # TODO: move this into a class property. 'empty': { 'type': 'boolean', 'description': 'True if the SQL is empty', }, 'tags': { 'type': 'array', 'items': { 'type': 'string', } }, # this is really nodes-only 'patch_path': { 'type': 'string', 'description': ('The path to the patch source if the node was patched'), }, 'build_path': { 'type': 'string', 'description': ('In seeds, the path to the source file used during build.'), }, }, 'required': ['empty', 'tags', 'alias'], })
'raw_sql': { 'type': 'string', 'description': ( 'For nodes defined in SQL files, this is just the contents ' 'of that file. For schema tests, archives, etc. this is ' 'generated by dbt.'), }, 'index': { 'type': 'integer', } }, 'required': ['raw_sql'] } UNPARSED_MACRO_CONTRACT = deep_merge( UNPARSED_BASE_CONTRACT, UNPARSED_HAS_SQL_CONTRACT ) UNPARSED_NODE_CONTRACT = deep_merge( UNPARSED_BASE_CONTRACT, UNPARSED_HAS_SQL_CONTRACT, { 'properties': { 'name': { 'type': 'string', 'description': ( 'Name of this node. For models, this is used as the ' 'identifier in the database.'), 'minLength': 1, }, 'resource_type': {
'raw_sql': { 'type': 'string', 'description': ('For nodes defined in SQL files, this is just the contents ' 'of that file. For schema tests, archives, etc. this is ' 'generated by dbt.'), }, 'index': { 'type': 'integer', } }, 'required': ['raw_sql'] } UNPARSED_MACRO_CONTRACT = deep_merge(UNPARSED_BASE_CONTRACT, UNPARSED_HAS_SQL_CONTRACT) UNPARSED_NODE_CONTRACT = deep_merge( UNPARSED_BASE_CONTRACT, UNPARSED_HAS_SQL_CONTRACT, { 'properties': { 'name': { 'type': 'string', 'description': ('Name of this node. For models, this is used as the ' 'identifier in the database.'), 'minLength': 1, },
required.extend(arg['required']) return required CONFIG_CONTRACT = deep_merge( PROJECT_CONTRACT, PACKAGE_FILE_CONTRACT, PROFILE_INFO_CONTRACT, { 'properties': { 'cli_vars': { 'type': 'object', 'additionalProperties': True, }, # override quoting: both 'identifier' and 'schema' must be # populated 'quoting': { 'required': ['identifier', 'schema'], }, }, 'required': _merge_requirements( ['cli_vars'], PROJECT_CONTRACT, PACKAGE_FILE_CONTRACT, PROFILE_INFO_CONTRACT ), }, ) class Configuration(APIObject): SCHEMA = CONFIG_CONTRACT