def validate_args(self, source_name: str, table_name: str): if not isinstance(source_name, str): raise CompilationException( f'The source name (first) argument to source() must be a ' f'string, got {type(source_name)}') if not isinstance(table_name, str): raise CompilationException( f'The table name (second) argument to source() must be a ' f'string, got {type(table_name)}')
def validate_args(self, name: str, package: Optional[str]): if not isinstance(name, str): raise CompilationException( f'The name argument to ref() must be a string, got ' f'{type(name)}') if package is not None and not isinstance(package, str): raise CompilationException( f'The package argument to ref() must be a string or None, got ' f'{type(package)}')
def catch_jinja(node=None) -> Iterator[None]: try: yield except jinja2.exceptions.TemplateSyntaxError as e: e.translated = False raise CompilationException(str(e), node) from e except jinja2.exceptions.UndefinedError as e: raise CompilationException(str(e), node) from e except CompilationException as exc: exc.add_node(node) raise
def __post_init__(self): # handle pesky jinja2.Undefined sneaking in here and messing up rende if not isinstance(self.database, (type(None), str)): raise CompilationException( 'Got an invalid path database: {}'.format(self.database)) if not isinstance(self.schema, (type(None), str)): raise CompilationException('Got an invalid path schema: {}'.format( self.schema)) if not isinstance(self.identifier, (type(None), str)): raise CompilationException( 'Got an invalid path identifier: {}'.format(self.identifier))
def dispatch( self, macro_name: str, packages: Optional[List[str]] = None ) -> MacroGenerator: search_packages: List[Optional[str]] if '.' in macro_name: suggest_package, suggest_macro_name = macro_name.split('.', 1) msg = ( f'In adapter.dispatch, got a macro name of "{macro_name}", ' f'but "." is not a valid macro name component. Did you mean ' f'`adapter.dispatch("{suggest_macro_name}", ' f'packages=["{suggest_package}"])`?' ) raise CompilationException(msg) if packages is None: search_packages = [None] elif isinstance(packages, str): raise CompilationException( f'In adapter.dispatch, got a string packages argument ' f'("{packages}"), but packages should be None or a list.' ) else: search_packages = packages attempts = [] for package_name in search_packages: for prefix in self._get_adapter_macro_prefixes(): search_name = f'{prefix}__{macro_name}' try: # this uses the namespace from the context macro = self._namespace.get_from_package( package_name, search_name ) except CompilationException as exc: raise CompilationException( f'In dispatch: {exc.msg}', ) from exc if package_name is None: attempts.append(search_name) else: attempts.append(f'{package_name}.{search_name}') if macro is not None: return macro searched = ', '.join(repr(a) for a in attempts) msg = ( f"In dispatch: No macro named '{macro_name}' found\n" f" Searched for: {searched}" ) raise CompilationException(msg)
def get_key_dicts(self) -> Iterable[Dict[str, Any]]: data = self.yaml.data.get(self.key, []) if not isinstance(data, list): raise CompilationException( '{} must be a list, got {} instead: ({})'.format( self.key, type(data), _trimmed(str(data)))) path = self.yaml.path.original_file_path for entry in data: if coerce_dict_str(entry) is not None: yield entry else: msg = error_context(path, self.key, data, 'expected a dict with string keys') raise CompilationException(msg)
def _expect_value(key: K_T, src: Mapping[K_T, V_T], old_file: SourceFile, name: str) -> V_T: if key not in src: raise CompilationException( 'Expected to find "{}" in cached "result.{}" based ' 'on cached file information: {}!'.format(key, name, old_file)) return src[key]
def _process_node( self, node_id: str, source_file: SourceFile, old_file: SourceFile, old_manifest: Any, ) -> None: """Nodes are a special kind of complicated - there can be multiple with the same name, as long as all but one are disabled. Only handle nodes where the matching node has the same resource type as the current parser. """ source_path = source_file.path.original_file_path found: bool = False if node_id in old_manifest.nodes: old_node = old_manifest.nodes[node_id] if old_node.original_file_path == source_path: self.add_node(source_file, old_node) found = True if node_id in old_manifest._disabled: matches = old_manifest._get_disabled(node_id, source_file) for match in matches: self.add_disabled(source_file, match) found = True if not found: raise CompilationException( 'Expected to find "{}" in cached "manifest.nodes" or ' '"manifest.disabled" based on cached file information: {}!'. format(node_id, old_file))
def parse_file(self, block: FileBlock) -> None: dct = self._yaml_from_file(block.file) # mark the file as seen, even if there are no macros in it self.results.get_file(block.file) if dct: try: dct = self.raw_renderer.render_data(dct) except CompilationException as exc: raise CompilationException( f'Failed to render {block.path.original_file_path} from ' f'project {self.project.project_name}: {exc}') from exc yaml_block = YamlBlock.from_file_block(block, dct) self._parse_format_version(yaml_block) parser: YamlDocsReader for key in NodeType.documentable(): plural = key.pluralize() if key == NodeType.Source: parser = SourceParser(self, yaml_block, plural) elif key == NodeType.Macro: parser = MacroPatchParser(self, yaml_block, plural) elif key == NodeType.Analysis: parser = AnalysisPatchParser(self, yaml_block, plural) else: parser = TestablePatchParser(self, yaml_block, plural) for test_block in parser.parse(): self.parse_tests(test_block)
def _target_from_dict(self, cls: Type[T], data: Dict[str, Any]) -> T: path = self.yaml.path.original_file_path try: return cls.from_dict(data) except (ValidationError, JSONValidationException) as exc: msg = error_context(path, self.key, data, exc) raise CompilationException(msg) from exc
def transform(self, node: IntermediateSnapshotNode) -> ParsedSnapshotNode: try: parsed_node = ParsedSnapshotNode.from_dict(node.to_dict()) self.set_snapshot_attributes(parsed_node) return parsed_node except ValidationError as exc: raise CompilationException(validator_error_message(exc), node)
def __delitem__(self, key): if hasattr(self, key): msg = ('Error, tried to delete config key "{}": Cannot delete ' 'built-in keys').format(key) raise CompilationException(msg) else: del self._extra[key]
def adapter_macro(self, name: str, *args, **kwargs): """Find the most appropriate macro for the name, considering the adapter type currently in use, and call that with the given arguments. If the name has a `.` in it, the first section before the `.` is interpreted as a package name, and the remainder as a macro name. If no adapter is found, raise a compiler exception. If an invalid package name is specified, raise a compiler exception. Some examples: {# dbt will call this macro by name, providing any arguments #} {% macro create_table_as(temporary, relation, sql) -%} {# dbt will dispatch the macro call to the relevant macro #} {{ adapter_macro('create_table_as', temporary, relation, sql) }} {%- endmacro %} {# If no macro matches the specified adapter, "default" will be used #} {% macro default__create_table_as(temporary, relation, sql) -%} ... {%- endmacro %} {# Example which defines special logic for Redshift #} {% macro redshift__create_table_as(temporary, relation, sql) -%} ... {%- endmacro %} {# Example which defines special logic for BigQuery #} {% macro bigquery__create_table_as(temporary, relation, sql) -%} ... {%- endmacro %} """ deprecations.warn('adapter-macro', macro_name=name) original_name = name package_names: Optional[List[str]] = None if '.' in name: package_name, name = name.split('.', 1) package_names = [package_name] try: macro = self.db_wrapper.dispatch(macro_name=name, packages=package_names) except CompilationException as exc: raise CompilationException( f'In adapter_macro: {exc.msg}\n' f" Original name: '{original_name}'", node=self.model) from exc return macro(*args, **kwargs)
def project_name_from_path(include_path: str) -> str: # avoid an import cycle from dbt.config.project import Project partial = Project.partial_load(include_path) if partial.project_name is None: raise CompilationException( f'Invalid project at {include_path}: name not set!') return partial.project_name
def _get_dicts_for(self, yaml: YamlBlock, key: str) -> Iterable[Dict[str, Any]]: data = yaml.data.get(key, []) if not isinstance(data, list): raise CompilationException( '{} must be a list, got {} instead: ({})'.format( key, type(data), _trimmed(str(data)))) path = yaml.path.original_file_path for entry in data: str_keys = (isinstance(entry, dict) and all(isinstance(k, str) for k in entry)) if str_keys: yield entry else: msg = error_context(path, key, data, 'expected a dict with string keys') raise CompilationException(msg)
def parse(self) -> Iterable[ParsedExposure]: for data in self.get_key_dicts(): try: unparsed = UnparsedExposure.from_dict(data) except (ValidationError, JSONValidationException) as exc: msg = error_context(self.yaml.path, self.key, data, exc) raise CompilationException(msg) from exc parsed = self.parse_exposure(unparsed) yield parsed
def render_update(self, node: IntermediateNode, config: ContextConfig) -> None: try: self.render_with_context(node, config) self.update_parsed_node(node, config) except ValidationError as exc: # we got a ValidationError - probably bad types in config() msg = validator_error_message(exc) raise CompilationException(msg, node=node) from exc
def read_yaml_models(self, yaml: YamlBlock) -> Iterable[ModelTarget]: path = yaml.path.original_file_path yaml_key = 'models' for data in self._get_dicts_for(yaml, yaml_key): try: model = UnparsedNodeUpdate.from_dict(data) except (ValidationError, JSONValidationException) as exc: msg = error_context(path, yaml_key, data, exc) raise CompilationException(msg) from exc else: yield model
def _yaml_from_file(self, source_file: SourceFile) -> Optional[Dict[str, Any]]: """If loading the yaml fails, raise an exception. """ path: str = source_file.path.relative_path try: return load_yaml_text(source_file.contents) except ValidationException as e: reason = validator_error_message(e) raise CompilationException('Error reading {}: {} - {}'.format( self.project.project_name, path, reason)) return None
def render_value(self, value: Any, keypath: Optional[Keypath] = None) -> Any: # keypath is ignored. # if it wasn't read as a string, ignore it if not isinstance(value, str): return value try: with catch_jinja(): return get_rendered(value, self.context, native=True) except CompilationException as exc: msg = f'Could not render {value}: {exc.msg}' raise CompilationException(msg) from exc
def _materialization_relations(self, result: Any, model) -> List[BaseRelation]: if isinstance(result, str): deprecations.warn('materialization-return', materialization=model.get_materialization()) return [self.adapter.Relation.create_from(self.config, model)] if isinstance(result, dict): return _validate_materialization_relations_dict(result, model) msg = ('Invalid return value from materialization, expected a dict ' 'with key "relations", got: {}'.format(str(result))) raise CompilationException(msg, node=model)
def read_yaml_sources(self, yaml: YamlBlock) -> Iterable[SourceTarget]: path = yaml.path.original_file_path yaml_key = 'sources' for data in self._get_dicts_for(yaml, yaml_key): try: data = self._renderer.render_schema_source(data) source = UnparsedSourceDefinition.from_dict(data) except (ValidationError, JSONValidationException) as exc: msg = error_context(path, yaml_key, data, exc) raise CompilationException(msg) from exc else: for table in source.tables: yield SourceTarget(source, table)
def sanitized_update( self, source_file: SourceFile, old_result: 'ParseResult', ) -> bool: """Perform a santized update. If the file can't be updated, invalidate it and return false. """ if isinstance(source_file.path, RemoteFile): return False old_file = old_result.get_file(source_file) for doc_id in old_file.docs: doc = _expect_value(doc_id, old_result.docs, old_file, "docs") self.add_doc(source_file, doc) for macro_id in old_file.macros: macro = _expect_value(macro_id, old_result.macros, old_file, "macros") self.add_macro(source_file, macro) for source_id in old_file.sources: source = _expect_value(source_id, old_result.sources, old_file, "sources") self.add_source(source_file, source) # because we know this is how we _parsed_ the node, we can safely # assume if it's disabled it was done by the project or file, and # we can keep our old data for node_id in old_file.nodes: if node_id in old_result.nodes: node = old_result.nodes[node_id] self.add_node(source_file, node) elif node_id in old_result.disabled: matches = old_result._get_disabled(node_id, source_file) for match in matches: self.add_disabled(source_file, match) else: raise CompilationException( 'Expected to find "{}" in cached "manifest.nodes" or ' '"manifest.disabled" based on cached file information: {}!' .format(node_id, old_file)) for name in old_file.patches: patch = _expect_value(name, old_result.patches, old_file, "patches") self.add_patch(source_file, patch) return True
def _validate_materialization_relations_dict(inp: Dict[Any, Any], model) -> List[BaseRelation]: try: relations_value = inp['relations'] except KeyError: msg = ('Invalid return value from materialization, "relations" ' 'not found, got keys: {}'.format(list(inp))) raise CompilationException(msg, node=model) from None if not isinstance(relations_value, list): msg = ('Invalid return value from materialization, "relations" ' 'not a list, got: {}'.format(relations_value)) raise CompilationException(msg, node=model) from None relations: List[BaseRelation] = [] for relation in relations_value: if not isinstance(relation, BaseRelation): msg = ('Invalid return value from materialization, ' '"relations" contains non-Relation: {}'.format(relation)) raise CompilationException(msg, node=model) assert isinstance(relation, BaseRelation) relations.append(relation) return relations
def get_unparsed_target(self) -> Iterable[NonSourceTarget]: path = self.yaml.path.original_file_path for data in self.get_key_dicts(): data.update({ 'original_file_path': path, 'yaml_key': self.key, 'package_name': self.project.project_name, }) try: model = self._target_type().from_dict(data) except (ValidationError, JSONValidationException) as exc: msg = error_context(path, self.key, data, exc) raise CompilationException(msg) from exc else: yield model
def create_test_node( self, target: Union[UnpatchedSourceDefinition, UnparsedNodeUpdate], path: str, config: ContextConfig, tags: List[str], fqn: List[str], name: str, raw_sql: str, test_metadata: Dict[str, Any], column_name: Optional[str], ) -> ParsedSchemaTestNode: dct = { 'alias': name, 'schema': self.default_schema, 'database': self.default_database, 'fqn': fqn, 'name': name, 'root_path': self.project.project_root, 'resource_type': self.resource_type, 'tags': tags, 'path': path, 'original_file_path': target.original_file_path, 'package_name': self.project.project_name, 'raw_sql': raw_sql, 'unique_id': self.generate_unique_id(name), 'config': self.config_dict(config), 'test_metadata': test_metadata, 'column_name': column_name, 'checksum': FileHash.empty().to_dict(omit_none=True), } try: ParsedSchemaTestNode.validate(dct) return ParsedSchemaTestNode.from_dict(dct) except ValidationError as exc: msg = validator_error_message(exc) # this is a bit silly, but build an UnparsedNode just for error # message reasons node = self._create_error_node( name=target.name, path=path, original_file_path=target.original_file_path, raw_sql=raw_sql, ) raise CompilationException(msg, node=node) from exc
def parse_test(self, target_block: TargetBlock, test: TestDef, column_name: Optional[str]) -> None: if isinstance(test, str): test = {test: {}} block = SchemaTestBlock.from_target_block(src=target_block, test=test, column_name=column_name) try: self.parse_node(block) except CompilationException as exc: context = _trimmed(str(block.target)) msg = ('Invalid test config given in {}:' '\n\t{}\n\t@: {}'.format(block.path.original_file_path, exc.msg, context)) raise CompilationException(msg) from exc
def _create_parsetime_node( self, block: ConfiguredBlockType, path: str, config: ContextConfig, fqn: List[str], name=None, **kwargs, ) -> IntermediateNode: """Create the node that will be passed in to the parser context for "rendering". Some information may be partial, as it'll be updated by config() and any ref()/source() calls discovered during rendering. """ if name is None: name = block.name dct = { 'alias': name, 'schema': self.default_schema, 'database': self.default_database, 'fqn': fqn, 'name': name, 'root_path': self.project.project_root, 'resource_type': self.resource_type, 'path': path, 'original_file_path': block.path.original_file_path, 'package_name': self.project.project_name, 'raw_sql': block.contents, 'unique_id': self.generate_unique_id(name), 'config': self.config_dict(config), 'checksum': block.file.checksum.to_dict(omit_none=True), } dct.update(kwargs) try: return self.parse_from_dict(dct, validate=True) except ValidationError as exc: msg = validator_error_message(exc) # this is a bit silly, but build an UnparsedNode just for error # message reasons node = self._create_error_node( name=block.name, path=path, original_file_path=block.path.original_file_path, raw_sql=block.contents, ) raise CompilationException(msg, node=node)
def render_test_update(self, node, config, builder): macro_unique_id = self.macro_resolver.get_macro_id( node.package_name, 'test_' + builder.name) # Add the depends_on here so we can limit the macros added # to the context in rendering processing node.depends_on.add_macro(macro_unique_id) if (macro_unique_id in ['macro.dbt.test_not_null', 'macro.dbt.test_unique']): self.update_parsed_node(node, config) if builder.severity() is not None: node.unrendered_config['severity'] = builder.severity() node.config['severity'] = builder.severity() if builder.enabled() is not None: node.config['enabled'] = builder.enabled() # source node tests are processed at patch_source time if isinstance(builder.target, UnpatchedSourceDefinition): sources = [builder.target.fqn[-2], builder.target.fqn[-1]] node.sources.append(sources) else: # all other nodes node.refs.append([builder.target.name]) else: try: # make a base context that doesn't have the magic kwargs field context = generate_test_context( node, self.root_project, self.manifest, config, self.macro_resolver, ) # update with rendered test kwargs (which collects any refs) add_rendered_test_kwargs(context, node, capture_macros=True) # the parsed node is not rendered in the native context. get_rendered(node.raw_sql, context, node, capture_macros=True) self.update_parsed_node(node, config) except ValidationError as exc: # we got a ValidationError - probably bad types in config() msg = validator_error_message(exc) raise CompilationException(msg, node=node) from exc
def parse_unparsed_macros( self, base_node: UnparsedMacro) -> Iterable[ParsedMacro]: try: blocks: List[jinja.BlockTag] = [ t for t in jinja.extract_toplevel_blocks( base_node.raw_sql, allowed_blocks={'macro', 'materialization'}, collect_raw_data=False, ) if isinstance(t, jinja.BlockTag) ] except CompilationException as exc: exc.add_node(base_node) raise for block in blocks: try: ast = jinja.parse(block.full_block) except CompilationException as e: e.add_node(base_node) raise macro_nodes = list(ast.find_all(jinja2.nodes.Macro)) if len(macro_nodes) != 1: # things have gone disastrously wrong, we thought we only # parsed one block! raise CompilationException( f'Found multiple macros in {block.full_block}, expected 1', node=base_node) macro_name = macro_nodes[0].name if not macro_name.startswith(MACRO_PREFIX): continue name: str = macro_name.replace(MACRO_PREFIX, '') node = self.parse_macro(block, base_node, name) yield node