def _process_sources_for_node( manifest: Manifest, current_project: str, node: ManifestNode ): target_source: Optional[Union[Disabled, ParsedSourceDefinition]] = None for source_name, table_name in node.sources: target_source = manifest.resolve_source( source_name, table_name, current_project, node.package_name, ) if target_source is None or isinstance(target_source, Disabled): # this folows the same pattern as refs node.config.enabled = False invalid_source_fail_unless_test( node, source_name, table_name, disabled=(isinstance(target_source, Disabled)) ) continue target_source_id = target_source.unique_id node.depends_on.nodes.append(target_source_id) manifest.update_node(node)
def _process_refs_for_node( manifest: Manifest, current_project: str, node: ManifestNode ): """Given a manifest and a node in that manifest, process its refs""" for ref in node.refs: target_model: Optional[Union[Disabled, ManifestNode]] = None target_model_name: str target_model_package: Optional[str] = None if len(ref) == 1: target_model_name = ref[0] elif len(ref) == 2: target_model_package, target_model_name = ref else: raise dbt.exceptions.InternalException( f'Refs should always be 1 or 2 arguments - got {len(ref)}' ) target_model = manifest.resolve_ref( target_model_name, target_model_package, current_project, node.package_name, ) if target_model is None or isinstance(target_model, Disabled): # This may raise. Even if it doesn't, we don't want to add # this node to the graph b/c there is no destination node node.config.enabled = False invalid_ref_fail_unless_test( node, target_model_name, target_model_package, disabled=(isinstance(target_model, Disabled)) ) continue target_model_id = target_model.unique_id node.depends_on.nodes.append(target_model_id) # TODO: I think this is extraneous, node should already be the same # as manifest.nodes[node.unique_id] (we're mutating node here, not # making a new one) # Q: could we stop doing this? manifest.update_node(node)
def _recursively_prepend_ctes( self, model: NonSourceCompiledNode, manifest: Manifest, extra_context: Dict[str, Any], ) -> Tuple[NonSourceCompiledNode, List[InjectedCTE]]: if model.extra_ctes_injected: return (model, model.extra_ctes) if flags.STRICT_MODE: if not isinstance(model, tuple(COMPILED_TYPES.values())): raise InternalException(f'Bad model type: {type(model)}') prepended_ctes: List[InjectedCTE] = [] dbt_test_name = self._get_dbt_test_name() for cte in model.extra_ctes: if cte.id == dbt_test_name: sql = cte.sql else: cte_model = self._get_compiled_model( manifest, cte.id, extra_context, ) cte_model, new_prepended_ctes = self._recursively_prepend_ctes( cte_model, manifest, extra_context) _extend_prepended_ctes(prepended_ctes, new_prepended_ctes) new_cte_name = self.add_ephemeral_prefix(cte_model.name) sql = f' {new_cte_name} as (\n{cte_model.compiled_sql}\n)' _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql)) model = self._model_prepend_ctes(model, prepended_ctes) manifest.update_node(model) return model, prepended_ctes
def _recursively_prepend_ctes( self, model: NonSourceCompiledNode, manifest: Manifest, extra_context: Optional[Dict[str, Any]], ) -> Tuple[NonSourceCompiledNode, List[InjectedCTE]]: """This method is called by the 'compile_node' method. Starting from the node that it is passed in, it will recursively call itself using the 'extra_ctes'. The 'ephemeral' models do not produce SQL that is executed directly, instead they are rolled up into the models that refer to them by inserting CTEs into the SQL. """ if model.compiled_sql is None: raise RuntimeException('Cannot inject ctes into an unparsed node', model) if model.extra_ctes_injected: return (model, model.extra_ctes) # Just to make it plain that nothing is actually injected for this case if not model.extra_ctes: model.extra_ctes_injected = True manifest.update_node(model) return (model, model.extra_ctes) # This stores the ctes which will all be recursively # gathered and then "injected" into the model. prepended_ctes: List[InjectedCTE] = [] dbt_test_name = self._get_dbt_test_name() # extra_ctes are added to the model by # RuntimeRefResolver.create_relation, which adds an # extra_cte for every model relation which is an # ephemeral model. for cte in model.extra_ctes: if cte.id == dbt_test_name: sql = cte.sql else: if cte.id not in manifest.nodes: raise InternalException( f'During compilation, found a cte reference that ' f'could not be resolved: {cte.id}') cte_model = manifest.nodes[cte.id] if not cte_model.is_ephemeral_model: raise InternalException(f'{cte.id} is not ephemeral') # This model has already been compiled, so it's been # through here before if getattr(cte_model, 'compiled', False): assert isinstance(cte_model, tuple(COMPILED_TYPES.values())) cte_model = cast(NonSourceCompiledNode, cte_model) new_prepended_ctes = cte_model.extra_ctes # if the cte_model isn't compiled, i.e. first time here else: # This is an ephemeral parsed model that we can compile. # Compile and update the node cte_model = self._compile_node(cte_model, manifest, extra_context) # recursively call this method cte_model, new_prepended_ctes = \ self._recursively_prepend_ctes( cte_model, manifest, extra_context ) # Save compiled SQL file and sync manifest self._write_node(cte_model) manifest.sync_update_node(cte_model) _extend_prepended_ctes(prepended_ctes, new_prepended_ctes) new_cte_name = self.add_ephemeral_prefix(cte_model.name) rendered_sql = (cte_model._pre_injected_sql or cte_model.compiled_sql) sql = f' {new_cte_name} as (\n{rendered_sql}\n)' _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql)) injected_sql = self._inject_ctes_into_sql( model.compiled_sql, prepended_ctes, ) model._pre_injected_sql = model.compiled_sql model.compiled_sql = injected_sql model.extra_ctes_injected = True model.extra_ctes = prepended_ctes model.validate(model.to_dict(omit_none=True)) manifest.update_node(model) return model, prepended_ctes
def _recursively_prepend_ctes( self, model: NonSourceCompiledNode, manifest: Manifest, extra_context: Optional[Dict[str, Any]], ) -> Tuple[NonSourceCompiledNode, List[InjectedCTE]]: if model.compiled_sql is None: raise RuntimeException('Cannot inject ctes into an unparsed node', model) if model.extra_ctes_injected: return (model, model.extra_ctes) # Just to make it plain that nothing is actually injected for this case if not model.extra_ctes: model.extra_ctes_injected = True manifest.update_node(model) return (model, model.extra_ctes) # This stores the ctes which will all be recursively # gathered and then "injected" into the model. prepended_ctes: List[InjectedCTE] = [] # extra_ctes are added to the model by # RuntimeRefResolver.create_relation, which adds an # extra_cte for every model relation which is an # ephemeral model. for cte in model.extra_ctes: if cte.id not in manifest.nodes: raise InternalException( f'During compilation, found a cte reference that ' f'could not be resolved: {cte.id}') cte_model = manifest.nodes[cte.id] if not cte_model.is_ephemeral_model: raise InternalException(f'{cte.id} is not ephemeral') # This model has already been compiled, so it's been # through here before if getattr(cte_model, 'compiled', False): assert isinstance(cte_model, tuple(COMPILED_TYPES.values())) cte_model = cast(NonSourceCompiledNode, cte_model) new_prepended_ctes = cte_model.extra_ctes # if the cte_model isn't compiled, i.e. first time here else: # This is an ephemeral parsed model that we can compile. # Compile and update the node cte_model = self._compile_node(cte_model, manifest, extra_context) # recursively call this method cte_model, new_prepended_ctes = \ self._recursively_prepend_ctes( cte_model, manifest, extra_context ) # Save compiled SQL file and sync manifest self._write_node(cte_model) manifest.sync_update_node(cte_model) _extend_prepended_ctes(prepended_ctes, new_prepended_ctes) new_cte_name = self.add_ephemeral_prefix(cte_model.name) sql = f' {new_cte_name} as (\n{cte_model.compiled_sql}\n)' _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql)) # We don't save injected_sql into compiled sql for ephemeral models # because it will cause problems with processing of subsequent models. # Ephemeral models do not produce executable SQL of their own. if not model.is_ephemeral_model: injected_sql = self._inject_ctes_into_sql( model.compiled_sql, prepended_ctes, ) model.compiled_sql = injected_sql model.extra_ctes_injected = True model.extra_ctes = prepended_ctes model.validate(model.to_dict(omit_none=True)) manifest.update_node(model) return model, prepended_ctes