def _process_docs_for_source( context: Dict[str, Any], source: ParsedSourceDefinition, ): table_description = source.description source_description = source.source_description table_description = get_rendered(table_description, context) source_description = get_rendered(source_description, context) source.description = table_description source.source_description = source_description for column in source.columns.values(): column_desc = column.description column_desc = get_rendered(column_desc, context) column.description = column_desc
def test__parse_basic_source(self): block = self.file_block_for(SINGLE_TABLE_SOURCE, 'test_one.yml') self.parser.parse_file(block) # self.parser.parse_yaml_sources(yaml_block=block) self.assert_has_results_length(self.parser.results, sources=1) src = list(self.parser.results.sources.values())[0] expected = ParsedSourceDefinition( package_name='snowplow', source_name='my_source', schema='my_source', name='my_table', loader='', freshness=FreshnessThreshold(), external=ExternalTable(), source_description='', identifier='my_table', fqn=['snowplow', 'my_source', 'my_table'], database='test', unique_id='source.snowplow.my_source.my_table', root_path=get_abs_os_path('./dbt_modules/snowplow'), path=normalize('models/test_one.yml'), original_file_path=normalize('models/test_one.yml'), resource_type=NodeType.Source, ) self.assertEqual(src, expected)
def parse_source( self, target: UnpatchedSourceDefinition) -> ParsedSourceDefinition: source = target.source table = target.table refs = ParserRef.from_target(table) unique_id = target.unique_id description = table.description or '' meta = table.meta or {} source_description = source.description or '' loaded_at_field = table.loaded_at_field or source.loaded_at_field freshness = merge_freshness(source.freshness, table.freshness) quoting = source.quoting.merged(table.quoting) # path = block.path.original_file_path source_meta = source.meta or {} # make sure we don't do duplicate tags from source + table tags = sorted(set(itertools.chain(source.tags, table.tags))) config = self.config_generator.calculate_node_config( config_calls=[], fqn=target.fqn, resource_type=NodeType.Source, project_name=self.project.project_name, base=False, ) if not isinstance(config, SourceConfig): raise InternalException( f'Calculated a {type(config)} for a source, but expected ' f'a SourceConfig') default_database = self.root_project.credentials.database return ParsedSourceDefinition( package_name=target.package_name, database=(source.database or default_database), schema=(source.schema or source.name), identifier=(table.identifier or table.name), root_path=target.root_path, path=target.path, original_file_path=target.original_file_path, columns=refs.column_info, unique_id=unique_id, name=table.name, description=description, external=table.external, source_name=source.name, source_description=source_description, source_meta=source_meta, meta=meta, loader=source.loader, loaded_at_field=loaded_at_field, freshness=freshness, quoting=quoting, resource_type=NodeType.Source, fqn=target.fqn, tags=tags, config=config, )
def generate_source_node(self, source, table, path, package_name, root_dir, refs): unique_id = self.get_path(NodeType.Source, package_name, source.name, table.name) context = {'doc': dbt.context.parser.docs(source, refs.docrefs)} description = table.get('description', '') source_description = source.get('description', '') get_rendered(description, context) get_rendered(source_description, context) freshness = dbt.utils.deep_merge(source.get('freshness', {}), table.get('freshness', {})) loaded_at_field = table.get('loaded_at_field', source.get('loaded_at_field')) # use 'or {}' to allow quoting: null source_quoting = source.get('quoting') or {} table_quoting = table.get('quoting') or {} quoting = dbt.utils.deep_merge(source_quoting, table_quoting) default_database = self.root_project_config.credentials.database return ParsedSourceDefinition( package_name=package_name, database=source.get('database', default_database), schema=source.get('schema', source.name), identifier=table.get('identifier', table.name), root_path=root_dir, path=path, original_file_path=path, columns=refs.column_info, unique_id=unique_id, name=table.name, description=description, source_name=source.name, source_description=source_description, loader=source.get('loader', ''), docrefs=refs.docrefs, loaded_at_field=loaded_at_field, freshness=freshness, quoting=quoting, resource_type=NodeType.Source, fqn=[package_name, source.name, table.name])
def generate_source_node(self, block: TargetBlock, refs: ParserRef) -> ParsedSourceDefinition: assert isinstance(block.target, SourceTarget) source = block.target.source table = block.target.table unique_id = '.'.join([ NodeType.Source, self.project.project_name, source.name, table.name ]) description = table.description or '' source_description = source.description or '' collect_docrefs(source, refs, None, description, source_description) loaded_at_field = table.loaded_at_field or source.loaded_at_field freshness = self._calculate_freshness(source, table) quoting = source.quoting.merged(table.quoting) path = block.path.original_file_path return ParsedSourceDefinition( package_name=self.project.project_name, database=(source.database or self.default_database), schema=(source.schema or source.name), identifier=(table.identifier or table.name), root_path=self.project.project_root, path=path, original_file_path=path, columns=refs.column_info, unique_id=unique_id, name=table.name, description=description, external=table.external, source_name=source.name, source_description=source_description, loader=source.loader, docrefs=refs.docrefs, loaded_at_field=loaded_at_field, freshness=freshness, quoting=quoting, resource_type=NodeType.Source, fqn=[self.project.project_name, source.name, table.name], )
def make_source(pkg, source_name, table_name, path=None, loader=None, identifier=None, fqn_extras=None): if path is None: path = 'models/schema.yml' if loader is None: loader = 'my_loader' if identifier is None: identifier = table_name if fqn_extras is None: fqn_extras = [] fqn = [pkg] + fqn_extras + [source_name, table_name] return ParsedSourceDefinition( fqn=fqn, database='dbt', schema='dbt_schema', unique_id=f'source.{pkg}.{source_name}.{table_name}', package_name=pkg, root_path='/usr/dbt/some-project', path=path, original_file_path=path, name=table_name, source_name=source_name, loader='my_loader', identifier=identifier, resource_type=NodeType.Source, loaded_at_field='loaded_at', tags=[], source_description='', )
def parse_manifest( manifest: dict[str, Any] ) -> tuple[dict[str, ParsedModelNode | CompiledModelNode] | None, dict[str, ParsedSeedNode | CompiledSeedNode] | None, dict[str, ParsedGenericTestNode | CompiledGenericTestNode] | None, dict[str, ParsedSourceDefinition] | None, ]: """ Parse the manifest. Only V4 manifest is supported. Parameters ---------- manifest : dict[str, Any] The raw manifest. Returns ------- dbt manifest parsed into a tuple of dicts containing: models_nodes, seed_nodes, test_nodes and source_nodes. Any of them may be `None` if they were not in the manifest. Raises ------ NotImplementedError : If the dbt schema is not equal to the V4 manifest Source ------ https://docs.getdbt.com/reference/artifacts/manifest-json """ if manifest.get('nodes') is not None: model_nodes = { node_name: CompiledModelNode(**node) if "compiled" in node.keys() else ParsedModelNode(**node) for node_name, node in manifest["nodes"].items() if node["resource_type"] == NodeType.Model } seed_nodes = { node_name: CompiledSeedNode(**node) if "compiled" in node.keys() else ParsedSeedNode(**node) for node_name, node in manifest["nodes"].items() if node["resource_type"] == NodeType.Seed } test_nodes = {} for node_name, node in manifest["nodes"].items(): if node["resource_type"] == NodeType.Test: if "test_metadata" in node.keys(): if "compiled" in node.keys(): node = CompiledGenericTestNode(**node) else: node = ParsedGenericTestNode(**node) test_nodes[node_name] = node else: logger.info(f"Ignoring unsupported {node_name}") else: model_nodes = None seed_nodes = None test_nodes = None if manifest.get('sources') is not None: source_nodes: Optional[dict] = { source_name: ParsedSourceDefinition(**source) for source_name, source in manifest["sources"].items() if source['resource_type'] == NodeType.Source } else: source_nodes = None return model_nodes, seed_nodes, test_nodes, source_nodes