def test_get_resource_fqns(self):
nodes = copy.copy(self.nested_nodes)
nodes['seed.root.seed'] = ParsedSeedNode(
name='seed',
database='dbt',
schema='analytics',
alias='seed',
resource_type='seed',
unique_id='seed.root.seed',
fqn=['root', 'seed'],
package_name='root',
refs=[['events']],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='seed.csv',
original_file_path='seed.csv',
root_path='',
raw_sql='-- csv --',
seed_file_path='data/seed.csv')
manifest = Manifest(nodes=nodes,
macros={},
docs={},
generated_at=datetime.utcnow(),
disabled=[],
files={})
expect = {
'models':
frozenset([
('snowplow', 'events'),
('root', 'events'),
('root', 'dep'),
('root', 'nested'),
('root', 'sibling'),
('root', 'multi'),
]),
'seeds':
frozenset([('root', 'seed')]),
}
resource_fqns = manifest.get_resource_fqns()
self.assertEqual(resource_fqns, expect)
def make_seed(pkg,
name,
path=None,
loader=None,
alias=None,
tags=None,
fqn_extras=None,
checksum=None):
if alias is None:
alias = name
if tags is None:
tags = []
if path is None:
path = f'{name}.csv'
if fqn_extras is None:
fqn_extras = []
if checksum is None:
checksum = FileHash.from_contents('')
fqn = [pkg] + fqn_extras + [name]
return ParsedSeedNode(
raw_sql='',
database='dbt',
schema='dbt_schema',
alias=alias,
name=name,
fqn=fqn,
unique_id=f'seed.{pkg}.{name}',
package_name=pkg,
root_path='/usr/dbt/some-project',
path=path,
original_file_path=f'data/{path}',
tags=tags,
resource_type=NodeType.Seed,
checksum=FileHash.from_contents(''),
)
def parse_from_dict(self, dct, validate=True) -> ParsedSeedNode:
if validate:
ParsedSeedNode.validate(dct)
return ParsedSeedNode.from_dict(dct)
def parse_from_dict(self, dct, validate=True) -> ParsedSeedNode:
return ParsedSeedNode.from_dict(dct, validate=validate)
def parse_manifest(
manifest: dict[str, Any]
) -> tuple[dict[str, ParsedModelNode | CompiledModelNode] | None,
dict[str, ParsedSeedNode | CompiledSeedNode] | None,
dict[str, ParsedGenericTestNode | CompiledGenericTestNode] | None,
dict[str, ParsedSourceDefinition] | None, ]:
"""
Parse the manifest.
Only V4 manifest is supported.
Parameters
----------
manifest : dict[str, Any]
The raw manifest.
Returns
-------
dbt manifest parsed into a tuple of dicts containing: models_nodes, seed_nodes, test_nodes and source_nodes.
Any of them may be `None` if they were not in the manifest.
Raises
------
NotImplementedError :
If the dbt schema is not equal to the V4 manifest
Source
------
https://docs.getdbt.com/reference/artifacts/manifest-json
"""
if manifest.get('nodes') is not None:
model_nodes = {
node_name: CompiledModelNode(**node)
if "compiled" in node.keys() else ParsedModelNode(**node)
for node_name, node in manifest["nodes"].items()
if node["resource_type"] == NodeType.Model
}
seed_nodes = {
node_name: CompiledSeedNode(**node)
if "compiled" in node.keys() else ParsedSeedNode(**node)
for node_name, node in manifest["nodes"].items()
if node["resource_type"] == NodeType.Seed
}
test_nodes = {}
for node_name, node in manifest["nodes"].items():
if node["resource_type"] == NodeType.Test:
if "test_metadata" in node.keys():
if "compiled" in node.keys():
node = CompiledGenericTestNode(**node)
else:
node = ParsedGenericTestNode(**node)
test_nodes[node_name] = node
else:
logger.info(f"Ignoring unsupported {node_name}")
else:
model_nodes = None
seed_nodes = None
test_nodes = None
if manifest.get('sources') is not None:
source_nodes: Optional[dict] = {
source_name: ParsedSourceDefinition(**source)
for source_name, source in manifest["sources"].items()
if source['resource_type'] == NodeType.Source
}
else:
source_nodes = None
return model_nodes, seed_nodes, test_nodes, source_nodes