def merge_from_artifact(
self,
adapter,
other: 'WritableManifest',
selected: AbstractSet[UniqueID],
) -> None:
"""Given the selected unique IDs and a writable manifest, update this
manifest by replacing any unselected nodes with their counterpart.
Only non-ephemeral refable nodes are examined.
"""
refables = set(NodeType.refable())
merged = set()
for unique_id, node in other.nodes.items():
current = self.nodes.get(unique_id)
if current and (
node.resource_type in refables and not node.is_ephemeral
and unique_id not in selected and not adapter.get_relation(
current.database, current.schema, current.identifier)):
merged.add(unique_id)
self.nodes[unique_id] = node.replace(deferred=True)
# log up to 5 items
sample = list(islice(merged, 5))
logger.debug(
f'Merged {len(merged)} items from state (sample: {sample})')
def _check_resource_uniqueness(manifest):
names_resources = {}
alias_resources = {}
for resource, node in manifest.nodes.items():
if node.resource_type not in NodeType.refable():
continue
name = node.name
alias = "{}.{}".format(node.schema, node.alias)
existing_node = names_resources.get(name)
if existing_node is not None:
dbt.exceptions.raise_duplicate_resource_name(
existing_node, node
)
existing_alias = alias_resources.get(alias)
if existing_alias is not None:
dbt.exceptions.raise_ambiguous_alias(
existing_alias, node
)
names_resources[name] = node
alias_resources[alias] = node
def _check_resource_uniqueness(
manifest: Manifest,
config: RuntimeConfig,
) -> None:
names_resources: Dict[str, ManifestNode] = {}
alias_resources: Dict[str, ManifestNode] = {}
for resource, node in manifest.nodes.items():
if node.resource_type not in NodeType.refable():
continue
# appease mypy - sources aren't refable!
assert not isinstance(node, ParsedSourceDefinition)
name = node.name
# the full node name is really defined by the adapter's relation
relation_cls = get_relation_class_by_name(config.credentials.type)
relation = relation_cls.create_from(config=config, node=node)
full_node_name = str(relation)
existing_node = names_resources.get(name)
if existing_node is not None:
dbt.exceptions.raise_duplicate_resource_name(existing_node, node)
existing_alias = alias_resources.get(full_node_name)
if existing_alias is not None:
dbt.exceptions.raise_ambiguous_alias(existing_alias, node,
full_node_name)
names_resources[name] = node
alias_resources[full_node_name] = node
def _get_relation_name(self, node: ParsedNode):
relation_name = None
if (node.resource_type in NodeType.refable()
and not node.is_ephemeral_model):
adapter = get_adapter(self.config)
relation_cls = adapter.Relation
relation_name = str(relation_cls.create_from(self.config, node))
return relation_name
def find_disabled_by_name(
self,
name: str,
package: Optional[str] = None) -> Optional[ManifestNode]:
searcher: NameSearcher = NameSearcher(name, package,
NodeType.refable())
result = searcher.search(self.disabled)
return result
def add_nodes(self, new_nodes: Mapping[str, ManifestNode]):
"""Add the given dict of new nodes to the manifest."""
for unique_id, node in new_nodes.items():
if unique_id in self.nodes:
raise_duplicate_resource_name(node, self.nodes[unique_id])
self.nodes[unique_id] = node
# fixup the cache if it exists.
if self._refs_cache is not None:
if node.resource_type in NodeType.refable():
self._refs_cache.add_node(node)
def raise_duplicate_resource_name(node_1, node_2):
duped_name = node_1.name
if node_1.resource_type in NodeType.refable():
get_func = 'ref("{}")'.format(duped_name)
elif node_1.resource_type == NodeType.Source:
duped_name = node_1.get_full_source_name()
get_func = node_1.get_source_representation()
elif node_1.resource_type == NodeType.Documentation:
get_func = 'doc("{}")'.format(duped_name)
elif node_1.resource_type == NodeType.Test and 'schema' in node_1.tags:
return
else:
get_func = '"{}"'.format(duped_name)
raise_compiler_error(
'dbt found two resources with the name "{}". Since these resources '
'have the same name,\ndbt will be unable to find the correct resource '
'when {} is used. To fix this,\nchange the name of one of '
'these resources:\n- {} ({})\n- {} ({})'.format(
duped_name, get_func, node_1.unique_id, node_1.original_file_path,
node_2.unique_id, node_2.original_file_path))
def _check_resource_uniqueness(manifest):
names_resources = {}
alias_resources = {}
for resource, node in manifest.nodes.items():
if node.resource_type not in NodeType.refable():
continue
name = node.name
alias = "{}.{}".format(node.schema, node.alias)
existing_node = names_resources.get(name)
if existing_node is not None:
dbt.exceptions.raise_duplicate_resource_name(
existing_node, node)
existing_alias = alias_resources.get(alias)
if existing_alias is not None:
dbt.exceptions.raise_ambiguous_alias(
existing_alias, node)
names_resources[name] = node
alias_resources[alias] = node
def is_refable(self):
return self.resource_type in NodeType.refable()
def __init__(self, manifest: 'Manifest'):
self._cached_types = set(NodeType.refable())
super().__init__(manifest)
def find_refable_by_name(self, name, package):
"""Find any valid target for "ref()" in the graph by its name and
package name, or None for any package.
"""
return self._find_by_name(name, package, 'nodes', NodeType.refable())
def is_refable(cls, node):
return node.get('resource_type') in NodeType.refable()
def run(self):
# Look up all of the relations in the DB
adapter = dbt.adapters.factory.get_adapter(self.config)
manifest = self._get_manifest()
schemas = set()
model_relations = set()
# Look up all of the relations dbt knows about
for node in manifest.nodes.values():
if node["resource_type"] != "source":
schema_info = (node["database"], node["schema"])
schemas.update([schema_info])
node = node.to_dict()
is_refable = (node["resource_type"] in NodeType.refable()
or node["resource_type"] == "archive")
is_enabled = check_is_enabled(node)
is_ephemeral = node["config"]["materialized"] == "ephemeral"
if is_refable and is_enabled and not is_ephemeral:
rel = (node["schema"].lower(), node["alias"].lower())
model_relations.add(rel)
db_relations = []
for database_name, schema_name in schemas:
db_relations.extend(
adapter.list_relations(database_name, schema_name))
database_relations = set()
database_relations_map = dict()
for relation in db_relations:
relation_id = (relation.schema.lower(),
relation.identifier.lower())
database_relations_map[relation_id] = relation
database_relations.add(relation_id)
logger.info("Comparing local models to the database catalog. "
"Checking schemas:")
for database_name, schema_name in schemas:
logger.info("- {}".format(schema_name))
problems = database_relations - model_relations
if len(problems) == 0:
logger.info(
dbt.ui.printer.green(
"All clear! There are no relations in the checked schemas in the database"
"that are not defined in dbt models."))
else:
logger.info(
dbt.ui.printer.yellow(
"Warning: The following relations do not match any models "
"found in this project:"))
problem_relation_list = [] # Get a list of relations to return
for relation_id in problems:
relation = database_relations_map[relation_id]
problem_relation_list.append(relation)
logger.info("{} {}".format(relation.type.upper(), relation))
# TODO: Fix this so that it doesn't break when type is None
# logger.info("{} {}".format(relation.type, relation))
return problem_relation_list
def find_refable_by_name(flat_graph, target_name, target_package):
return find_by_name(flat_graph, target_name, target_package, 'nodes',
NodeType.refable())
def find_disabled_by_name(self, name, package=None):
return dbt.utils.find_in_list_by_name(self.disabled, name, package,
NodeType.refable())
def find_refable_by_name(self, name, package):
"""Find any valid target for "ref()" in the graph by its name and
package name, or None for any package.
"""
return self._find_by_name(name, package, 'nodes', NodeType.refable())
def find_disabled_by_name(self, name, package=None):
return dbt.utils.find_in_list_by_name(self.disabled, name, package,
NodeType.refable())