def _pluralize(string: Union[str, NodeType]) -> str: try: convert = NodeType(string) except ValueError: return f'{string}s' else: return convert.pluralize()
def merge_from_artifact( self, adapter, other: 'WritableManifest', selected: AbstractSet[UniqueID], ) -> None: """Given the selected unique IDs and a writable manifest, update this manifest by replacing any unselected nodes with their counterpart. Only non-ephemeral refable nodes are examined. """ refables = set(NodeType.refable()) merged = set() for unique_id, node in other.nodes.items(): current = self.nodes.get(unique_id) if current and ( node.resource_type in refables and not node.is_ephemeral and unique_id not in selected and not adapter.get_relation( current.database, current.schema, current.identifier)): merged.add(unique_id) self.nodes[unique_id] = node.replace(deferred=True) # log up to 5 items sample = list(islice(merged, 5)) logger.debug( f'Merged {len(merged)} items from state (sample: {sample})')
def _check_resource_uniqueness(manifest): names_resources = {} alias_resources = {} for resource, node in manifest.nodes.items(): if node.resource_type not in NodeType.refable(): continue name = node.name alias = "{}.{}".format(node.schema, node.alias) existing_node = names_resources.get(name) if existing_node is not None: dbt.exceptions.raise_duplicate_resource_name( existing_node, node ) existing_alias = alias_resources.get(alias) if existing_alias is not None: dbt.exceptions.raise_ambiguous_alias( existing_alias, node ) names_resources[name] = node alias_resources[alias] = node
def parse_file(self, block: FileBlock) -> None: dct = self._yaml_from_file(block.file) # mark the file as seen, even if there are no macros in it self.results.get_file(block.file) if dct: try: dct = self.raw_renderer.render_data(dct) except CompilationException as exc: raise CompilationException( f'Failed to render {block.path.original_file_path} from ' f'project {self.project.project_name}: {exc}') from exc yaml_block = YamlBlock.from_file_block(block, dct) self._parse_format_version(yaml_block) parser: YamlDocsReader for key in NodeType.documentable(): plural = key.pluralize() if key == NodeType.Source: parser = SourceParser(self, yaml_block, plural) elif key == NodeType.Macro: parser = MacroPatchParser(self, yaml_block, plural) elif key == NodeType.Analysis: parser = AnalysisPatchParser(self, yaml_block, plural) else: parser = TestablePatchParser(self, yaml_block, plural) for test_block in parser.parse(): self.parse_tests(test_block)
def build_query(self): return { "include": self.args.models, "exclude": self.args.exclude, "resource_types": NodeType.executable(), "tags": [], }
def _check_resource_uniqueness( manifest: Manifest, config: RuntimeConfig, ) -> None: names_resources: Dict[str, ManifestNode] = {} alias_resources: Dict[str, ManifestNode] = {} for resource, node in manifest.nodes.items(): if node.resource_type not in NodeType.refable(): continue # appease mypy - sources aren't refable! assert not isinstance(node, ParsedSourceDefinition) name = node.name # the full node name is really defined by the adapter's relation relation_cls = get_relation_class_by_name(config.credentials.type) relation = relation_cls.create_from(config=config, node=node) full_node_name = str(relation) existing_node = names_resources.get(name) if existing_node is not None: dbt.exceptions.raise_duplicate_resource_name(existing_node, node) existing_alias = alias_resources.get(full_node_name) if existing_alias is not None: dbt.exceptions.raise_ambiguous_alias(existing_alias, node, full_node_name) names_resources[name] = node alias_resources[full_node_name] = node
def _get_relation_name(self, node: ParsedNode): relation_name = None if (node.resource_type in NodeType.refable() and not node.is_ephemeral_model): adapter = get_adapter(self.config) relation_cls = adapter.Relation relation_name = str(relation_cls.create_from(self.config, node)) return relation_name
def find_disabled_by_name( self, name: str, package: Optional[str] = None) -> Optional[ManifestNode]: searcher: NameSearcher = NameSearcher(name, package, NodeType.refable()) result = searcher.search(self.disabled) return result
class SchemaYamlRenderer(BaseRenderer): DOCUMENTABLE_NODES = frozenset(n.pluralize() for n in NodeType.documentable()) @property def name(self): return 'Rendering yaml' def _is_norender_key(self, keypath: Keypath) -> bool: """ models: - name: blah - description: blah tests: ... - columns: - name: - description: blah tests: ... Return True if it's tests or description - those aren't rendered """ if len(keypath) >= 2 and keypath[1] in ('tests', 'description'): return True if (len(keypath) >= 4 and keypath[1] == 'columns' and keypath[3] in ('tests', 'description')): return True return False # don't render descriptions or test keyword arguments def should_render_keypath(self, keypath: Keypath) -> bool: if len(keypath) < 2: return True if keypath[0] not in self.DOCUMENTABLE_NODES: return True if len(keypath) < 3: return True if keypath[0] == NodeType.Source.pluralize(): if keypath[2] == 'description': return False if keypath[2] == 'tables': if self._is_norender_key(keypath[3:]): return False elif keypath[0] == NodeType.Macro.pluralize(): if keypath[2] == 'arguments': if self._is_norender_key(keypath[3:]): return False elif self._is_norender_key(keypath[1:]): return False else: # keypath[0] in self.DOCUMENTABLE_NODES: if self._is_norender_key(keypath[1:]): return False return True
def _get_cache_schemas(self, manifest: Manifest) -> Set[BaseRelation]: """Get the set of schema relations that the cache logic needs to populate. This means only executable nodes are included. """ # the cache only cares about executable nodes return { self.Relation.create_from(self.config, node).without_identifier() for node in manifest.nodes.values() if node.resource_type in NodeType.executable() }
def get_node_selector(self) -> ResourceTypeSelector: if self.manifest is None or self.graph is None: raise InternalException( 'manifest and graph must be set to get perform node selection') return ResourceTypeSelector( graph=self.graph, manifest=self.manifest, previous_state=self.previous_state, resource_types=NodeType.executable(), )
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: try: resource_type = NodeType(selector) except ValueError as exc: raise RuntimeException( f'Invalid resource_type selector "{selector}"') from exc for node, real_node in self.parsed_nodes(included_nodes): if real_node.resource_type == resource_type: yield node
def add_nodes(self, new_nodes: Mapping[str, ManifestNode]): """Add the given dict of new nodes to the manifest.""" for unique_id, node in new_nodes.items(): if unique_id in self.nodes: raise_duplicate_resource_name(node, self.nodes[unique_id]) self.nodes[unique_id] = node # fixup the cache if it exists. if self._refs_cache is not None: if node.resource_type in NodeType.refable(): self._refs_cache.add_node(node)
def run(self): query = { "include": self.args.models, "exclude": self.args.exclude, "resource_types": NodeType.executable(), "tags": [], } results = RunManager(self.config, query, CompileRunner).run() dbt.ui.printer.print_timestamped_line('Done.') return results
def _get_cache_schemas(self, manifest, exec_only=False): info_schema_name_map = SchemaSearchMap() for node in manifest.nodes.values(): if exec_only and node.resource_type not in NodeType.executable(): continue relation = self.Relation.create( database=node.database, schema=node.schema, identifier='information_schema', quote_policy=self.config.quoting, ) key = relation.information_schema_only() info_schema_name_map[key] = {node.schema} return info_schema_name_map
def run(self): runner = RunManager(self.project, self.project['target-path'], self.args) query = { "include": self.args.models, "exclude": self.args.exclude, "resource_types": NodeType.executable(), "tags": set() } results = runner.run(query, CompileRunner) dbt.ui.printer.print_timestamped_line('Done.') return results
def _get_cache_schemas(self, manifest, exec_only=False): """Get a mapping of each node's "information_schema" relations to a set of all schemas expected in that information_schema. There may be keys that are technically duplicates on the database side, for example all of '"foo", 'foo', '"FOO"' and 'FOO' could coexist as databases, and values could overlap as appropriate. All values are lowercase strings. """ info_schema_name_map = SchemaSearchMap() for node in manifest.nodes.values(): if exec_only and node.resource_type not in NodeType.executable(): continue relation = self.Relation.create_from(self.config, node) info_schema_name_map.add(relation) # result is a map whose keys are information_schema Relations without # identifiers that have appropriate database prefixes, and whose values # are sets of lowercase schema names that are valid members of those # schemas return info_schema_name_map
def _get_cache_schemas(self, manifest, exec_only=False): """Get a mapping of each node's "information_schema" relations to a set of all schemas expected in that information_schema. There may be keys that are technically duplicates on the database side, for example all of '"foo", 'foo', '"FOO"' and 'FOO' could coexist as databases, and values could overlap as appropriate. All values are lowercase strings. """ info_schema_name_map = SchemaSearchMap() for node in manifest.nodes.values(): if exec_only and node.resource_type not in NodeType.executable(): continue relation = self.Relation.create_from(self.config, node) info_schema_name_map.add(relation) # result is a map whose keys are information_schema Relations without # identifiers that have appropriate database prefixes, and whose values # are sets of lowercase schema names that are valid members of those # schemas return info_schema_name_map
def raise_duplicate_resource_name(node_1, node_2): duped_name = node_1.name if node_1.resource_type in NodeType.refable(): get_func = 'ref("{}")'.format(duped_name) elif node_1.resource_type == NodeType.Source: duped_name = node_1.get_full_source_name() get_func = node_1.get_source_representation() elif node_1.resource_type == NodeType.Documentation: get_func = 'doc("{}")'.format(duped_name) elif node_1.resource_type == NodeType.Test and 'schema' in node_1.tags: return else: get_func = '"{}"'.format(duped_name) raise_compiler_error( 'dbt found two resources with the name "{}". Since these resources ' 'have the same name,\ndbt will be unable to find the correct resource ' 'when {} is used. To fix this,\nchange the name of one of ' 'these resources:\n- {} ({})\n- {} ({})'.format( duped_name, get_func, node_1.unique_id, node_1.original_file_path, node_2.unique_id, node_2.original_file_path))
def _check_resource_uniqueness(manifest): names_resources = {} alias_resources = {} for resource, node in manifest.nodes.items(): if node.resource_type not in NodeType.refable(): continue name = node.name alias = "{}.{}".format(node.schema, node.alias) existing_node = names_resources.get(name) if existing_node is not None: dbt.exceptions.raise_duplicate_resource_name( existing_node, node) existing_alias = alias_resources.get(alias) if existing_alias is not None: dbt.exceptions.raise_ambiguous_alias( existing_alias, node) names_resources[name] = node alias_resources[alias] = node
def find_refable_by_name(self, name, package): """Find any valid target for "ref()" in the graph by its name and package name, or None for any package. """ return self._find_by_name(name, package, 'nodes', NodeType.refable())
def is_refable(self): return self.resource_type in NodeType.refable()
def run(self): # Look up all of the relations in the DB adapter = dbt.adapters.factory.get_adapter(self.config) manifest = self._get_manifest() schemas = set() model_relations = set() # Look up all of the relations dbt knows about for node in manifest.nodes.values(): if node["resource_type"] != "source": schema_info = (node["database"], node["schema"]) schemas.update([schema_info]) node = node.to_dict() is_refable = (node["resource_type"] in NodeType.refable() or node["resource_type"] == "archive") is_enabled = check_is_enabled(node) is_ephemeral = node["config"]["materialized"] == "ephemeral" if is_refable and is_enabled and not is_ephemeral: rel = (node["schema"].lower(), node["alias"].lower()) model_relations.add(rel) db_relations = [] for database_name, schema_name in schemas: db_relations.extend( adapter.list_relations(database_name, schema_name)) database_relations = set() database_relations_map = dict() for relation in db_relations: relation_id = (relation.schema.lower(), relation.identifier.lower()) database_relations_map[relation_id] = relation database_relations.add(relation_id) logger.info("Comparing local models to the database catalog. " "Checking schemas:") for database_name, schema_name in schemas: logger.info("- {}".format(schema_name)) problems = database_relations - model_relations if len(problems) == 0: logger.info( dbt.ui.printer.green( "All clear! There are no relations in the checked schemas in the database" "that are not defined in dbt models.")) else: logger.info( dbt.ui.printer.yellow( "Warning: The following relations do not match any models " "found in this project:")) problem_relation_list = [] # Get a list of relations to return for relation_id in problems: relation = database_relations_map[relation_id] problem_relation_list.append(relation) logger.info("{} {}".format(relation.type.upper(), relation)) # TODO: Fix this so that it doesn't break when type is None # logger.info("{} {}".format(relation.type, relation)) return problem_relation_list
def is_refable(cls, node): return node.get('resource_type') in NodeType.refable()
def find_refable_by_name(flat_graph, target_name, target_package): return find_by_name(flat_graph, target_name, target_package, 'nodes', NodeType.refable())
def find_disabled_by_name(self, name, package=None): return dbt.utils.find_in_list_by_name(self.disabled, name, package, NodeType.refable())
def find_refable_by_name(self, name, package): """Find any valid target for "ref()" in the graph by its name and package name, or None for any package. """ return self._find_by_name(name, package, 'nodes', NodeType.refable())
def __init__(self, manifest: 'Manifest'): self._cached_types = set(NodeType.refable()) super().__init__(manifest)
def find_disabled_by_name(self, name, package=None): return dbt.utils.find_in_list_by_name(self.disabled, name, package, NodeType.refable())