def setUp(self): self.cache = RelationsCache() inputs = [ ('foo', 'table1'), ('bar', 'table2'), ('foo', 'table3'), ('foo', 'table4'), ('bar', 'table3'), ] self.inputs = [make_relation(s, i) for s, i in inputs] for relation in self.inputs: self.cache.add(relation) # foo.table3 references foo.table1 # (create view table3 as (select * from table1...)) self.cache.add_link(make_relation('foo', 'table1'), make_relation('foo', 'table3')) # bar.table3 references foo.table3 # (create view bar.table5 as (select * from foo.table3...)) self.cache.add_link(make_relation('foo', 'table3'), make_relation('bar', 'table3')) # foo.table2 also references foo.table1 self.cache.add_link(make_relation('foo', 'table1'), make_relation('foo', 'table4'))
def setUp(self): self.cache = RelationsCache() inputs = [ ('dbt', 'foo', 'table1'), ('dbt', 'foo', 'table3'), ('dbt', 'foo', 'table4'), ('dbt', 'bar', 'table2'), ('dbt', 'bar', 'table3'), ('dbt_2', 'foo', 'table1'), ('dbt_2', 'foo', 'table2'), ] self.inputs = [make_relation(d, s, i) for d, s, i in inputs] for relation in self.inputs: self.cache.add(relation) # dbt.foo.table3 references dbt.foo.table1 # (create view dbt.foo.table3 as (select * from dbt.foo.table1...)) self.cache.add_link(make_relation('dbt', 'foo', 'table1'), make_relation('dbt', 'foo', 'table3')) # dbt.bar.table3 references dbt.foo.table3 # (create view dbt.bar.table5 as (select * from dbt.foo.table3...)) self.cache.add_link(make_relation('dbt', 'foo', 'table3'), make_relation('dbt', 'bar', 'table3')) # dbt.foo.table4 also references dbt.foo.table1 self.cache.add_link(make_relation('dbt', 'foo', 'table1'), make_relation('dbt', 'foo', 'table4')) # and dbt_2.foo.table1 references dbt.foo.table1 self.cache.add_link( make_relation('dbt', 'foo', 'table1'), make_relation('dbt_2', 'foo', 'table1'), )
class TestCache(TestCase): def setUp(self): self.cache = RelationsCache() def assert_relations_state(self, database, schema, identifiers): relations = self.cache.get_relations(database, schema) for identifier, expect in identifiers.items(): found = any( (r.identifier == identifier and \ r.schema == schema and \ r.database == database) for r in relations ) msg = '{}.{}.{} was{} found in the cache!'.format( database, schema, identifier, '' if found else ' not') self.assertEqual(expect, found, msg) def assert_relations_exist(self, database, schema, *identifiers): self.assert_relations_state(database, schema, {k: True for k in identifiers}) def assert_relations_do_not_exist(self, database, schema, *identifiers): self.assert_relations_state(database, schema, {k: False for k in identifiers})
def setUp(self): self.cache = RelationsCache() self._sleep = True # add a bunch of cache entries for ident in 'abcdef': self.cache.add(make_relation('dbt', 'schema', ident)) # 'b' references 'a' self.cache.add_link(make_relation('dbt', 'schema', 'a'), make_relation('dbt', 'schema', 'b')) # and 'c' references 'b' self.cache.add_link(make_relation('dbt', 'schema', 'b'), make_relation('dbt', 'schema', 'c')) # and 'd' references 'b' self.cache.add_link(make_relation('dbt', 'schema', 'b'), make_relation('dbt', 'schema', 'd')) # and 'e' references 'a' self.cache.add_link(make_relation('dbt', 'schema', 'a'), make_relation('dbt', 'schema', 'e')) # and 'f' references 'd' self.cache.add_link(make_relation('dbt', 'schema', 'd'), make_relation('dbt', 'schema', 'f'))
def setUp(self): self.cache = RelationsCache() inputs = [ ('dbt', 'foo', 'table1'), ('dbt', 'foo', 'table3'), ('dbt', 'foo', 'table4'), ('dbt', 'bar', 'table2'), ('dbt', 'bar', 'table3'), ('dbt_2', 'foo', 'table1'), ('dbt_2', 'foo', 'table2'), ] self.inputs = [make_relation(d, s, i) for d, s, i in inputs] for relation in self.inputs: self.cache.add(relation) # dbt.foo.table3 references dbt.foo.table1 # (create view dbt.foo.table3 as (select * from dbt.foo.table1...)) self.cache.add_link( make_relation('dbt', 'foo', 'table1'), make_relation('dbt', 'foo', 'table3') ) # dbt.bar.table3 references dbt.foo.table3 # (create view dbt.bar.table5 as (select * from dbt.foo.table3...)) self.cache.add_link( make_relation('dbt', 'foo', 'table3'), make_relation('dbt', 'bar', 'table3') ) # dbt.foo.table4 also references dbt.foo.table1 self.cache.add_link( make_relation('dbt', 'foo', 'table1'), make_relation('dbt', 'foo', 'table4') ) # and dbt_2.foo.table1 references dbt.foo.table1 self.cache.add_link( make_relation('dbt', 'foo', 'table1'), make_relation('dbt_2', 'foo', 'table1'), )
def setUp(self): self.cache = RelationsCache() self._sleep = True # add a bunch of cache entries for ident in 'abcdef': self.cache.add(make_relation('dbt', 'schema', ident)) # 'b' references 'a' self.cache.add_link(make_relation('dbt', 'schema', 'a'), make_relation('dbt', 'schema', 'b')) # and 'c' references 'b' self.cache.add_link(make_relation('dbt', 'schema', 'b'), make_relation('dbt', 'schema', 'c')) # and 'd' references 'b' self.cache.add_link(make_relation('dbt', 'schema', 'b'), make_relation('dbt', 'schema', 'd')) # and 'e' references 'a' self.cache.add_link(make_relation('dbt', 'schema', 'a'), make_relation('dbt', 'schema', 'e')) # and 'f' references 'd' self.cache.add_link(make_relation('dbt', 'schema', 'd'), make_relation('dbt', 'schema', 'f'))
class TestCache(TestCase): def setUp(self): self.cache = RelationsCache() def assert_relations_state(self, database, schema, identifiers): relations = self.cache.get_relations(database, schema) for identifier, expect in identifiers.items(): found = any( (r.identifier == identifier and \ r.schema == schema and \ r.database == database) for r in relations ) msg = '{}.{}.{} was{} found in the cache!'.format( database, schema, identifier, '' if found else ' not' ) self.assertEqual(expect, found, msg) def assert_relations_exist(self, database, schema, *identifiers): self.assert_relations_state(database, schema, {k: True for k in identifiers}) def assert_relations_do_not_exist(self, database, schema, *identifiers): self.assert_relations_state(database, schema, {k: False for k in identifiers})
class TestComplexCache(TestCase): def setUp(self): self.cache = RelationsCache() inputs = [ ('dbt', 'foo', 'table1'), ('dbt', 'foo', 'table3'), ('dbt', 'foo', 'table4'), ('dbt', 'bar', 'table2'), ('dbt', 'bar', 'table3'), ('dbt_2', 'foo', 'table1'), ('dbt_2', 'foo', 'table2'), ] self.inputs = [make_relation(d, s, i) for d, s, i in inputs] for relation in self.inputs: self.cache.add(relation) # dbt.foo.table3 references dbt.foo.table1 # (create view dbt.foo.table3 as (select * from dbt.foo.table1...)) self.cache.add_link(make_relation('dbt', 'foo', 'table1'), make_relation('dbt', 'foo', 'table3')) # dbt.bar.table3 references dbt.foo.table3 # (create view dbt.bar.table5 as (select * from dbt.foo.table3...)) self.cache.add_link(make_relation('dbt', 'foo', 'table3'), make_relation('dbt', 'bar', 'table3')) # dbt.foo.table4 also references dbt.foo.table1 self.cache.add_link(make_relation('dbt', 'foo', 'table1'), make_relation('dbt', 'foo', 'table4')) # and dbt_2.foo.table1 references dbt.foo.table1 self.cache.add_link( make_relation('dbt', 'foo', 'table1'), make_relation('dbt_2', 'foo', 'table1'), ) def test_get_relations(self): self.assertEqual(len(self.cache.get_relations('dbt', 'foo')), 3) self.assertEqual(len(self.cache.get_relations('dbt', 'bar')), 2) self.assertEqual(len(self.cache.get_relations('dbt_2', 'foo')), 2) self.assertEqual(len(self.cache.relations), 7) def test_drop_one(self): # dropping dbt.bar.table2 should only drop itself self.cache.drop(make_relation('dbt', 'bar', 'table2')) self.assertEqual(len(self.cache.get_relations('dbt', 'foo')), 3) self.assertEqual(len(self.cache.get_relations('dbt', 'bar')), 1) self.assertEqual(len(self.cache.get_relations('dbt_2', 'foo')), 2) self.assertEqual(len(self.cache.relations), 6) def test_drop_many(self): # dropping dbt.foo.table1 should drop everything but dbt.bar.table2 and # dbt_2.foo.table2 self.cache.drop(make_relation('dbt', 'foo', 'table1')) self.assertEqual(len(self.cache.get_relations('dbt', 'foo')), 0) self.assertEqual(len(self.cache.get_relations('dbt', 'bar')), 1) self.assertEqual(len(self.cache.get_relations('dbt_2', 'foo')), 1) self.assertEqual(len(self.cache.relations), 2) def test_rename_root(self): self.cache.rename(make_relation('dbt', 'foo', 'table1'), make_relation('dbt', 'bar', 'table1')) retrieved = self.cache.relations[('dbt', 'bar', 'table1')].inner self.assertEqual(retrieved.schema, 'bar') self.assertEqual(retrieved.identifier, 'table1') self.assertEqual(len(self.cache.get_relations('dbt', 'foo')), 2) self.assertEqual(len(self.cache.get_relations('dbt', 'bar')), 3) self.assertEqual(len(self.cache.get_relations('dbt_2', 'foo')), 2) self.assertEqual(len(self.cache.relations), 7) # make sure drops still cascade from the renamed table self.cache.drop(make_relation('dbt', 'bar', 'table1')) self.assertEqual(len(self.cache.get_relations('dbt', 'foo')), 0) self.assertEqual(len(self.cache.get_relations('dbt', 'bar')), 1) self.assertEqual(len(self.cache.get_relations('dbt_2', 'foo')), 1) self.assertEqual(len(self.cache.relations), 2) def test_rename_branch(self): self.cache.rename(make_relation('dbt', 'foo', 'table3'), make_relation('dbt', 'foo', 'table2')) self.assertEqual(len(self.cache.get_relations('dbt', 'foo')), 3) self.assertEqual(len(self.cache.get_relations('dbt', 'bar')), 2) self.assertEqual(len(self.cache.get_relations('dbt_2', 'foo')), 2) # make sure drops still cascade through the renamed table self.cache.drop(make_relation('dbt', 'foo', 'table1')) self.assertEqual(len(self.cache.get_relations('dbt', 'foo')), 0) self.assertEqual(len(self.cache.get_relations('dbt', 'bar')), 1) self.assertEqual(len(self.cache.get_relations('dbt_2', 'foo')), 1) self.assertEqual(len(self.cache.relations), 2)
class BaseAdapter(object): """The BaseAdapter provides an abstract base class for adapters. Adapters must implement the following methods and macros. Some of the methods can be safely overridden as a noop, where it makes sense (transactions on databases that don't support them, for instance). Those methods are marked with a (passable) in their docstrings. Check docstrings for type information, etc. To implement a macro, implement "${adapter_type}__${macro_name}". in the adapter's internal project. Methods: - exception_handler - date_function - list_schemas - drop_relation - truncate_relation - rename_relation - get_columns_in_relation - expand_column_types - list_relations_without_caching - is_cancelable - create_schema - drop_schema - quote - convert_text_type - convert_number_type - convert_boolean_type - convert_datetime_type - convert_date_type - convert_time_type Macros: - get_catalog """ requires = {} Relation = BaseRelation Column = Column # This should be an implementation of BaseConnectionManager ConnectionManager = None # A set of clobber config fields accepted by this adapter # for use in materializations AdapterSpecificConfigs = frozenset() def __init__(self, config): self.config = config self.cache = RelationsCache() self.connections = self.ConnectionManager(config) self._internal_manifest_lazy = None ### # Methods that pass through to the connection manager ### def acquire_connection(self, name): return self.connections.get(name) def release_connection(self, name): return self.connections.release(name) def cleanup_connections(self): return self.connections.cleanup_all() def clear_transaction(self, conn_name='master'): return self.connections.clear_transaction(conn_name) def commit_if_has_connection(self, name): return self.connections.commit_if_has_connection(name) @available def execute(self, sql, model_name=None, auto_begin=False, fetch=False): """Execute the given SQL. This is a thin wrapper around ConnectionManager.execute. :param str sql: The sql to execute. :param Optional[str] model_name: The model name to use for the connection. :param bool auto_begin: If set, and dbt is not currently inside a transaction, automatically begin one. :param bool fetch: If set, fetch results. :return: A tuple of the status and the results (empty if fetch=False). :rtype: Tuple[str, agate.Table] """ return self.connections.execute( sql=sql, name=model_name, auto_begin=auto_begin, fetch=fetch ) ### # Methods that should never be overridden ### @classmethod def type(cls): """Get the type of this adapter. Types must be class-unique and consistent. :return: The type name :rtype: str """ return cls.ConnectionManager.TYPE @property def _internal_manifest(self): if self._internal_manifest_lazy is None: manifest = GraphLoader.load_internal(self.config) self._internal_manifest_lazy = manifest return self._internal_manifest_lazy def check_internal_manifest(self): """Return the internal manifest (used for executing macros) if it's been initialized, otherwise return None. """ return self._internal_manifest_lazy ### # Caching methods ### def _schema_is_cached(self, database, schema, model_name=None): """Check if the schema is cached, and by default logs if it is not.""" if dbt.flags.USE_CACHE is False: return False elif (database, schema) not in self.cache: logger.debug( 'On "{}": cache miss for schema "{}.{}", this is inefficient' .format(model_name or '<None>', database, schema) ) return False else: return True @classmethod def _relations_filter_table(cls, table, schemas): """Filter the table as appropriate for relations table entries. Subclasses can override this to change filtering rules on a per-adapter basis. """ return table.where(_relations_filter_schemas(schemas)) def _get_cache_schemas(self, manifest, exec_only=False): """Get a mapping of each node's "information_schema" relations to a set of all schemas expected in that information_schema. There may be keys that are technically duplicates on the database side, for example all of '"foo", 'foo', '"FOO"' and 'FOO' could coexist as databases, and values could overlap as appropriate. All values are lowercase strings. """ info_schema_name_map = SchemaSearchMap() for node in manifest.nodes.values(): if exec_only and node.resource_type not in NodeType.executable(): continue relation = self.Relation.create_from(self.config, node) info_schema_name_map.add(relation) # result is a map whose keys are information_schema Relations without # identifiers that have appropriate database prefixes, and whose values # are sets of lowercase schema names that are valid members of those # schemas return info_schema_name_map def _relations_cache_for_schemas(self, manifest): """Populate the relations cache for the given schemas. Returns an iteratble of the schemas populated, as strings. """ if not dbt.flags.USE_CACHE: return info_schema_name_map = self._get_cache_schemas(manifest, exec_only=True) for db, schema in info_schema_name_map.search(): for relation in self.list_relations_without_caching(db, schema): self.cache.add(relation) # it's possible that there were no relations in some schemas. We want # to insert the schemas we query into the cache's `.schemas` attribute # so we can check it later self.cache.update_schemas(info_schema_name_map.schemas_searched()) def set_relations_cache(self, manifest, clear=False): """Run a query that gets a populated cache of the relations in the database and set the cache on this adapter. """ if not dbt.flags.USE_CACHE: return with self.cache.lock: if clear: self.cache.clear() self._relations_cache_for_schemas(manifest) def cache_new_relation(self, relation, model_name=None): """Cache a new relation in dbt. It will show up in `list relations`.""" if relation is None: dbt.exceptions.raise_compiler_error( 'Attempted to cache a null relation for {}'.format(model_name) ) if dbt.flags.USE_CACHE: self.cache.add(relation) # so jinja doesn't render things return '' ### # Abstract methods for database-specific values, attributes, and types ### @abstractclassmethod def date_function(cls): """Get the date function used by this adapter's database. :return: The date function :rtype: str """ raise dbt.exceptions.NotImplementedException( '`date_function` is not implemented for this adapter!') @abstractclassmethod def is_cancelable(cls): raise dbt.exceptions.NotImplementedException( '`is_cancelable` is not implemented for this adapter!' ) ### # Abstract methods about schemas ### @abc.abstractmethod def list_schemas(self, database, model_name=None): """Get a list of existing schemas. :param str database: The name of the database to list under. :param Optional[str] model_name: The name of the connection to query as :return: All schemas that currently exist in the database :rtype: List[str] """ raise dbt.exceptions.NotImplementedException( '`list_schemas` is not implemented for this adapter!' ) def check_schema_exists(self, database, schema, model_name=None): """Check if a schema exists. The default implementation of this is potentially unnecessarily slow, and adapters should implement it if there is an optimized path (and there probably is) """ search = ( s.lower() for s in self.list_schemas(database=database, model_name=model_name) ) return schema.lower() in search ### # Abstract methods about relations ### @abc.abstractmethod @available def drop_relation(self, relation, model_name=None): """Drop the given relation. *Implementors must call self.cache.drop() to preserve cache state!* :param self.Relation relation: The relation to drop :param Optional[str] model_name: The name of the model to use for the connection. """ raise dbt.exceptions.NotImplementedException( '`drop_relation` is not implemented for this adapter!' ) @abc.abstractmethod @available def truncate_relation(self, relation, model_name=None): """Truncate the given relation. :param self.Relation relation: The relation to truncate :param Optional[str] model_name: The name of the model to use for the connection.""" raise dbt.exceptions.NotImplementedException( '`truncate_relation` is not implemented for this adapter!' ) @abc.abstractmethod @available def rename_relation(self, from_relation, to_relation, model_name=None): """Rename the relation from from_relation to to_relation. Implementors must call self.cache.rename() to preserve cache state. :param self.Relation from_relation: The original relation name :param self.Relation to_relation: The new relation name :param Optional[str] model_name: The name of the model to use for the connection. """ raise dbt.exceptions.NotImplementedException( '`rename_relation` is not implemented for this adapter!' ) @abc.abstractmethod @available def get_columns_in_relation(self, relation, model_name=None): """Get a list of the columns in the given Relation. :param self.Relation relation: The relation to query for. :param Optional[str] model_name: The name of the model to use for the connection. :return: Information about all columns in the given relation. :rtype: List[self.Column] """ raise dbt.exceptions.NotImplementedException( '`get_columns_in_relation` is not implemented for this adapter!' ) @available_deprecated('get_columns_in_relation') def get_columns_in_table(self, schema, identifier, model_name=None): """DEPRECATED: Get a list of the columns in the given table.""" relation = self.Relation.create( database=self.config.credentials.database, schema=schema, identifier=identifier, quote_policy=self.config.quoting ) return self.get_columns_in_relation(relation, model_name=model_name) @abc.abstractmethod def expand_column_types(self, goal, current, model_name=None): """Expand the current table's types to match the goal table. (passable) :param self.Relation goal: A relation that currently exists in the database with columns of the desired types. :param self.Relation current: A relation that currently exists in the database with columns of unspecified types. :param Optional[str] model_name: The name of the model to use for the connection. """ raise dbt.exceptions.NotImplementedException( '`expand_target_column_types` is not implemented for this adapter!' ) @abc.abstractmethod def list_relations_without_caching(self, information_schema, schema, model_name=None): """List relations in the given schema, bypassing the cache. This is used as the underlying behavior to fill the cache. :param Relation information_schema: The information schema to list relations from. :param str schema: The name of the schema to list relations from. :param Optional[str] model_name: The name of the model to use for the connection. :return: The relations in schema :retype: List[self.Relation] """ raise dbt.exceptions.NotImplementedException( '`list_relations_without_caching` is not implemented for this ' 'adapter!' ) ### # Provided methods about relations ### @available def get_missing_columns(self, from_relation, to_relation, model_name=None): """Returns dict of {column:type} for columns in from_table that are missing from to_relation """ if not isinstance(from_relation, self.Relation): dbt.exceptions.invalid_type_error( method_name='get_missing_columns', arg_name='from_relation', got_value=from_relation, expected_type=self.Relation) if not isinstance(to_relation, self.Relation): dbt.exceptions.invalid_type_error( method_name='get_missing_columns', arg_name='to_relation', got_value=to_relation, expected_type=self.Relation) from_columns = { col.name: col for col in self.get_columns_in_relation(from_relation, model_name=model_name) } to_columns = { col.name: col for col in self.get_columns_in_relation(to_relation, model_name=model_name) } missing_columns = set(from_columns.keys()) - set(to_columns.keys()) return [ col for (col_name, col) in from_columns.items() if col_name in missing_columns ] @available def expand_target_column_types(self, temp_table, to_relation, model_name=None): if not isinstance(to_relation, self.Relation): dbt.exceptions.invalid_type_error( method_name='expand_target_column_types', arg_name='to_relation', got_value=to_relation, expected_type=self.Relation) goal = self.Relation.create( database=None, schema=None, identifier=temp_table, type='table', quote_policy=self.config.quoting ) self.expand_column_types(goal, to_relation, model_name) def list_relations(self, database, schema, model_name=None): if self._schema_is_cached(database, schema, model_name): return self.cache.get_relations(database, schema) information_schema = self.Relation.create( database=database, schema=schema, model_name='').information_schema() # we can't build the relations cache because we don't have a # manifest so we can't run any operations. relations = self.list_relations_without_caching( information_schema, schema, model_name=model_name ) logger.debug('with schema={}, model_name={}, relations={}' .format(schema, model_name, relations)) return relations def _make_match_kwargs(self, database, schema, identifier): quoting = self.config.quoting if identifier is not None and quoting['identifier'] is False: identifier = identifier.lower() if schema is not None and quoting['schema'] is False: schema = schema.lower() if database is not None and quoting['schema'] is False: database = database.lower() return filter_null_values({ 'database': database, 'identifier': identifier, 'schema': schema, }) def _make_match(self, relations_list, database, schema, identifier): matches = [] search = self._make_match_kwargs(database, schema, identifier) for relation in relations_list: if relation.matches(**search): matches.append(relation) return matches @available def get_relation(self, database, schema, identifier, model_name=None): relations_list = self.list_relations(database, schema, model_name) matches = self._make_match(relations_list, database, schema, identifier) if len(matches) > 1: kwargs = { 'identifier': identifier, 'schema': schema, 'database': database, } dbt.exceptions.get_relation_returned_multiple_results( kwargs, matches ) elif matches: return matches[0] return None @available_deprecated('get_relation') def already_exists(self, schema, name, model_name=None): """DEPRECATED: Return if a model already exists in the database""" database = self.config.credentials.database relation = self.get_relation(database, schema, name, model_name=model_name) return relation is not None ### # ODBC FUNCTIONS -- these should not need to change for every adapter, # although some adapters may override them ### @abc.abstractmethod @available def create_schema(self, database, schema, model_name=None): """Create the given schema if it does not exist. :param str schema: The schema name to create. :param Optional[str] model_name: The name of the model to use for the connection. """ raise dbt.exceptions.NotImplementedException( '`create_schema` is not implemented for this adapter!' ) @abc.abstractmethod def drop_schema(self, database, schema, model_name=None): """Drop the given schema (and everything in it) if it exists. :param str schema: The schema name to drop. :param Optional[str] model_name: The name of the model to use for the connection. """ raise dbt.exceptions.NotImplementedException( '`drop_schema` is not implemented for this adapter!' ) @available_raw @abstractclassmethod def quote(cls, identifier): """Quote the given identifier, as appropriate for the database. :param str identifier: The identifier to quote :return: The quoted identifier :rtype: str """ raise dbt.exceptions.NotImplementedException( '`quote` is not implemented for this adapter!' ) @available def quote_as_configured(self, identifier, quote_key, model_name=None): """Quote or do not quote the given identifer as configured in the project config for the quote key. The quote key should be one of 'database' (on bigquery, 'profile'), 'identifier', or 'schema', or it will be treated as if you set `True`. """ default = self.Relation.DEFAULTS['quote_policy'].get(quote_key) if self.config.quoting.get(quote_key, default): return self.quote(identifier) else: return identifier ### # Conversions: These must be implemented by concrete implementations, for # converting agate types into their sql equivalents. ### @abstractclassmethod def convert_text_type(cls, agate_table, col_idx): """Return the type in the database that best maps to the agate.Text type for the given agate table and column index. :param agate.Table agate_table: The table :param int col_idx: The index into the agate table for the column. :return: The name of the type in the database :rtype: str """ raise dbt.exceptions.NotImplementedException( '`convert_text_type` is not implemented for this adapter!') @abstractclassmethod def convert_number_type(cls, agate_table, col_idx): """Return the type in the database that best maps to the agate.Number type for the given agate table and column index. :param agate.Table agate_table: The table :param int col_idx: The index into the agate table for the column. :return: The name of the type in the database :rtype: str """ raise dbt.exceptions.NotImplementedException( '`convert_number_type` is not implemented for this adapter!') @abstractclassmethod def convert_boolean_type(cls, agate_table, col_idx): """Return the type in the database that best maps to the agate.Boolean type for the given agate table and column index. :param agate.Table agate_table: The table :param int col_idx: The index into the agate table for the column. :return: The name of the type in the database :rtype: str """ raise dbt.exceptions.NotImplementedException( '`convert_boolean_type` is not implemented for this adapter!') @abstractclassmethod def convert_datetime_type(cls, agate_table, col_idx): """Return the type in the database that best maps to the agate.DateTime type for the given agate table and column index. :param agate.Table agate_table: The table :param int col_idx: The index into the agate table for the column. :return: The name of the type in the database :rtype: str """ raise dbt.exceptions.NotImplementedException( '`convert_datetime_type` is not implemented for this adapter!') @abstractclassmethod def convert_date_type(cls, agate_table, col_idx): """Return the type in the database that best maps to the agate.Date type for the given agate table and column index. :param agate.Table agate_table: The table :param int col_idx: The index into the agate table for the column. :return: The name of the type in the database :rtype: str """ raise dbt.exceptions.NotImplementedException( '`convert_date_type` is not implemented for this adapter!') @abstractclassmethod def convert_time_type(cls, agate_table, col_idx): """Return the type in the database that best maps to the agate.TimeDelta type for the given agate table and column index. :param agate.Table agate_table: The table :param int col_idx: The index into the agate table for the column. :return: The name of the type in the database :rtype: str """ raise dbt.exceptions.NotImplementedException( '`convert_time_type` is not implemented for this adapter!') @available_raw @classmethod def convert_type(cls, agate_table, col_idx): return cls.convert_agate_type(agate_table, col_idx) @classmethod def convert_agate_type(cls, agate_table, col_idx): agate_type = agate_table.column_types[col_idx] conversions = [ (agate.Text, cls.convert_text_type), (agate.Number, cls.convert_number_type), (agate.Boolean, cls.convert_boolean_type), (agate.DateTime, cls.convert_datetime_type), (agate.Date, cls.convert_date_type), (agate.TimeDelta, cls.convert_time_type), ] for agate_cls, func in conversions: if isinstance(agate_type, agate_cls): return func(agate_table, col_idx) ### # Operations involving the manifest ### def execute_macro(self, macro_name, manifest=None, project=None, context_override=None, kwargs=None, release=False, connection_name=None): """Look macro_name up in the manifest and execute its results. :param str macro_name: The name of the macro to execute. :param Optional[Manifest] manifest: The manifest to use for generating the base macro execution context. If none is provided, use the internal manifest. :param Optional[str] project: The name of the project to search in, or None for the first match. :param Optional[dict] context_override: An optional dict to update() the macro execution context. :param Optional[dict] kwargs: An optional dict of keyword args used to pass to the macro. :param bool release: If True, release the connection after executing. :param Optional[str] connection_name: The connection name to use, or use the macro name. Return an an AttrDict with three attributes: 'table', 'data', and 'status'. 'table' is an agate.Table. """ if kwargs is None: kwargs = {} if context_override is None: context_override = {} if connection_name is None: connection_name = macro_name if manifest is None: manifest = self._internal_manifest macro = manifest.find_macro_by_name(macro_name, project) if macro is None: if project is None: package_name = 'any package' else: package_name = 'the "{}" package'.format(project) # The import of dbt.context.runtime below shadows 'dbt' import dbt.exceptions raise dbt.exceptions.RuntimeException( 'dbt could not find a macro with the name "{}" in {}' .format(macro_name, package_name) ) # This causes a reference cycle, as dbt.context.runtime.generate() # ends up calling get_adapter, so the import has to be here. import dbt.context.runtime macro_context = dbt.context.runtime.generate_macro( macro, self.config, manifest, connection_name ) macro_context.update(context_override) macro_function = macro.generator(macro_context) try: result = macro_function(**kwargs) finally: if release: self.release_connection(connection_name) return result @classmethod def _catalog_filter_table(cls, table, manifest): """Filter the table as appropriate for catalog entries. Subclasses can override this to change filtering rules on a per-adapter basis. """ return table.where(_catalog_filter_schemas(manifest)) def get_catalog(self, manifest): """Get the catalog for this manifest by running the get catalog macro. Returns an agate.Table of catalog information. """ information_schemas = list(self._get_cache_schemas(manifest).keys()) # make it a list so macros can index into it. kwargs = {'information_schemas': information_schemas} table = self.execute_macro(GET_CATALOG_MACRO_NAME, kwargs=kwargs, release=True) results = self._catalog_filter_table(table, manifest) return results def cancel_open_connections(self): """Cancel all open connections.""" return self.connections.cancel_open() def calculate_freshness(self, source, loaded_at_field, manifest=None, connection_name=None): """Calculate the freshness of sources in dbt, and return it""" # in the future `source` will be a Relation instead of a string kwargs = { 'source': source, 'loaded_at_field': loaded_at_field } # run the macro table = self.execute_macro( FRESHNESS_MACRO_NAME, kwargs=kwargs, release=True, manifest=manifest, connection_name=connection_name ) # now we have a 1-row table of the maximum `loaded_at_field` value and # the current time according to the db. if len(table) != 1 or len(table[0]) != 2: dbt.exceptions.raise_compiler_error( 'Got an invalid result from "{}" macro: {}'.format( FRESHNESS_MACRO_NAME, [tuple(r) for r in table] ) ) max_loaded_at = _utc(table[0][0], source, loaded_at_field) snapshotted_at = _utc(table[0][1], source, loaded_at_field) age = (snapshotted_at - max_loaded_at).total_seconds() return { 'max_loaded_at': max_loaded_at, 'snapshotted_at': snapshotted_at, 'age': age, }
class TestLikeDbt(TestCase): def setUp(self): self.cache = RelationsCache() self._sleep = True # add a bunch of cache entries for ident in 'abcdef': self.cache.add(make_relation('dbt', 'schema', ident)) # 'b' references 'a' self.cache.add_link(make_relation('dbt', 'schema', 'a'), make_relation('dbt', 'schema', 'b')) # and 'c' references 'b' self.cache.add_link(make_relation('dbt', 'schema', 'b'), make_relation('dbt', 'schema', 'c')) # and 'd' references 'b' self.cache.add_link(make_relation('dbt', 'schema', 'b'), make_relation('dbt', 'schema', 'd')) # and 'e' references 'a' self.cache.add_link(make_relation('dbt', 'schema', 'a'), make_relation('dbt', 'schema', 'e')) # and 'f' references 'd' self.cache.add_link(make_relation('dbt', 'schema', 'd'), make_relation('dbt', 'schema', 'f')) # so drop propagation goes (a -> (b -> (c (d -> f))) e) def assert_has_relations(self, expected): current = set(r.identifier for r in self.cache.get_relations('dbt', 'schema')) self.assertEqual(current, expected) def test_drop_inner(self): self.assert_has_relations(set('abcdef')) self.cache.drop(make_relation('dbt', 'schema', 'b')) self.assert_has_relations({'a', 'e'}) def test_rename_and_drop(self): self.assert_has_relations(set('abcdef')) # drop the backup/tmp self.cache.drop(make_relation('dbt', 'schema', 'b__backup')) self.cache.drop(make_relation('dbt', 'schema', 'b__tmp')) self.assert_has_relations(set('abcdef')) # create a new b__tmp self.cache.add(make_relation( 'dbt', 'schema', 'b__tmp', )) self.assert_has_relations(set('abcdef') | {'b__tmp'}) # rename b -> b__backup self.cache.rename(make_relation('dbt', 'schema', 'b'), make_relation('dbt', 'schema', 'b__backup')) self.assert_has_relations(set('acdef') | {'b__tmp', 'b__backup'}) # rename temp to b self.cache.rename(make_relation('dbt', 'schema', 'b__tmp'), make_relation('dbt', 'schema', 'b')) self.assert_has_relations(set('abcdef') | {'b__backup'}) # drop backup, everything that used to depend on b should be gone, but # b itself should still exist self.cache.drop(make_relation('dbt', 'schema', 'b__backup')) self.assert_has_relations(set('abe')) relation = self.cache.relations[('dbt', 'schema', 'a')] self.assertEqual(len(relation.referenced_by), 1) def _rand_sleep(self): if not self._sleep: return time.sleep(random.random() * 0.1) def _target(self, ident): self._rand_sleep() self.cache.rename(make_relation('dbt', 'schema', ident), make_relation('dbt', 'schema', ident + '__backup')) self._rand_sleep() self.cache.add(make_relation('dbt', 'schema', ident + '__tmp')) self._rand_sleep() self.cache.rename(make_relation('dbt', 'schema', ident + '__tmp'), make_relation('dbt', 'schema', ident)) self._rand_sleep() self.cache.drop(make_relation('dbt', 'schema', ident + '__backup')) return ident, self.cache.get_relations('dbt', 'schema') def test_threaded(self): # add three more short subchains for threads to test on for ident in 'ghijklmno': obj = make_mock_relationship('test_db', 'schema', ident) self.cache.add(make_relation('dbt', 'schema', ident)) self.cache.add_link(make_relation('dbt', 'schema', 'a'), make_relation('dbt', 'schema', 'g')) self.cache.add_link(make_relation('dbt', 'schema', 'g'), make_relation('dbt', 'schema', 'h')) self.cache.add_link(make_relation('dbt', 'schema', 'h'), make_relation('dbt', 'schema', 'i')) self.cache.add_link(make_relation('dbt', 'schema', 'a'), make_relation('dbt', 'schema', 'j')) self.cache.add_link(make_relation('dbt', 'schema', 'j'), make_relation('dbt', 'schema', 'k')) self.cache.add_link(make_relation('dbt', 'schema', 'k'), make_relation('dbt', 'schema', 'l')) self.cache.add_link(make_relation('dbt', 'schema', 'a'), make_relation('dbt', 'schema', 'm')) self.cache.add_link(make_relation('dbt', 'schema', 'm'), make_relation('dbt', 'schema', 'n')) self.cache.add_link(make_relation('dbt', 'schema', 'n'), make_relation('dbt', 'schema', 'o')) pool = ThreadPool(4) results = list(pool.imap_unordered(self._target, ('b', 'g', 'j', 'm'))) pool.close() pool.join() # at a minimum, we expect each table to "see" itself, its parent ('a'), # and the unrelated table ('a') min_expect = { 'b': {'a', 'b', 'e'}, 'g': {'a', 'g', 'e'}, 'j': {'a', 'j', 'e'}, 'm': {'a', 'm', 'e'}, } for ident, relations in results: seen = set(r.identifier for r in relations) self.assertTrue(min_expect[ident].issubset(seen)) self.assert_has_relations(set('abgjme')) def test_threaded_repeated(self): for _ in range(10): self.setUp() self._sleep = False self.test_threaded()
class TestLikeDbt(TestCase): def setUp(self): self.cache = RelationsCache() self._sleep = True # add a bunch of cache entries for ident in 'abcdef': self.cache.add(make_relation('dbt', 'schema', ident)) # 'b' references 'a' self.cache.add_link(make_relation('dbt', 'schema', 'a'), make_relation('dbt', 'schema', 'b')) # and 'c' references 'b' self.cache.add_link(make_relation('dbt', 'schema', 'b'), make_relation('dbt', 'schema', 'c')) # and 'd' references 'b' self.cache.add_link(make_relation('dbt', 'schema', 'b'), make_relation('dbt', 'schema', 'd')) # and 'e' references 'a' self.cache.add_link(make_relation('dbt', 'schema', 'a'), make_relation('dbt', 'schema', 'e')) # and 'f' references 'd' self.cache.add_link(make_relation('dbt', 'schema', 'd'), make_relation('dbt', 'schema', 'f')) # so drop propagation goes (a -> (b -> (c (d -> f))) e) def assert_has_relations(self, expected): current = set(r.identifier for r in self.cache.get_relations('dbt', 'schema')) self.assertEqual(current, expected) def test_drop_inner(self): self.assert_has_relations(set('abcdef')) self.cache.drop(make_relation('dbt', 'schema', 'b')) self.assert_has_relations({'a', 'e'}) def test_rename_and_drop(self): self.assert_has_relations(set('abcdef')) # drop the backup/tmp self.cache.drop(make_relation('dbt', 'schema', 'b__backup')) self.cache.drop(make_relation('dbt', 'schema', 'b__tmp')) self.assert_has_relations(set('abcdef')) # create a new b__tmp self.cache.add(make_relation('dbt', 'schema', 'b__tmp',)) self.assert_has_relations(set('abcdef') | {'b__tmp'}) # rename b -> b__backup self.cache.rename(make_relation('dbt', 'schema', 'b'), make_relation('dbt', 'schema', 'b__backup')) self.assert_has_relations(set('acdef') | {'b__tmp', 'b__backup'}) # rename temp to b self.cache.rename(make_relation('dbt', 'schema', 'b__tmp'), make_relation('dbt', 'schema', 'b')) self.assert_has_relations(set('abcdef') | {'b__backup'}) # drop backup, everything that used to depend on b should be gone, but # b itself should still exist self.cache.drop(make_relation('dbt', 'schema', 'b__backup')) self.assert_has_relations(set('abe')) relation = self.cache.relations[('dbt', 'schema', 'a')] self.assertEqual(len(relation.referenced_by), 1) def _rand_sleep(self): if not self._sleep: return time.sleep(random.random() * 0.1) def _target(self, ident): self._rand_sleep() self.cache.rename(make_relation('dbt', 'schema', ident), make_relation('dbt', 'schema', ident+'__backup')) self._rand_sleep() self.cache.add(make_relation('dbt', 'schema', ident+'__tmp') ) self._rand_sleep() self.cache.rename(make_relation('dbt', 'schema', ident+'__tmp'), make_relation('dbt', 'schema', ident)) self._rand_sleep() self.cache.drop(make_relation('dbt', 'schema', ident+'__backup')) return ident, self.cache.get_relations('dbt', 'schema') def test_threaded(self): # add three more short subchains for threads to test on for ident in 'ghijklmno': obj = make_mock_relationship('test_db', 'schema', ident) self.cache.add(make_relation('dbt', 'schema', ident)) self.cache.add_link(make_relation('dbt', 'schema', 'a'), make_relation('dbt', 'schema', 'g')) self.cache.add_link(make_relation('dbt', 'schema', 'g'), make_relation('dbt', 'schema', 'h')) self.cache.add_link(make_relation('dbt', 'schema', 'h'), make_relation('dbt', 'schema', 'i')) self.cache.add_link(make_relation('dbt', 'schema', 'a'), make_relation('dbt', 'schema', 'j')) self.cache.add_link(make_relation('dbt', 'schema', 'j'), make_relation('dbt', 'schema', 'k')) self.cache.add_link(make_relation('dbt', 'schema', 'k'), make_relation('dbt', 'schema', 'l')) self.cache.add_link(make_relation('dbt', 'schema', 'a'), make_relation('dbt', 'schema', 'm')) self.cache.add_link(make_relation('dbt', 'schema', 'm'), make_relation('dbt', 'schema', 'n')) self.cache.add_link(make_relation('dbt', 'schema', 'n'), make_relation('dbt', 'schema', 'o')) pool = ThreadPool(4) results = list(pool.imap_unordered(self._target, ('b', 'g', 'j', 'm'))) pool.close() pool.join() # at a minimum, we expect each table to "see" itself, its parent ('a'), # and the unrelated table ('a') min_expect = { 'b': {'a', 'b', 'e'}, 'g': {'a', 'g', 'e'}, 'j': {'a', 'j', 'e'}, 'm': {'a', 'm', 'e'}, } for ident, relations in results: seen = set(r.identifier for r in relations) self.assertTrue(min_expect[ident].issubset(seen)) self.assert_has_relations(set('abgjme')) def test_threaded_repeated(self): for _ in range(10): self.setUp() self._sleep = False self.test_threaded()
class TestComplexCache(TestCase): def setUp(self): self.cache = RelationsCache() inputs = [ ('dbt', 'foo', 'table1'), ('dbt', 'foo', 'table3'), ('dbt', 'foo', 'table4'), ('dbt', 'bar', 'table2'), ('dbt', 'bar', 'table3'), ('dbt_2', 'foo', 'table1'), ('dbt_2', 'foo', 'table2'), ] self.inputs = [make_relation(d, s, i) for d, s, i in inputs] for relation in self.inputs: self.cache.add(relation) # dbt.foo.table3 references dbt.foo.table1 # (create view dbt.foo.table3 as (select * from dbt.foo.table1...)) self.cache.add_link( make_relation('dbt', 'foo', 'table1'), make_relation('dbt', 'foo', 'table3') ) # dbt.bar.table3 references dbt.foo.table3 # (create view dbt.bar.table5 as (select * from dbt.foo.table3...)) self.cache.add_link( make_relation('dbt', 'foo', 'table3'), make_relation('dbt', 'bar', 'table3') ) # dbt.foo.table4 also references dbt.foo.table1 self.cache.add_link( make_relation('dbt', 'foo', 'table1'), make_relation('dbt', 'foo', 'table4') ) # and dbt_2.foo.table1 references dbt.foo.table1 self.cache.add_link( make_relation('dbt', 'foo', 'table1'), make_relation('dbt_2', 'foo', 'table1'), ) def test_get_relations(self): self.assertEqual(len(self.cache.get_relations('dbt', 'foo')), 3) self.assertEqual(len(self.cache.get_relations('dbt', 'bar')), 2) self.assertEqual(len(self.cache.get_relations('dbt_2', 'foo')), 2) self.assertEqual(len(self.cache.relations), 7) def test_drop_one(self): # dropping dbt.bar.table2 should only drop itself self.cache.drop(make_relation('dbt', 'bar', 'table2')) self.assertEqual(len(self.cache.get_relations('dbt', 'foo')), 3) self.assertEqual(len(self.cache.get_relations('dbt', 'bar')), 1) self.assertEqual(len(self.cache.get_relations('dbt_2', 'foo')), 2) self.assertEqual(len(self.cache.relations), 6) def test_drop_many(self): # dropping dbt.foo.table1 should drop everything but dbt.bar.table2 and # dbt_2.foo.table2 self.cache.drop(make_relation('dbt', 'foo', 'table1')) self.assertEqual(len(self.cache.get_relations('dbt', 'foo')), 0) self.assertEqual(len(self.cache.get_relations('dbt', 'bar')), 1) self.assertEqual(len(self.cache.get_relations('dbt_2', 'foo')), 1) self.assertEqual(len(self.cache.relations), 2) def test_rename_root(self): self.cache.rename(make_relation('dbt', 'foo', 'table1'), make_relation('dbt', 'bar', 'table1')) retrieved = self.cache.relations[('dbt', 'bar', 'table1')].inner self.assertEqual(retrieved.schema, 'bar') self.assertEqual(retrieved.identifier, 'table1') self.assertEqual(len(self.cache.get_relations('dbt', 'foo')), 2) self.assertEqual(len(self.cache.get_relations('dbt', 'bar')), 3) self.assertEqual(len(self.cache.get_relations('dbt_2', 'foo')), 2) self.assertEqual(len(self.cache.relations), 7) # make sure drops still cascade from the renamed table self.cache.drop(make_relation('dbt', 'bar', 'table1')) self.assertEqual(len(self.cache.get_relations('dbt', 'foo')), 0) self.assertEqual(len(self.cache.get_relations('dbt', 'bar')), 1) self.assertEqual(len(self.cache.get_relations('dbt_2', 'foo')), 1) self.assertEqual(len(self.cache.relations), 2) def test_rename_branch(self): self.cache.rename(make_relation('dbt', 'foo', 'table3'), make_relation('dbt', 'foo', 'table2')) self.assertEqual(len(self.cache.get_relations('dbt', 'foo')), 3) self.assertEqual(len(self.cache.get_relations('dbt', 'bar')), 2) self.assertEqual(len(self.cache.get_relations('dbt_2', 'foo')), 2) # make sure drops still cascade through the renamed table self.cache.drop(make_relation('dbt', 'foo', 'table1')) self.assertEqual(len(self.cache.get_relations('dbt', 'foo')), 0) self.assertEqual(len(self.cache.get_relations('dbt', 'bar')), 1) self.assertEqual(len(self.cache.get_relations('dbt_2', 'foo')), 1) self.assertEqual(len(self.cache.relations), 2)
def setUp(self): self.cache = RelationsCache()
class TestCache(TestCase): def setUp(self): self.cache = RelationsCache() def test_empty(self): self.assertEqual(len(self.cache.relations), 0) relations = self.cache.get_relations('test') self.assertEqual(len(relations), 0) def test_bad_drop(self): self.cache.drop(make_relation('foo', 'bar')) def test_bad_link(self): self.cache.add(make_relation('schema', 'foo')) # src does not exist with self.assertRaises(dbt.exceptions.InternalException): self.cache.add_link(make_relation('schema', 'bar'), make_relation('schema', 'foo')) # dst does not exist with self.assertRaises(dbt.exceptions.InternalException): self.cache.add_link(make_relation('schema', 'foo'), make_relation('schema', 'bar')) def test_bad_rename(self): # foo does not exist - should be ignored self.cache.rename(make_relation('schema', 'foo'), make_relation('schema', 'bar')) self.cache.add(make_relation('schema', 'foo')) self.cache.add(make_relation('schema', 'bar')) # bar exists with self.assertRaises(dbt.exceptions.InternalException): self.cache.rename(make_relation('schema', 'foo'), make_relation('schema', 'bar')) def test_get_relations(self): relation = make_relation('foo', 'bar') self.cache.add(relation) self.assertEqual(len(self.cache.relations), 1) relations = self.cache.get_relations('foo') self.assertEqual(len(relations), 1) self.assertIs(relations[0], relation) relations = self.cache.get_relations('FOO') self.assertEqual(len(relations), 1) self.assertIs(relations[0], relation) def test_add(self): rel = make_relation('foo', 'bar') self.cache.add(rel) relations = self.cache.get_relations('foo') self.assertEqual(len(relations), 1) self.assertIs(relations[0], rel) # add a new relation with same name self.cache.add(make_relation('foo', 'bar')) self.assertEqual(len(self.cache.relations), 1) self.assertEqual(self.cache.schemas, {'foo'}) relations = self.cache.get_relations('foo') self.assertEqual(len(relations), 1) self.assertIs(relations[0], rel) self.cache.add(make_relation('FOO', 'baz')) self.assertEqual(len(self.cache.relations), 2) relations = self.cache.get_relations('foo') self.assertEqual(len(relations), 2) self.assertEqual(self.cache.schemas, {'foo'}) self.assertIsNot(self.cache.relations[('foo', 'bar')].inner, None) self.assertIsNot(self.cache.relations[('foo', 'baz')].inner, None) def test_rename(self): self.cache.add(make_relation('foo', 'bar')) self.assertIsNot(self.cache.relations[('foo', 'bar')].inner, None) self.cache.rename(make_relation('foo', 'bar'), make_relation('foo', 'baz')) relations = self.cache.get_relations('foo') self.assertEqual(len(relations), 1) self.assertEqual(relations[0].schema, 'foo') self.assertEqual(relations[0].identifier, 'baz') relation = self.cache.relations[('foo', 'baz')] self.assertEqual(relation.inner.schema, 'foo') self.assertEqual(relation.inner.identifier, 'baz') self.assertEqual(relation.schema, 'foo') self.assertEqual(relation.identifier, 'baz') with self.assertRaises(KeyError): self.cache.relations[('foo', 'bar')]
def __init__(self, config): self.config = config self.cache = RelationsCache()
def setUp(self): self.cache = RelationsCache()
class BaseAdapter(object): """The BaseAdapter provides an abstract base class for adapters. Adapters must implement the following methods and macros. Some of the methods can be safely overridden as a noop, where it makes sense (transactions on databases that don't support them, for instance). Those methods are marked with a (passable) in their docstrings. Check docstrings for type information, etc. To implement a macro, implement "${adapter_type}__${macro_name}". in the adapter's internal project. Methods: - exception_handler - date_function - list_schemas - drop_relation - truncate_relation - rename_relation - get_columns_in_relation - expand_column_types - list_relations_without_caching - is_cancelable - create_schema - drop_schema - quote - convert_text_type - convert_number_type - convert_boolean_type - convert_datetime_type - convert_date_type - convert_time_type Macros: - get_catalog """ requires = {} Relation = BaseRelation Column = Column # This should be an implementation of BaseConnectionManager ConnectionManager = None # A set of clobber config fields accepted by this adapter # for use in materializations AdapterSpecificConfigs = frozenset() def __init__(self, config): self.config = config self.cache = RelationsCache() self.connections = self.ConnectionManager(config) self._internal_manifest_lazy = None ### # Methods that pass through to the connection manager ### def acquire_connection(self, name=None): return self.connections.set_connection_name(name) def release_connection(self): return self.connections.release() def cleanup_connections(self): return self.connections.cleanup_all() def clear_transaction(self): self.connections.clear_transaction() def commit_if_has_connection(self): return self.connections.commit_if_has_connection() def nice_connection_name(self): conn = self.connections.get_thread_connection() if conn is None or conn.name is None: return '<None>' return conn.name @contextmanager def connection_named(self, name): try: yield self.acquire_connection(name) finally: self.release_connection() @available.parse(lambda *a, **k: ('', dbt.clients.agate_helper())) def execute(self, sql, auto_begin=False, fetch=False): """Execute the given SQL. This is a thin wrapper around ConnectionManager.execute. :param str sql: The sql to execute. :param bool auto_begin: If set, and dbt is not currently inside a transaction, automatically begin one. :param bool fetch: If set, fetch results. :return: A tuple of the status and the results (empty if fetch=False). :rtype: Tuple[str, agate.Table] """ return self.connections.execute(sql=sql, auto_begin=auto_begin, fetch=fetch) ### # Methods that should never be overridden ### @classmethod def type(cls): """Get the type of this adapter. Types must be class-unique and consistent. :return: The type name :rtype: str """ return cls.ConnectionManager.TYPE @property def _internal_manifest(self): if self._internal_manifest_lazy is None: manifest = GraphLoader.load_internal(self.config) self._internal_manifest_lazy = manifest return self._internal_manifest_lazy def check_internal_manifest(self): """Return the internal manifest (used for executing macros) if it's been initialized, otherwise return None. """ return self._internal_manifest_lazy ### # Caching methods ### def _schema_is_cached(self, database, schema): """Check if the schema is cached, and by default logs if it is not.""" if dbt.flags.USE_CACHE is False: return False elif (database, schema) not in self.cache: logger.debug( 'On "{}": cache miss for schema "{}.{}", this is inefficient'. format(self.nice_connection_name(), database, schema)) return False else: return True @classmethod def _relations_filter_table(cls, table, schemas): """Filter the table as appropriate for relations table entries. Subclasses can override this to change filtering rules on a per-adapter basis. """ return table.where(_relations_filter_schemas(schemas)) def _get_cache_schemas(self, manifest, exec_only=False): """Get a mapping of each node's "information_schema" relations to a set of all schemas expected in that information_schema. There may be keys that are technically duplicates on the database side, for example all of '"foo", 'foo', '"FOO"' and 'FOO' could coexist as databases, and values could overlap as appropriate. All values are lowercase strings. """ info_schema_name_map = SchemaSearchMap() for node in manifest.nodes.values(): if exec_only and node.resource_type not in NodeType.executable(): continue relation = self.Relation.create_from(self.config, node) info_schema_name_map.add(relation) # result is a map whose keys are information_schema Relations without # identifiers that have appropriate database prefixes, and whose values # are sets of lowercase schema names that are valid members of those # schemas return info_schema_name_map def _relations_cache_for_schemas(self, manifest): """Populate the relations cache for the given schemas. Returns an iteratble of the schemas populated, as strings. """ if not dbt.flags.USE_CACHE: return info_schema_name_map = self._get_cache_schemas(manifest, exec_only=True) for db, schema in info_schema_name_map.search(): for relation in self.list_relations_without_caching(db, schema): self.cache.add(relation) # it's possible that there were no relations in some schemas. We want # to insert the schemas we query into the cache's `.schemas` attribute # so we can check it later self.cache.update_schemas(info_schema_name_map.schemas_searched()) def set_relations_cache(self, manifest, clear=False): """Run a query that gets a populated cache of the relations in the database and set the cache on this adapter. """ if not dbt.flags.USE_CACHE: return with self.cache.lock: if clear: self.cache.clear() self._relations_cache_for_schemas(manifest) def cache_new_relation(self, relation): """Cache a new relation in dbt. It will show up in `list relations`.""" if relation is None: name = self.nice_connection_name() dbt.exceptions.raise_compiler_error( 'Attempted to cache a null relation for {}'.format(name)) if dbt.flags.USE_CACHE: self.cache.add(relation) # so jinja doesn't render things return '' ### # Abstract methods for database-specific values, attributes, and types ### @abstractclassmethod def date_function(cls): """Get the date function used by this adapter's database. :return: The date function :rtype: str """ raise dbt.exceptions.NotImplementedException( '`date_function` is not implemented for this adapter!') @abstractclassmethod def is_cancelable(cls): raise dbt.exceptions.NotImplementedException( '`is_cancelable` is not implemented for this adapter!') ### # Abstract methods about schemas ### @abc.abstractmethod def list_schemas(self, database): """Get a list of existing schemas. :param str database: The name of the database to list under. :return: All schemas that currently exist in the database :rtype: List[str] """ raise dbt.exceptions.NotImplementedException( '`list_schemas` is not implemented for this adapter!') @available.parse(lambda *a, **k: False) def check_schema_exists(self, database, schema): """Check if a schema exists. The default implementation of this is potentially unnecessarily slow, and adapters should implement it if there is an optimized path (and there probably is) """ search = (s.lower() for s in self.list_schemas(database=database)) return schema.lower() in search ### # Abstract methods about relations ### @abc.abstractmethod @available.parse_none def drop_relation(self, relation): """Drop the given relation. *Implementors must call self.cache.drop() to preserve cache state!* :param self.Relation relation: The relation to drop """ raise dbt.exceptions.NotImplementedException( '`drop_relation` is not implemented for this adapter!') @abc.abstractmethod @available.parse_none def truncate_relation(self, relation): """Truncate the given relation. :param self.Relation relation: The relation to truncate """ raise dbt.exceptions.NotImplementedException( '`truncate_relation` is not implemented for this adapter!') @abc.abstractmethod @available.parse_none def rename_relation(self, from_relation, to_relation): """Rename the relation from from_relation to to_relation. Implementors must call self.cache.rename() to preserve cache state. :param self.Relation from_relation: The original relation name :param self.Relation to_relation: The new relation name """ raise dbt.exceptions.NotImplementedException( '`rename_relation` is not implemented for this adapter!') @abc.abstractmethod @available.parse_list def get_columns_in_relation(self, relation): """Get a list of the columns in the given Relation. :param self.Relation relation: The relation to query for. :return: Information about all columns in the given relation. :rtype: List[self.Column] """ raise dbt.exceptions.NotImplementedException( '`get_columns_in_relation` is not implemented for this adapter!') @available.deprecated('get_columns_in_relation', lambda *a, **k: []) def get_columns_in_table(self, schema, identifier): """DEPRECATED: Get a list of the columns in the given table.""" relation = self.Relation.create( database=self.config.credentials.database, schema=schema, identifier=identifier, quote_policy=self.config.quoting) return self.get_columns_in_relation(relation) @abc.abstractmethod def expand_column_types(self, goal, current): """Expand the current table's types to match the goal table. (passable) :param self.Relation goal: A relation that currently exists in the database with columns of the desired types. :param self.Relation current: A relation that currently exists in the database with columns of unspecified types. """ raise dbt.exceptions.NotImplementedException( '`expand_target_column_types` is not implemented for this adapter!' ) @abc.abstractmethod def list_relations_without_caching(self, information_schema, schema): """List relations in the given schema, bypassing the cache. This is used as the underlying behavior to fill the cache. :param Relation information_schema: The information schema to list relations from. :param str schema: The name of the schema to list relations from. :return: The relations in schema :rtype: List[self.Relation] """ raise dbt.exceptions.NotImplementedException( '`list_relations_without_caching` is not implemented for this ' 'adapter!') ### # Provided methods about relations ### @available.parse_list def get_missing_columns(self, from_relation, to_relation): """Returns a list of Columns in from_relation that are missing from to_relation. :param Relation from_relation: The relation that might have extra columns :param Relation to_relation: The realtion that might have columns missing :return: The columns in from_relation that are missing from to_relation :rtype: List[self.Relation] """ if not isinstance(from_relation, self.Relation): dbt.exceptions.invalid_type_error( method_name='get_missing_columns', arg_name='from_relation', got_value=from_relation, expected_type=self.Relation) if not isinstance(to_relation, self.Relation): dbt.exceptions.invalid_type_error( method_name='get_missing_columns', arg_name='to_relation', got_value=to_relation, expected_type=self.Relation) from_columns = { col.name: col for col in self.get_columns_in_relation(from_relation) } to_columns = { col.name: col for col in self.get_columns_in_relation(to_relation) } missing_columns = set(from_columns.keys()) - set(to_columns.keys()) return [ col for (col_name, col) in from_columns.items() if col_name in missing_columns ] @available.parse_none def valid_snapshot_target(self, relation): """Ensure that the target relation is valid, by making sure it has the expected columns. :param Relation relation: The relation to check :raises dbt.exceptions.CompilationException: If the columns are incorrect. """ if not isinstance(relation, self.Relation): dbt.exceptions.invalid_type_error( method_name='valid_snapshot_target', arg_name='relation', got_value=relation, expected_type=self.Relation) columns = self.get_columns_in_relation(relation) names = set(c.name.lower() for c in columns) expanded_keys = ('scd_id', 'valid_from', 'valid_to') extra = [] missing = [] for legacy in expanded_keys: desired = 'dbt_' + legacy if desired not in names: missing.append(desired) if legacy in names: extra.append(legacy) if missing: if extra: msg = ('Snapshot target has ("{}") but not ("{}") - is it an ' 'unmigrated previous version archive?'.format( '", "'.join(extra), '", "'.join(missing))) else: msg = ('Snapshot target is not a snapshot table (missing "{}")' .format('", "'.join(missing))) dbt.exceptions.raise_compiler_error(msg) @available.parse_none def expand_target_column_types(self, from_relation, to_relation): if not isinstance(from_relation, self.Relation): dbt.exceptions.invalid_type_error( method_name='expand_target_column_types', arg_name='from_relation', got_value=from_relation, expected_type=self.Relation) if not isinstance(to_relation, self.Relation): dbt.exceptions.invalid_type_error( method_name='expand_target_column_types', arg_name='to_relation', got_value=to_relation, expected_type=self.Relation) self.expand_column_types(from_relation, to_relation) def list_relations(self, database, schema): if self._schema_is_cached(database, schema): return self.cache.get_relations(database, schema) information_schema = self.Relation.create( database=database, schema=schema, model_name='', quote_policy=self.config.quoting).information_schema() # we can't build the relations cache because we don't have a # manifest so we can't run any operations. relations = self.list_relations_without_caching( information_schema, schema) logger.debug('with database={}, schema={}, relations={}'.format( database, schema, relations)) return relations def _make_match_kwargs(self, database, schema, identifier): quoting = self.config.quoting if identifier is not None and quoting['identifier'] is False: identifier = identifier.lower() if schema is not None and quoting['schema'] is False: schema = schema.lower() if database is not None and quoting['database'] is False: database = database.lower() return filter_null_values({ 'database': database, 'identifier': identifier, 'schema': schema, }) def _make_match(self, relations_list, database, schema, identifier): matches = [] search = self._make_match_kwargs(database, schema, identifier) for relation in relations_list: if relation.matches(**search): matches.append(relation) return matches @available.parse_none def get_relation(self, database, schema, identifier): relations_list = self.list_relations(database, schema) matches = self._make_match(relations_list, database, schema, identifier) if len(matches) > 1: kwargs = { 'identifier': identifier, 'schema': schema, 'database': database, } dbt.exceptions.get_relation_returned_multiple_results( kwargs, matches) elif matches: return matches[0] return None @available.deprecated('get_relation', lambda *a, **k: False) def already_exists(self, schema, name): """DEPRECATED: Return if a model already exists in the database""" database = self.config.credentials.database relation = self.get_relation(database, schema, name) return relation is not None ### # ODBC FUNCTIONS -- these should not need to change for every adapter, # although some adapters may override them ### @abc.abstractmethod @available.parse_none def create_schema(self, database, schema): """Create the given schema if it does not exist. :param str schema: The schema name to create. """ raise dbt.exceptions.NotImplementedException( '`create_schema` is not implemented for this adapter!') @abc.abstractmethod def drop_schema(self, database, schema): """Drop the given schema (and everything in it) if it exists. :param str schema: The schema name to drop. """ raise dbt.exceptions.NotImplementedException( '`drop_schema` is not implemented for this adapter!') @available @abstractclassmethod def quote(cls, identifier): """Quote the given identifier, as appropriate for the database. :param str identifier: The identifier to quote :return: The quoted identifier :rtype: str """ raise dbt.exceptions.NotImplementedException( '`quote` is not implemented for this adapter!') @available def quote_as_configured(self, identifier, quote_key): """Quote or do not quote the given identifer as configured in the project config for the quote key. The quote key should be one of 'database' (on bigquery, 'profile'), 'identifier', or 'schema', or it will be treated as if you set `True`. """ default = self.Relation.DEFAULTS['quote_policy'].get(quote_key) if self.config.quoting.get(quote_key, default): return self.quote(identifier) else: return identifier ### # Conversions: These must be implemented by concrete implementations, for # converting agate types into their sql equivalents. ### @abstractclassmethod def convert_text_type(cls, agate_table, col_idx): """Return the type in the database that best maps to the agate.Text type for the given agate table and column index. :param agate.Table agate_table: The table :param int col_idx: The index into the agate table for the column. :return: The name of the type in the database :rtype: str """ raise dbt.exceptions.NotImplementedException( '`convert_text_type` is not implemented for this adapter!') @abstractclassmethod def convert_number_type(cls, agate_table, col_idx): """Return the type in the database that best maps to the agate.Number type for the given agate table and column index. :param agate.Table agate_table: The table :param int col_idx: The index into the agate table for the column. :return: The name of the type in the database :rtype: str """ raise dbt.exceptions.NotImplementedException( '`convert_number_type` is not implemented for this adapter!') @abstractclassmethod def convert_boolean_type(cls, agate_table, col_idx): """Return the type in the database that best maps to the agate.Boolean type for the given agate table and column index. :param agate.Table agate_table: The table :param int col_idx: The index into the agate table for the column. :return: The name of the type in the database :rtype: str """ raise dbt.exceptions.NotImplementedException( '`convert_boolean_type` is not implemented for this adapter!') @abstractclassmethod def convert_datetime_type(cls, agate_table, col_idx): """Return the type in the database that best maps to the agate.DateTime type for the given agate table and column index. :param agate.Table agate_table: The table :param int col_idx: The index into the agate table for the column. :return: The name of the type in the database :rtype: str """ raise dbt.exceptions.NotImplementedException( '`convert_datetime_type` is not implemented for this adapter!') @abstractclassmethod def convert_date_type(cls, agate_table, col_idx): """Return the type in the database that best maps to the agate.Date type for the given agate table and column index. :param agate.Table agate_table: The table :param int col_idx: The index into the agate table for the column. :return: The name of the type in the database :rtype: str """ raise dbt.exceptions.NotImplementedException( '`convert_date_type` is not implemented for this adapter!') @abstractclassmethod def convert_time_type(cls, agate_table, col_idx): """Return the type in the database that best maps to the agate.TimeDelta type for the given agate table and column index. :param agate.Table agate_table: The table :param int col_idx: The index into the agate table for the column. :return: The name of the type in the database :rtype: str """ raise dbt.exceptions.NotImplementedException( '`convert_time_type` is not implemented for this adapter!') @available @classmethod def convert_type(cls, agate_table, col_idx): return cls.convert_agate_type(agate_table, col_idx) @classmethod def convert_agate_type(cls, agate_table, col_idx): agate_type = agate_table.column_types[col_idx] conversions = [ (agate.Text, cls.convert_text_type), (agate.Number, cls.convert_number_type), (agate.Boolean, cls.convert_boolean_type), (agate.DateTime, cls.convert_datetime_type), (agate.Date, cls.convert_date_type), (agate.TimeDelta, cls.convert_time_type), ] for agate_cls, func in conversions: if isinstance(agate_type, agate_cls): return func(agate_table, col_idx) ### # Operations involving the manifest ### def execute_macro(self, macro_name, manifest=None, project=None, context_override=None, kwargs=None, release=False): """Look macro_name up in the manifest and execute its results. :param str macro_name: The name of the macro to execute. :param Optional[Manifest] manifest: The manifest to use for generating the base macro execution context. If none is provided, use the internal manifest. :param Optional[str] project: The name of the project to search in, or None for the first match. :param Optional[dict] context_override: An optional dict to update() the macro execution context. :param Optional[dict] kwargs: An optional dict of keyword args used to pass to the macro. :param bool release: If True, release the connection after executing. Return an an AttrDict with three attributes: 'table', 'data', and 'status'. 'table' is an agate.Table. """ if kwargs is None: kwargs = {} if context_override is None: context_override = {} if manifest is None: manifest = self._internal_manifest macro = manifest.find_macro_by_name(macro_name, project) if macro is None: if project is None: package_name = 'any package' else: package_name = 'the "{}" package'.format(project) # The import of dbt.context.runtime below shadows 'dbt' import dbt.exceptions raise dbt.exceptions.RuntimeException( 'dbt could not find a macro with the name "{}" in {}'.format( macro_name, package_name)) # This causes a reference cycle, as dbt.context.runtime.generate() # ends up calling get_adapter, so the import has to be here. import dbt.context.operation macro_context = dbt.context.operation.generate(macro, self.config, manifest) macro_context.update(context_override) macro_function = macro.generator(macro_context) try: result = macro_function(**kwargs) finally: if release: self.release_connection() return result @classmethod def _catalog_filter_table(cls, table, manifest): """Filter the table as appropriate for catalog entries. Subclasses can override this to change filtering rules on a per-adapter basis. """ return table.where(_catalog_filter_schemas(manifest)) def get_catalog(self, manifest): """Get the catalog for this manifest by running the get catalog macro. Returns an agate.Table of catalog information. """ information_schemas = list(self._get_cache_schemas(manifest).keys()) # make it a list so macros can index into it. kwargs = {'information_schemas': information_schemas} table = self.execute_macro(GET_CATALOG_MACRO_NAME, kwargs=kwargs, release=True) results = self._catalog_filter_table(table, manifest) return results def cancel_open_connections(self): """Cancel all open connections.""" return self.connections.cancel_open() def calculate_freshness(self, source, loaded_at_field, manifest=None): """Calculate the freshness of sources in dbt, and return it""" # in the future `source` will be a Relation instead of a string kwargs = {'source': source, 'loaded_at_field': loaded_at_field} # run the macro table = self.execute_macro(FRESHNESS_MACRO_NAME, kwargs=kwargs, release=True, manifest=manifest) # now we have a 1-row table of the maximum `loaded_at_field` value and # the current time according to the db. if len(table) != 1 or len(table[0]) != 2: dbt.exceptions.raise_compiler_error( 'Got an invalid result from "{}" macro: {}'.format( FRESHNESS_MACRO_NAME, [tuple(r) for r in table])) max_loaded_at = _utc(table[0][0], source, loaded_at_field) snapshotted_at = _utc(table[0][1], source, loaded_at_field) age = (snapshotted_at - max_loaded_at).total_seconds() return { 'max_loaded_at': max_loaded_at, 'snapshotted_at': snapshotted_at, 'age': age, }
class BaseAdapter(metaclass=AdapterMeta): """The BaseAdapter provides an abstract base class for adapters. Adapters must implement the following methods and macros. Some of the methods can be safely overridden as a noop, where it makes sense (transactions on databases that don't support them, for instance). Those methods are marked with a (passable) in their docstrings. Check docstrings for type information, etc. To implement a macro, implement "${adapter_type}__${macro_name}". in the adapter's internal project. Methods: - exception_handler - date_function - list_schemas - drop_relation - truncate_relation - rename_relation - get_columns_in_relation - expand_column_types - list_relations_without_caching - is_cancelable - create_schema - drop_schema - quote - convert_text_type - convert_number_type - convert_boolean_type - convert_datetime_type - convert_date_type - convert_time_type Macros: - get_catalog """ Relation: Type[BaseRelation] = BaseRelation Column: Type[BaseColumn] = BaseColumn ConnectionManager: Type[BaseConnectionManager] # A set of clobber config fields accepted by this adapter # for use in materializations AdapterSpecificConfigs: Type[AdapterConfig] = AdapterConfig def __init__(self, config): self.config = config self.cache = RelationsCache() self.connections = self.ConnectionManager(config) self._internal_manifest_lazy: Optional[Manifest] = None ### # Methods that pass through to the connection manager ### def acquire_connection(self, name=None) -> Connection: return self.connections.set_connection_name(name) def release_connection(self) -> None: self.connections.release() def cleanup_connections(self) -> None: self.connections.cleanup_all() def clear_transaction(self) -> None: self.connections.clear_transaction() def commit_if_has_connection(self) -> None: self.connections.commit_if_has_connection() def nice_connection_name(self) -> str: conn = self.connections.get_if_exists() if conn is None or conn.name is None: return '<None>' return conn.name @contextmanager def connection_named( self, name: str, node: Optional[CompileResultNode] = None) -> Iterator[None]: try: if self.connections.query_header is not None: self.connections.query_header.set(name, node) self.acquire_connection(name) yield finally: self.release_connection() if self.connections.query_header is not None: self.connections.query_header.reset() @contextmanager def connection_for(self, node: CompileResultNode) -> Iterator[None]: with self.connection_named(node.unique_id, node): yield @available.parse(lambda *a, **k: ('', empty_table())) def execute(self, sql: str, auto_begin: bool = False, fetch: bool = False) -> Tuple[str, agate.Table]: """Execute the given SQL. This is a thin wrapper around ConnectionManager.execute. :param str sql: The sql to execute. :param bool auto_begin: If set, and dbt is not currently inside a transaction, automatically begin one. :param bool fetch: If set, fetch results. :return: A tuple of the status and the results (empty if fetch=False). :rtype: Tuple[str, agate.Table] """ return self.connections.execute(sql=sql, auto_begin=auto_begin, fetch=fetch) ### # Methods that should never be overridden ### @classmethod def type(cls) -> str: """Get the type of this adapter. Types must be class-unique and consistent. :return: The type name :rtype: str """ return cls.ConnectionManager.TYPE @property def _internal_manifest(self) -> Manifest: if self._internal_manifest_lazy is None: return self.load_internal_manifest() return self._internal_manifest_lazy def check_internal_manifest(self) -> Optional[Manifest]: """Return the internal manifest (used for executing macros) if it's been initialized, otherwise return None. """ return self._internal_manifest_lazy def load_internal_manifest(self) -> Manifest: if self._internal_manifest_lazy is None: # avoid a circular import from dbt.parser.manifest import load_internal_manifest manifest = load_internal_manifest(self.config) self._internal_manifest_lazy = manifest return self._internal_manifest_lazy ### # Caching methods ### def _schema_is_cached(self, database: Optional[str], schema: str) -> bool: """Check if the schema is cached, and by default logs if it is not.""" if dbt.flags.USE_CACHE is False: return False elif (database, schema) not in self.cache: logger.debug( 'On "{}": cache miss for schema "{}.{}", this is inefficient'. format(self.nice_connection_name(), database, schema)) return False else: return True def _get_cache_schemas(self, manifest: Manifest) -> Set[BaseRelation]: """Get the set of schema relations that the cache logic needs to populate. This means only executable nodes are included. """ # the cache only cares about executable nodes return { self.Relation.create_from(self.config, node).without_identifier() for node in manifest.nodes.values() if node.resource_type in NodeType.executable() } def _get_catalog_schemas(self, manifest: Manifest) -> SchemaSearchMap: """Get a mapping of each node's "information_schema" relations to a set of all schemas expected in that information_schema. There may be keys that are technically duplicates on the database side, for example all of '"foo", 'foo', '"FOO"' and 'FOO' could coexist as databases, and values could overlap as appropriate. All values are lowercase strings. """ info_schema_name_map = SchemaSearchMap() nodes: Iterator[CompileResultNode] = chain( manifest.nodes.values(), manifest.sources.values(), ) for node in nodes: relation = self.Relation.create_from(self.config, node) info_schema_name_map.add(relation) # result is a map whose keys are information_schema Relations without # identifiers that have appropriate database prefixes, and whose values # are sets of lowercase schema names that are valid members of those # databases return info_schema_name_map def _list_relations_get_connection( self, schema_relation: BaseRelation) -> List[BaseRelation]: name = f'list_{schema_relation.database}_{schema_relation.schema}' with self.connection_named(name): return self.list_relations_without_caching(schema_relation) def _relations_cache_for_schemas(self, manifest: Manifest) -> None: """Populate the relations cache for the given schemas. Returns an iterable of the schemas populated, as strings. """ if not dbt.flags.USE_CACHE: return cache_schemas = self._get_cache_schemas(manifest) with executor(self.config) as tpe: futures: List[Future[List[BaseRelation]]] = [ tpe.submit(self._list_relations_get_connection, cache_schema) for cache_schema in cache_schemas ] for future in as_completed(futures): # if we can't read the relations we need to just raise anyway, # so just call future.result() and let that raise on failure for relation in future.result(): self.cache.add(relation) # it's possible that there were no relations in some schemas. We want # to insert the schemas we query into the cache's `.schemas` attribute # so we can check it later cache_update: Set[Tuple[Optional[str], Optional[str]]] = set() for relation in cache_schemas: cache_update.add((relation.database, relation.schema)) self.cache.update_schemas(cache_update) def set_relations_cache(self, manifest: Manifest, clear: bool = False) -> None: """Run a query that gets a populated cache of the relations in the database and set the cache on this adapter. """ if not dbt.flags.USE_CACHE: return with self.cache.lock: if clear: self.cache.clear() self._relations_cache_for_schemas(manifest) @available def cache_added(self, relation: Optional[BaseRelation]) -> str: """Cache a new relation in dbt. It will show up in `list relations`.""" if relation is None: name = self.nice_connection_name() raise_compiler_error( 'Attempted to cache a null relation for {}'.format(name)) if dbt.flags.USE_CACHE: self.cache.add(relation) # so jinja doesn't render things return '' @available def cache_dropped(self, relation: Optional[BaseRelation]) -> str: """Drop a relation in dbt. It will no longer show up in `list relations`, and any bound views will be dropped from the cache """ if relation is None: name = self.nice_connection_name() raise_compiler_error( 'Attempted to drop a null relation for {}'.format(name)) if dbt.flags.USE_CACHE: self.cache.drop(relation) return '' @available def cache_renamed( self, from_relation: Optional[BaseRelation], to_relation: Optional[BaseRelation], ) -> str: """Rename a relation in dbt. It will show up with a new name in `list_relations`, but bound views will remain bound. """ if from_relation is None or to_relation is None: name = self.nice_connection_name() src_name = _relation_name(from_relation) dst_name = _relation_name(to_relation) raise_compiler_error('Attempted to rename {} to {} for {}'.format( src_name, dst_name, name)) if dbt.flags.USE_CACHE: self.cache.rename(from_relation, to_relation) return '' ### # Abstract methods for database-specific values, attributes, and types ### @abc.abstractclassmethod def date_function(cls) -> str: """Get the date function used by this adapter's database.""" raise NotImplementedException( '`date_function` is not implemented for this adapter!') @abc.abstractclassmethod def is_cancelable(cls) -> bool: raise NotImplementedException( '`is_cancelable` is not implemented for this adapter!') ### # Abstract methods about schemas ### @abc.abstractmethod def list_schemas(self, database: str) -> List[str]: """Get a list of existing schemas in database""" raise NotImplementedException( '`list_schemas` is not implemented for this adapter!') @available.parse(lambda *a, **k: False) def check_schema_exists(self, database: str, schema: str) -> bool: """Check if a schema exists. The default implementation of this is potentially unnecessarily slow, and adapters should implement it if there is an optimized path (and there probably is) """ search = (s.lower() for s in self.list_schemas(database=database)) return schema.lower() in search ### # Abstract methods about relations ### @abc.abstractmethod @available.parse_none def drop_relation(self, relation: BaseRelation) -> None: """Drop the given relation. *Implementors must call self.cache.drop() to preserve cache state!* """ raise NotImplementedException( '`drop_relation` is not implemented for this adapter!') @abc.abstractmethod @available.parse_none def truncate_relation(self, relation: BaseRelation) -> None: """Truncate the given relation.""" raise NotImplementedException( '`truncate_relation` is not implemented for this adapter!') @abc.abstractmethod @available.parse_none def rename_relation(self, from_relation: BaseRelation, to_relation: BaseRelation) -> None: """Rename the relation from from_relation to to_relation. Implementors must call self.cache.rename() to preserve cache state. """ raise NotImplementedException( '`rename_relation` is not implemented for this adapter!') @abc.abstractmethod @available.parse_list def get_columns_in_relation(self, relation: BaseRelation) -> List[BaseColumn]: """Get a list of the columns in the given Relation.""" raise NotImplementedException( '`get_columns_in_relation` is not implemented for this adapter!') @available.deprecated('get_columns_in_relation', lambda *a, **k: []) def get_columns_in_table(self, schema: str, identifier: str) -> List[BaseColumn]: """DEPRECATED: Get a list of the columns in the given table.""" relation = self.Relation.create( database=self.config.credentials.database, schema=schema, identifier=identifier, quote_policy=self.config.quoting) return self.get_columns_in_relation(relation) @abc.abstractmethod def expand_column_types(self, goal: BaseRelation, current: BaseRelation) -> None: """Expand the current table's types to match the goal table. (passable) :param self.Relation goal: A relation that currently exists in the database with columns of the desired types. :param self.Relation current: A relation that currently exists in the database with columns of unspecified types. """ raise NotImplementedException( '`expand_target_column_types` is not implemented for this adapter!' ) @abc.abstractmethod def list_relations_without_caching( self, schema_relation: BaseRelation) -> List[BaseRelation]: """List relations in the given schema, bypassing the cache. This is used as the underlying behavior to fill the cache. :param schema_relation: A relation containing the database and schema as appropraite for the underlying data warehouse :return: The relations in schema :rtype: List[self.Relation] """ raise NotImplementedException( '`list_relations_without_caching` is not implemented for this ' 'adapter!') ### # Provided methods about relations ### @available.parse_list def get_missing_columns(self, from_relation: BaseRelation, to_relation: BaseRelation) -> List[BaseColumn]: """Returns a list of Columns in from_relation that are missing from to_relation. """ if not isinstance(from_relation, self.Relation): invalid_type_error(method_name='get_missing_columns', arg_name='from_relation', got_value=from_relation, expected_type=self.Relation) if not isinstance(to_relation, self.Relation): invalid_type_error(method_name='get_missing_columns', arg_name='to_relation', got_value=to_relation, expected_type=self.Relation) from_columns = { col.name: col for col in self.get_columns_in_relation(from_relation) } to_columns = { col.name: col for col in self.get_columns_in_relation(to_relation) } missing_columns = set(from_columns.keys()) - set(to_columns.keys()) return [ col for (col_name, col) in from_columns.items() if col_name in missing_columns ] @available.parse_none def valid_snapshot_target(self, relation: BaseRelation) -> None: """Ensure that the target relation is valid, by making sure it has the expected columns. :param Relation relation: The relation to check :raises CompilationException: If the columns are incorrect. """ if not isinstance(relation, self.Relation): invalid_type_error(method_name='valid_snapshot_target', arg_name='relation', got_value=relation, expected_type=self.Relation) columns = self.get_columns_in_relation(relation) names = set(c.name.lower() for c in columns) expanded_keys = ('scd_id', 'valid_from', 'valid_to') extra = [] missing = [] for legacy in expanded_keys: desired = 'dbt_' + legacy if desired not in names: missing.append(desired) if legacy in names: extra.append(legacy) if missing: if extra: msg = ('Snapshot target has ("{}") but not ("{}") - is it an ' 'unmigrated previous version archive?'.format( '", "'.join(extra), '", "'.join(missing))) else: msg = ('Snapshot target is not a snapshot table (missing "{}")' .format('", "'.join(missing))) raise_compiler_error(msg) @available.parse_none def expand_target_column_types(self, from_relation: BaseRelation, to_relation: BaseRelation) -> None: if not isinstance(from_relation, self.Relation): invalid_type_error(method_name='expand_target_column_types', arg_name='from_relation', got_value=from_relation, expected_type=self.Relation) if not isinstance(to_relation, self.Relation): invalid_type_error(method_name='expand_target_column_types', arg_name='to_relation', got_value=to_relation, expected_type=self.Relation) self.expand_column_types(from_relation, to_relation) def list_relations(self, database: Optional[str], schema: str) -> List[BaseRelation]: if self._schema_is_cached(database, schema): return self.cache.get_relations(database, schema) schema_relation = self.Relation.create( database=database, schema=schema, identifier='', quote_policy=self.config.quoting).without_identifier() # we can't build the relations cache because we don't have a # manifest so we can't run any operations. relations = self.list_relations_without_caching(schema_relation) logger.debug('with database={}, schema={}, relations={}'.format( database, schema, relations)) return relations def _make_match_kwargs(self, database: str, schema: str, identifier: str) -> Dict[str, str]: quoting = self.config.quoting if identifier is not None and quoting['identifier'] is False: identifier = identifier.lower() if schema is not None and quoting['schema'] is False: schema = schema.lower() if database is not None and quoting['database'] is False: database = database.lower() return filter_null_values({ 'database': database, 'identifier': identifier, 'schema': schema, }) def _make_match( self, relations_list: List[BaseRelation], database: str, schema: str, identifier: str, ) -> List[BaseRelation]: matches = [] search = self._make_match_kwargs(database, schema, identifier) for relation in relations_list: if relation.matches(**search): matches.append(relation) return matches @available.parse_none def get_relation(self, database: str, schema: str, identifier: str) -> Optional[BaseRelation]: relations_list = self.list_relations(database, schema) matches = self._make_match(relations_list, database, schema, identifier) if len(matches) > 1: kwargs = { 'identifier': identifier, 'schema': schema, 'database': database, } get_relation_returned_multiple_results(kwargs, matches) elif matches: return matches[0] return None @available.deprecated('get_relation', lambda *a, **k: False) def already_exists(self, schema: str, name: str) -> bool: """DEPRECATED: Return if a model already exists in the database""" database = self.config.credentials.database relation = self.get_relation(database, schema, name) return relation is not None ### # ODBC FUNCTIONS -- these should not need to change for every adapter, # although some adapters may override them ### @abc.abstractmethod @available.parse_none def create_schema(self, relation: BaseRelation): """Create the given schema if it does not exist.""" raise NotImplementedException( '`create_schema` is not implemented for this adapter!') @abc.abstractmethod @available.parse_none def drop_schema(self, relation: BaseRelation): """Drop the given schema (and everything in it) if it exists.""" raise NotImplementedException( '`drop_schema` is not implemented for this adapter!') @available @abc.abstractclassmethod def quote(cls, identifier: str) -> str: """Quote the given identifier, as appropriate for the database.""" raise NotImplementedException( '`quote` is not implemented for this adapter!') @available def quote_as_configured(self, identifier: str, quote_key: str) -> str: """Quote or do not quote the given identifer as configured in the project config for the quote key. The quote key should be one of 'database' (on bigquery, 'profile'), 'identifier', or 'schema', or it will be treated as if you set `True`. """ try: key = ComponentName(quote_key) except ValueError: return identifier default = self.Relation.get_default_quote_policy().get_part(key) if self.config.quoting.get(key, default): return self.quote(identifier) else: return identifier @available def quote_seed_column(self, column: str, quote_config: Optional[bool]) -> str: # this is the default for now quote_columns: bool = False if isinstance(quote_config, bool): quote_columns = quote_config elif quote_config is None: deprecations.warn('column-quoting-unset') else: raise_compiler_error( f'The seed configuration value of "quote_columns" has an ' f'invalid type {type(quote_config)}') if quote_columns: return self.quote(column) else: return column ### # Conversions: These must be implemented by concrete implementations, for # converting agate types into their sql equivalents. ### @abc.abstractclassmethod def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str: """Return the type in the database that best maps to the agate.Text type for the given agate table and column index. :param agate_table: The table :param col_idx: The index into the agate table for the column. :return: The name of the type in the database """ raise NotImplementedException( '`convert_text_type` is not implemented for this adapter!') @abc.abstractclassmethod def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: """Return the type in the database that best maps to the agate.Number type for the given agate table and column index. :param agate_table: The table :param col_idx: The index into the agate table for the column. :return: The name of the type in the database """ raise NotImplementedException( '`convert_number_type` is not implemented for this adapter!') @abc.abstractclassmethod def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str: """Return the type in the database that best maps to the agate.Boolean type for the given agate table and column index. :param agate_table: The table :param col_idx: The index into the agate table for the column. :return: The name of the type in the database """ raise NotImplementedException( '`convert_boolean_type` is not implemented for this adapter!') @abc.abstractclassmethod def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: """Return the type in the database that best maps to the agate.DateTime type for the given agate table and column index. :param agate_table: The table :param col_idx: The index into the agate table for the column. :return: The name of the type in the database """ raise NotImplementedException( '`convert_datetime_type` is not implemented for this adapter!') @abc.abstractclassmethod def convert_date_type(cls, agate_table: agate.Table, col_idx: int) -> str: """Return the type in the database that best maps to the agate.Date type for the given agate table and column index. :param agate_table: The table :param col_idx: The index into the agate table for the column. :return: The name of the type in the database """ raise NotImplementedException( '`convert_date_type` is not implemented for this adapter!') @abc.abstractclassmethod def convert_time_type(cls, agate_table: agate.Table, col_idx: int) -> str: """Return the type in the database that best maps to the agate.TimeDelta type for the given agate table and column index. :param agate_table: The table :param col_idx: The index into the agate table for the column. :return: The name of the type in the database """ raise NotImplementedException( '`convert_time_type` is not implemented for this adapter!') @available @classmethod def convert_type(cls, agate_table: agate.Table, col_idx: int) -> Optional[str]: return cls.convert_agate_type(agate_table, col_idx) @classmethod def convert_agate_type(cls, agate_table: agate.Table, col_idx: int) -> Optional[str]: agate_type: Type = agate_table.column_types[col_idx] conversions: List[Tuple[Type, Callable[..., str]]] = [ (agate.Text, cls.convert_text_type), (agate.Number, cls.convert_number_type), (agate.Boolean, cls.convert_boolean_type), (agate.DateTime, cls.convert_datetime_type), (agate.Date, cls.convert_date_type), (agate.TimeDelta, cls.convert_time_type), ] for agate_cls, func in conversions: if isinstance(agate_type, agate_cls): return func(agate_table, col_idx) return None ### # Operations involving the manifest ### def execute_macro( self, macro_name: str, manifest: Optional[Manifest] = None, project: Optional[str] = None, context_override: Optional[Dict[str, Any]] = None, kwargs: Dict[str, Any] = None, release: bool = False, text_only_columns: Optional[Iterable[str]] = None, ) -> agate.Table: """Look macro_name up in the manifest and execute its results. :param macro_name: The name of the macro to execute. :param manifest: The manifest to use for generating the base macro execution context. If none is provided, use the internal manifest. :param project: The name of the project to search in, or None for the first match. :param context_override: An optional dict to update() the macro execution context. :param kwargs: An optional dict of keyword args used to pass to the macro. :param release: If True, release the connection after executing. """ if kwargs is None: kwargs = {} if context_override is None: context_override = {} if manifest is None: manifest = self._internal_manifest macro = manifest.find_macro_by_name(macro_name, self.config.project_name, project) if macro is None: if project is None: package_name = 'any package' else: package_name = 'the "{}" package'.format(project) raise RuntimeException( 'dbt could not find a macro with the name "{}" in {}'.format( macro_name, package_name)) # This causes a reference cycle, as generate_runtime_macro() # ends up calling get_adapter, so the import has to be here. from dbt.context.providers import generate_runtime_macro macro_context = generate_runtime_macro(macro=macro, config=self.config, manifest=manifest, package_name=project) macro_context.update(context_override) macro_function = MacroGenerator(macro, macro_context) with self.connections.exception_handler(f'macro {macro_name}'): try: result = macro_function(**kwargs) finally: if release: self.release_connection() return result @classmethod def _catalog_filter_table(cls, table: agate.Table, manifest: Manifest) -> agate.Table: """Filter the table as appropriate for catalog entries. Subclasses can override this to change filtering rules on a per-adapter basis. """ # force database + schema to be strings table = table_from_rows( table.rows, table.column_names, text_only_columns=['table_database', 'table_schema', 'table_name']) return table.where(_catalog_filter_schemas(manifest)) def _get_one_catalog( self, information_schema: InformationSchema, schemas: Set[str], manifest: Manifest, ) -> agate.Table: name = '.'.join( [str(information_schema.database), 'information_schema']) with self.connection_named(name): kwargs = { 'information_schema': information_schema, 'schemas': schemas } table = self.execute_macro( GET_CATALOG_MACRO_NAME, kwargs=kwargs, release=True, # pass in the full manifest so we get any local project # overrides manifest=manifest, ) results = self._catalog_filter_table(table, manifest) return results def get_catalog(self, manifest: Manifest) -> Tuple[agate.Table, List[Exception]]: schema_map = self._get_catalog_schemas(manifest) with executor(self.config) as tpe: futures: List[Future[agate.Table]] = [ tpe.submit(self._get_one_catalog, info, schemas, manifest) for info, schemas in schema_map.items() if len(schemas) > 0 ] catalogs, exceptions = catch_as_completed(futures) return catalogs, exceptions def cancel_open_connections(self): """Cancel all open connections.""" return self.connections.cancel_open() def calculate_freshness( self, source: BaseRelation, loaded_at_field: str, filter: Optional[str], manifest: Optional[Manifest] = None) -> Dict[str, Any]: """Calculate the freshness of sources in dbt, and return it""" kwargs: Dict[str, Any] = { 'source': source, 'loaded_at_field': loaded_at_field, 'filter': filter, } # run the macro table = self.execute_macro(FRESHNESS_MACRO_NAME, kwargs=kwargs, release=True, manifest=manifest) # now we have a 1-row table of the maximum `loaded_at_field` value and # the current time according to the db. if len(table) != 1 or len(table[0]) != 2: raise_compiler_error( 'Got an invalid result from "{}" macro: {}'.format( FRESHNESS_MACRO_NAME, [tuple(r) for r in table])) if table[0][0] is None: # no records in the table, so really the max_loaded_at was # infinitely long ago. Just call it 0:00 January 1 year UTC max_loaded_at = datetime(1, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) else: max_loaded_at = _utc(table[0][0], source, loaded_at_field) snapshotted_at = _utc(table[0][1], source, loaded_at_field) age = (snapshotted_at - max_loaded_at).total_seconds() return { 'max_loaded_at': max_loaded_at, 'snapshotted_at': snapshotted_at, 'age': age, } def pre_model_hook(self, config: Mapping[str, Any]) -> Any: """A hook for running some operation before the model materialization runs. The hook can assume it has a connection available. The only parameter is a configuration dictionary (the same one available in the materialization context). It should be considered read-only. The pre-model hook may return anything as a context, which will be passed to the post-model hook. """ pass def post_model_hook(self, config: Mapping[str, Any], context: Any) -> None: """A hook for running some operation after the model materialization runs. The hook can assume it has a connection available. The first parameter is a configuration dictionary (the same one available in the materialization context). It should be considered read-only. The second parameter is the value returned by pre_mdoel_hook. """ pass
def __init__(self, config): self.config = config self.cache = RelationsCache() self.connections = self.ConnectionManager(config) self._macro_manifest_lazy: Optional[MacroManifest] = None
class DefaultAdapter(object): requires = {} config_functions = [ "get_columns_in_table", "get_missing_columns", "expand_target_column_types", "create_schema", "quote_as_configured", "cache_new_relation", # deprecated -- use versions that take relations instead "already_exists", "query_for_existing", "rename", "drop", "truncate", # just deprecated. going away in a future release "quote_schema_and_table", # versions of adapter functions that take / return Relations "get_relation", "drop_relation", "rename_relation", "truncate_relation", # formerly profile functions "execute", "add_query", ] raw_functions = [ "get_status", "get_result_from_cursor", "quote", "convert_type", ] Relation = DefaultRelation Column = Column def __init__(self, config): self.config = config self.cache = RelationsCache() ### # ADAPTER-SPECIFIC FUNCTIONS -- each of these must be overridden in # every adapter ### @contextmanager def exception_handler(self, sql, model_name=None, connection_name=None): raise dbt.exceptions.NotImplementedException( '`exception_handler` is not implemented for this adapter!') @classmethod def type(cls): raise dbt.exceptions.NotImplementedException( '`type` is not implemented for this adapter!') @classmethod def date_function(cls): raise dbt.exceptions.NotImplementedException( '`date_function` is not implemented for this adapter!') @classmethod def get_status(cls, cursor): raise dbt.exceptions.NotImplementedException( '`get_status` is not implemented for this adapter!') def alter_column_type(self, schema, table, column_name, new_column_type, model_name=None): raise dbt.exceptions.NotImplementedException( '`alter_column_type` is not implemented for this adapter!') def query_for_existing(self, schemas, model_name=None): if not isinstance(schemas, (list, tuple)): schemas = [schemas] all_relations = [] for schema in schemas: all_relations.extend(self.list_relations(schema, model_name)) return { relation.identifier: relation.type for relation in all_relations } def get_existing_schemas(self, model_name=None): raise dbt.exceptions.NotImplementedException( '`get_existing_schemas` is not implemented for this adapter!') def check_schema_exists(self, schema): raise dbt.exceptions.NotImplementedException( '`check_schema_exists` is not implemented for this adapter!') def cancel_connection(self, connection): raise dbt.exceptions.NotImplementedException( '`cancel_connection` is not implemented for this adapter!') ### # FUNCTIONS THAT SHOULD BE ABSTRACT ### def cache_new_relation(self, relation, model_name=None): """Cache a new relation in dbt. It will show up in `list relations`.""" if relation is None: dbt.exceptions.raise_compiler_error( 'Attempted to cache a null relation for {}'.format(model_name)) if dbt.flags.USE_CACHE: self.cache.add(relation) # so jinja doesn't render things return '' @classmethod def get_result_from_cursor(cls, cursor): data = [] column_names = [] if cursor.description is not None: column_names = [col[0] for col in cursor.description] raw_results = cursor.fetchall() data = [dict(zip(column_names, row)) for row in raw_results] return dbt.clients.agate_helper.table_from_data(data, column_names) def drop(self, schema, relation, relation_type, model_name=None): identifier = relation relation = self.Relation.create(schema=schema, identifier=identifier, type=relation_type, quote_policy=self.config.quoting) return self.drop_relation(relation, model_name) def drop_relation(self, relation, model_name=None): if dbt.flags.USE_CACHE: self.cache.drop(relation) if relation.type is None: dbt.exceptions.raise_compiler_error( 'Tried to drop relation {}, but its type is null.'.format( relation)) sql = 'drop {} if exists {} cascade'.format(relation.type, relation) connection, cursor = self.add_query(sql, model_name, auto_begin=False) def truncate(self, schema, table, model_name=None): relation = self.Relation.create(schema=schema, identifier=table, type='table', quote_policy=self.config.quoting) return self.truncate_relation(relation, model_name) def truncate_relation(self, relation, model_name=None): sql = 'truncate table {}'.format(relation) connection, cursor = self.add_query(sql, model_name) def rename(self, schema, from_name, to_name, model_name=None): quote_policy = self.config.quoting from_relation = self.Relation.create(schema=schema, identifier=from_name, quote_policy=quote_policy) to_relation = self.Relation.create(identifier=to_name, quote_policy=quote_policy) return self.rename_relation(from_relation=from_relation, to_relation=to_relation, model_name=model_name) def rename_relation(self, from_relation, to_relation, model_name=None): if dbt.flags.USE_CACHE: self.cache.rename(from_relation, to_relation) sql = 'alter {} {} rename to {}'.format( to_relation.type, from_relation, to_relation.include(schema=False)) connection, cursor = self.add_query(sql, model_name) @classmethod def is_cancelable(cls): return True def get_missing_columns(self, from_schema, from_table, to_schema, to_table, model_name=None): """Returns dict of {column:type} for columns in from_table that are missing from to_table""" from_columns = { col.name: col for col in self.get_columns_in_table( from_schema, from_table, model_name=model_name) } to_columns = { col.name: col for col in self.get_columns_in_table( to_schema, to_table, model_name=model_name) } missing_columns = set(from_columns.keys()) - set(to_columns.keys()) return [ col for (col_name, col) in from_columns.items() if col_name in missing_columns ] @classmethod def _get_columns_in_table_sql(cls, schema_name, table_name, database): schema_filter = '1=1' if schema_name is not None: schema_filter = "table_schema = '{}'".format(schema_name) db_prefix = '' if database is None else '{}.'.format(database) sql = """ select column_name, data_type, character_maximum_length, numeric_precision || ',' || numeric_scale as numeric_size from {db_prefix}information_schema.columns where table_name = '{table_name}' and {schema_filter} order by ordinal_position """.format(db_prefix=db_prefix, table_name=table_name, schema_filter=schema_filter).strip() return sql def get_columns_in_table(self, schema_name, table_name, database=None, model_name=None): sql = self._get_columns_in_table_sql(schema_name, table_name, database) connection, cursor = self.add_query(sql, model_name) data = cursor.fetchall() columns = [] for row in data: name, data_type, char_size, numeric_size = row column = self.Column(name, data_type, char_size, numeric_size) columns.append(column) return columns @classmethod def _table_columns_to_dict(cls, columns): return {col.name: col for col in columns} def expand_target_column_types(self, temp_table, to_schema, to_table, model_name=None): reference_columns = self._table_columns_to_dict( self.get_columns_in_table(None, temp_table, model_name=model_name)) target_columns = self._table_columns_to_dict( self.get_columns_in_table(to_schema, to_table, model_name=model_name)) for column_name, reference_column in reference_columns.items(): target_column = target_columns.get(column_name) if target_column is not None and \ target_column.can_expand_to(reference_column): col_string_size = reference_column.string_size() new_type = self.Column.string_type(col_string_size) logger.debug("Changing col type from %s to %s in table %s.%s", target_column.data_type, new_type, to_schema, to_table) self.alter_column_type(to_schema, to_table, column_name, new_type, model_name) ### # RELATIONS ### def _schema_is_cached(self, schema, model_name=None, debug_on_missing=True): """Check if the schema is cached, and by default logs if it is not.""" if dbt.flags.USE_CACHE is False: return False elif schema not in self.cache: if debug_on_missing: logger.debug( 'On "{}": cache miss for schema "{}", this is inefficient'. format(model_name or '<None>', schema)) return False else: return True def _list_relations(self, schema, model_name=None): raise dbt.exceptions.NotImplementedException( '`list_relations` is not implemented for this adapter!') def list_relations(self, schema, model_name=None): if self._schema_is_cached(schema, model_name): return self.cache.get_relations(schema) # we can't build the relations cache because we don't have a # manifest so we can't run any operations. relations = self._list_relations(schema, model_name=model_name) logger.debug('with schema={}, model_name={}, relations={}'.format( schema, model_name, relations)) return relations def _make_match_kwargs(self, schema, identifier): quoting = self.config.quoting if identifier is not None and quoting['identifier'] is False: identifier = identifier.lower() if schema is not None and quoting['schema'] is False: schema = schema.lower() return filter_null_values({'identifier': identifier, 'schema': schema}) def _make_match(self, relations_list, schema, identifier): matches = [] search = self._make_match_kwargs(schema, identifier) for relation in relations_list: if relation.matches(**search): matches.append(relation) return matches def get_relation(self, schema, identifier, model_name=None): relations_list = self.list_relations(schema, model_name) matches = self._make_match(relations_list, schema, identifier) if len(matches) > 1: dbt.exceptions.get_relation_returned_multiple_results( { 'identifier': identifier, 'schema': schema }, matches) elif matches: return matches[0] return None ### # SANE ANSI SQL DEFAULTS ### def get_create_schema_sql(self, schema): schema = self.quote_as_configured(schema, 'schema') return ('create schema if not exists {schema}'.format(schema=schema)) def get_drop_schema_sql(self, schema): schema = self.quote_as_configured(schema, 'schema') return ('drop schema if exists {schema} cascade'.format(schema=schema)) ### # ODBC FUNCTIONS -- these should not need to change for every adapter, # although some adapters may override them ### def get_default_schema(self): return self.config.credentials.schema def get_connection(self, name=None, recache_if_missing=True): global connections_in_use if name is None: # if a name isn't specified, we'll re-use a single handle # named 'master' name = 'master' if connections_in_use.get(name): return connections_in_use.get(name) if not recache_if_missing: raise dbt.exceptions.InternalException( 'Tried to get a connection "{}" which does not exist ' '(recache_if_missing is off).'.format(name)) logger.debug('Acquiring new {} connection "{}".'.format( self.type(), name)) connection = self.acquire_connection(name) connections_in_use[name] = connection return self.get_connection(name) def cancel_open_connections(self): global connections_in_use for name, connection in connections_in_use.items(): if name == 'master': continue self.cancel_connection(connection) yield name @classmethod def total_connections_allocated(cls): global connections_in_use, connections_available return len(connections_in_use) + len(connections_available) def acquire_connection(self, name): global connections_available, lock # we add a magic number, 2 because there are overhead connections, # one for pre- and post-run hooks and other misc operations that occur # before the run starts, and one for integration tests. max_connections = self.config.threads + 2 with lock: num_allocated = self.total_connections_allocated() if len(connections_available) > 0: logger.debug('Re-using an available connection from the pool.') to_return = connections_available.pop() to_return.name = name return to_return elif num_allocated >= max_connections: raise dbt.exceptions.InternalException( 'Tried to request a new connection "{}" but ' 'the maximum number of connections are already ' 'allocated!'.format(name)) logger.debug( 'Opening a new connection ({} currently allocated)'.format( num_allocated)) result = Connection(type=self.type(), name=name, state='init', transaction_open=False, handle=None, credentials=self.config.credentials) return self.open_connection(result) def release_connection(self, name): global connections_in_use, connections_available, lock with lock: if name not in connections_in_use: return to_release = self.get_connection(name, recache_if_missing=False) if to_release.state == 'open': if to_release.transaction_open is True: self.rollback(to_release) to_release.name = None connections_available.append(to_release) else: self.close(to_release) del connections_in_use[name] @classmethod def cleanup_connections(cls): global connections_in_use, connections_available, lock with lock: for name, connection in connections_in_use.items(): if connection.get('state') != 'closed': logger.debug("Connection '{}' was left open.".format(name)) else: logger.debug( "Connection '{}' was properly closed.".format(name)) conns_in_use = list(connections_in_use.values()) for conn in conns_in_use + connections_available: cls.close(conn) # garbage collect these connections connections_in_use = {} connections_available = [] def reload(self, connection): return self.get_connection(connection.name) def add_begin_query(self, name): return self.add_query('BEGIN', name, auto_begin=False) def add_commit_query(self, name): return self.add_query('COMMIT', name, auto_begin=False) def begin(self, name): global connections_in_use connection = self.get_connection(name) if dbt.flags.STRICT_MODE: assert isinstance(connection, Connection) if connection.transaction_open is True: raise dbt.exceptions.InternalException( 'Tried to begin a new transaction on connection "{}", but ' 'it already had one open!'.format(connection.get('name'))) self.add_begin_query(name) connection.transaction_open = True connections_in_use[name] = connection return connection def commit_if_has_connection(self, name): global connections_in_use if name is None: name = 'master' if connections_in_use.get(name) is None: return connection = self.get_connection(name, False) return self.commit(connection) def commit(self, connection): global connections_in_use if dbt.flags.STRICT_MODE: assert isinstance(connection, Connection) connection = self.reload(connection) if connection.transaction_open is False: raise dbt.exceptions.InternalException( 'Tried to commit transaction on connection "{}", but ' 'it does not have one open!'.format(connection.name)) logger.debug('On {}: COMMIT'.format(connection.name)) self.add_commit_query(connection.name) connection.transaction_open = False connections_in_use[connection.name] = connection return connection def rollback(self, connection): if dbt.flags.STRICT_MODE: Connection(**connection) connection = self.reload(connection) if connection.transaction_open is False: raise dbt.exceptions.InternalException( 'Tried to rollback transaction on connection "{}", but ' 'it does not have one open!'.format(connection.name)) logger.debug('On {}: ROLLBACK'.format(connection.name)) connection.handle.rollback() connection.transaction_open = False connections_in_use[connection.name] = connection return connection @classmethod def close(cls, connection): if dbt.flags.STRICT_MODE: assert isinstance(connection, Connection) # On windows, sometimes connection handles don't have a close() attr. if hasattr(connection.handle, 'close'): connection.handle.close() connection.state = 'closed' return connection def add_query(self, sql, model_name=None, auto_begin=True, bindings=None, abridge_sql_log=False): connection = self.get_connection(model_name) connection_name = connection.name if auto_begin and connection.transaction_open is False: self.begin(connection_name) logger.debug('Using {} connection "{}".'.format( self.type(), connection_name)) with self.exception_handler(sql, model_name, connection_name): if abridge_sql_log: logger.debug('On %s: %s....', connection_name, sql[0:512]) else: logger.debug('On %s: %s', connection_name, sql) pre = time.time() cursor = connection.handle.cursor() cursor.execute(sql, bindings) logger.debug("SQL status: %s in %0.2f seconds", self.get_status(cursor), (time.time() - pre)) return connection, cursor def clear_transaction(self, conn_name='master'): conn = self.begin(conn_name) self.commit(conn) return conn_name def execute_one(self, sql, model_name=None, auto_begin=False): self.get_connection(model_name) return self.add_query(sql, model_name, auto_begin) def execute_and_fetch(self, sql, model_name=None, auto_begin=False): _, cursor = self.execute_one(sql, model_name, auto_begin) status = self.get_status(cursor) table = self.get_result_from_cursor(cursor) return status, table def execute(self, sql, model_name=None, auto_begin=False, fetch=False): if fetch: return self.execute_and_fetch(sql, model_name, auto_begin) else: _, cursor = self.execute_one(sql, model_name, auto_begin) status = self.get_status(cursor) return status, dbt.clients.agate_helper.empty_table() def execute_all(self, sqls, model_name=None): connection = self.get_connection(model_name) if len(sqls) == 0: return connection for i, sql in enumerate(sqls): connection, _ = self.add_query(sql, model_name) return connection def create_schema(self, schema, model_name=None): logger.debug('Creating schema "%s".', schema) sql = self.get_create_schema_sql(schema) res = self.add_query(sql, model_name) self.commit_if_has_connection(model_name) return res def drop_schema(self, schema, model_name=None): logger.debug('Dropping schema "%s".', schema) sql = self.get_drop_schema_sql(schema) return self.add_query(sql, model_name) def already_exists(self, schema, table, model_name=None): relation = self.get_relation(schema=schema, identifier=table) return relation is not None @classmethod def quote(cls, identifier): return '"{}"'.format(identifier) def quote_as_configured(self, identifier, quote_key, model_name=None): """Quote or do not quote the given identifer as configured in the project config for the quote key. The quote key should be one of 'database' (on bigquery, 'profile'), 'identifier', or 'schema', or it will be treated as if you set `True`. """ default = self.Relation.DEFAULTS['quote_policy'].get(quote_key) if self.config.quoting.get(quote_key, default): return self.quote(identifier) else: return identifier @classmethod def convert_text_type(cls, agate_table, col_idx): raise dbt.exceptions.NotImplementedException( '`convert_text_type` is not implemented for this adapter!') @classmethod def convert_number_type(cls, agate_table, col_idx): raise dbt.exceptions.NotImplementedException( '`convert_number_type` is not implemented for this adapter!') @classmethod def convert_boolean_type(cls, agate_table, col_idx): raise dbt.exceptions.NotImplementedException( '`convert_boolean_type` is not implemented for this adapter!') @classmethod def convert_datetime_type(cls, agate_table, col_idx): raise dbt.exceptions.NotImplementedException( '`convert_datetime_type` is not implemented for this adapter!') @classmethod def convert_date_type(cls, agate_table, col_idx): raise dbt.exceptions.NotImplementedException( '`convert_date_type` is not implemented for this adapter!') @classmethod def convert_time_type(cls, agate_table, col_idx): raise dbt.exceptions.NotImplementedException( '`convert_time_type` is not implemented for this adapter!') @classmethod def convert_type(cls, agate_table, col_idx): return cls.convert_agate_type(agate_table, col_idx) @classmethod def convert_agate_type(cls, agate_table, col_idx): agate_type = agate_table.column_types[col_idx] conversions = [ (agate.Text, cls.convert_text_type), (agate.Number, cls.convert_number_type), (agate.Boolean, cls.convert_boolean_type), (agate.DateTime, cls.convert_datetime_type), (agate.Date, cls.convert_date_type), (agate.TimeDelta, cls.convert_time_type), ] for agate_cls, func in conversions: if isinstance(agate_type, agate_cls): return func(agate_table, col_idx) ### # Operations involving the manifest ### def run_operation(self, manifest, operation_name): """Look the operation identified by operation_name up in the manifest and run it. Return an an AttrDict with three attributes: 'table', 'data', and 'status'. 'table' is an agate.Table. """ operation = manifest.find_operation_by_name(operation_name, 'dbt') # This causes a reference cycle, as dbt.context.runtime.generate() # ends up calling get_adapter, so the import has to be here. import dbt.context.runtime context = dbt.context.runtime.generate( operation, self.config, manifest, ) result = operation.generator(context)() return result ### # Abstract methods involving the manifest ### @classmethod def _catalog_filter_table(cls, table, manifest): return table.where(_catalog_filter_schemas(manifest)) def get_catalog(self, manifest): try: table = self.run_operation(manifest, GET_CATALOG_OPERATION_NAME) finally: self.release_connection(GET_CATALOG_OPERATION_NAME) results = self._catalog_filter_table(table, manifest) return results @classmethod def _relations_filter_table(cls, table, schemas): return table.where(_relations_filter_schemas(schemas)) def _link_cached_relations(self, manifest, schemas): """This method has to exist because BigQueryAdapter and SnowflakeAdapter inherit from the PostgresAdapter, so they need something to override in order to disable linking. """ pass def _relations_cache_for_schemas(self, manifest, schemas=None): if not dbt.flags.USE_CACHE: return if schemas is None: schemas = manifest.get_used_schemas() relations = [] # add all relations for schema in schemas: # bypass the cache, of course! for relation in self._list_relations(schema): self.cache.add(relation) self._link_cached_relations(manifest, schemas) # it's possible that there were no relations in some schemas. We want # to insert the schemas we query into the cache's `.schemas` attribute # so we can check it later self.cache.update_schemas(schemas) def set_relations_cache(self, manifest, clear=False): """Run a query that gets a populated cache of the relations in the database and set the cache on this adapter. """ if not dbt.flags.USE_CACHE: return with self.cache.lock: if clear: self.cache.clear() self._relations_cache_for_schemas(manifest)
def __init__(self, config): self.config = config self.cache = RelationsCache() self.connections = self.ConnectionManager(config) self._internal_manifest_lazy = None
def __init__(self, config): self.config = config self.cache = RelationsCache() self.connections = self.ConnectionManager(config) self._internal_manifest_lazy = None