def _relations_cache_for_schemas(self, manifest: Manifest) -> None: """Populate the relations cache for the given schemas. Returns an iterable of the schemas populated, as strings. """ if not dbt.flags.USE_CACHE: return cache_schemas = self._get_cache_schemas(manifest) with executor(self.config) as tpe: futures: List[Future[List[BaseRelation]]] = [ tpe.submit(self._list_relations_get_connection, cache_schema) for cache_schema in cache_schemas ] for future in as_completed(futures): # if we can't read the relations we need to just raise anyway, # so just call future.result() and let that raise on failure for relation in future.result(): self.cache.add(relation) # it's possible that there were no relations in some schemas. We want # to insert the schemas we query into the cache's `.schemas` attribute # so we can check it later cache_update: Set[Tuple[Optional[str], Optional[str]]] = set() for relation in cache_schemas: cache_update.add((relation.database, relation.schema)) self.cache.update_schemas(cache_update)
def get_catalog(self, manifest: Manifest) -> Tuple[agate.Table, List[Exception]]: schema_map = self._get_catalog_schemas(manifest) with executor(self.config) as tpe: futures: List[Future[agate.Table]] = [ tpe.submit(self._get_one_catalog, info, schemas, manifest) for info, schemas in schema_map.items() if len(schemas) > 0 ] catalogs, exceptions = catch_as_completed(futures) return catalogs, exceptions
def get_catalog(self, manifest): schema_map = self._get_catalog_schemas(manifest) if len(schema_map) != 1: dbt.exceptions.raise_compiler_error( f'Expected only one database in get_catalog, found ' f'{list(schema_map)}') with executor(self.config) as tpe: futures: List[Future[agate.Table]] = [] for info, schemas in schema_map.items(): for schema in schemas: futures.append( tpe.submit(self._get_one_catalog, info, [schema], manifest)) catalogs, exceptions = catch_as_completed(futures) return catalogs, exceptions
def get_catalog(self, manifest: Manifest) -> Tuple[agate.Table, List[Exception]]: schema_map = self._get_catalog_schemas(manifest) with executor(self.config) as tpe: futures: List[Future[agate.Table]] = [] for info, schemas in schema_map.items(): if len(schemas) == 0: continue name = '.'.join([str(info.database), 'information_schema']) fut = tpe.submit_connected(self, name, self._get_one_catalog, info, schemas, manifest) futures.append(fut) catalogs, exceptions = catch_as_completed(futures) return catalogs, exceptions