def execute(self, model, manifest):
context = generate_runtime_model(model, self.config, manifest)
materialization_macro = manifest.find_materialization_macro_by_name(
self.config.project_name, model.get_materialization(),
self.adapter.type())
if materialization_macro is None:
missing_materialization(model, self.adapter.type())
if 'config' not in context:
raise InternalException(
'Invalid materialization context generated, missing config: {}'
.format(context))
context_config = context['config']
hook_ctx = self.adapter.pre_model_hook(context_config)
try:
result = MacroGenerator(materialization_macro, context)()
finally:
self.adapter.post_model_hook(context_config, hook_ctx)
for relation in self._materialization_relations(result, model):
self.adapter.cache_added(relation.incorporate(dbt_created=True))
return self._build_run_model_result(model, context)
def execute_macro(
self,
macro_name: str,
manifest: Optional[Manifest] = None,
project: Optional[str] = None,
context_override: Optional[Dict[str, Any]] = None,
kwargs: Dict[str, Any] = None,
release: bool = False,
text_only_columns: Optional[Iterable[str]] = None,
) -> agate.Table:
"""Look macro_name up in the manifest and execute its results.
:param macro_name: The name of the macro to execute.
:param manifest: The manifest to use for generating the base macro
execution context. If none is provided, use the internal manifest.
:param project: The name of the project to search in, or None for the
first match.
:param context_override: An optional dict to update() the macro
execution context.
:param kwargs: An optional dict of keyword args used to pass to the
macro.
:param release: If True, release the connection after executing.
"""
if kwargs is None:
kwargs = {}
if context_override is None:
context_override = {}
if manifest is None:
manifest = self._internal_manifest
macro = manifest.find_macro_by_name(macro_name,
self.config.project_name, project)
if macro is None:
if project is None:
package_name = 'any package'
else:
package_name = 'the "{}" package'.format(project)
raise RuntimeException(
'dbt could not find a macro with the name "{}" in {}'.format(
macro_name, package_name))
# This causes a reference cycle, as generate_runtime_macro()
# ends up calling get_adapter, so the import has to be here.
from dbt.context.providers import generate_runtime_macro
macro_context = generate_runtime_macro(macro=macro,
config=self.config,
manifest=manifest,
package_name=project)
macro_context.update(context_override)
macro_function = MacroGenerator(macro, macro_context)
with self.connections.exception_handler(f'macro {macro_name}'):
try:
result = macro_function(**kwargs)
finally:
if release:
self.release_connection()
return result
def __init__(self, config: RuntimeConfig, macro_manifest: MacroManifest,
component: str) -> None:
macro = macro_manifest.find_generate_macro_by_name(
component=component,
root_project_name=config.project_name,
)
if macro is None:
raise InternalException(
f'No macro with name generate_{component}_name found')
root_context = generate_generate_component_name_macro(
macro, config, macro_manifest)
self.updater = MacroGenerator(macro, root_context)
self.component = component
def get_from_package(self, package_name: Optional[str],
name: str) -> Optional[MacroGenerator]:
macro = None
if package_name is None:
macro = self.macro_resolver.macros_by_name.get(name)
elif package_name == GLOBAL_PROJECT_NAME:
macro = self.macro_resolver.internal_packages_namespace.get(name)
elif package_name in self.resolver.packages:
macro = self.macro_resolver.packages[package_name].get(name)
else:
raise_compiler_error(f"Could not find package '{package_name}'")
macro_func = MacroGenerator(macro, self.ctx, self.node,
self.thread_ctx)
return macro_func
def __init__(self, macro_resolver, ctx, node, thread_ctx,
depends_on_macros):
self.macro_resolver = macro_resolver
self.ctx = ctx
self.node = node
self.thread_ctx = thread_ctx
local_namespace = {}
if depends_on_macros:
for macro_unique_id in depends_on_macros:
macro = self.manifest.macros[macro_unique_id]
local_namespace[macro.name] = MacroGenerator(
macro,
self.ctx,
self.node,
self.thread_ctx,
)
self.local_namespace = local_namespace
def add_macro(self, macro: ParsedMacro, ctx: Dict[str, Any]):
macro_name: str = macro.name
macro_func: MacroGenerator = MacroGenerator(
macro, ctx, self.node, self.thread_ctx
)
# internal macros (from plugins) will be processed separately from
# project macros, so store them in a different place
if macro.package_name in self.internal_package_names:
self._add_macro_to(self.internal_packages, macro, macro_func)
else:
self._add_macro_to(self.packages, macro, macro_func)
if macro.package_name == self.search_package:
self.locals[macro_name] = macro_func
elif macro.package_name == self.root_package:
self.globals[macro_name] = macro_func
def add_macro(self, macro: ParsedMacro, ctx: Dict[str, Any]):
macro_name: str = macro.name
# MacroGenerator is in clients/jinja.py
# a MacroGenerator object is a callable object that will
# execute the MacroGenerator.__call__ function
macro_func: MacroGenerator = MacroGenerator(macro, ctx, self.node,
self.thread_ctx)
# internal macros (from plugins) will be processed separately from
# project macros, so store them in a different place
if macro.package_name in self.internal_package_names:
self._add_macro_to(self.internal_packages, macro, macro_func)
else:
# if it's not an internal package
self._add_macro_to(self.packages, macro, macro_func)
# add to locals if it's the package this node is in
if macro.package_name == self.search_package:
self.locals[macro_name] = macro_func
# add to globals if it's in the root package
elif macro.package_name == self.root_package:
self.globals[macro_name] = macro_func
def add_macro(self, macro: ParsedMacro, ctx: Dict[str, Any]):
macro_name: str = macro.name
macro_func: MacroGenerator = MacroGenerator(macro, ctx, self.node,
self.thread_ctx)
# put plugin macros into the root namespace
if macro.package_name in PACKAGES:
namespace: str = GLOBAL_PROJECT_NAME
else:
namespace = macro.package_name
if namespace not in self.packages:
value: Dict[str, MacroGenerator] = {}
self.packages[namespace] = value
if macro_name in self.packages[namespace]:
raise_duplicate_macro_name(macro_func.macro, macro, namespace)
self.packages[namespace][macro_name] = macro_func
if namespace == self.search_package:
self.locals[macro_name] = macro_func
elif namespace in {self.root_package, GLOBAL_PROJECT_NAME}:
self.globals[macro_name] = macro_func
def execute_test(self, test: Union[CompiledDataTestNode,
CompiledSchemaTestNode],
manifest: Manifest) -> int:
context = generate_runtime_model(test, self.config, manifest)
materialization_macro = manifest.find_materialization_macro_by_name(
self.config.project_name, test.get_materialization(),
self.adapter.type())
if materialization_macro is None:
missing_materialization(test, self.adapter.type())
if 'config' not in context:
raise InternalException(
'Invalid materialization context generated, missing config: {}'
.format(context))
# generate materialization macro
# simple `select(*)` of the compiled test node
macro_func = MacroGenerator(materialization_macro, context)
# execute materialization macro
macro_func()
# load results from context
# could eventually be returned directly by materialization
result = context['load_result']('main')
table = result['table']
num_rows = len(table.rows)
if num_rows != 1:
num_cols = len(table.columns)
# since we just wrapped our query in `select count(*)`, we are in
# big trouble!
raise InternalException(
f"dbt internally failed to execute {test.unique_id}: "
f"Returned {num_rows} rows and {num_cols} cols, but expected "
f"1 row and 1 column")
return int(table[0][0])
def generator(self) -> Callable[[Dict[str, Any]], Callable]:
"""
Returns a function that can be called to render the macro results.
"""
return MacroGenerator(self)