def __init__(self, config, query, Runner):
"""
Runner is a type (not instance!) derived from
dbt.node_runners.BaseRunner
"""
self.config = config
self.query = query
self.Runner = Runner
manifest, linker = self.compile(self.config)
self.manifest = manifest
self.linker = linker
selector = dbt.graph.selector.NodeSelector(linker, manifest)
selected_nodes = selector.select(query)
self.job_queue = self.linker.as_graph_queue(manifest, selected_nodes)
# we use this a couple times. order does not matter.
self._flattened_nodes = [
self.manifest.nodes[uid] for uid in selected_nodes
]
self.run_count = 0
self.num_nodes = len([
n for n in self._flattened_nodes
if not Runner.is_ephemeral_model(n)
])
self.node_results = []
self._skipped_children = {}
self._raise_next_tick = None
def run_from_graph(self, Selector, Runner, query):
flat_graph, linker = self.compile(self.project)
selector = Selector(linker, flat_graph)
selected_nodes = selector.select(query)
dep_list = selector.as_node_list(selected_nodes)
profile = self.project.run_environment()
adapter = get_adapter(profile)
flat_nodes = dbt.utils.flatten_nodes(dep_list)
if len(flat_nodes) == 0:
logger.info("WARNING: Nothing to do. Try checking your model "
"configs and model specification args")
return []
elif Runner.print_header:
stat_line = dbt.ui.printer.get_counts(flat_nodes)
logger.info("")
dbt.ui.printer.print_timestamped_line(stat_line)
dbt.ui.printer.print_timestamped_line("")
else:
logger.info("")
try:
Runner.before_run(self.project, adapter, flat_graph)
started = time.time()
res = self.execute_nodes(linker, Runner, flat_graph, dep_list)
elapsed = time.time() - started
Runner.after_run(self.project, adapter, res, flat_graph, elapsed)
finally:
adapter.cleanup_connections()
return res
def select_nodes(self):
if self.manifest is None or self.linker is None:
raise InternalException(
'select_nodes called before manifest and linker were loaded'
)
selector = dbt.graph.selector.NodeSelector(
self.linker.graph, self.manifest
)
selected_nodes = selector.select(self.build_query())
return selected_nodes
def run_from_graph(self, Selector, Runner, query):
"""
Run dbt for the query, based on the graph.
Selector is a type (not instance!) derived from
dbt.graph.selector.NodeSelector
Runner is a type (not instance!) derived from
dbt.node_runners.BaseRunner
"""
manifest, linker = self.compile(self.project)
selector = Selector(linker, manifest)
selected_nodes = selector.select(query)
dep_list = selector.as_node_list(selected_nodes)
profile = self.project.run_environment()
adapter = get_adapter(profile)
flat_nodes = dbt.utils.flatten_nodes(dep_list)
if len(flat_nodes) == 0:
logger.info("WARNING: Nothing to do. Try checking your model "
"configs and model specification args")
return []
elif Runner.print_header:
stat_line = dbt.ui.printer.get_counts(flat_nodes)
logger.info("")
dbt.ui.printer.print_timestamped_line(stat_line)
dbt.ui.printer.print_timestamped_line("")
else:
logger.info("")
try:
Runner.before_hooks(self.project, adapter, manifest)
started = time.time()
Runner.before_run(self.project, adapter, manifest)
res = self.execute_nodes(linker, Runner, manifest, dep_list)
Runner.after_run(self.project, adapter, res, manifest)
elapsed = time.time() - started
Runner.after_hooks(self.project, adapter, res, manifest, elapsed)
finally:
adapter.cleanup_connections()
result = ExecutionResult(
results=res,
elapsed_time=elapsed,
generated_at=dbt.utils.timestring(),
)
self.write_results(result)
return res
def _runtime_initialize(self):
self.manifest = load_manifest(self.config)
self.linker = compile_manifest(self.config, self.manifest)
selector = dbt.graph.selector.NodeSelector(self.linker, self.manifest)
selected_nodes = selector.select(self.build_query())
self.job_queue = self.linker.as_graph_queue(self.manifest,
selected_nodes)
# we use this a couple times. order does not matter.
self._flattened_nodes = [
self.manifest.nodes[uid] for uid in selected_nodes
]
self.num_nodes = len([
n for n in self._flattened_nodes
if not n.is_ephemeral_model
])
def select_nodes(self):
selector = dbt.graph.selector.NodeSelector(self.linker.graph,
self.manifest)
selected_nodes = selector.select(self.build_query())
return selected_nodes