def do_ref(*args):
if len(args) == 1:
other_model_name = self.create_template.model_name(args[0])
other_model = find_model_by_name(all_models, other_model_name)
elif len(args) == 2:
other_model_package, other_model_name = args
other_model_name = self.create_template.model_name(other_model_name)
other_model = find_model_by_name(all_models, other_model_name, package_namespace=other_model_package)
else:
compiler_error(model, "ref() takes at most two arguments ({} given)".format(len(args)))
other_model_fqn = tuple(other_model.fqn[:-1] + [other_model_name])
src_fqn = ".".join(source_model)
ref_fqn = ".".join(other_model_fqn)
#if not self.model_can_reference(model, other_model):
# compiler_error(model, "Model '{}' exists but cannot be referenced from dependency model '{}'".format(ref_fqn, src_fqn))
if not other_model.is_enabled:
raise RuntimeError("Model '{}' depends on model '{}' which is disabled in the project config".format(src_fqn, ref_fqn))
# this creates a trivial cycle -- should this be a compiler error?
# we can still interpolate the name w/o making a self-cycle
if source_model == other_model_fqn or not add_dependency:
pass
else:
linker.dependency(source_model, other_model_fqn)
if other_model.is_ephemeral:
linker.inject_cte(model, other_model)
return other_model.cte_name
else:
return '"{}"."{}"'.format(schema, other_model_name)
def run(self, specified_models=None):
linker = self.deserialize_graph()
compiled_models = self.get_compiled_models()
limit_to = None
if specified_models is not None:
limit_to = []
for model_name in specified_models:
try:
model = find_model_by_name(compiled_models, model_name)
limit_to.append(tuple(model.fqn))
except RuntimeError as e:
print("ERROR: {}".format(str(e)))
print("Exiting")
return[]
target_cfg = self.project.run_environment()
schema_name = target_cfg['schema']
try:
schemas = self.get_schemas()
if schema_name not in schemas:
self.create_schema_or_exit(schema_name)
return self.execute_models(linker, compiled_models, limit_to)
except psycopg2.OperationalError as e:
print("ERROR: Could not connect to the target database. Try `dbt debug` for more information")
print(str(e))
sys.exit(1)
def do_ref(*args):
if len(args) == 1:
other_model_name = self.create_template.model_name(args[0])
other_model = find_model_by_name(all_models, other_model_name)
elif len(args) == 2:
other_model_package, other_model_name = args
other_model_name = self.create_template.model_name(other_model_name)
other_model = find_model_by_name(all_models, other_model_name, package_namespace=other_model_package)
other_model_fqn = tuple(other_model.fqn[:-1] + [other_model_name])
other_model_config = other_model.get_config(self.project)
if not other_model_config['enabled']:
src_fqn = ".".join(source_model)
ref_fqn = ".".join(other_model_fqn)
raise RuntimeError("Model '{}' depends on model '{}' which is disabled in the project config".format(src_fqn, ref_fqn))
linker.dependency(source_model, other_model_fqn)
return '"{}"."{}"'.format(schema, other_model_name)
def run_from_graph(self, runner, limit_to):
logger.info("Loading dependency graph file")
linker = self.deserialize_graph()
compiled_models = [
make_compiled_model(fqn, linker.get_node(fqn))
for fqn in linker.nodes()
]
relevant_compiled_models = [
m for m in compiled_models if m.is_type(runner.run_type)
]
for m in relevant_compiled_models:
if m.should_execute(self.args, existing=[]):
context = self.context.copy()
context.update(m.context())
m.compile(context)
schema_name = self.target.schema
logger.info("Connecting to redshift")
try:
self.schema.create_schema_if_not_exists(schema_name)
except psycopg2.OperationalError as e:
logger.info("ERROR: Could not connect to the target database. Try"
"`dbt debug` for more information")
logger.info(str(e))
sys.exit(1)
existing = self.schema.query_for_existing(schema_name)
if limit_to is None:
specified_models = None
else:
specified_models = [
find_model_by_name(relevant_compiled_models, name).fqn
for name in limit_to
]
model_dependency_list = self.as_concurrent_dep_list(
linker, relevant_compiled_models, existing, self.target,
specified_models)
on_failure = self.on_model_failure(linker, relevant_compiled_models)
results = self.execute_models(runner, model_dependency_list,
on_failure)
return results
def compile_schema_tests(self, linker, models):
all_schema_specs = self.get_local_and_package_sources(
self.project, self.project_schemas)
schema_tests = []
for schema in all_schema_specs:
# compiling a SchemaFile returns >= 0 SchemaTest models
try:
schema_tests.extend(schema.compile())
except RuntimeError as e:
logger.info("\n" + str(e))
schema_test_path = schema.filepath
logger.info("Skipping compilation for {}...\n".format(
schema_test_path))
written_tests = []
for schema_test in schema_tests:
# show a warning if the model being tested doesn't exist
try:
source_model = find_model_by_name(models,
schema_test.model_name)
except RuntimeError as e:
dbt.utils.compiler_warning(schema_test, str(e))
continue
if not source_model.is_enabled:
continue
serialized = schema_test.serialize()
model_node = tuple(source_model.fqn)
test_node = tuple(schema_test.fqn)
linker.dependency(test_node, model_node)
linker.update_node_data(test_node, serialized)
query = schema_test.render()
self.__write(schema_test.build_path(), query)
written_tests.append(schema_test)
return written_tests
def run(self, specified_models=None):
linker = self.deserialize_graph()
compiled_models = self.get_compiled_models()
limit_to = None
if specified_models is not None:
limit_to = []
for model_name in specified_models:
try:
model = find_model_by_name(compiled_models, model_name)
limit_to.append(tuple(model.fqn))
except RuntimeError as e:
print("ERROR: {}".format(str(e)))
print("Exiting")
return
try:
self.create_schema()
for model in self.execute_models(linker, compiled_models, limit_to):
yield model, True
except psycopg2.OperationalError as e:
print("ERROR: Could not connect to the target database. Try `dbt debug` for more information")
print(str(e))
sys.exit(1)