def do_ref(*args): if len(args) == 1: other_model_name = self.create_template.model_name(args[0]) other_model = find_model_by_name(all_models, other_model_name) elif len(args) == 2: other_model_package, other_model_name = args other_model_name = self.create_template.model_name(other_model_name) other_model = find_model_by_name(all_models, other_model_name, package_namespace=other_model_package) else: compiler_error(model, "ref() takes at most two arguments ({} given)".format(len(args))) other_model_fqn = tuple(other_model.fqn[:-1] + [other_model_name]) src_fqn = ".".join(source_model) ref_fqn = ".".join(other_model_fqn) #if not self.model_can_reference(model, other_model): # compiler_error(model, "Model '{}' exists but cannot be referenced from dependency model '{}'".format(ref_fqn, src_fqn)) if not other_model.is_enabled: raise RuntimeError("Model '{}' depends on model '{}' which is disabled in the project config".format(src_fqn, ref_fqn)) # this creates a trivial cycle -- should this be a compiler error? # we can still interpolate the name w/o making a self-cycle if source_model == other_model_fqn or not add_dependency: pass else: linker.dependency(source_model, other_model_fqn) if other_model.is_ephemeral: linker.inject_cte(model, other_model) return other_model.cte_name else: return '"{}"."{}"'.format(schema, other_model_name)
def run(self, specified_models=None): linker = self.deserialize_graph() compiled_models = self.get_compiled_models() limit_to = None if specified_models is not None: limit_to = [] for model_name in specified_models: try: model = find_model_by_name(compiled_models, model_name) limit_to.append(tuple(model.fqn)) except RuntimeError as e: print("ERROR: {}".format(str(e))) print("Exiting") return[] target_cfg = self.project.run_environment() schema_name = target_cfg['schema'] try: schemas = self.get_schemas() if schema_name not in schemas: self.create_schema_or_exit(schema_name) return self.execute_models(linker, compiled_models, limit_to) except psycopg2.OperationalError as e: print("ERROR: Could not connect to the target database. Try `dbt debug` for more information") print(str(e)) sys.exit(1)
def do_ref(*args): if len(args) == 1: other_model_name = self.create_template.model_name(args[0]) other_model = find_model_by_name(all_models, other_model_name) elif len(args) == 2: other_model_package, other_model_name = args other_model_name = self.create_template.model_name(other_model_name) other_model = find_model_by_name(all_models, other_model_name, package_namespace=other_model_package) other_model_fqn = tuple(other_model.fqn[:-1] + [other_model_name]) other_model_config = other_model.get_config(self.project) if not other_model_config['enabled']: src_fqn = ".".join(source_model) ref_fqn = ".".join(other_model_fqn) raise RuntimeError("Model '{}' depends on model '{}' which is disabled in the project config".format(src_fqn, ref_fqn)) linker.dependency(source_model, other_model_fqn) return '"{}"."{}"'.format(schema, other_model_name)
def run_from_graph(self, runner, limit_to): logger.info("Loading dependency graph file") linker = self.deserialize_graph() compiled_models = [ make_compiled_model(fqn, linker.get_node(fqn)) for fqn in linker.nodes() ] relevant_compiled_models = [ m for m in compiled_models if m.is_type(runner.run_type) ] for m in relevant_compiled_models: if m.should_execute(self.args, existing=[]): context = self.context.copy() context.update(m.context()) m.compile(context) schema_name = self.target.schema logger.info("Connecting to redshift") try: self.schema.create_schema_if_not_exists(schema_name) except psycopg2.OperationalError as e: logger.info("ERROR: Could not connect to the target database. Try" "`dbt debug` for more information") logger.info(str(e)) sys.exit(1) existing = self.schema.query_for_existing(schema_name) if limit_to is None: specified_models = None else: specified_models = [ find_model_by_name(relevant_compiled_models, name).fqn for name in limit_to ] model_dependency_list = self.as_concurrent_dep_list( linker, relevant_compiled_models, existing, self.target, specified_models) on_failure = self.on_model_failure(linker, relevant_compiled_models) results = self.execute_models(runner, model_dependency_list, on_failure) return results
def compile_schema_tests(self, linker, models): all_schema_specs = self.get_local_and_package_sources( self.project, self.project_schemas) schema_tests = [] for schema in all_schema_specs: # compiling a SchemaFile returns >= 0 SchemaTest models try: schema_tests.extend(schema.compile()) except RuntimeError as e: logger.info("\n" + str(e)) schema_test_path = schema.filepath logger.info("Skipping compilation for {}...\n".format( schema_test_path)) written_tests = [] for schema_test in schema_tests: # show a warning if the model being tested doesn't exist try: source_model = find_model_by_name(models, schema_test.model_name) except RuntimeError as e: dbt.utils.compiler_warning(schema_test, str(e)) continue if not source_model.is_enabled: continue serialized = schema_test.serialize() model_node = tuple(source_model.fqn) test_node = tuple(schema_test.fqn) linker.dependency(test_node, model_node) linker.update_node_data(test_node, serialized) query = schema_test.render() self.__write(schema_test.build_path(), query) written_tests.append(schema_test) return written_tests
def run(self, specified_models=None): linker = self.deserialize_graph() compiled_models = self.get_compiled_models() limit_to = None if specified_models is not None: limit_to = [] for model_name in specified_models: try: model = find_model_by_name(compiled_models, model_name) limit_to.append(tuple(model.fqn)) except RuntimeError as e: print("ERROR: {}".format(str(e))) print("Exiting") return try: self.create_schema() for model in self.execute_models(linker, compiled_models, limit_to): yield model, True except psycopg2.OperationalError as e: print("ERROR: Could not connect to the target database. Try `dbt debug` for more information") print(str(e)) sys.exit(1)