def sort_qualifier(self, model_config): if 'sort' not in model_config: return '' if (self.is_view or self.is_ephemeral) and 'sort' in model_config: return '' sort_keys = model_config['sort'] sort_type = model_config.get('sort_type', 'compound') if type(sort_type) != str: compiler_error( self, "The provided sort_type '{}' is not valid!".format(sort_type)) sort_type = sort_type.strip().lower() valid_sort_types = ['compound', 'interleaved'] if sort_type not in valid_sort_types: compiler_error( self, "Invalid sort_type given: {} -- must be one of {}".format( sort_type, valid_sort_types)) if type(sort_keys) == str: sort_keys = [sort_keys] # remove existing quotes in field name, then wrap in quotes formatted_sort_keys = [ '"{}"'.format(sort_key.replace('"', '')) for sort_key in sort_keys ] keys_csv = ', '.join(formatted_sort_keys) return "{sort_type} sortkey ({keys_csv})".format(sort_type=sort_type, keys_csv=keys_csv)
def compile(self): try: return self.do_compile() except TypeError as e: compiler_error(self, str(e)) except AttributeError as e: compiler_error(self, str(e))
def do_ref(*args): if len(args) == 1: other_model_name = self.create_template.model_name(args[0]) other_model = find_model_by_name(all_models, other_model_name) elif len(args) == 2: other_model_package, other_model_name = args other_model_name = self.create_template.model_name(other_model_name) other_model = find_model_by_name(all_models, other_model_name, package_namespace=other_model_package) else: compiler_error(model, "ref() takes at most two arguments ({} given)".format(len(args))) other_model_fqn = tuple(other_model.fqn[:-1] + [other_model_name]) src_fqn = ".".join(source_model) ref_fqn = ".".join(other_model_fqn) #if not self.model_can_reference(model, other_model): # compiler_error(model, "Model '{}' exists but cannot be referenced from dependency model '{}'".format(ref_fqn, src_fqn)) if not other_model.is_enabled: raise RuntimeError("Model '{}' depends on model '{}' which is disabled in the project config".format(src_fqn, ref_fqn)) # this creates a trivial cycle -- should this be a compiler error? # we can still interpolate the name w/o making a self-cycle if source_model == other_model_fqn or not add_dependency: pass else: linker.dependency(source_model, other_model_fqn) if other_model.is_ephemeral: linker.inject_cte(model, other_model) return other_model.cte_name else: return '"{}"."{}"'.format(schema, other_model_name)
def compile(self, context): contents = self.contents try: env = jinja2.Environment() self.compiled_contents = env.from_string(contents).render(context) return self.compiled_contents except jinja2.exceptions.TemplateSyntaxError as e: compiler_error(self, str(e))
def is_enabled(self): enabled = self.config['enabled'] if enabled not in (True, False): compiler_error( self, "'enabled' config must be either True or False. '{}' given.". format(enabled)) return enabled
def compile_string(string, ctx): try: env = jinja2.Environment() template = env.from_string(str(string), globals=ctx) return template.render(ctx) except jinja2.exceptions.TemplateSyntaxError as e: compiler_error(None, str(e)) except jinja2.exceptions.UndefinedError as e: compiler_error(None, str(e))
def get_test(self, test_type): if test_type in SchemaFile.SchemaTestMap: return SchemaFile.SchemaTestMap[test_type] else: possible_types = ", ".join(SchemaFile.SchemaTestMap.keys()) compiler_error( self, "Invalid validation type given in {}: '{}'. Possible: {}". format(self.filepath, test_type, possible_types))
def get_macros(self, ctx): env = jinja2.Environment() try: template = env.from_string(self.contents, globals=ctx) except jinja2.exceptions.TemplateSyntaxError as e: compiler_error(self, str(e)) for key, item in template.module.__dict__.items(): if type(item) == jinja2.runtime.Macro: yield {"project": self.own_project, "name": key, "macro": item}
def compile(self, rendered_query, project, ctx): model_config = self.config if self.materialization not in SourceConfig.Materializations: compiler_error( self, "Invalid materialize option given: '{}'. Must be one of {}". format(self.materialization, SourceConfig.Materializations)) schema = ctx['env'].get('schema', 'public') # these are empty strings if configs aren't provided dist_qualifier = self.dist_qualifier(model_config) sort_qualifier = self.sort_qualifier(model_config) if self.materialization == 'incremental': identifier = self.name if 'sql_where' not in model_config: compiler_error( self, # TODO - this probably won't be an error now? "sql_where not specified in model materialized " "as incremental") raw_sql_where = model_config['sql_where'] sql_where = self.compile_string(ctx, raw_sql_where) unique_key = model_config.get('unique_key', None) else: identifier = self.tmp_name() sql_where = None unique_key = None pre_hooks = self.get_hooks(ctx, 'pre-hook') post_hooks = self.get_hooks(ctx, 'post-hook') opts = { "materialization": self.materialization, "schema": schema, "identifier": identifier, "query": rendered_query, "dist_qualifier": dist_qualifier, "sort_qualifier": sort_qualifier, "sql_where": sql_where, "prologue": self.get_prologue_string(), "unique_key": unique_key, "pre-hooks": pre_hooks, "post-hooks": post_hooks, "non_destructive": self.is_non_destructive() } return self.template.wrap(opts)
def remove_node_from_graph(self, linker, model, models): # remove the node children = linker.remove_node(tuple(model.fqn)) # check if we bricked the graph. if so: throw compilation error for child in children: other_model = find_model_by_fqn(models, child) if other_model.is_enabled: this_fqn = ".".join(model.fqn) that_fqn = ".".join(other_model.fqn) compiler_error( model, "Model '{}' depends on model '{}' which is " "disabled".format(that_fqn, this_fqn))
def compile_model(self, linker, model, models): try: jinja = jinja2.Environment(loader=jinja2.FileSystemLoader( searchpath=model.root_dir)) # this is a dumb jinja2 bug -- on windows, forward slashes are EXPECTED posix_filepath = '/'.join(split_path(model.rel_filepath)) template = jinja.get_template(posix_filepath) context = self.get_context(linker, model, models) rendered = template.render(context) except jinja2.exceptions.TemplateSyntaxError as e: compiler_error(model, str(e)) return rendered
def validate(self, data): required = [ 'source_schema', 'target_schema', 'source_table', 'target_table', 'unique_key', 'updated_at', ] for key in required: if data.get(key, None) is None: compiler_error( "Invalid archive config: missing required field '{}'". format(key))
def do_compile(self): schema_tests = [] for model_name, constraint_blob in self.schema.items(): constraints = constraint_blob.get('constraints', {}) for constraint_type, constraint_data in constraints.items(): if constraint_data is None: compiler_error( self, "no constraints given to test: '{}.{}'".format( model_name, constraint_type)) for params in constraint_data: schema_test_klass = self.get_test(constraint_type) schema_test = schema_test_klass(self.project, self.og_target_dir, self.rel_filepath, model_name, params) schema_tests.append(schema_test) return schema_tests
def compile_model(self, linker, model, models): try: fs_loader = jinja2.FileSystemLoader(searchpath=model.root_dir) jinja = jinja2.Environment(loader=fs_loader) template_contents = dbt.clients.system.load_file_contents( model.absolute_path) template = jinja.from_string(template_contents) context = self.get_context(linker, model, models) rendered = template.render(context) except jinja2.exceptions.TemplateSyntaxError as e: compiler_error(model, str(e)) except jinja2.exceptions.UndefinedError as e: compiler_error(model, str(e)) return rendered
def __get_hooks(self, relevant_configs, key): hooks = [] if key not in relevant_configs: return [] new_hooks = relevant_configs[key] if type(new_hooks) not in [list, tuple]: new_hooks = [new_hooks] for hook in new_hooks: if type(hook) != str: name = ".".join(self.fqn) compiler_error( None, "{} for model {} is not a string!".format(key, name)) hooks.append(hook) return hooks
def dist_qualifier(self, model_config): if 'dist' not in model_config: return '' if (self.is_view or self.is_ephemeral) and 'dist' in model_config: return '' dist_key = model_config['dist'] if type(dist_key) != str: compiler_error( self, "The provided distkey '{}' is not valid!".format(dist_key)) dist_key = dist_key.strip().lower() adapter = get_adapter(self.project.run_environment()) return adapter.dist_qualifier(dist_key)
def get_dist_qualifier(model, project): model_config = model.get('config', {}) if 'dist' not in model_config: return '' if get_materialization(model) not in ('table', 'incremental'): return '' dist_key = model_config.get('dist') if not isinstance(dist_key, basestring): compiler_error( model, "The provided distkey '{}' is not valid!".format(dist_key)) dist_key = dist_key.strip().lower() adapter = get_adapter(project.run_environment()) return adapter.dist_qualifier(dist_key)
def sort_qualifier(self, model_config): if 'sort' not in model_config: return '' if (self.is_view or self.is_ephemeral) and 'sort' in model_config: return '' sort_keys = model_config['sort'] sort_type = model_config.get('sort_type', 'compound') if type(sort_type) != str: compiler_error( self, "The provided sort_type '{}' is not valid!".format(sort_type)) sort_type = sort_type.strip().lower() adapter = get_adapter(self.project.run_environment()) return adapter.sort_qualifier(sort_type, sort_keys)
def dist_qualifier(self, model_config): if 'dist' not in model_config: return '' if (self.is_view or self.is_ephemeral) and 'dist' in model_config: return '' dist_key = model_config['dist'] if type(dist_key) != str: compiler_error( self, "The provided distkey '{}' is not valid!".format(dist_key)) dist_key = dist_key.strip().lower() if dist_key in ['all', 'even']: return 'diststyle {}'.format(dist_key) else: return 'diststyle key distkey ("{}")'.format(dist_key)
def get_sort_qualifier(model, project): model_config = model.get('config', {}) if 'sort' not in model['config']: return '' if get_materialization(model) not in ('table', 'incremental'): return '' sort_keys = model_config.get('sort') sort_type = model_config.get('sort_type', 'compound') if not isinstance(sort_type, basestring): compiler_error( model, "The provided sort_type '{}' is not valid!".format(sort_type)) sort_type = sort_type.strip().lower() adapter = get_adapter(project.run_environment()) return adapter.sort_qualifier(sort_type, sort_keys)
def compile_string(self, ctx, string): # python 2+3 check for stringiness try: basestring except NameError: basestring = str # if bool/int/float/etc are passed in, don't compile anything if not isinstance(string, basestring): return string try: fs_loader = jinja2.FileSystemLoader( searchpath=self.project['macro-paths']) env = jinja2.Environment(loader=fs_loader) template = env.from_string(string, globals=ctx) return template.render(ctx) except jinja2.exceptions.TemplateSyntaxError as e: compiler_error(self, str(e)) except jinja2.exceptions.UndefinedError as e: compiler_error(self, str(e))
def compile_string(self, ctx, string): try: env = jinja2.Environment() return env.from_string(string).render(ctx) except jinja2.exceptions.TemplateSyntaxError as e: compiler_error(self, str(e))
def __getattr__(self, attr): if attr in self: return self.get(attr) else: compiler_error(self.model, "'{}' is undefined".format(attr))