def sort_qualifier(self, model_config):
if 'sort' not in model_config:
return ''
if (self.is_view or self.is_ephemeral) and 'sort' in model_config:
return ''
sort_keys = model_config['sort']
sort_type = model_config.get('sort_type', 'compound')
if type(sort_type) != str:
compiler_error(
self,
"The provided sort_type '{}' is not valid!".format(sort_type))
sort_type = sort_type.strip().lower()
valid_sort_types = ['compound', 'interleaved']
if sort_type not in valid_sort_types:
compiler_error(
self,
"Invalid sort_type given: {} -- must be one of {}".format(
sort_type, valid_sort_types))
if type(sort_keys) == str:
sort_keys = [sort_keys]
# remove existing quotes in field name, then wrap in quotes
formatted_sort_keys = [
'"{}"'.format(sort_key.replace('"', '')) for sort_key in sort_keys
]
keys_csv = ', '.join(formatted_sort_keys)
return "{sort_type} sortkey ({keys_csv})".format(sort_type=sort_type,
keys_csv=keys_csv)
def compile(self):
try:
return self.do_compile()
except TypeError as e:
compiler_error(self, str(e))
except AttributeError as e:
compiler_error(self, str(e))
def do_ref(*args):
if len(args) == 1:
other_model_name = self.create_template.model_name(args[0])
other_model = find_model_by_name(all_models, other_model_name)
elif len(args) == 2:
other_model_package, other_model_name = args
other_model_name = self.create_template.model_name(other_model_name)
other_model = find_model_by_name(all_models, other_model_name, package_namespace=other_model_package)
else:
compiler_error(model, "ref() takes at most two arguments ({} given)".format(len(args)))
other_model_fqn = tuple(other_model.fqn[:-1] + [other_model_name])
src_fqn = ".".join(source_model)
ref_fqn = ".".join(other_model_fqn)
#if not self.model_can_reference(model, other_model):
# compiler_error(model, "Model '{}' exists but cannot be referenced from dependency model '{}'".format(ref_fqn, src_fqn))
if not other_model.is_enabled:
raise RuntimeError("Model '{}' depends on model '{}' which is disabled in the project config".format(src_fqn, ref_fqn))
# this creates a trivial cycle -- should this be a compiler error?
# we can still interpolate the name w/o making a self-cycle
if source_model == other_model_fqn or not add_dependency:
pass
else:
linker.dependency(source_model, other_model_fqn)
if other_model.is_ephemeral:
linker.inject_cte(model, other_model)
return other_model.cte_name
else:
return '"{}"."{}"'.format(schema, other_model_name)
def compile(self, context):
contents = self.contents
try:
env = jinja2.Environment()
self.compiled_contents = env.from_string(contents).render(context)
return self.compiled_contents
except jinja2.exceptions.TemplateSyntaxError as e:
compiler_error(self, str(e))
def is_enabled(self):
enabled = self.config['enabled']
if enabled not in (True, False):
compiler_error(
self,
"'enabled' config must be either True or False. '{}' given.".
format(enabled))
return enabled
def compile_string(string, ctx):
try:
env = jinja2.Environment()
template = env.from_string(str(string), globals=ctx)
return template.render(ctx)
except jinja2.exceptions.TemplateSyntaxError as e:
compiler_error(None, str(e))
except jinja2.exceptions.UndefinedError as e:
compiler_error(None, str(e))
def get_test(self, test_type):
if test_type in SchemaFile.SchemaTestMap:
return SchemaFile.SchemaTestMap[test_type]
else:
possible_types = ", ".join(SchemaFile.SchemaTestMap.keys())
compiler_error(
self,
"Invalid validation type given in {}: '{}'. Possible: {}".
format(self.filepath, test_type, possible_types))
def get_macros(self, ctx):
env = jinja2.Environment()
try:
template = env.from_string(self.contents, globals=ctx)
except jinja2.exceptions.TemplateSyntaxError as e:
compiler_error(self, str(e))
for key, item in template.module.__dict__.items():
if type(item) == jinja2.runtime.Macro:
yield {"project": self.own_project, "name": key, "macro": item}
def compile(self, rendered_query, project, ctx):
model_config = self.config
if self.materialization not in SourceConfig.Materializations:
compiler_error(
self,
"Invalid materialize option given: '{}'. Must be one of {}".
format(self.materialization, SourceConfig.Materializations))
schema = ctx['env'].get('schema', 'public')
# these are empty strings if configs aren't provided
dist_qualifier = self.dist_qualifier(model_config)
sort_qualifier = self.sort_qualifier(model_config)
if self.materialization == 'incremental':
identifier = self.name
if 'sql_where' not in model_config:
compiler_error(
self,
# TODO - this probably won't be an error now?
"sql_where not specified in model materialized "
"as incremental")
raw_sql_where = model_config['sql_where']
sql_where = self.compile_string(ctx, raw_sql_where)
unique_key = model_config.get('unique_key', None)
else:
identifier = self.tmp_name()
sql_where = None
unique_key = None
pre_hooks = self.get_hooks(ctx, 'pre-hook')
post_hooks = self.get_hooks(ctx, 'post-hook')
opts = {
"materialization": self.materialization,
"schema": schema,
"identifier": identifier,
"query": rendered_query,
"dist_qualifier": dist_qualifier,
"sort_qualifier": sort_qualifier,
"sql_where": sql_where,
"prologue": self.get_prologue_string(),
"unique_key": unique_key,
"pre-hooks": pre_hooks,
"post-hooks": post_hooks,
"non_destructive": self.is_non_destructive()
}
return self.template.wrap(opts)
def remove_node_from_graph(self, linker, model, models):
# remove the node
children = linker.remove_node(tuple(model.fqn))
# check if we bricked the graph. if so: throw compilation error
for child in children:
other_model = find_model_by_fqn(models, child)
if other_model.is_enabled:
this_fqn = ".".join(model.fqn)
that_fqn = ".".join(other_model.fqn)
compiler_error(
model, "Model '{}' depends on model '{}' which is "
"disabled".format(that_fqn, this_fqn))
def compile_model(self, linker, model, models):
try:
jinja = jinja2.Environment(loader=jinja2.FileSystemLoader(
searchpath=model.root_dir))
# this is a dumb jinja2 bug -- on windows, forward slashes are EXPECTED
posix_filepath = '/'.join(split_path(model.rel_filepath))
template = jinja.get_template(posix_filepath)
context = self.get_context(linker, model, models)
rendered = template.render(context)
except jinja2.exceptions.TemplateSyntaxError as e:
compiler_error(model, str(e))
return rendered
def validate(self, data):
required = [
'source_schema',
'target_schema',
'source_table',
'target_table',
'unique_key',
'updated_at',
]
for key in required:
if data.get(key, None) is None:
compiler_error(
"Invalid archive config: missing required field '{}'".
format(key))
def do_compile(self):
schema_tests = []
for model_name, constraint_blob in self.schema.items():
constraints = constraint_blob.get('constraints', {})
for constraint_type, constraint_data in constraints.items():
if constraint_data is None:
compiler_error(
self, "no constraints given to test: '{}.{}'".format(
model_name, constraint_type))
for params in constraint_data:
schema_test_klass = self.get_test(constraint_type)
schema_test = schema_test_klass(self.project,
self.og_target_dir,
self.rel_filepath,
model_name, params)
schema_tests.append(schema_test)
return schema_tests
def compile_model(self, linker, model, models):
try:
fs_loader = jinja2.FileSystemLoader(searchpath=model.root_dir)
jinja = jinja2.Environment(loader=fs_loader)
template_contents = dbt.clients.system.load_file_contents(
model.absolute_path)
template = jinja.from_string(template_contents)
context = self.get_context(linker, model, models)
rendered = template.render(context)
except jinja2.exceptions.TemplateSyntaxError as e:
compiler_error(model, str(e))
except jinja2.exceptions.UndefinedError as e:
compiler_error(model, str(e))
return rendered
def __get_hooks(self, relevant_configs, key):
hooks = []
if key not in relevant_configs:
return []
new_hooks = relevant_configs[key]
if type(new_hooks) not in [list, tuple]:
new_hooks = [new_hooks]
for hook in new_hooks:
if type(hook) != str:
name = ".".join(self.fqn)
compiler_error(
None, "{} for model {} is not a string!".format(key, name))
hooks.append(hook)
return hooks
def dist_qualifier(self, model_config):
if 'dist' not in model_config:
return ''
if (self.is_view or self.is_ephemeral) and 'dist' in model_config:
return ''
dist_key = model_config['dist']
if type(dist_key) != str:
compiler_error(
self,
"The provided distkey '{}' is not valid!".format(dist_key))
dist_key = dist_key.strip().lower()
adapter = get_adapter(self.project.run_environment())
return adapter.dist_qualifier(dist_key)
def get_dist_qualifier(model, project):
model_config = model.get('config', {})
if 'dist' not in model_config:
return ''
if get_materialization(model) not in ('table', 'incremental'):
return ''
dist_key = model_config.get('dist')
if not isinstance(dist_key, basestring):
compiler_error(
model, "The provided distkey '{}' is not valid!".format(dist_key))
dist_key = dist_key.strip().lower()
adapter = get_adapter(project.run_environment())
return adapter.dist_qualifier(dist_key)
def sort_qualifier(self, model_config):
if 'sort' not in model_config:
return ''
if (self.is_view or self.is_ephemeral) and 'sort' in model_config:
return ''
sort_keys = model_config['sort']
sort_type = model_config.get('sort_type', 'compound')
if type(sort_type) != str:
compiler_error(
self,
"The provided sort_type '{}' is not valid!".format(sort_type))
sort_type = sort_type.strip().lower()
adapter = get_adapter(self.project.run_environment())
return adapter.sort_qualifier(sort_type, sort_keys)
def dist_qualifier(self, model_config):
if 'dist' not in model_config:
return ''
if (self.is_view or self.is_ephemeral) and 'dist' in model_config:
return ''
dist_key = model_config['dist']
if type(dist_key) != str:
compiler_error(
self,
"The provided distkey '{}' is not valid!".format(dist_key))
dist_key = dist_key.strip().lower()
if dist_key in ['all', 'even']:
return 'diststyle {}'.format(dist_key)
else:
return 'diststyle key distkey ("{}")'.format(dist_key)
def get_sort_qualifier(model, project):
model_config = model.get('config', {})
if 'sort' not in model['config']:
return ''
if get_materialization(model) not in ('table', 'incremental'):
return ''
sort_keys = model_config.get('sort')
sort_type = model_config.get('sort_type', 'compound')
if not isinstance(sort_type, basestring):
compiler_error(
model,
"The provided sort_type '{}' is not valid!".format(sort_type))
sort_type = sort_type.strip().lower()
adapter = get_adapter(project.run_environment())
return adapter.sort_qualifier(sort_type, sort_keys)
def compile_string(self, ctx, string):
# python 2+3 check for stringiness
try:
basestring
except NameError:
basestring = str
# if bool/int/float/etc are passed in, don't compile anything
if not isinstance(string, basestring):
return string
try:
fs_loader = jinja2.FileSystemLoader(
searchpath=self.project['macro-paths'])
env = jinja2.Environment(loader=fs_loader)
template = env.from_string(string, globals=ctx)
return template.render(ctx)
except jinja2.exceptions.TemplateSyntaxError as e:
compiler_error(self, str(e))
except jinja2.exceptions.UndefinedError as e:
compiler_error(self, str(e))
def compile_string(self, ctx, string):
try:
env = jinja2.Environment()
return env.from_string(string).render(ctx)
except jinja2.exceptions.TemplateSyntaxError as e:
compiler_error(self, str(e))
def __getattr__(self, attr):
if attr in self:
return self.get(attr)
else:
compiler_error(self.model, "'{}' is undefined".format(attr))
