def print_model_start_line(model, schema_name, index, total):
msg = "START {model_type} model {schema}.{relation}".format(
model_type=get_materialization(model),
schema=schema_name,
relation=model.get('name'))
print_fancy_output_line(msg, 'RUN', index, total)
def config(*args, **kwargs):
if len(args) == 1 and len(kwargs) == 0:
opts = args[0]
elif len(args) == 0 and len(kwargs) > 0:
opts = kwargs
else:
dbt.utils.compiler_error(
model.get('name'),
"Invalid model config given inline in {}".format(model))
cfg.update_in_model_config(opts)
def print_archive_result_line(result, index, total):
model = result.node
info = 'OK archived'
if result.errored:
info = 'ERROR archiving'
cfg = model.get('config', {})
print_fancy_output_line(
"{info} {source_schema}.{source_table} --> "
"{target_schema}.{target_table}".format(info=info, **cfg),
result.status, index, total, result.execution_time)
def print_model_result_line(result, schema_name, index, total):
model = result.node
info = 'OK created'
if result.errored:
info = 'ERROR creating'
print_fancy_output_line(
"{info} {model_type} model {schema}.{relation}".format(
info=info,
model_type=get_materialization(model),
schema=schema_name,
relation=model.get('name')), result.status, index, total,
result.execution_time)
def print_test_result_line(result, schema_name, index, total):
model = result.node
info = 'PASS'
if result.errored:
info = "ERROR"
elif result.status > 0:
info = 'FAIL {}'.format(result.status)
result.fail = True
elif result.status == 0:
info = 'PASS'
else:
raise RuntimeError("unexpected status: {}".format(result.status))
print_fancy_output_line(
"{info} {name}".format(info=info, name=model.get('name')), info, index,
total, result.execution_time)
def execute_model(profile, model, existing):
adapter = get_adapter(profile)
schema = adapter.get_default_schema(profile)
tmp_name = '{}__dbt_tmp'.format(model.get('name'))
if dbt.flags.NON_DESTRUCTIVE:
# for non destructive mode, we only look at the already existing table.
tmp_name = model.get('name')
result = None
# TRUNCATE / DROP
if get_materialization(model) == 'table' and \
dbt.flags.NON_DESTRUCTIVE and \
existing.get(tmp_name) == 'table':
# tables get truncated instead of dropped in non-destructive mode.
adapter.truncate(profile=profile,
table=tmp_name,
model_name=model.get('name'))
elif dbt.flags.NON_DESTRUCTIVE:
# never drop existing relations in non destructive mode.
pass
elif (get_materialization(model) != 'incremental'
and existing.get(tmp_name) is not None):
# otherwise, for non-incremental things, drop them with IF EXISTS
adapter.drop(profile=profile,
relation=tmp_name,
relation_type=existing.get(tmp_name),
model_name=model.get('name'))
# and update the list of what exists
existing = adapter.query_for_existing(profile,
schema,
model_name=model.get('name'))
# EXECUTE
if get_materialization(model) == 'view' and dbt.flags.NON_DESTRUCTIVE and \
model.get('name') in existing:
# views don't need to be recreated in non destructive mode since they
# will repopulate automatically. note that we won't run DDL for these
# views either.
pass
elif is_enabled(model) and get_materialization(model) != 'ephemeral':
result = adapter.execute_model(profile, model)
# DROP OLD RELATION AND RENAME
if dbt.flags.NON_DESTRUCTIVE:
# in non-destructive mode, we truncate and repopulate tables, and
# don't modify views.
pass
elif get_materialization(model) in ['table', 'view']:
# otherwise, drop tables and views, and rename tmp tables/views to
# their new names
if existing.get(model.get('name')) is not None:
adapter.drop(profile=profile,
relation=model.get('name'),
relation_type=existing.get(model.get('name')),
model_name=model.get('name'))
adapter.rename(profile=profile,
from_name=tmp_name,
to_name=model.get('name'),
model_name=model.get('name'))
return result
def is_enabled(model):
return model.get('config', {}).get('enabled') is True
def get_hashed_contents(model):
return hashlib.md5(model.get('raw_sql').encode('utf-8')).hexdigest()
def get_hash(model):
return hashlib.md5(model.get('unique_id').encode('utf-8')).hexdigest()
def print_test_start_line(model, schema_name, index, total):
msg = "START test {name}".format(name=model.get('name'))
print_fancy_output_line(msg, 'RUN', index, total)
def print_archive_start_line(model, index, total):
cfg = model.get('config', {})
msg = "START archive {source_schema}.{source_table} --> "\
"{target_schema}.{target_table}".format(**cfg)
print_fancy_output_line(msg, 'RUN', index, total)