def test_placeholder_initialized_with_placeholder(env_init, path_to_test_pkg):
env = env_init(path_to_test_pkg)
placeholder = Placeholder(env.get_template('query.sql'))
placeholder_new = Placeholder(placeholder)
assert placeholder_new._raw == placeholder._raw
assert placeholder_new.path == placeholder.path
assert placeholder_new is not placeholder
assert placeholder_new._loader_init is not None
assert placeholder_new._loader_init == placeholder._loader_init
assert placeholder_new._loader_init is not placeholder._loader_init
def test_error_if_initialized_with_unsupported_type():
with pytest.raises(TypeError) as excinfo:
Placeholder(None)
assert ('Placeholder must be initialized with a Template, '
'Placeholder, pathlib.Path or str, got NoneType instead') == str(
excinfo.value)
def execute(self, code):
"""Run code
"""
fd, path_to_tmp = tempfile.mkstemp()
os.close(fd)
Path(path_to_tmp).write_text(code)
run_template = Placeholder(self.run_template)
# we need quoting to make windows paths keep their \\ separators when
# running shlex.split
source = run_template.render(
dict(path_to_code=shlex.quote(path_to_tmp)))
res = subprocess.run(shlex.split(source), **self.subprocess_run_kwargs)
Path(path_to_tmp).unlink()
stdout = res.stdout.decode('utf-8')
stderr = res.stderr.decode('utf-8')
if res.returncode != 0:
# log source code without expanded params
self._logger.info(
('{} returned stdout: '
'{}\nstderr: {}\n'
'exit status {}').format(code, stdout, stderr,
res.returncode))
raise RuntimeError(
('Error executing code.\nReturned stdout: '
'{}\nstderr: {}\n'
'exit status {}').format(stdout, stderr, res.returncode))
else:
self._logger.info(('Finished running {}. stdout: {},'
' stderr: {}').format(self, stdout, stderr))
def get_template(self, name):
"""Load a template by name
"""
try:
template = self.env.get_template(str(name))
except exceptions.TemplateNotFound as e:
exception = e
else:
exception = None
if exception is not None:
expected_path = str(Path(self.path_full, name))
# user saved the template locally, but the source loader is
# configured to load from a different place
if Path(name).exists():
raise exceptions.TemplateNotFound(
f'{str(name)!r} template does not exist. '
'However such a file exists in the current working '
'directory, if you want to load it as a template, move it '
f'to {self.path_full!r} or remove the source_loader')
# no template and the file does not exist, raise a generic message
else:
raise exceptions.TemplateNotFound(
f'{str(name)!r} template does not exist. '
'Based on your configuration, if should be located '
f'at: {expected_path!r}')
return Placeholder(template)
def test_raise(tmp_directory):
p = Placeholder("{% raise 'some error message' %}")
with pytest.raises(TemplateRuntimeError) as excinfo:
p.render({})
assert str(excinfo.value) == 'some error message'
def test_macros_with_template_environment(env_init, path_to_test_pkg):
env = env_init(path_to_test_pkg)
# this template contains a macro
placeholder = Placeholder(env.get_template('query.sql'))
placeholder.render({})
assert str(placeholder) == 'SELECT * FROM table'
def test_error_if_missing_upstream():
p = Placeholder('SELECT * FROM {{upstream["name"]}}')
upstream = Upstream({'a': 1}, name='task')
with pytest.raises(UpstreamKeyError) as excinfo:
p.render({'upstream': upstream})
assert ('Cannot obtain upstream dependency "name" for task "task"'
in str(excinfo.value))
def __init__(self, value, hot_reload=False, optional=None, required=None):
self._primitive = value
# rename, and make it private
self._placeholder = Placeholder(value,
hot_reload=hot_reload,
required=required)
self._post_init_validation(self._placeholder)
self._optional = optional
self._required = required
def test_task_init_source_with_placeholder_obj(Task, prod, source):
"""
Testing we can initialize a task with a Placeholder as the source argument
"""
dag = DAG()
dag.clients[Task] = Mock()
dag.clients[type(prod)] = Mock()
Task(Placeholder(source), prod, dag, name='task')
def test_error_if_no_upstream():
p = Placeholder('SELECT * FROM {{upstream["name"]}}')
upstream = Upstream({}, name='task')
with pytest.raises(UpstreamKeyError) as excinfo:
p.render({'upstream': upstream})
msg = ('Cannot obtain upstream dependency "name". '
'Task "task" has no upstream dependencies')
assert msg == str(excinfo.value)
def test_error_on_read_before_render():
placeholder = Placeholder('some template {{variable}}')
with pytest.raises(RuntimeError) as excinfo:
str(placeholder)
assert (
'Tried to read Placeholder '
'Placeholder(\'some template {{variable}}\') without rendering first'
) == str(excinfo.value)
def test_can_copy_placeholders(path_to_assets):
path = str(path_to_assets / 'templates')
env = Environment(loader=FileSystemLoader(path), undefined=StrictUndefined)
st = Placeholder(env.get_template('template.sql'))
cst = copy(st)
dpst = deepcopy(st)
assert cst.render({'file': 'a_file'}) == '\n\na_file'
assert str(cst) == '\n\na_file'
assert dpst.render({'file': 'a_file2'}) == '\n\na_file2'
assert str(dpst) == '\n\na_file2'
def test_hot_reload_with_with_path(tmp_directory):
query_path = Path(tmp_directory, 'simple_query.sql')
query_path.write_text('SELECT * FROM {{tag}}')
placeholder = Placeholder(Path(query_path), hot_reload=True)
placeholder.render({'tag': 'table'})
assert str(placeholder) == 'SELECT * FROM table'
query_path.write_text('SELECT * FROM {{tag}} WHERE x = 10')
placeholder.render({'tag': 'table'})
assert str(placeholder) == 'SELECT * FROM table WHERE x = 10'
def test_string_identifier_initialized_with_template_from_env():
tmp = tempfile.mkdtemp()
Path(tmp, 'template.sql').write_text('{{key}}')
env = Environment(loader=FileSystemLoader(tmp), undefined=StrictUndefined)
template = env.get_template('template.sql')
si = Placeholder(template).render(params=dict(key='things'))
assert str(si) == 'things'
def test_placeholder_is_copied_upon_initialization():
dag = DAG()
dag.clients[SQLScript] = Mock()
dag.clients[PostgresRelation] = Mock()
p = Placeholder('CREATE TABLE {{product}} AS SELECT * FROM TABLE')
t1 = SQLScript(p,
PostgresRelation(('schema', 'a_table', 'table')),
dag,
name='t1')
t2 = SQLScript(p,
PostgresRelation(('schema', 'another_table', 'table')),
dag,
name='t2')
assert t1.source._placeholder is not t2.source._placeholder
def execute(self, code):
"""Run code
"""
ftp = self.connection.open_sftp()
path_remote = self.path_to_directory + self._random_name()
fd, path_to_tmp = tempfile.mkstemp()
os.close(fd)
Path(path_to_tmp).write_text(code)
ftp.put(path_to_tmp, path_remote)
ftp.close()
run_template = Placeholder(self.run_template)
source = run_template.render(dict(path_to_code=path_remote))
# stream stdout. related: https://stackoverflow.com/q/31834743
# using pty is not ideal, fabric has a clean implementation for this
# worth checking out
stdin, stdout, stderr = self.connection.exec_command(source,
get_pty=True)
for line in iter(stdout.readline, ""):
self._logger.info('(STDOUT): {}'.format(line))
returncode = stdout.channel.recv_exit_status()
stdout = ''.join(stdout)
stderr = ''.join(stderr)
if returncode != 0:
# log source code without expanded params
self._logger.info(
'%s returned stdout: '
'%s and stderr: %s '
'and exit status %s', code, stdout, stderr, returncode)
raise subprocess.CalledProcessError(returncode, code)
else:
self._logger.info('Finished running %s. stdout: %s,'
' stderr: %s', self, stdout, stderr)
return {'returncode': returncode, 'stdout': stdout, 'stderr': stderr}
def test_hot_reload_with_template_env(env_init, path_to_test_pkg):
query_path = Path(path_to_test_pkg, 'templates', 'query.sql')
query_original = query_path.read_text()
env = env_init(path_to_test_pkg)
placeholder = Placeholder(env.get_template('query.sql'), hot_reload=True)
placeholder.render({})
assert str(placeholder) == 'SELECT * FROM table'
# use a macro to make sure the template loader is correctly initialized
query = ('{% import "macros.sql" as m %}SELECT * FROM {{m.my_macro()}}'
' WHERE x = 10')
query_path.write_text(query)
placeholder.render({})
assert str(placeholder) == 'SELECT * FROM table WHERE x = 10'
# revert query to their original value
query_path.write_text(query_original)
def __init__(self, value, hot_reload=False):
# hot_reload does not apply here, ignored
value = str(value)
super().__init__(Placeholder(value))
def test_error_when_init_from_string_and_hot_reload():
with pytest.raises(ValueError) as excinfo:
Placeholder('SELECT * FROM table', hot_reload=True)
m = 'hot_reload only works when Placeholder is initialized from a file'
assert str(excinfo.value) == m
def __init__(self, value, hot_reload=False):
self._primitive = value
# rename, and make it private
self._placeholder = Placeholder(value, hot_reload=hot_reload)
self._post_init_validation(self._placeholder)
def _init_identifier(self, identifier):
return Placeholder(identifier)
def test_verify_if_strict_template_is_literal():
assert not Placeholder('no need for rendering').needs_render
def test_raises_error_if_extra_parameter():
with pytest.raises(TypeError):
(Placeholder('SELECT * FROM {{table}}').render(table=1, not_a_param=1))
def test_placeholder_is_picklable():
p = Placeholder('{{hi}}')
pickle.loads(pickle.dumps(p))
def _init_identifier(self, identifier):
if not isinstance(identifier, (str, Path)):
raise TypeError('File must be initialized with a str or a '
'pathlib.Path')
return Placeholder(str(identifier))
def test_strict_templates_raises_error_if_not_strictundefined(path_to_assets):
path = str(path_to_assets / 'templates')
env = Environment(loader=FileSystemLoader(path))
with pytest.raises(ValueError):
Placeholder(env.get_template('template.sql'))
def test_strict_templates_initialized_from_jinja_template(path_to_assets):
path = str(path_to_assets / 'templates')
env = Environment(loader=FileSystemLoader(path), undefined=StrictUndefined)
st = Placeholder(env.get_template('template.sql'))
assert st.render({'file': 1})
def test_error_if_raw_source_cant_be_retrieved():
with pytest.raises(ValueError) as excinfo:
Placeholder(Template('some template', undefined=StrictUndefined))
assert 'Could not load raw source from jinja2.Template' in str(
excinfo.value)
def test_verify_if_strict_template_needs_render():
assert Placeholder('I need {{params}}').needs_render
def test_raises_error_if_missing_parameter():
with pytest.raises(TypeError):
Placeholder('SELECT * FROM {{table}}').render()
