def execute(self, code):
"""Run code
"""
_, path_to_tmp = tempfile.mkstemp()
Path(path_to_tmp).write_text(code)
run_template = Placeholder(self.run_template)
source = run_template.render(dict(path_to_code=path_to_tmp))
res = subprocess.run(shlex.split(source), **self.subprocess_run_kwargs)
stdout = res.stdout.decode('utf-8')
stderr = res.stderr.decode('utf-8')
if res.returncode != 0:
# log source code without expanded params
self._logger.info(
('{} returned stdout: '
'{}\nstderr: {}\n'
'exit status {}').format(code, stdout, stderr,
res.returncode))
raise RuntimeError(
('Error executing code.\nReturned stdout: '
'{}\nstderr: {}\n'
'exit status {}').format(stdout, stderr, res.returncode))
else:
self._logger.info(('Finished running {}. stdout: {},'
' stderr: {}').format(self, stdout, stderr))
class Source(abc.ABC):
def __init__(self, value):
self.value = Placeholder(value)
self._post_init_validation(self.value)
@property
def needs_render(self):
return self.value.needs_render
def render(self, params):
self.value.render(params)
self._post_render_validation(self.value.value, params)
def __str__(self):
return str(self.value)
# required by subclasses
@property
@abc.abstractmethod
def doc(self):
pass
@property
@abc.abstractmethod
def doc_short(self):
pass
@property
@abc.abstractmethod
def language(self):
pass
@property
@abc.abstractmethod
def loc(self):
pass
# optional validation
def _post_render_validation(self, rendered_value, params):
pass
def _post_init_validation(self, value):
pass
def test_can_copy_placeholders(path_to_assets):
path = str(path_to_assets / 'templates')
env = Environment(loader=FileSystemLoader(path), undefined=StrictUndefined)
st = Placeholder(env.get_template('template.sql'))
cst = copy(st)
dpst = deepcopy(st)
assert cst.render({'file': 'a_file'}) == '\n\na_file'
assert str(cst) == '\n\na_file'
assert dpst.render({'file': 'a_file2'}) == '\n\na_file2'
assert str(dpst) == '\n\na_file2'
def execute(self, code):
"""Run code
"""
ftp = self.connection.open_sftp()
path_remote = self.path_to_directory + self._random_name()
_, path_to_tmp = tempfile.mkstemp()
Path(path_to_tmp).write_text(code)
ftp.put(path_to_tmp, path_remote)
ftp.close()
run_template = Placeholder(self.run_template)
source = run_template.render(dict(path_to_code=path_remote))
# stream stdout. related: https://stackoverflow.com/q/31834743
# using pty is not ideal, fabric has a clean implementation for this
# worth checking out
stdin, stdout, stderr = self.connection.exec_command(source,
get_pty=True)
for line in iter(stdout.readline, ""):
self._logger.info('(STDOUT): {}'.format(line))
returncode = stdout.channel.recv_exit_status()
stdout = ''.join(stdout)
stderr = ''.join(stderr)
if returncode != 0:
# log source code without expanded params
self._logger.info(f'{code} returned stdout: '
f'{stdout} and stderr: {stderr} '
f'and exit status {returncode}')
raise CalledProcessError(returncode, code)
else:
self._logger.info(f'Finished running {self}. stdout: {stdout},'
f' stderr: {stderr}')
return {'returncode': returncode, 'stdout': stdout, 'stderr': stderr}
def test_string_identifier_initialized_with_template_from_env():
tmp = tempfile.mkdtemp()
Path(tmp, 'template.sql').write_text('{{key}}')
env = Environment(loader=FileSystemLoader(tmp), undefined=StrictUndefined)
template = env.get_template('template.sql')
si = Placeholder(template).render(params=dict(key='things'))
assert str(si) == 'things'
def __init__(self,
identifier,
path_to_metadata,
exists_command,
delete_command,
client=None):
self._identifier = Placeholder(str(identifier))
self._path_to_metadata = path_to_metadata
self._client = client
self.exists_command = Placeholder(str(exists_command))
self.delete_command = Placeholder(str(delete_command))
self.did_download_metadata = False
self.task = None
self._logger = logging.getLogger(__name__)
def _init_identifier(self, identifier):
if not isinstance(identifier, (str, Path)):
raise TypeError('File must be initialized with a str or a '
'pathlib.Path')
return Placeholder(str(identifier))
def test_strict_templates_initialized_from_strict_template(path_to_assets):
path = str(path_to_assets / 'templates')
env = Environment(loader=FileSystemLoader(path), undefined=StrictUndefined)
st = Placeholder(env.get_template('template.sql'))
assert Placeholder(st).render({'file': 1})
def test_strict_templates_raises_error_if_not_strictundefined(path_to_assets):
path = str(path_to_assets / 'templates')
env = Environment(loader=FileSystemLoader(path))
with pytest.raises(ValueError):
Placeholder(env.get_template('template.sql'))
def test_raises_error_if_extra_parameter():
with pytest.raises(TypeError):
(Placeholder('SELECT * FROM {{table}}').render(table=1, not_a_param=1))
def test_raises_error_if_missing_parameter():
with pytest.raises(TypeError):
Placeholder('SELECT * FROM {{table}}').render()
def test_verify_if_strict_template_needs_render():
assert Placeholder('I need {{params}}').needs_render
def __init__(self, value):
self.value = Placeholder(value)
self._post_init_validation(self.value)
def test_string_identifier_initialized_with_template_raises_error():
with pytest.raises(ValueError):
Placeholder(Template('{{key}}')).render(params=dict(key='things'))
def test_string_identifier_initialized_with_str():
si = Placeholder('things').render({})
# assert repr(si) == "StringPlaceholder('things')"
assert str(si) == 'things'
class GenericProduct(Product):
def __init__(self,
identifier,
path_to_metadata,
exists_command,
delete_command,
client=None):
self._identifier = Placeholder(str(identifier))
self._path_to_metadata = path_to_metadata
self._client = client
self.exists_command = Placeholder(str(exists_command))
self.delete_command = Placeholder(str(delete_command))
self.did_download_metadata = False
self.task = None
self._logger = logging.getLogger(__name__)
def _init_identifier(self, identifier):
pass
# TODO: create a mixing with this so all client-based tasks can include it
@property
def client(self):
if self._client is None:
default = self.task.dag.clients.get(type(self))
if default is None:
raise ValueError('{} must be initialized with a client'.format(
type(self).__name__))
else:
self._client = default
return self._client
def render(self, params, **kwargs):
# overriding parent implementation since this product also needs
# render for other variables
self._identifier.render(params, **kwargs)
self.exists_command.render(params, **kwargs)
self.delete_command.render(params, **kwargs)
@property
def _path_to_metadata_file(self):
return self._path_to_metadata + str(self._identifier) + '.json'
def fetch_metadata(self):
try:
meta = self.client.read_file(self._path_to_metadata_file)
except Exception as e:
self._logger.exception(e)
return {}
else:
return json.loads(meta)
def save_metadata(self):
metadata_str = json.dumps(self.metadata)
self.client.write_to_file(metadata_str, self._path_to_metadata_file)
# TODO: implement
def exists(self):
return True
def delete(self, force=False):
pass
@property
def name(self):
return Path(str(self._path_to_metadata)).with_suffix('').name
def get_template(self, name):
template = self.env.get_template(name)
return Placeholder(template)
def test_placeholder_is_picklable():
p = Placeholder('{{hi}}')
pickle.loads(pickle.dumps(p))
def test_string_identifier_initialized_with_str_with_tags():
si = Placeholder('{{key}}').render(params=dict(key='things'))
# assert repr(si) == "StringPlaceholder('things')"
assert str(si) == 'things'
def test_init_placeholder_with_placeholder():
t = Placeholder('{{file}}')
tt = Placeholder(t)
assert tt.render({'file': 'some file'})
def test_verify_if_strict_template_is_literal():
assert not Placeholder('no need for rendering').needs_render
def __init__(self, value):
value = str(value)
super().__init__(Placeholder(value))
