def execute(self, code): """Run code """ _, path_to_tmp = tempfile.mkstemp() Path(path_to_tmp).write_text(code) run_template = Placeholder(self.run_template) source = run_template.render(dict(path_to_code=path_to_tmp)) res = subprocess.run(shlex.split(source), **self.subprocess_run_kwargs) stdout = res.stdout.decode('utf-8') stderr = res.stderr.decode('utf-8') if res.returncode != 0: # log source code without expanded params self._logger.info( ('{} returned stdout: ' '{}\nstderr: {}\n' 'exit status {}').format(code, stdout, stderr, res.returncode)) raise RuntimeError( ('Error executing code.\nReturned stdout: ' '{}\nstderr: {}\n' 'exit status {}').format(stdout, stderr, res.returncode)) else: self._logger.info(('Finished running {}. stdout: {},' ' stderr: {}').format(self, stdout, stderr))
class Source(abc.ABC): def __init__(self, value): self.value = Placeholder(value) self._post_init_validation(self.value) @property def needs_render(self): return self.value.needs_render def render(self, params): self.value.render(params) self._post_render_validation(self.value.value, params) def __str__(self): return str(self.value) # required by subclasses @property @abc.abstractmethod def doc(self): pass @property @abc.abstractmethod def doc_short(self): pass @property @abc.abstractmethod def language(self): pass @property @abc.abstractmethod def loc(self): pass # optional validation def _post_render_validation(self, rendered_value, params): pass def _post_init_validation(self, value): pass
def test_can_copy_placeholders(path_to_assets): path = str(path_to_assets / 'templates') env = Environment(loader=FileSystemLoader(path), undefined=StrictUndefined) st = Placeholder(env.get_template('template.sql')) cst = copy(st) dpst = deepcopy(st) assert cst.render({'file': 'a_file'}) == '\n\na_file' assert str(cst) == '\n\na_file' assert dpst.render({'file': 'a_file2'}) == '\n\na_file2' assert str(dpst) == '\n\na_file2'
def execute(self, code): """Run code """ ftp = self.connection.open_sftp() path_remote = self.path_to_directory + self._random_name() _, path_to_tmp = tempfile.mkstemp() Path(path_to_tmp).write_text(code) ftp.put(path_to_tmp, path_remote) ftp.close() run_template = Placeholder(self.run_template) source = run_template.render(dict(path_to_code=path_remote)) # stream stdout. related: https://stackoverflow.com/q/31834743 # using pty is not ideal, fabric has a clean implementation for this # worth checking out stdin, stdout, stderr = self.connection.exec_command(source, get_pty=True) for line in iter(stdout.readline, ""): self._logger.info('(STDOUT): {}'.format(line)) returncode = stdout.channel.recv_exit_status() stdout = ''.join(stdout) stderr = ''.join(stderr) if returncode != 0: # log source code without expanded params self._logger.info(f'{code} returned stdout: ' f'{stdout} and stderr: {stderr} ' f'and exit status {returncode}') raise CalledProcessError(returncode, code) else: self._logger.info(f'Finished running {self}. stdout: {stdout},' f' stderr: {stderr}') return {'returncode': returncode, 'stdout': stdout, 'stderr': stderr}
def test_string_identifier_initialized_with_template_from_env(): tmp = tempfile.mkdtemp() Path(tmp, 'template.sql').write_text('{{key}}') env = Environment(loader=FileSystemLoader(tmp), undefined=StrictUndefined) template = env.get_template('template.sql') si = Placeholder(template).render(params=dict(key='things')) assert str(si) == 'things'
def __init__(self, identifier, path_to_metadata, exists_command, delete_command, client=None): self._identifier = Placeholder(str(identifier)) self._path_to_metadata = path_to_metadata self._client = client self.exists_command = Placeholder(str(exists_command)) self.delete_command = Placeholder(str(delete_command)) self.did_download_metadata = False self.task = None self._logger = logging.getLogger(__name__)
def _init_identifier(self, identifier): if not isinstance(identifier, (str, Path)): raise TypeError('File must be initialized with a str or a ' 'pathlib.Path') return Placeholder(str(identifier))
def test_strict_templates_initialized_from_strict_template(path_to_assets): path = str(path_to_assets / 'templates') env = Environment(loader=FileSystemLoader(path), undefined=StrictUndefined) st = Placeholder(env.get_template('template.sql')) assert Placeholder(st).render({'file': 1})
def test_strict_templates_raises_error_if_not_strictundefined(path_to_assets): path = str(path_to_assets / 'templates') env = Environment(loader=FileSystemLoader(path)) with pytest.raises(ValueError): Placeholder(env.get_template('template.sql'))
def test_raises_error_if_extra_parameter(): with pytest.raises(TypeError): (Placeholder('SELECT * FROM {{table}}').render(table=1, not_a_param=1))
def test_raises_error_if_missing_parameter(): with pytest.raises(TypeError): Placeholder('SELECT * FROM {{table}}').render()
def test_verify_if_strict_template_needs_render(): assert Placeholder('I need {{params}}').needs_render
def __init__(self, value): self.value = Placeholder(value) self._post_init_validation(self.value)
def test_string_identifier_initialized_with_template_raises_error(): with pytest.raises(ValueError): Placeholder(Template('{{key}}')).render(params=dict(key='things'))
def test_string_identifier_initialized_with_str(): si = Placeholder('things').render({}) # assert repr(si) == "StringPlaceholder('things')" assert str(si) == 'things'
class GenericProduct(Product): def __init__(self, identifier, path_to_metadata, exists_command, delete_command, client=None): self._identifier = Placeholder(str(identifier)) self._path_to_metadata = path_to_metadata self._client = client self.exists_command = Placeholder(str(exists_command)) self.delete_command = Placeholder(str(delete_command)) self.did_download_metadata = False self.task = None self._logger = logging.getLogger(__name__) def _init_identifier(self, identifier): pass # TODO: create a mixing with this so all client-based tasks can include it @property def client(self): if self._client is None: default = self.task.dag.clients.get(type(self)) if default is None: raise ValueError('{} must be initialized with a client'.format( type(self).__name__)) else: self._client = default return self._client def render(self, params, **kwargs): # overriding parent implementation since this product also needs # render for other variables self._identifier.render(params, **kwargs) self.exists_command.render(params, **kwargs) self.delete_command.render(params, **kwargs) @property def _path_to_metadata_file(self): return self._path_to_metadata + str(self._identifier) + '.json' def fetch_metadata(self): try: meta = self.client.read_file(self._path_to_metadata_file) except Exception as e: self._logger.exception(e) return {} else: return json.loads(meta) def save_metadata(self): metadata_str = json.dumps(self.metadata) self.client.write_to_file(metadata_str, self._path_to_metadata_file) # TODO: implement def exists(self): return True def delete(self, force=False): pass @property def name(self): return Path(str(self._path_to_metadata)).with_suffix('').name
def get_template(self, name): template = self.env.get_template(name) return Placeholder(template)
def test_placeholder_is_picklable(): p = Placeholder('{{hi}}') pickle.loads(pickle.dumps(p))
def test_string_identifier_initialized_with_str_with_tags(): si = Placeholder('{{key}}').render(params=dict(key='things')) # assert repr(si) == "StringPlaceholder('things')" assert str(si) == 'things'
def test_init_placeholder_with_placeholder(): t = Placeholder('{{file}}') tt = Placeholder(t) assert tt.render({'file': 'some file'})
def test_verify_if_strict_template_is_literal(): assert not Placeholder('no need for rendering').needs_render
def __init__(self, value): value = str(value) super().__init__(Placeholder(value))