def process(self, operation: Operation):
filepath = self.get_valid_filepath(operation.filename)
file_dir = os.path.dirname(filepath)
file_name = os.path.basename(filepath)
self.log.debug(f'Processing {self._script_type} script: {filepath}')
argv = self.get_argv(filepath)
envs = os.environ.copy(
) # Make sure this process's env is "available" in subprocess
envs.update(operation.env_vars_as_dict())
t0 = time.time()
try:
run(argv, cwd=file_dir, env=envs, check=True, stderr=PIPE)
except CalledProcessError as cpe:
error_msg = str(cpe.stderr.decode())
self.log.error(f'Error executing {file_name}: {error_msg}')
error_trim_index = error_msg.rfind('\n', 0,
error_msg.rfind('Error'))
if error_trim_index != -1:
raise RuntimeError(
f'({file_name}): {error_msg[error_trim_index:].strip()}'
) from cpe
else:
raise RuntimeError(f'({file_name})') from cpe
except Exception as ex:
self.log_and_raise(file_name, ex)
t1 = time.time()
duration = (t1 - t0)
self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def process(self, operation: Operation):
filepath = self.get_valid_filepath(operation.filename)
file_dir = os.path.dirname(filepath)
file_name = os.path.basename(filepath)
self.log.debug(f'Processing notebook: {filepath}')
# We'll always use the ElyraEngine. This engine is essentially the default Papermill engine
# but allows for environment variables to be passed to the kernel process (via 'kernel_env').
# If the current notebook server is running with Enterprise Gateway configured, we will also
# point the 'kernel_manager_class' to our HTTPKernelManager so that notebooks run as they
# would outside of Elyra. Current working directory (cwd) is specified both for where papermill
# runs the notebook (cwd) and where the directory of the kernel process (kernel_cwd). The latter
# of which is important when EG is configured.
additional_kwargs = dict()
additional_kwargs['engine_name'] = "ElyraEngine"
additional_kwargs[
'cwd'] = file_dir # For local operations, papermill runs from this dir
additional_kwargs['kernel_cwd'] = file_dir
additional_kwargs['kernel_env'] = operation.env_vars_as_dict()
if GatewayClient.instance().gateway_enabled:
additional_kwargs[
'kernel_manager_class'] = 'elyra.pipeline.http_kernel_manager.HTTPKernelManager'
t0 = time.time()
try:
papermill.execute_notebook(filepath, filepath, **additional_kwargs)
except Exception as ex:
raise RuntimeError(
f'Internal error executing {filepath}: {ex}') from ex
t1 = time.time()
duration = (t1 - t0)
self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def test_env_list_to_dict_function():
env_variables = ['FOO=Bar', 'BAR=Foo', None, '', ' =Dog', 'DOG= ']
env_variables_dict = {"FOO": "Bar", "BAR": "Foo", "DOG": " "}
test_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
env_vars=env_variables,
runtime_image='tensorflow/tensorflow:latest')
assert test_operation.env_vars_as_dict() == env_variables_dict
def test_env_list_to_dict_function():
env_variables = ['KEY=value', None, '', ' =empty_key', '=no_key', 'EMPTY_VALUE= ',
'NO_VALUE=', 'KEY2=value2', 'TWO_EQUALS=KEY=value', '==']
env_variables_dict = {"KEY": "value", "KEY2": "value2", "EMPTY_VALUE": " ", "TWO_EQUALS": "KEY=value"}
test_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
name='test',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
env_vars=env_variables,
runtime_image='tensorflow/tensorflow:latest')
assert test_operation.env_vars_as_dict() == env_variables_dict
def process(self, operation: Operation):
filepath = self.get_valid_filepath(operation.filename)
file_dir = os.path.dirname(filepath)
file_name = os.path.basename(filepath)
self.log.debug(f'Processing notebook: {filepath}')
# We'll always use the ElyraEngine. This engine is essentially the default Papermill engine
# but allows for environment variables to be passed to the kernel process (via 'kernel_env').
# If the current notebook server is running with Enterprise Gateway configured, we will also
# point the 'kernel_manager_class' to GatewayKernelManager so that notebooks run as they
# would outside of Elyra. Current working directory (cwd) is specified both for where papermill
# runs the notebook (cwd) and where the directory of the kernel process (kernel_cwd). The latter
# of which is important when EG is configured.
additional_kwargs = dict()
additional_kwargs['engine_name'] = "ElyraEngine"
additional_kwargs[
'cwd'] = file_dir # For local operations, papermill runs from this dir
additional_kwargs['kernel_cwd'] = file_dir
envs = os.environ.copy(
) # Make sure this process's env is "available" in the kernel subprocess
envs.update(operation.env_vars_as_dict())
additional_kwargs['kernel_env'] = envs
if GatewayClient.instance().gateway_enabled:
additional_kwargs[
'kernel_manager_class'] = 'jupyter_server.gateway.managers.GatewayKernelManager'
t0 = time.time()
try:
papermill.execute_notebook(filepath, filepath, **additional_kwargs)
except papermill.PapermillExecutionError as pmee:
self.log.error(
f'Error executing {file_name} in cell {pmee.exec_count}: ' +
f'{str(pmee.ename)} {str(pmee.evalue)}')
raise RuntimeError(
f'({file_name}) in cell {pmee.exec_count}: ' +
f'{str(pmee.ename)} {str(pmee.evalue)}') from pmee
except Exception as ex:
self.log_and_raise(file_name, ex)
t1 = time.time()
duration = (t1 - t0)
self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def process(self, operation: Operation):
filepath = self.get_valid_filepath(operation.filename)
file_dir = os.path.dirname(filepath)
file_name = os.path.basename(filepath)
self.log.debug(f'Processing python script: {filepath}')
argv = ['python3', filepath, '--PYTHONHOME', file_dir]
envs = operation.env_vars_as_dict()
t0 = time.time()
try:
run(argv, cwd=file_dir, env=envs, check=True)
except Exception as ex:
raise RuntimeError(
f'Internal error executing {filepath}: {ex}') from ex
t1 = time.time()
duration = (t1 - t0)
self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def process(self, operation: Operation):
filepath = self.get_valid_filepath(operation.filename)
file_dir = os.path.dirname(filepath)
file_name = os.path.basename(filepath)
self.log.debug(f'Processing R script: {filepath}')
argv = ['Rscript', filepath]
envs = os.environ.copy(
) # Make sure this process's env is "available" in subprocess
envs.update(operation.env_vars_as_dict())
t0 = time.time()
try:
run(argv, cwd=file_dir, env=envs, check=True)
except Exception as ex:
raise RuntimeError(
f'Internal error executing {filepath}: {ex}') from ex
t1 = time.time()
duration = (t1 - t0)
self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def _collect_envs(operation: Operation) -> Dict:
envs = os.environ.copy(
) # Make sure this process's env is "available" in the kernel subprocess
envs.update(operation.env_vars_as_dict())
envs['ELYRA_RUNTIME_ENV'] = "local" # Special case
return envs