def test_env_list_to_dict_function():
env_variables = ['FOO=Bar', 'BAR=Foo', None, '', ' =Dog', 'DOG= ']
env_variables_dict = {"FOO": "Bar", "BAR": "Foo", "DOG": " "}
test_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
env_vars=env_variables,
runtime_image='tensorflow/tensorflow:latest')
assert test_operation.env_vars_as_dict() == env_variables_dict
def test_env_list_to_dict_function():
env_variables = ['KEY=value', None, '', ' =empty_key', '=no_key', 'EMPTY_VALUE= ',
'NO_VALUE=', 'KEY2=value2', 'TWO_EQUALS=KEY=value', '==']
env_variables_dict = {"KEY": "value", "KEY2": "value2", "EMPTY_VALUE": " ", "TWO_EQUALS": "KEY=value"}
test_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
name='test',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
env_vars=env_variables,
runtime_image='tensorflow/tensorflow:latest')
assert test_operation.env_vars_as_dict() == env_variables_dict
def test_fail_create_operation_missing_name():
with pytest.raises(TypeError):
Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
runtime_image='tensorflow/tensorflow:latest')
def process(self, operation: Operation):
filepath = self.get_valid_filepath(operation.filename)
file_dir = os.path.dirname(filepath)
file_name = os.path.basename(filepath)
self.log.debug(f'Processing {self._script_type} script: {filepath}')
argv = self.get_argv(filepath)
envs = os.environ.copy(
) # Make sure this process's env is "available" in subprocess
envs.update(operation.env_vars_as_dict())
t0 = time.time()
try:
run(argv, cwd=file_dir, env=envs, check=True, stderr=PIPE)
except CalledProcessError as cpe:
error_msg = str(cpe.stderr.decode())
self.log.error(f'Error executing {file_name}: {error_msg}')
error_trim_index = error_msg.rfind('\n', 0,
error_msg.rfind('Error'))
if error_trim_index != -1:
raise RuntimeError(
f'({file_name}): {error_msg[error_trim_index:].strip()}'
) from cpe
else:
raise RuntimeError(f'({file_name})') from cpe
except Exception as ex:
self.log_and_raise(file_name, ex)
t1 = time.time()
duration = (t1 - t0)
self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def process(self, operation: Operation):
filepath = self.get_valid_filepath(operation.filename)
file_dir = os.path.dirname(filepath)
file_name = os.path.basename(filepath)
self.log.debug(f'Processing notebook: {filepath}')
# We'll always use the ElyraEngine. This engine is essentially the default Papermill engine
# but allows for environment variables to be passed to the kernel process (via 'kernel_env').
# If the current notebook server is running with Enterprise Gateway configured, we will also
# point the 'kernel_manager_class' to our HTTPKernelManager so that notebooks run as they
# would outside of Elyra. Current working directory (cwd) is specified both for where papermill
# runs the notebook (cwd) and where the directory of the kernel process (kernel_cwd). The latter
# of which is important when EG is configured.
additional_kwargs = dict()
additional_kwargs['engine_name'] = "ElyraEngine"
additional_kwargs[
'cwd'] = file_dir # For local operations, papermill runs from this dir
additional_kwargs['kernel_cwd'] = file_dir
additional_kwargs['kernel_env'] = operation.env_vars_as_dict()
if GatewayClient.instance().gateway_enabled:
additional_kwargs[
'kernel_manager_class'] = 'elyra.pipeline.http_kernel_manager.HTTPKernelManager'
t0 = time.time()
try:
papermill.execute_notebook(filepath, filepath, **additional_kwargs)
except Exception as ex:
raise RuntimeError(
f'Internal error executing {filepath}: {ex}') from ex
t1 = time.time()
duration = (t1 - t0)
self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def good_operation():
test_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
runtime_image='tensorflow/tensorflow:latest')
return test_operation
def test_operations_are_equal(good_operation):
compare_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
runtime_image='tensorflow/tensorflow:latest')
assert compare_operation == good_operation
def test_fail_creating_operation_with_negative_memory_resources():
with pytest.raises(ValueError):
Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
memory='-1',
runtime_image='tensorflow/tensorflow:latest')
def test_fail_operations_are_equal(good_operation):
parent_operation_ids = ['id-123123-123123-123123', 'id-456456-456456-456456']
compare_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
parent_operations=parent_operation_ids,
runtime_image='tensorflow/tensorflow:latest')
with pytest.raises(AssertionError):
assert compare_operation == good_operation
def valid_operation():
return Operation(id='{{uuid}}',
type='execution_node',
classifier='execute-notebook-node',
filename='{{filename}}',
runtime_image='{{runtime_image}}',
env_vars=["var1=var1", "var2=var2"],
dependencies=["a.txt", "b.txt", "c.txt"],
outputs=["d.txt", "e.txt", "f.txt"],
)
def test_fail_validate_max_resource_value_exceeded():
system_max_size = str(sys.maxsize)
with pytest.raises(ValueError):
Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
memory=system_max_size,
runtime_image='tensorflow/tensorflow:latest')
def test_validate_max_resource_value():
system_max_size = str(sys.maxsize - 1)
test_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
memory=system_max_size,
runtime_image='tensorflow/tensorflow:latest')
assert test_operation.memory == system_max_size
def test_validate_resource_values_as_none():
test_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
runtime_image='tensorflow/tensorflow:latest')
assert test_operation.cpu is None
assert test_operation.gpu is None
assert test_operation.memory is None
def test_create_operation_with_dependencies():
dependencies = ['elyra/pipline/tests/resources', 'elyra/pipline/tests/resources/archive']
test_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
dependencies=dependencies,
runtime_image='tensorflow/tensorflow:latest')
assert test_operation.dependencies == dependencies
def test_create_operation_with_inputs():
inputs = ["input1.txt", "input2.txt"]
test_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
inputs=inputs,
runtime_image='tensorflow/tensorflow:latest')
assert test_operation.inputs == inputs
def test_create_operation_with_parent_operations():
parent_operation_ids = ['id-123123-123123-123123', 'id-456456-456456-456456']
test_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
parent_operations=parent_operation_ids,
runtime_image='tensorflow/tensorflow:latest')
assert test_operation.parent_operations == parent_operation_ids
def test_create_operation_with_environmental_variables():
env_variables = ['FOO="Bar"', 'BAR="Foo']
test_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
env_vars=env_variables,
runtime_image='tensorflow/tensorflow:latest')
assert test_operation.env_vars == env_variables
def test_create_operation_include_subdirectories():
include_subdirectories = True
test_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
include_subdirectories=include_subdirectories,
runtime_image='tensorflow/tensorflow:latest')
assert test_operation.include_subdirectories == include_subdirectories
def test_validate_gpu_accepts_zero_as_value():
test_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
filename='elyra/pipeline/tests/resources/archive/test.ipynb',
cpu='4',
gpu='0',
memory='10',
runtime_image='tensorflow/tensorflow:latest')
assert test_operation.gpu == '0'
def test_create_operation_correct_naming():
label = 'test.ipynb'
filename = 'elyra/pipeline/tests/resources/archive/' + label
test_operation = Operation(id='test-id',
type='test',
classifier='execution-node',
name=label,
filename=filename,
runtime_image='tensorflow/tensorflow:latest')
assert test_operation.name == label.split('.')[0]
def process(self, operation: Operation):
filepath = self.get_valid_filepath(operation.filename)
file_dir = os.path.dirname(filepath)
file_name = os.path.basename(filepath)
self.log.debug(f'Processing notebook: {filepath}')
# We'll always use the ElyraEngine. This engine is essentially the default Papermill engine
# but allows for environment variables to be passed to the kernel process (via 'kernel_env').
# If the current notebook server is running with Enterprise Gateway configured, we will also
# point the 'kernel_manager_class' to GatewayKernelManager so that notebooks run as they
# would outside of Elyra. Current working directory (cwd) is specified both for where papermill
# runs the notebook (cwd) and where the directory of the kernel process (kernel_cwd). The latter
# of which is important when EG is configured.
additional_kwargs = dict()
additional_kwargs['engine_name'] = "ElyraEngine"
additional_kwargs[
'cwd'] = file_dir # For local operations, papermill runs from this dir
additional_kwargs['kernel_cwd'] = file_dir
envs = os.environ.copy(
) # Make sure this process's env is "available" in the kernel subprocess
envs.update(operation.env_vars_as_dict())
additional_kwargs['kernel_env'] = envs
if GatewayClient.instance().gateway_enabled:
additional_kwargs[
'kernel_manager_class'] = 'jupyter_server.gateway.managers.GatewayKernelManager'
t0 = time.time()
try:
papermill.execute_notebook(filepath, filepath, **additional_kwargs)
except papermill.PapermillExecutionError as pmee:
self.log.error(
f'Error executing {file_name} in cell {pmee.exec_count}: ' +
f'{str(pmee.ename)} {str(pmee.evalue)}')
raise RuntimeError(
f'({file_name}) in cell {pmee.exec_count}: ' +
f'{str(pmee.ename)} {str(pmee.evalue)}') from pmee
except Exception as ex:
self.log_and_raise(file_name, ex)
t1 = time.time()
duration = (t1 - t0)
self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def process(self, operation: Operation):
filepath = self.get_valid_filepath(operation.filename)
file_dir = os.path.dirname(filepath)
file_name = os.path.basename(filepath)
self.log.debug(f'Processing python script: {filepath}')
argv = ['python3', filepath, '--PYTHONHOME', file_dir]
envs = operation.env_vars_as_dict()
t0 = time.time()
try:
run(argv, cwd=file_dir, env=envs, check=True)
except Exception as ex:
raise RuntimeError(
f'Internal error executing {filepath}: {ex}') from ex
t1 = time.time()
duration = (t1 - t0)
self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def process(self, operation: Operation):
filepath = self.get_valid_filepath(operation.filename)
file_dir = os.path.dirname(filepath)
file_name = os.path.basename(filepath)
self.log.debug(f'Processing R script: {filepath}')
argv = ['Rscript', filepath]
envs = os.environ.copy(
) # Make sure this process's env is "available" in subprocess
envs.update(operation.env_vars_as_dict())
t0 = time.time()
try:
run(argv, cwd=file_dir, env=envs, check=True)
except Exception as ex:
raise RuntimeError(
f'Internal error executing {filepath}: {ex}') from ex
t1 = time.time()
duration = (t1 - t0)
self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def valid_operation():
return Operation(id='{{uuid}}',
type='{{type}}',
title='{{title}}',
artifact='{{artifact}}',
image='{{image}}')
def _collect_envs(operation: Operation) -> Dict:
envs = os.environ.copy(
) # Make sure this process's env is "available" in the kernel subprocess
envs.update(operation.env_vars_as_dict())
envs['ELYRA_RUNTIME_ENV'] = "local" # Special case
return envs
