def test_env_list_to_dict_function(): env_variables = ['FOO=Bar', 'BAR=Foo', None, '', ' =Dog', 'DOG= '] env_variables_dict = {"FOO": "Bar", "BAR": "Foo", "DOG": " "} test_operation = Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', env_vars=env_variables, runtime_image='tensorflow/tensorflow:latest') assert test_operation.env_vars_as_dict() == env_variables_dict
def test_env_list_to_dict_function(): env_variables = ['KEY=value', None, '', ' =empty_key', '=no_key', 'EMPTY_VALUE= ', 'NO_VALUE=', 'KEY2=value2', 'TWO_EQUALS=KEY=value', '=='] env_variables_dict = {"KEY": "value", "KEY2": "value2", "EMPTY_VALUE": " ", "TWO_EQUALS": "KEY=value"} test_operation = Operation(id='test-id', type='test', classifier='execution-node', name='test', filename='elyra/pipeline/tests/resources/archive/test.ipynb', env_vars=env_variables, runtime_image='tensorflow/tensorflow:latest') assert test_operation.env_vars_as_dict() == env_variables_dict
def test_fail_create_operation_missing_name(): with pytest.raises(TypeError): Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', runtime_image='tensorflow/tensorflow:latest')
def process(self, operation: Operation): filepath = self.get_valid_filepath(operation.filename) file_dir = os.path.dirname(filepath) file_name = os.path.basename(filepath) self.log.debug(f'Processing {self._script_type} script: {filepath}') argv = self.get_argv(filepath) envs = os.environ.copy( ) # Make sure this process's env is "available" in subprocess envs.update(operation.env_vars_as_dict()) t0 = time.time() try: run(argv, cwd=file_dir, env=envs, check=True, stderr=PIPE) except CalledProcessError as cpe: error_msg = str(cpe.stderr.decode()) self.log.error(f'Error executing {file_name}: {error_msg}') error_trim_index = error_msg.rfind('\n', 0, error_msg.rfind('Error')) if error_trim_index != -1: raise RuntimeError( f'({file_name}): {error_msg[error_trim_index:].strip()}' ) from cpe else: raise RuntimeError(f'({file_name})') from cpe except Exception as ex: self.log_and_raise(file_name, ex) t1 = time.time() duration = (t1 - t0) self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def process(self, operation: Operation): filepath = self.get_valid_filepath(operation.filename) file_dir = os.path.dirname(filepath) file_name = os.path.basename(filepath) self.log.debug(f'Processing notebook: {filepath}') # We'll always use the ElyraEngine. This engine is essentially the default Papermill engine # but allows for environment variables to be passed to the kernel process (via 'kernel_env'). # If the current notebook server is running with Enterprise Gateway configured, we will also # point the 'kernel_manager_class' to our HTTPKernelManager so that notebooks run as they # would outside of Elyra. Current working directory (cwd) is specified both for where papermill # runs the notebook (cwd) and where the directory of the kernel process (kernel_cwd). The latter # of which is important when EG is configured. additional_kwargs = dict() additional_kwargs['engine_name'] = "ElyraEngine" additional_kwargs[ 'cwd'] = file_dir # For local operations, papermill runs from this dir additional_kwargs['kernel_cwd'] = file_dir additional_kwargs['kernel_env'] = operation.env_vars_as_dict() if GatewayClient.instance().gateway_enabled: additional_kwargs[ 'kernel_manager_class'] = 'elyra.pipeline.http_kernel_manager.HTTPKernelManager' t0 = time.time() try: papermill.execute_notebook(filepath, filepath, **additional_kwargs) except Exception as ex: raise RuntimeError( f'Internal error executing {filepath}: {ex}') from ex t1 = time.time() duration = (t1 - t0) self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def good_operation(): test_operation = Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', runtime_image='tensorflow/tensorflow:latest') return test_operation
def test_operations_are_equal(good_operation): compare_operation = Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', runtime_image='tensorflow/tensorflow:latest') assert compare_operation == good_operation
def test_fail_creating_operation_with_negative_memory_resources(): with pytest.raises(ValueError): Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', memory='-1', runtime_image='tensorflow/tensorflow:latest')
def test_fail_operations_are_equal(good_operation): parent_operation_ids = ['id-123123-123123-123123', 'id-456456-456456-456456'] compare_operation = Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', parent_operations=parent_operation_ids, runtime_image='tensorflow/tensorflow:latest') with pytest.raises(AssertionError): assert compare_operation == good_operation
def valid_operation(): return Operation(id='{{uuid}}', type='execution_node', classifier='execute-notebook-node', filename='{{filename}}', runtime_image='{{runtime_image}}', env_vars=["var1=var1", "var2=var2"], dependencies=["a.txt", "b.txt", "c.txt"], outputs=["d.txt", "e.txt", "f.txt"], )
def test_fail_validate_max_resource_value_exceeded(): system_max_size = str(sys.maxsize) with pytest.raises(ValueError): Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', memory=system_max_size, runtime_image='tensorflow/tensorflow:latest')
def test_validate_max_resource_value(): system_max_size = str(sys.maxsize - 1) test_operation = Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', memory=system_max_size, runtime_image='tensorflow/tensorflow:latest') assert test_operation.memory == system_max_size
def test_validate_resource_values_as_none(): test_operation = Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', runtime_image='tensorflow/tensorflow:latest') assert test_operation.cpu is None assert test_operation.gpu is None assert test_operation.memory is None
def test_create_operation_with_dependencies(): dependencies = ['elyra/pipline/tests/resources', 'elyra/pipline/tests/resources/archive'] test_operation = Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', dependencies=dependencies, runtime_image='tensorflow/tensorflow:latest') assert test_operation.dependencies == dependencies
def test_create_operation_with_inputs(): inputs = ["input1.txt", "input2.txt"] test_operation = Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', inputs=inputs, runtime_image='tensorflow/tensorflow:latest') assert test_operation.inputs == inputs
def test_create_operation_with_parent_operations(): parent_operation_ids = ['id-123123-123123-123123', 'id-456456-456456-456456'] test_operation = Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', parent_operations=parent_operation_ids, runtime_image='tensorflow/tensorflow:latest') assert test_operation.parent_operations == parent_operation_ids
def test_create_operation_with_environmental_variables(): env_variables = ['FOO="Bar"', 'BAR="Foo'] test_operation = Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', env_vars=env_variables, runtime_image='tensorflow/tensorflow:latest') assert test_operation.env_vars == env_variables
def test_create_operation_include_subdirectories(): include_subdirectories = True test_operation = Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', include_subdirectories=include_subdirectories, runtime_image='tensorflow/tensorflow:latest') assert test_operation.include_subdirectories == include_subdirectories
def test_validate_gpu_accepts_zero_as_value(): test_operation = Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', cpu='4', gpu='0', memory='10', runtime_image='tensorflow/tensorflow:latest') assert test_operation.gpu == '0'
def test_create_operation_correct_naming(): label = 'test.ipynb' filename = 'elyra/pipeline/tests/resources/archive/' + label test_operation = Operation(id='test-id', type='test', classifier='execution-node', name=label, filename=filename, runtime_image='tensorflow/tensorflow:latest') assert test_operation.name == label.split('.')[0]
def process(self, operation: Operation): filepath = self.get_valid_filepath(operation.filename) file_dir = os.path.dirname(filepath) file_name = os.path.basename(filepath) self.log.debug(f'Processing notebook: {filepath}') # We'll always use the ElyraEngine. This engine is essentially the default Papermill engine # but allows for environment variables to be passed to the kernel process (via 'kernel_env'). # If the current notebook server is running with Enterprise Gateway configured, we will also # point the 'kernel_manager_class' to GatewayKernelManager so that notebooks run as they # would outside of Elyra. Current working directory (cwd) is specified both for where papermill # runs the notebook (cwd) and where the directory of the kernel process (kernel_cwd). The latter # of which is important when EG is configured. additional_kwargs = dict() additional_kwargs['engine_name'] = "ElyraEngine" additional_kwargs[ 'cwd'] = file_dir # For local operations, papermill runs from this dir additional_kwargs['kernel_cwd'] = file_dir envs = os.environ.copy( ) # Make sure this process's env is "available" in the kernel subprocess envs.update(operation.env_vars_as_dict()) additional_kwargs['kernel_env'] = envs if GatewayClient.instance().gateway_enabled: additional_kwargs[ 'kernel_manager_class'] = 'jupyter_server.gateway.managers.GatewayKernelManager' t0 = time.time() try: papermill.execute_notebook(filepath, filepath, **additional_kwargs) except papermill.PapermillExecutionError as pmee: self.log.error( f'Error executing {file_name} in cell {pmee.exec_count}: ' + f'{str(pmee.ename)} {str(pmee.evalue)}') raise RuntimeError( f'({file_name}) in cell {pmee.exec_count}: ' + f'{str(pmee.ename)} {str(pmee.evalue)}') from pmee except Exception as ex: self.log_and_raise(file_name, ex) t1 = time.time() duration = (t1 - t0) self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def process(self, operation: Operation): filepath = self.get_valid_filepath(operation.filename) file_dir = os.path.dirname(filepath) file_name = os.path.basename(filepath) self.log.debug(f'Processing python script: {filepath}') argv = ['python3', filepath, '--PYTHONHOME', file_dir] envs = operation.env_vars_as_dict() t0 = time.time() try: run(argv, cwd=file_dir, env=envs, check=True) except Exception as ex: raise RuntimeError( f'Internal error executing {filepath}: {ex}') from ex t1 = time.time() duration = (t1 - t0) self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def process(self, operation: Operation): filepath = self.get_valid_filepath(operation.filename) file_dir = os.path.dirname(filepath) file_name = os.path.basename(filepath) self.log.debug(f'Processing R script: {filepath}') argv = ['Rscript', filepath] envs = os.environ.copy( ) # Make sure this process's env is "available" in subprocess envs.update(operation.env_vars_as_dict()) t0 = time.time() try: run(argv, cwd=file_dir, env=envs, check=True) except Exception as ex: raise RuntimeError( f'Internal error executing {filepath}: {ex}') from ex t1 = time.time() duration = (t1 - t0) self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def valid_operation(): return Operation(id='{{uuid}}', type='{{type}}', title='{{title}}', artifact='{{artifact}}', image='{{image}}')
def _collect_envs(operation: Operation) -> Dict: envs = os.environ.copy( ) # Make sure this process's env is "available" in the kernel subprocess envs.update(operation.env_vars_as_dict()) envs['ELYRA_RUNTIME_ENV'] = "local" # Special case return envs