def process(self, operation: Operation): filepath = self.get_valid_filepath(operation.filename) file_dir = os.path.dirname(filepath) file_name = os.path.basename(filepath) self.log.debug(f'Processing {self._script_type} script: {filepath}') argv = self.get_argv(filepath) envs = os.environ.copy( ) # Make sure this process's env is "available" in subprocess envs.update(operation.env_vars_as_dict()) t0 = time.time() try: run(argv, cwd=file_dir, env=envs, check=True, stderr=PIPE) except CalledProcessError as cpe: error_msg = str(cpe.stderr.decode()) self.log.error(f'Error executing {file_name}: {error_msg}') error_trim_index = error_msg.rfind('\n', 0, error_msg.rfind('Error')) if error_trim_index != -1: raise RuntimeError( f'({file_name}): {error_msg[error_trim_index:].strip()}' ) from cpe else: raise RuntimeError(f'({file_name})') from cpe except Exception as ex: self.log_and_raise(file_name, ex) t1 = time.time() duration = (t1 - t0) self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def process(self, operation: Operation): filepath = self.get_valid_filepath(operation.filename) file_dir = os.path.dirname(filepath) file_name = os.path.basename(filepath) self.log.debug(f'Processing notebook: {filepath}') # We'll always use the ElyraEngine. This engine is essentially the default Papermill engine # but allows for environment variables to be passed to the kernel process (via 'kernel_env'). # If the current notebook server is running with Enterprise Gateway configured, we will also # point the 'kernel_manager_class' to our HTTPKernelManager so that notebooks run as they # would outside of Elyra. Current working directory (cwd) is specified both for where papermill # runs the notebook (cwd) and where the directory of the kernel process (kernel_cwd). The latter # of which is important when EG is configured. additional_kwargs = dict() additional_kwargs['engine_name'] = "ElyraEngine" additional_kwargs[ 'cwd'] = file_dir # For local operations, papermill runs from this dir additional_kwargs['kernel_cwd'] = file_dir additional_kwargs['kernel_env'] = operation.env_vars_as_dict() if GatewayClient.instance().gateway_enabled: additional_kwargs[ 'kernel_manager_class'] = 'elyra.pipeline.http_kernel_manager.HTTPKernelManager' t0 = time.time() try: papermill.execute_notebook(filepath, filepath, **additional_kwargs) except Exception as ex: raise RuntimeError( f'Internal error executing {filepath}: {ex}') from ex t1 = time.time() duration = (t1 - t0) self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def test_env_list_to_dict_function(): env_variables = ['FOO=Bar', 'BAR=Foo', None, '', ' =Dog', 'DOG= '] env_variables_dict = {"FOO": "Bar", "BAR": "Foo", "DOG": " "} test_operation = Operation(id='test-id', type='test', classifier='execution-node', filename='elyra/pipeline/tests/resources/archive/test.ipynb', env_vars=env_variables, runtime_image='tensorflow/tensorflow:latest') assert test_operation.env_vars_as_dict() == env_variables_dict
def test_env_list_to_dict_function(): env_variables = ['KEY=value', None, '', ' =empty_key', '=no_key', 'EMPTY_VALUE= ', 'NO_VALUE=', 'KEY2=value2', 'TWO_EQUALS=KEY=value', '=='] env_variables_dict = {"KEY": "value", "KEY2": "value2", "EMPTY_VALUE": " ", "TWO_EQUALS": "KEY=value"} test_operation = Operation(id='test-id', type='test', classifier='execution-node', name='test', filename='elyra/pipeline/tests/resources/archive/test.ipynb', env_vars=env_variables, runtime_image='tensorflow/tensorflow:latest') assert test_operation.env_vars_as_dict() == env_variables_dict
def process(self, operation: Operation): filepath = self.get_valid_filepath(operation.filename) file_dir = os.path.dirname(filepath) file_name = os.path.basename(filepath) self.log.debug(f'Processing notebook: {filepath}') # We'll always use the ElyraEngine. This engine is essentially the default Papermill engine # but allows for environment variables to be passed to the kernel process (via 'kernel_env'). # If the current notebook server is running with Enterprise Gateway configured, we will also # point the 'kernel_manager_class' to GatewayKernelManager so that notebooks run as they # would outside of Elyra. Current working directory (cwd) is specified both for where papermill # runs the notebook (cwd) and where the directory of the kernel process (kernel_cwd). The latter # of which is important when EG is configured. additional_kwargs = dict() additional_kwargs['engine_name'] = "ElyraEngine" additional_kwargs[ 'cwd'] = file_dir # For local operations, papermill runs from this dir additional_kwargs['kernel_cwd'] = file_dir envs = os.environ.copy( ) # Make sure this process's env is "available" in the kernel subprocess envs.update(operation.env_vars_as_dict()) additional_kwargs['kernel_env'] = envs if GatewayClient.instance().gateway_enabled: additional_kwargs[ 'kernel_manager_class'] = 'jupyter_server.gateway.managers.GatewayKernelManager' t0 = time.time() try: papermill.execute_notebook(filepath, filepath, **additional_kwargs) except papermill.PapermillExecutionError as pmee: self.log.error( f'Error executing {file_name} in cell {pmee.exec_count}: ' + f'{str(pmee.ename)} {str(pmee.evalue)}') raise RuntimeError( f'({file_name}) in cell {pmee.exec_count}: ' + f'{str(pmee.ename)} {str(pmee.evalue)}') from pmee except Exception as ex: self.log_and_raise(file_name, ex) t1 = time.time() duration = (t1 - t0) self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def process(self, operation: Operation): filepath = self.get_valid_filepath(operation.filename) file_dir = os.path.dirname(filepath) file_name = os.path.basename(filepath) self.log.debug(f'Processing python script: {filepath}') argv = ['python3', filepath, '--PYTHONHOME', file_dir] envs = operation.env_vars_as_dict() t0 = time.time() try: run(argv, cwd=file_dir, env=envs, check=True) except Exception as ex: raise RuntimeError( f'Internal error executing {filepath}: {ex}') from ex t1 = time.time() duration = (t1 - t0) self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def process(self, operation: Operation): filepath = self.get_valid_filepath(operation.filename) file_dir = os.path.dirname(filepath) file_name = os.path.basename(filepath) self.log.debug(f'Processing R script: {filepath}') argv = ['Rscript', filepath] envs = os.environ.copy( ) # Make sure this process's env is "available" in subprocess envs.update(operation.env_vars_as_dict()) t0 = time.time() try: run(argv, cwd=file_dir, env=envs, check=True) except Exception as ex: raise RuntimeError( f'Internal error executing {filepath}: {ex}') from ex t1 = time.time() duration = (t1 - t0) self.log.debug(f'Execution of {file_name} took {duration:.3f} secs.')
def _collect_envs(operation: Operation) -> Dict: envs = os.environ.copy( ) # Make sure this process's env is "available" in the kernel subprocess envs.update(operation.env_vars_as_dict()) envs['ELYRA_RUNTIME_ENV'] = "local" # Special case return envs