コード例 #1
0
 def test_get_logs_from_past_kernels(self):
     new_dir = tempfile.mkdtemp()
     logger = CWLLogger(new_dir)
     logger.save()
     self.assertListEqual([logger.to_dict()['process_id']],
                          [l['process_id'] for l in logger.load()])
コード例 #2
0
class CWLKernel(Kernel):
    implementation = 'CWLKernel'
    implementation_version = '0.1'
    language_version = '1.0'
    language_info = {
        'name': 'yaml',
        'mimetype': 'text/x-cwl',
        'file_extension': '.cwl',
    }
    banner = "Common Workflow Language"

    _magic_commands = frozenset(['logs', 'data', 'display_data'])

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        conf = CWLExecuteConfigurator()
        self._yaml_input_data: List[str] = []
        self._results_manager = IOFileManager(
            os.sep.join([conf.CWLKERNEL_BOOT_DIRECTORY, 'results']))
        runtime_file_manager = IOFileManager(
            os.sep.join([conf.CWLKERNEL_BOOT_DIRECTORY, 'runtime_data']))
        self._cwl_executor = CoreExecutor(runtime_file_manager)
        self._pid = (os.getpid(), os.getppid())
        self._cwl_logger = CWLLogger(
            os.path.join(conf.CWLKERNEL_BOOT_DIRECTORY, 'logs'))
        self._set_process_ids()
        self._cwl_logger.save()

    def _set_process_ids(self):
        self._cwl_logger.process_id = {
            "process_id": os.getpid(),
            "parent_process_id": os.getppid()
        }

    def _code_is_valid_yaml(self, code) -> Optional[Dict]:
        yaml = YAML(typ='safe')
        try:
            return yaml.load(code)
        except Exception:
            return None

    def _is_magic_command(self, code: str) -> bool:
        split_code = code.split()
        if len(split_code) < 2:
            return False
        if code.startswith("% ") and code.split()[1] in self._magic_commands:
            return True
        return False

    def do_execute(self,
                   code: str,
                   silent=False,
                   store_history: bool = True,
                   user_expressions=None,
                   allow_stdin: bool = False) -> Dict:
        if self._is_magic_command(code):
            self._execute_magic_command(code)
            return {
                'status': 'ok',
                # The base class increments the execution count
                'execution_count': self.execution_count,
                'payload': [],
                'user_expressions': {},
            }
        else:
            dict_code = self._code_is_valid_yaml(code)
            if dict_code is None:
                return {
                    'status': 'error',
                    # The base class increments the execution count
                    'execution_count': self.execution_count,
                    'payload': [],
                    'user_expressions': {},
                }

        if not self._is_cwl(dict_code):
            exception = self._accumulate_data(code)
        else:
            exception = self._execute_workflow(code)
            self._clear_data()

        status = 'ok' if exception is None else 'error'
        if exception is not None:
            self.send_response(
                self.iopub_socket, 'stream', {
                    'name': 'stderr',
                    'text': f'{type(exception).__name__}: {exception}'
                })
        return {
            'status': status,
            # The base class increments the execution count
            'execution_count': self.execution_count,
            'payload': [],
            'user_expressions': {},
        }

    def _execute_magic_command(self, command: str):
        command = command.split()[1:]
        command_name = command[0]
        args = command[1:]
        getattr(self, f'_execute_magic_{command_name}')(args)

    def _execute_magic_display_data(self, data_name):
        if len(data_name) != 1 or not isinstance(data_name[0], str):
            self._send_error_response(
                'ERROR: you must select an output to display. Correct format:\n % display_data [output name]'
            )
            return
        results = list(
            filter(lambda item: item[1]['id'] == data_name[0],
                   self._results_manager.get_files_registry().items()))
        if len(results) != 1:
            self.send_response(self.iopub_socket, 'stream', {
                'name': 'stderr',
                'text': 'Result not found'
            })
            return
        results = results[0]
        with open(results[0]) as f:
            data = f.read()
        self.send_response(self.iopub_socket, 'stream', {
            'name': 'stdout',
            'text': data
        })

    def _send_error_response(self, text):
        self.send_response(self.iopub_socket, 'stream', {
            'name': 'stdout',
            'text': text
        })

    def _send_json_response(self, json_data: Union[Dict, List]):
        self.send_response(
            self.iopub_socket, 'display_data', {
                'data': {
                    'text/plain': '<IPython.core.display.JSON object>',
                    'application/json': json_data
                },
                'metadata': {
                    'application/json': {
                        'expanded': False,
                        'root': 'root'
                    }
                }
            })

    def _execute_magic_logs(self, limit=None):
        logger.error('Execute logs magic command')
        limit_len = len(limit)
        if limit_len == 0:
            limit = None
        if limit_len > 0:
            limit = limit[0]
        if isinstance(limit, str):
            limit = int(limit)
        self.send_response(
            self.iopub_socket, 'display_data', {
                'data': {
                    'text/plain': '<IPython.core.display.JSON object>',
                    'application/json': list(self._cwl_logger.load(limit))
                },
                'metadata': {
                    'application/json': {
                        'expanded': False,
                        'root': 'root'
                    }
                }
            })

    def _execute_magic_data(self, *args):
        data = "<ul>\n" + '\n'.join([
            f'\t<li><a href="file://{d}" target="_empty">{d}</a></li>'
            for d in self.get_past_results()
        ]) + "\n</ul>"
        self.send_response(
            self.iopub_socket, 'display_data', {
                'data': {
                    'text/html': data
                },
                'metadata': {
                    'application/json': {
                        'expanded': False,
                        'root': 'root'
                    }
                }
            })

    def _accumulate_data(self, code: str) -> Optional[Exception]:
        cwl = self._cwl_executor.file_manager.get_files_uri().path
        try:
            self._cwl_executor.validate_input_files(
                yaml.load(code, Loader=yaml.Loader), cwl)
        except FileNotFoundError as e:
            return e
        self._yaml_input_data.append(code)
        self.send_response(self.iopub_socket, 'stream', {
            'name': 'stdout',
            'text': 'Add data in memory'
        })

    def _clear_data(self):
        self._yaml_input_data = []

    def _execute_workflow(self, code) -> Optional[Exception]:
        self._cwl_executor.set_data(self._yaml_input_data)
        self._cwl_executor.set_workflow(code)
        logger.debug('starting executing workflow ...')
        run_id, results, exception = self._cwl_executor.execute()
        logger.debug(f'\texecution results: {run_id}, {results}, {exception}')
        output_directory_for_that_run = str(run_id)
        for output in results:
            if isinstance(results[output], list):
                for i, output_i in enumerate(results[output]):
                    results[output][i]['id'] = f'{output}_{i+1}'
                    self._results_manager.append_files(
                        [results[output][i]['location']],
                        output_directory_for_that_run,
                        metadata=results[output][i])
            else:
                results[output]['id'] = output
                self._results_manager.append_files(
                    [results[output]['location']],
                    output_directory_for_that_run,
                    metadata=results[output])
        self.send_response(
            self.iopub_socket, 'display_data', {
                'data': {
                    'text/plain': '<IPython.core.display.JSON object>',
                    'application/json': results
                },
                'metadata': {
                    'application/json': {
                        'expanded': False,
                        'root': 'root'
                    }
                }
            })
        if exception is not None:
            logger.debug(f'execution error: {exception}')
            self.send_response(self.iopub_socket, 'stream', {
                'name': 'stderr',
                'text': str(exception)
            })
            return exception

    def get_past_results(self) -> List[str]:
        return self._results_manager.get_files()

    def _is_cwl(self, code: Dict):
        return 'cwlVersion' in code.keys()

    def get_pid(self) -> Tuple[int, int]:
        """
        :return: The process id and his parents id
        """
        return self._pid