def _setup_pipes(task_inputs, inputs, task_outputs, outputs, tempdir): """ Returns a 2 tuple of input and output pipe mappings. The first element is a dict mapping input file descriptors to the corresponding stream adapters, the second is a dict mapping output file descriptors to the corresponding stream adapters. This also handles the special cases of STDIN, STDOUT, and STDERR mappings, and in the case of non-streaming standard IO pipes, will create default bindings for those as well. """ ipipes = {} opipes = {} def make_pipe(id, spec, bindings): """ Helper to make a pipe conditionally for valid streaming IO specs. If the given spec is not a streaming spec, returns False. If it is, returns the path to the pipe file that was created. """ if (spec.get('stream') and id in bindings and spec.get('target') == 'filepath'): path = spec.get('path', id) if path.startswith('/'): raise Exception('Streaming filepaths must be relative.') path = os.path.join(tempdir, path) os.mkfifo(path) return path return False # handle stream inputs for id, spec in task_inputs.iteritems(): pipe = make_pipe(id, spec, inputs) if pipe: # Don't open from this side, must be opened for reading first! ipipes[pipe] = make_stream_fetch_adapter(inputs[id]) # handle stream outputs for id, spec in task_outputs.iteritems(): pipe = make_pipe(id, spec, outputs) if pipe: opipes[os.open(pipe, os.O_RDONLY | os.O_NONBLOCK)] = \ make_stream_push_adapter(outputs[id]) # special handling for stdin, stdout, and stderr pipes if '_stdin' in task_inputs and '_stdin' in inputs: if task_inputs['_stdin'].get('stream'): ipipes['_stdin'] = make_stream_fetch_adapter(inputs['_stdin']) else: ipipes['_stdin'] = utils.MemoryFetchAdapter( inputs[id], inputs[id]['data']) for id in ('_stdout', '_stderr'): if id in task_outputs and id in outputs: if task_outputs[id].get('stream'): opipes[id] = make_stream_push_adapter(outputs[id]) else: opipes[id] = utils.AccumulateDictAdapter( outputs[id], 'script_data') return ipipes, opipes
def _setup_streams(task_inputs, inputs, task_outputs, outputs, tempdir, job_mgr, progress_pipe): """ Returns a 2 tuple of input and output pipe mappings. The first element is a dict mapping input file descriptors to the corresponding stream adapters, the second is a dict mapping output file descriptors to the corresponding stream adapters. This also handles the special cases of STDIN, STDOUT, and STDERR mappings, and in the case of non-streaming standard IO pipes, will create default bindings for those as well. """ stream_connectors = [] def stream_pipe_path(id, spec, bindings): """ Helper to check for a valid streaming IO specs. If the given spec is not a streaming spec, returns None. If it is, returns the path. """ if spec.get('stream') and id in bindings and spec.get( 'target') == 'filepath': path = spec.get('path', id) if path.startswith('/'): raise Exception('Streaming filepaths must be relative.') path = os.path.join(tempdir, path) return path return None # handle stream inputs for id, spec in task_inputs.iteritems(): path = stream_pipe_path(id, spec, inputs) # We have a streaming input if path is not None: writer = NamedPipeWriter(NamedPipe(path)) connector = FDWriteStreamConnector( make_stream_fetch_adapter(inputs[id]), writer) stream_connectors.append(connector) # Don't open from this side, must be opened for reading first! # handle stream outputs for id, spec in task_outputs.iteritems(): path = stream_pipe_path(id, spec, outputs) if path is not None: reader = NamedPipeReader(NamedPipe(path)) connector = FDReadStreamConnector( reader, make_stream_push_adapter(outputs[id])) stream_connectors.append(connector) # handle special stream output for job progress if progress_pipe and job_mgr: progress_pipe = ProgressPipe(os.path.join(tempdir, '.girder_progress')) stream_connectors.append(progress_pipe.open()) return stream_connectors
def testInputStreams(self): input_spec = {"mode": "http", "method": "GET", "url": "http://mockedhost"} @httmock.urlmatch(netloc="^mockedhost$", method="GET") def mock_fetch(url, request): return "hello\nworld" adapters = {_pipepath: make_stream_fetch_adapter(input_spec)} cmd = [sys.executable, _iscript, _pipepath] try: with captureOutput() as stdpipes, httmock.HTTMock(mock_fetch): run_process(cmd, input_pipes=adapters) except Exception: print("Stdout/stderr from exception: ") print(stdpipes) raise self.assertEqual(stdpipes, ["olleh\ndlrow\n", ""])
def testInputStreams(self): input_spec = { 'mode': 'http', 'method': 'GET', 'url': 'http://mockedhost' } @httmock.urlmatch(netloc='^mockedhost$', method='GET') def mock_fetch(url, request): return 'hello\nworld' adapters = {_pipepath: make_stream_fetch_adapter(input_spec)} cmd = [sys.executable, _iscript, _pipepath] try: with captureOutput() as stdpipes, httmock.HTTMock(mock_fetch): run_process(cmd, input_pipes=adapters) except Exception: print('Stdout/stderr from exception: ') print(stdpipes) raise self.assertEqual(stdpipes, ['olleh\ndlrow\n', ''])