def communicate(self): self.storlet_logger = StorletLogger(self.storlet_logger_path, 'storlet_invoke') self.storlet_logger.open() self._prepare_invocation_descriptors() try: self._invoke() except Exception as e: raise e finally: self._close_remote_side_descriptors() self.storlet_logger.close() self._wait_for_write_with_timeout(self.input_data_write_fd) # We do the writing in a different thread. # Otherwise, we can run into the following deadlock # 1. middleware writes to Storlet # 2. Storlet reads and starts to write metadata and then data # 3. middleware continues writing # 4. Storlet continues writing and gets stuck as middleware # is busy writing, but still not consuming the reader end # of the Storlet writer. eventlet.spawn_n(self._write_input_data) out_md = self._read_metadata() self._wait_for_read_with_timeout(self.data_read_fd) return out_md, self.data_read_fd
class StorletInvocationGETProtocol(StorletInvocationProtocol): def _add_input_stream(self): self.fds.append(self.srequest.stream) # TODO: Break request metadata and systemmetadata md = dict() md['type'] = SBUS_FD_INPUT_OBJECT if self.srequest.user_metadata is not None: for key, val in self.srequest.user_metadata.iteritems(): md[key] = val self.fdmd.append(md) def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout): StorletInvocationProtocol.__init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout) def communicate(self): self.storlet_logger = StorletLogger(self.storlet_logger_path, 'storlet_invoke') self.storlet_logger.open() self._prepare_invocation_descriptors() try: self._invoke() except Exception as e: raise e finally: self._close_remote_side_descriptors() self.storlet_logger.close() out_md = self._read_metadata() os.close(self.metadata_read_fd) self._wait_for_read_with_timeout(self.data_read_fd) return out_md, self.data_read_fd
class StorletInvocationGETProtocol(StorletInvocationProtocol): def _add_input_stream(self): StorletInvocationProtocol._add_input_stream(self, self.srequest.stream) def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout): StorletInvocationProtocol.__init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout) def communicate(self): self.storlet_logger = StorletLogger(self.storlet_logger_path, 'storlet_invoke') self.storlet_logger.open() self._prepare_invocation_descriptors() try: self._invoke() except Exception as e: raise e finally: self._close_remote_side_descriptors() self.storlet_logger.close() out_md = self._read_metadata() os.close(self.metadata_read_fd) self._wait_for_read_with_timeout(self.data_read_fd) os.close(self.execution_str_read_fd) return out_md, self.data_read_fd
class StorletInvocationGETProtocol(StorletInvocationProtocol): def _add_input_stream(self): StorletInvocationProtocol._add_input_stream(self, self.srequest.stream) def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout): StorletInvocationProtocol.__init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout) def communicate(self): self.storlet_logger = StorletLogger(self.storlet_logger_path, 'storlet_invoke') self.storlet_logger.open() self._prepare_invocation_descriptors() try: self._invoke() except Exception as e: raise e finally: self._close_remote_side_descriptors() self.storlet_logger.close() out_md = self._read_metadata() os.close(self.metadata_read_fd) self._wait_for_read_with_timeout(self.data_read_fd) return out_md, self.data_read_fd
class StorletInvocationProxyProtocol(StorletInvocationProtocol): def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout): StorletInvocationProtocol.__init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout) self.input_data_read_fd, self.input_data_write_fd = os.pipe() # YM this pipe permits to take data from srequest.stream to input_data_write_fd # YM the write side stays with us, the read side is sent to storlet def _add_input_stream(self): StorletInvocationProtocol._add_input_stream(self, self.input_data_read_fd) def _wait_for_write_with_timeout(self,fd): r, w, e = select.select([ ], [ fd ], [ ], self.timeout) if len(w) == 0: raise Timeout('Timeout while waiting for storlet to read') if fd in w: return def _write_with_timeout(self, writer, chunk): timeout = Timeout(self.timeout) try: writer.write(chunk) except Timeout as t: if t is timeout: writer.close() raise t except Exception as e: raise e finally: timeout.cancel() def communicate(self): self.storlet_logger = StorletLogger(self.storlet_logger_path, 'storlet_invoke') self.storlet_logger.open() self._prepare_invocation_descriptors() try: self._invoke() except Exception as e: raise e finally: self._close_remote_side_descriptors() self.storlet_logger.close() self._wait_for_write_with_timeout(self.input_data_write_fd) # We do the writing in a different thread. # Otherwise, we can run into the following deadlock # 1. middleware writes to Storlet # 2. Storlet reads and starts to write metadata and then data # 3. middleware continues writing # 4. Storlet continues writing and gets stuck as middleware # is busy writing, but still not consuming the reader end # of the Storlet writer. eventlet.spawn_n(self._write_input_data) out_md = self._read_metadata() self._wait_for_read_with_timeout(self.data_read_fd) return out_md, self.data_read_fd
def communicate(self): self.storlet_logger = StorletLogger(self.storlet_logger_path, 'storlet_invoke') self.storlet_logger.open() self._prepare_invocation_descriptors() try: self._invoke() except Exception as e: raise e finally: self._close_remote_side_descriptors() self.storlet_logger.close() self._wait_for_write_with_timeout(self.input_data_write_fd) # We do the writing in a different thread. # Otherwise, we can run into the following deadlock # 1. md writeds to Storlet # 2. Storlet reads and starts to write md and thed data # 3. md continues writing # 4. Storlet continues writing and gets stuck as md is busy writing, # not consuming the reader end of the Storlet writer. eventlet.spawn_n(self._write_input_data) out_md = self._read_metadata() self._wait_for_read_with_timeout(self.data_read_fd) return out_md, self.data_read_fd
def communicate(self): self.storlet_logger = StorletLogger(self.storlet_logger_path, 'storlet_invoke') self.storlet_logger.open() self._prepare_invocation_descriptors() try: self._invoke() except Exception as e: raise e finally: self._close_remote_side_descriptors() self.storlet_logger.close() out_md = self._read_metadata() os.close(self.metadata_read_fd) self._wait_for_read_with_timeout(self.data_read_fd) os.close(self.execution_str_read_fd) return out_md, self.data_read_fd
def communicate(self): self.storlet_logger = StorletLogger(self.storlet_logger_path, 'storlet_invoke') self.storlet_logger.open() self._prepare_invocation_descriptors() try: self._invoke() except Exception as e: raise e finally: self._close_remote_side_descriptors() self.storlet_logger.close() out_md = self._read_metadata() os.close(self.metadata_read_fd) self._wait_for_read_with_timeout(self.data_read_fd) return out_md, self.data_read_fd
class StorletInvocationPUTProtocol(StorletInvocationProtocol): def _add_input_stream(self): self.fds.append(self.input_data_read_fd) # TODO: Break request metadata and systemmetadata md = dict() md['type'] = SBUS_FD_INPUT_OBJECT if self.srequest.user_metadata is not None: for key, val in self.srequest.user_metadata.iteritems(): md[key] = val self.fdmd.append(md) def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout): StorletInvocationProtocol.__init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout) self.input_data_read_fd, self.input_data_write_fd = os.pipe() # YM this pipe permits to take data from srequest.stream to input_data_write_fd # YM the write side stays with us, the read side is sent to storlet def _wait_for_write_with_timeout(self,fd): r, w, e = select.select([ ], [ fd ], [ ], self.timeout) if len(w) == 0: raise Timeout('Timeout while waiting for storlet to read') if fd in w: return def _write_with_timeout(self, writer, chunk): timeout = Timeout(self.timeout) try: writer.write(chunk) except Timeout as t: if t is timeout: writer.close() raise t except Exception as e: raise e finally: timeout.cancel() def _write_input_data(self): writer = os.fdopen(self.input_data_write_fd, 'w') reader = self.srequest.stream for chunk in iter(lambda: reader(65536), ''): self._write_with_timeout(writer, chunk) writer.close() def communicate(self): self.storlet_logger = StorletLogger(self.storlet_logger_path, 'storlet_invoke') self.storlet_logger.open() self._prepare_invocation_descriptors() try: self._invoke() except Exception as e: raise e finally: self._close_remote_side_descriptors() self.storlet_logger.close() self._wait_for_write_with_timeout(self.input_data_write_fd) # We do the writing in a different thread. # Otherwise, we can run into the following deadlock # 1. md writeds to Storlet # 2. Storlet reads and starts to write md and thed data # 3. md continues writing # 4. Storlet continues writing and gets stuck as md is busy writing, # not consuming the reader end of the Storlet writer. eventlet.spawn_n(self._write_input_data) out_md = self._read_metadata() self._wait_for_read_with_timeout(self.data_read_fd) return out_md, self.data_read_fd
class StorletInvocationProxyProtocol(StorletInvocationProtocol): def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout): StorletInvocationProtocol.__init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout) self.input_data_read_fd, self.input_data_write_fd = os.pipe() # YM this pipe permits to take data from srequest.stream to input_data_write_fd # YM the write side stays with us, the read side is sent to storlet def _add_input_stream(self): StorletInvocationProtocol._add_input_stream(self, self.input_data_read_fd) def _wait_for_write_with_timeout(self, fd): r, w, e = select.select([], [fd], [], self.timeout) if len(w) == 0: raise Timeout('Timeout while waiting for storlet to read') if fd in w: return def _write_with_timeout(self, writer, chunk): timeout = Timeout(self.timeout) try: writer.write(chunk) except Timeout as t: if t is timeout: writer.close() raise t except Exception as e: raise e finally: timeout.cancel() def communicate(self): self.storlet_logger = StorletLogger(self.storlet_logger_path, 'storlet_invoke') self.storlet_logger.open() self._prepare_invocation_descriptors() try: self._invoke() except Exception as e: raise e finally: self._close_remote_side_descriptors() self.storlet_logger.close() self._wait_for_write_with_timeout(self.input_data_write_fd) # We do the writing in a different thread. # Otherwise, we can run into the following deadlock # 1. middleware writes to Storlet # 2. Storlet reads and starts to write metadata and then data # 3. middleware continues writing # 4. Storlet continues writing and gets stuck as middleware # is busy writing, but still not consuming the reader end # of the Storlet writer. eventlet.spawn_n(self._write_input_data) out_md = self._read_metadata() self._wait_for_read_with_timeout(self.data_read_fd) return out_md, self.data_read_fd