def setup(): global _rfile, _wfile, _threads, _coro, _setup_already, _reqq, _rspq if _setup_already: return else: _setup_already = True try: _rpipe, _wpipe = os.pipe() _wfile = greenio.GreenPipe(_wpipe, 'wb', 0) _rfile = greenio.GreenPipe(_rpipe, 'rb', 0) except ImportError: # This is Windows compatibility -- use a socket instead of a pipe because # pipes don't really exist on Windows. import socket from eventlet import util sock = util.__original_socket__(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('localhost', 0)) sock.listen(50) csock = util.__original_socket__(socket.AF_INET, socket.SOCK_STREAM) csock.connect(('localhost', sock.getsockname()[1])) nsock, addr = sock.accept() _rfile = greenio.GreenSocket(csock).makefile('rb', 0) _wfile = nsock.makefile('wb', 0) _reqq = Queue(maxsize=-1) _rspq = Queue(maxsize=-1) for i in range(0, _nthreads): t = threading.Thread(target=tworker) t.setDaemon(True) t.start() _threads.add(t) _coro = greenthread.spawn_n(tpool_trampoline)
def _init_events_pipe(self): """Create a self-pipe for the native thread to synchronize on. This code is taken from the eventlet tpool module, under terms of the Apache License v2.0. """ self._event_queue = native_Queue.Queue() try: rpipe, wpipe = os.pipe() self._event_notify_send = greenio.GreenPipe(wpipe, 'wb', 0) self._event_notify_recv = greenio.GreenPipe(rpipe, 'rb', 0) except (ImportError, NotImplementedError): # This is Windows compatibility -- use a socket instead # of a pipe because pipes don't really exist on Windows. sock = eventlet_util.__original_socket__(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('localhost', 0)) sock.listen(50) csock = eventlet_util.__original_socket__(socket.AF_INET, socket.SOCK_STREAM) csock.connect(('localhost', sock.getsockname()[1])) nsock, addr = sock.accept() self._event_notify_send = nsock.makefile('wb', 0) gsock = greenio.GreenSocket(csock) self._event_notify_recv = gsock.makefile('rb', 0)
def test_timeout_and_final_write(self): # This test verifies that a write on a socket that we've # stopped listening for doesn't result in an incorrect switch rpipe, wpipe = os.pipe() rfile = os.fdopen(rpipe, "r", 0) wrap_rfile = greenio.GreenPipe(rfile) wfile = os.fdopen(wpipe, "w", 0) wrap_wfile = greenio.GreenPipe(wfile) def sender(evt): api.sleep(0.02) wrap_wfile.write('hi') evt.send('sent via event') from eventlet import coros evt = coros.event() api.spawn(sender, evt) try: # try and get some data off of this pipe # but bail before any is sent api.exc_after(0.01, api.TimeoutError) _c = wrap_rfile.read(1) self.fail() except api.TimeoutError: pass result = evt.wait() self.assertEquals(result, 'sent via event')
def test_pipe_read(self): # ensure that 'readline' works properly on GreenPipes when data is not # immediately available (fd is nonblocking, was raising EAGAIN) # also ensures that readline() terminates on '\n' and '\r\n' r, w = os.pipe() r = greenio.GreenPipe(r, 'rb') w = greenio.GreenPipe(w, 'wb') def writer(): eventlet.sleep(.1) w.write(b'line\n') w.flush() w.write(b'line\r\n') w.flush() gt = eventlet.spawn(writer) eventlet.sleep(0) line = r.readline() self.assertEqual(line, b'line\n') line = r.readline() self.assertEqual(line, b'line\r\n') gt.wait()
def create_green_pipe(): """Create a green pipe for threads to synchronize on. """ rpipe, wpipe = os.pipe() write_file_obj = greenio.GreenPipe(wpipe, 'wb', 0) read_file_obj = greenio.GreenPipe(rpipe, 'rb', 0) return (read_file_obj, write_file_obj)
def _init_pipe(self): """ The pipe is for synchronization, the queue is used for store the packets """ self._event_queue = native_queue.Queue() r_pipe, w_pipe = os.pipe() self._event_notify_send = greenio.GreenPipe(w_pipe, 'wb', 0) self._event_notify_recv = greenio.GreenPipe(r_pipe, 'rb', 0)
def test_pip_read_until_end(self): # similar to test_pip_read above but reading until eof r, w = os.pipe() r = greenio.GreenPipe(r, 'rb') w = greenio.GreenPipe(w, 'wb') w.write(b'c' * DEFAULT_BUFFER_SIZE * 2) w.close() buf = r.read() # no chunk size specified; read until end self.assertEqual(len(buf), 2 * DEFAULT_BUFFER_SIZE) self.assertEqual(buf[:3], b'ccc')
def test_pipe_context(): # ensure using a pipe as a context actually closes it. r, w = os.pipe() r = greenio.GreenPipe(r) w = greenio.GreenPipe(w, 'w') with r: pass assert r.closed and not w.closed with w as f: assert f == w assert r.closed and w.closed
def test_pipe_read_unbuffered(self): # Ensure that seting the buffer size works properly on GreenPipes, # it used to be ignored on Python 2 and the test would hang on r.readline() # below. r, w = os.pipe() r = greenio.GreenPipe(r, 'rb', 0) w = greenio.GreenPipe(w, 'wb', 0) w.write(b'line\n') line = r.readline() self.assertEqual(line, b'line\n') r.close() w.close()
def run(self, action_parameters): pack = self.action.pack if self.action else None action_wrapper = ActionWrapper(pack=pack, entry_point=self.entry_point, action_parameters=action_parameters) # We manually create a non-duplex pipe since multiprocessing.Pipe # doesn't play along nicely and work with eventlet rfd, wfd = os.pipe() parent_conn = greenio.GreenPipe(rfd, 'r') child_conn = greenio.GreenPipe(wfd, 'w', 0) p = Process(target=action_wrapper.run, args=(child_conn,)) p.daemon = True try: p.start() p.join(self._timeout) if p.is_alive(): # Process is still alive meaning the timeout has been reached p.terminate() message = 'Action failed to complete in %s seconds' % (self._timeout) raise Exception(message) output = parent_conn.readline() try: output = json.loads(output) except Exception: pass exit_code = p.exitcode except: LOG.exception('Failed to run action.') _, e, tb = sys.exc_info() exit_code = 1 output = {'error': str(e), 'traceback': ''.join(traceback.format_tb(tb, 20))} finally: parent_conn.close() status = ACTIONEXEC_STATUS_SUCCEEDED if exit_code == 0 else ACTIONEXEC_STATUS_FAILED self.container_service.report_result(output) self.container_service.report_status(status) LOG.info('Action output : %s. exit_code : %s. status : %s', str(output), exit_code, status) return output is not None
def run(self): self.dead = False self.started = False self.popen4 = None ## We use popen4 so that read() will read from either stdout or stderr self.popen4 = popen2.Popen4([self.command] + self.args) child_stdout_stderr = self.popen4.fromchild child_stdin = self.popen4.tochild self.child_stdout_stderr = greenio.GreenPipe(child_stdout_stderr, child_stdout_stderr.mode, 0) self.child_stdin = greenio.GreenPipe(child_stdin, child_stdin.mode, 0) self.sendall = self.child_stdin.write self.send = self.child_stdin.write self.recv = self.child_stdout_stderr.read self.readline = self.child_stdout_stderr.readline self._read_first_result = False
def test_pipe(self): r,w = os.pipe() rf = greenio.GreenPipe(r, 'r'); wf = greenio.GreenPipe(w, 'w', 0); def sender(f, content): for ch in content: eventlet.sleep(0.0001) f.write(ch) f.close() one_line = "12345\n"; eventlet.spawn(sender, wf, one_line*5) for i in xrange(5): line = rf.readline() eventlet.sleep(0.01) self.assertEquals(line, one_line) self.assertEquals(rf.readline(), '')
def test_pipe(self): r, w = os.pipe() rf = greenio.GreenPipe(r, 'rb') wf = greenio.GreenPipe(w, 'wb', 0) def sender(f, content): for ch in map(six.int2byte, six.iterbytes(content)): eventlet.sleep(0.0001) f.write(ch) f.close() one_line = b"12345\n" eventlet.spawn(sender, wf, one_line * 5) for i in range(5): line = rf.readline() eventlet.sleep(0.01) self.assertEqual(line, one_line) self.assertEqual(rf.readline(), b'')
def call(self, func_name, *args): func = getattr(self.__lib, func_name) __rawpipe, __wpipe = os.pipe() __rpipe = greenio.GreenPipe(__rawpipe, 'rb', bufsize=0) ret = func(*(args + (__wpipe, ))) _ = __rpipe.read(4) __rpipe.close() os.close(__wpipe) return ret
def __init__(self, args, bufsize=0, *argss, **kwds): # Forward the call to base-class constructor subprocess_orig.Popen.__init__(self, args, 0, *argss, **kwds) # Now wrap the pipes, if any. This logic is loosely borrowed from # eventlet.processes.Process.run() method. for attr in "stdin", "stdout", "stderr": pipe = getattr(self, attr) if pipe is not None: wrapped_pipe = greenio.GreenPipe(pipe, pipe.mode, bufsize) setattr(self, attr, wrapped_pipe)
def fdopen(fd, *args, **kw): """fdopen(fd [, mode='r' [, bufsize]]) -> file_object Return an open file object connected to a file descriptor.""" if not isinstance(fd, int): raise TypeError('fd should be int, not %r' % fd) try: return greenio.GreenPipe(fd, *args, **kw) except IOError as e: raise OSError(*e.args)
def setup(): global _rfile, _wfile, _threads, _coro, _setup_already, _rspq, _reqq if _setup_already: return else: _setup_already = True try: _rpipe, _wpipe = os.pipe() _wfile = greenio.GreenPipe(_wpipe, 'wb', 0) _rfile = greenio.GreenPipe(_rpipe, 'rb', 0) except (ImportError, NotImplementedError): # This is Windows compatibility -- use a socket instead of a pipe because # pipes don't really exist on Windows. import socket from eventlet import util sock = util.__original_socket__(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('localhost', 0)) sock.listen(50) csock = util.__original_socket__(socket.AF_INET, socket.SOCK_STREAM) csock.connect(('localhost', sock.getsockname()[1])) nsock, addr = sock.accept() _rfile = greenio.GreenSocket(csock).makefile('rb', 0) _wfile = nsock.makefile('wb',0) _rspq = Queue(maxsize=-1) _reqq = Queue(maxsize=-1) assert _nthreads >= 0, "Can't specify negative number of threads" if _nthreads == 0: import warnings warnings.warn("Zero threads in tpool. All tpool.execute calls will\ execute in main thread. Check the value of the environment \ variable EVENTLET_THREADPOOL_SIZE.", RuntimeWarning) for i in xrange(_nthreads): t = threading.Thread(target=tworker, name="tpool_thread_%s" % i, args=(_reqq,)) t.setDaemon(True) t.start() _threads.append(t) _coro = greenthread.spawn_n(tpool_trampoline)
def test_pipe_writes_large_messages(self): r, w = os.pipe() r = greenio.GreenPipe(r) w = greenio.GreenPipe(w, 'w') large_message = "".join([1024*chr(i) for i in xrange(65)]) def writer(): w.write(large_message) w.close() gt = eventlet.spawn(writer) for i in xrange(65): buf = r.read(1024) expected = 1024*chr(i) self.assertEquals(buf, expected, "expected=%r..%r, found=%r..%r iter=%d" % (expected[:4], expected[-4:], buf[:4], buf[-4:], i)) gt.wait()
def __init__(self, nthreads=2): self.nthreads = nthreads self._run_queue = Queue() self._result_queue = Queue() self._threads = [] self._alive = True if nthreads <= 0: return # We spawn a greenthread whose job it is to pull results from the # worker threads via a real Queue and send them to eventlet Events so # that the calling greenthreads can be awoken. # # Since each OS thread has its own collection of greenthreads, it # doesn't work to have the worker thread send stuff to the event, as # it then notifies its own thread-local eventlet hub to wake up, which # doesn't do anything to help out the actual calling greenthread over # in the main thread. # # Thus, each worker sticks its results into a result queue and then # writes a byte to a pipe, signaling the result-consuming greenlet (in # the main thread) to wake up and consume results. # # This is all stuff that eventlet.tpool does, but that code can't have # multiple instances instantiated. Since the object server uses one # pool per disk, we have to reimplement this stuff. _raw_rpipe, self.wpipe = os.pipe() self.rpipe = greenio.GreenPipe(_raw_rpipe, 'rb', bufsize=0) for _junk in xrange(nthreads): thr = stdlib_threading.Thread( target=self._worker, args=(self._run_queue, self._result_queue)) thr.daemon = True thr.start() self._threads.append(thr) # This is the result-consuming greenthread that runs in the main OS # thread, as described above. self._consumer_coro = greenthread.spawn_n(self._consume_results, self._result_queue)
def test_seek_on_buffered_pipe(self): f = greenio.GreenPipe(self.tempdir+"/TestFile", 'w+', 1024) self.assertEquals(f.tell(),0) f.seek(0,2) self.assertEquals(f.tell(),0) f.write('1234567890') f.seek(0,2) self.assertEquals(f.tell(),10) f.seek(0) value = f.read(1) self.assertEqual(value, '1') self.assertEquals(f.tell(),1) value = f.read(1) self.assertEqual(value, '2') self.assertEquals(f.tell(),2) f.seek(0, 1) self.assertEqual(f.readline(), '34567890') f.seek(0) self.assertEqual(f.readline(), '1234567890') f.seek(0, 2) self.assertEqual(f.readline(), '')
def __init__(self, args, bufsize=0, *argss, **kwds): self.args = args # Forward the call to base-class constructor subprocess_orig.Popen.__init__(self, args, 0, *argss, **kwds) # Now wrap the pipes, if any. This logic is loosely borrowed from # eventlet.processes.Process.run() method. for attr in "stdin", "stdout", "stderr": pipe = getattr(self, attr) if pipe is not None and type(pipe) != greenio.GreenPipe: # https://github.com/eventlet/eventlet/issues/243 # AttributeError: '_io.TextIOWrapper' object has no attribute 'mode' mode = getattr(pipe, 'mode', '') if not mode: if pipe.readable(): mode += 'r' if pipe.writable(): mode += 'w' # ValueError: can't have unbuffered text I/O if bufsize == 0: bufsize = -1 wrapped_pipe = greenio.GreenPipe(pipe, mode, bufsize) setattr(self, attr, wrapped_pipe)
def create(self): """ Context manager to create a data file and metadata file . We create a temporary file first, and then return a DiskFileWriter object to encapsulate the state. .. note:: An implementation is not required to perform on-disk preallocations even if the parameter is specified. But if it does and it fails, it must raise a `DiskFileNoSpace` exception. """ #if not exists(self._tmpdir): # mkdirs(self._tmpdir) tmp_data_file = '_'.join([hash_path(self.account), \ hash_path(self.account, self.container), \ self._name_hash]) + self._data_ext tmp_meta_file = '_'.join([hash_path(self.account), \ hash_path(self.account, self.container), \ self._name_hash]) + self._meta_ext #need to check if we need to generate the error def _create_tmp_file(tmpdir, tmp_data_file, tmp_meta_file): if not exists(tmpdir): mkdirs(tmpdir) fd_data, tmppath_data = create_tmp_file(tmpdir, tmp_data_file) fd_meta, tmppath_meta = create_tmp_file(tmpdir, tmp_meta_file) return (fd_data, tmppath_data, fd_meta, tmppath_meta) (fd_data, tmppath_data, fd_meta, tmppath_meta) = \ self._threadpool_write.force_run_in_thread( \ _create_tmp_file, self._tmpdir, tmp_data_file, tmp_meta_file) _raw_rpipe, wpipe = os.pipe() rpipe = greenio.GreenPipe(_raw_rpipe, 'rb', bufsize=0) #fd_data, tmppath_data = create_tmp_file(self._tmpdir, tmp_data_file) #rfd_meta, tmppath_meta = create_tmp_file(self._tmpdir, tmp_meta_file) try: yield DiskFileWriter(self._name, self._name_hash, self._datadir, self._metadir, fd_data, fd_meta, tmppath_data, tmppath_meta, rpipe, wpipe, self._threadpool_write, self._object_lib, self._logger) finally: def _finally(fd_data, fd_meta, tmppath_data, tmppath_meta): try: os.close(fd_data) os.close(fd_meta) except OSError: pass try: os.unlink(tmppath_data) os.unlink(tmppath_meta) except OSError: pass self._threadpool_write.force_run_in_thread(_finally, fd_data, fd_meta, tmppath_data, tmppath_meta) rpipe.close() os.close(wpipe)
def test_truncate(self): f = greenio.GreenPipe(self.tempdir + "/TestFile", 'wb+', 1024) f.write(b'1234567890') f.truncate(9) self.assertEqual(f.tell(), 9)
def _create_pipe(self): rpipe, wpipe = os.pipe() rfile = greenio.GreenPipe(rpipe, 'rb', 0) wfile = greenio.GreenPipe(wpipe, 'wb', 0) return rfile, wfile
def __iter__(self): """Returns an iterator over the data file.""" _raw_rpipe, self.wpipe = os.pipe() self.rpipe = greenio.GreenPipe(_raw_rpipe, 'rb', bufsize=0) try: dropped_cache = 0 self._bytes_read = 0 self._started_at_0 = False self._read_to_eof = False use_hash = False if self._fp.tell() == 0: self._started_at_0 = True use_hash = True self._object_lib.init_md5_hash(self._fp.fileno()) chunk = "".zfill(self._disk_chunk_size) #Performance Start time start_time = time.time() while True: self._object_lib.read_chunk(self._fp.fileno(), chunk, self._disk_chunk_size, self.wpipe) ret = self.rpipe.read(4) readBytes, = struct.unpack("i", ret) if readBytes == -1: raise IOError("read data file error") if readBytes != 0: self._bytes_read += readBytes chunk = chunk[:readBytes] if self._bytes_read - dropped_cache > (1024 * 1024): self._drop_cache(self._fp.fileno(), dropped_cache, self._bytes_read - dropped_cache) dropped_cache = self._bytes_read yield chunk if readBytes != self._disk_chunk_size: self._read_to_eof = True self._drop_cache(self._fp.fileno(), dropped_cache, self._bytes_read - dropped_cache) break #Performance Stat for GET Request size_in_mb = self._bytes_read / (1024 * 1024) elapsed_time = time.time() - start_time self.get_object_size += size_in_mb self.get_time += elapsed_time if self.statInstance: if elapsed_time: band = self._bytes_read / elapsed_time self.statInstance.avgBandForDownloadObject = self.get_object_size / self.get_time if self.statInstance.maxBandForDownloadObject < band: self.statInstance.maxBandForDownloadObject = band if self.statInstance.minBandForDownloadObject == 0: self.statInstance.minBandForDownloadObject = band else: if self.statInstance.minBandForDownloadObject > band: self.statInstance.minBandForDownloadObject = band except (Exception, Timeout) as e: self._logger.error( _('ERROR DiskFile %(data_file)s' ' read failure: %(exc)s : %(stack)s'), { 'exc': e, 'stack': ''.join(traceback.format_stack()), 'data_file': self._data_file }) raise finally: if use_hash: self._iter_etag = self._object_lib.get_md5_hash( self._fp.fileno()) if not self._suppress_file_closing: self.close() self.rpipe.close() os.close(self.wpipe)