def setup():
global _rfile, _wfile, _threads, _coro, _setup_already, _reqq, _rspq
if _setup_already:
return
else:
_setup_already = True
try:
_rpipe, _wpipe = os.pipe()
_wfile = greenio.GreenPipe(_wpipe, 'wb', 0)
_rfile = greenio.GreenPipe(_rpipe, 'rb', 0)
except ImportError:
# This is Windows compatibility -- use a socket instead of a pipe because
# pipes don't really exist on Windows.
import socket
from eventlet import util
sock = util.__original_socket__(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('localhost', 0))
sock.listen(50)
csock = util.__original_socket__(socket.AF_INET, socket.SOCK_STREAM)
csock.connect(('localhost', sock.getsockname()[1]))
nsock, addr = sock.accept()
_rfile = greenio.GreenSocket(csock).makefile('rb', 0)
_wfile = nsock.makefile('wb', 0)
_reqq = Queue(maxsize=-1)
_rspq = Queue(maxsize=-1)
for i in range(0, _nthreads):
t = threading.Thread(target=tworker)
t.setDaemon(True)
t.start()
_threads.add(t)
_coro = greenthread.spawn_n(tpool_trampoline)
def _init_events_pipe(self):
"""Create a self-pipe for the native thread to synchronize on.
This code is taken from the eventlet tpool module, under terms
of the Apache License v2.0.
"""
self._event_queue = native_Queue.Queue()
try:
rpipe, wpipe = os.pipe()
self._event_notify_send = greenio.GreenPipe(wpipe, 'wb', 0)
self._event_notify_recv = greenio.GreenPipe(rpipe, 'rb', 0)
except (ImportError, NotImplementedError):
# This is Windows compatibility -- use a socket instead
# of a pipe because pipes don't really exist on Windows.
sock = eventlet_util.__original_socket__(socket.AF_INET,
socket.SOCK_STREAM)
sock.bind(('localhost', 0))
sock.listen(50)
csock = eventlet_util.__original_socket__(socket.AF_INET,
socket.SOCK_STREAM)
csock.connect(('localhost', sock.getsockname()[1]))
nsock, addr = sock.accept()
self._event_notify_send = nsock.makefile('wb', 0)
gsock = greenio.GreenSocket(csock)
self._event_notify_recv = gsock.makefile('rb', 0)
def test_timeout_and_final_write(self):
# This test verifies that a write on a socket that we've
# stopped listening for doesn't result in an incorrect switch
rpipe, wpipe = os.pipe()
rfile = os.fdopen(rpipe, "r", 0)
wrap_rfile = greenio.GreenPipe(rfile)
wfile = os.fdopen(wpipe, "w", 0)
wrap_wfile = greenio.GreenPipe(wfile)
def sender(evt):
api.sleep(0.02)
wrap_wfile.write('hi')
evt.send('sent via event')
from eventlet import coros
evt = coros.event()
api.spawn(sender, evt)
try:
# try and get some data off of this pipe
# but bail before any is sent
api.exc_after(0.01, api.TimeoutError)
_c = wrap_rfile.read(1)
self.fail()
except api.TimeoutError:
pass
result = evt.wait()
self.assertEquals(result, 'sent via event')
def test_pipe_read(self):
# ensure that 'readline' works properly on GreenPipes when data is not
# immediately available (fd is nonblocking, was raising EAGAIN)
# also ensures that readline() terminates on '\n' and '\r\n'
r, w = os.pipe()
r = greenio.GreenPipe(r, 'rb')
w = greenio.GreenPipe(w, 'wb')
def writer():
eventlet.sleep(.1)
w.write(b'line\n')
w.flush()
w.write(b'line\r\n')
w.flush()
gt = eventlet.spawn(writer)
eventlet.sleep(0)
line = r.readline()
self.assertEqual(line, b'line\n')
line = r.readline()
self.assertEqual(line, b'line\r\n')
gt.wait()
def create_green_pipe():
"""Create a green pipe for threads to synchronize on.
"""
rpipe, wpipe = os.pipe()
write_file_obj = greenio.GreenPipe(wpipe, 'wb', 0)
read_file_obj = greenio.GreenPipe(rpipe, 'rb', 0)
return (read_file_obj, write_file_obj)
def _init_pipe(self):
"""
The pipe is for synchronization, the queue is used for store
the packets
"""
self._event_queue = native_queue.Queue()
r_pipe, w_pipe = os.pipe()
self._event_notify_send = greenio.GreenPipe(w_pipe, 'wb', 0)
self._event_notify_recv = greenio.GreenPipe(r_pipe, 'rb', 0)
def test_pip_read_until_end(self):
# similar to test_pip_read above but reading until eof
r, w = os.pipe()
r = greenio.GreenPipe(r, 'rb')
w = greenio.GreenPipe(w, 'wb')
w.write(b'c' * DEFAULT_BUFFER_SIZE * 2)
w.close()
buf = r.read() # no chunk size specified; read until end
self.assertEqual(len(buf), 2 * DEFAULT_BUFFER_SIZE)
self.assertEqual(buf[:3], b'ccc')
def test_pipe_context():
# ensure using a pipe as a context actually closes it.
r, w = os.pipe()
r = greenio.GreenPipe(r)
w = greenio.GreenPipe(w, 'w')
with r:
pass
assert r.closed and not w.closed
with w as f:
assert f == w
assert r.closed and w.closed
def test_pipe_read_unbuffered(self):
# Ensure that seting the buffer size works properly on GreenPipes,
# it used to be ignored on Python 2 and the test would hang on r.readline()
# below.
r, w = os.pipe()
r = greenio.GreenPipe(r, 'rb', 0)
w = greenio.GreenPipe(w, 'wb', 0)
w.write(b'line\n')
line = r.readline()
self.assertEqual(line, b'line\n')
r.close()
w.close()
def run(self, action_parameters):
pack = self.action.pack if self.action else None
action_wrapper = ActionWrapper(pack=pack,
entry_point=self.entry_point,
action_parameters=action_parameters)
# We manually create a non-duplex pipe since multiprocessing.Pipe
# doesn't play along nicely and work with eventlet
rfd, wfd = os.pipe()
parent_conn = greenio.GreenPipe(rfd, 'r')
child_conn = greenio.GreenPipe(wfd, 'w', 0)
p = Process(target=action_wrapper.run, args=(child_conn,))
p.daemon = True
try:
p.start()
p.join(self._timeout)
if p.is_alive():
# Process is still alive meaning the timeout has been reached
p.terminate()
message = 'Action failed to complete in %s seconds' % (self._timeout)
raise Exception(message)
output = parent_conn.readline()
try:
output = json.loads(output)
except Exception:
pass
exit_code = p.exitcode
except:
LOG.exception('Failed to run action.')
_, e, tb = sys.exc_info()
exit_code = 1
output = {'error': str(e), 'traceback': ''.join(traceback.format_tb(tb, 20))}
finally:
parent_conn.close()
status = ACTIONEXEC_STATUS_SUCCEEDED if exit_code == 0 else ACTIONEXEC_STATUS_FAILED
self.container_service.report_result(output)
self.container_service.report_status(status)
LOG.info('Action output : %s. exit_code : %s. status : %s', str(output), exit_code, status)
return output is not None
def run(self):
self.dead = False
self.started = False
self.popen4 = None
## We use popen4 so that read() will read from either stdout or stderr
self.popen4 = popen2.Popen4([self.command] + self.args)
child_stdout_stderr = self.popen4.fromchild
child_stdin = self.popen4.tochild
self.child_stdout_stderr = greenio.GreenPipe(child_stdout_stderr, child_stdout_stderr.mode, 0)
self.child_stdin = greenio.GreenPipe(child_stdin, child_stdin.mode, 0)
self.sendall = self.child_stdin.write
self.send = self.child_stdin.write
self.recv = self.child_stdout_stderr.read
self.readline = self.child_stdout_stderr.readline
self._read_first_result = False
def test_pipe(self):
r,w = os.pipe()
rf = greenio.GreenPipe(r, 'r');
wf = greenio.GreenPipe(w, 'w', 0);
def sender(f, content):
for ch in content:
eventlet.sleep(0.0001)
f.write(ch)
f.close()
one_line = "12345\n";
eventlet.spawn(sender, wf, one_line*5)
for i in xrange(5):
line = rf.readline()
eventlet.sleep(0.01)
self.assertEquals(line, one_line)
self.assertEquals(rf.readline(), '')
def test_pipe(self):
r, w = os.pipe()
rf = greenio.GreenPipe(r, 'rb')
wf = greenio.GreenPipe(w, 'wb', 0)
def sender(f, content):
for ch in map(six.int2byte, six.iterbytes(content)):
eventlet.sleep(0.0001)
f.write(ch)
f.close()
one_line = b"12345\n"
eventlet.spawn(sender, wf, one_line * 5)
for i in range(5):
line = rf.readline()
eventlet.sleep(0.01)
self.assertEqual(line, one_line)
self.assertEqual(rf.readline(), b'')
def call(self, func_name, *args):
func = getattr(self.__lib, func_name)
__rawpipe, __wpipe = os.pipe()
__rpipe = greenio.GreenPipe(__rawpipe, 'rb', bufsize=0)
ret = func(*(args + (__wpipe, )))
_ = __rpipe.read(4)
__rpipe.close()
os.close(__wpipe)
return ret
def __init__(self, args, bufsize=0, *argss, **kwds):
# Forward the call to base-class constructor
subprocess_orig.Popen.__init__(self, args, 0, *argss, **kwds)
# Now wrap the pipes, if any. This logic is loosely borrowed from
# eventlet.processes.Process.run() method.
for attr in "stdin", "stdout", "stderr":
pipe = getattr(self, attr)
if pipe is not None:
wrapped_pipe = greenio.GreenPipe(pipe, pipe.mode, bufsize)
setattr(self, attr, wrapped_pipe)
def fdopen(fd, *args, **kw):
"""fdopen(fd [, mode='r' [, bufsize]]) -> file_object
Return an open file object connected to a file descriptor."""
if not isinstance(fd, int):
raise TypeError('fd should be int, not %r' % fd)
try:
return greenio.GreenPipe(fd, *args, **kw)
except IOError as e:
raise OSError(*e.args)
def setup():
global _rfile, _wfile, _threads, _coro, _setup_already, _rspq, _reqq
if _setup_already:
return
else:
_setup_already = True
try:
_rpipe, _wpipe = os.pipe()
_wfile = greenio.GreenPipe(_wpipe, 'wb', 0)
_rfile = greenio.GreenPipe(_rpipe, 'rb', 0)
except (ImportError, NotImplementedError):
# This is Windows compatibility -- use a socket instead of a pipe because
# pipes don't really exist on Windows.
import socket
from eventlet import util
sock = util.__original_socket__(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('localhost', 0))
sock.listen(50)
csock = util.__original_socket__(socket.AF_INET, socket.SOCK_STREAM)
csock.connect(('localhost', sock.getsockname()[1]))
nsock, addr = sock.accept()
_rfile = greenio.GreenSocket(csock).makefile('rb', 0)
_wfile = nsock.makefile('wb',0)
_rspq = Queue(maxsize=-1)
_reqq = Queue(maxsize=-1)
assert _nthreads >= 0, "Can't specify negative number of threads"
if _nthreads == 0:
import warnings
warnings.warn("Zero threads in tpool. All tpool.execute calls will\
execute in main thread. Check the value of the environment \
variable EVENTLET_THREADPOOL_SIZE.", RuntimeWarning)
for i in xrange(_nthreads):
t = threading.Thread(target=tworker,
name="tpool_thread_%s" % i,
args=(_reqq,))
t.setDaemon(True)
t.start()
_threads.append(t)
_coro = greenthread.spawn_n(tpool_trampoline)
def test_pipe_writes_large_messages(self):
r, w = os.pipe()
r = greenio.GreenPipe(r)
w = greenio.GreenPipe(w, 'w')
large_message = "".join([1024*chr(i) for i in xrange(65)])
def writer():
w.write(large_message)
w.close()
gt = eventlet.spawn(writer)
for i in xrange(65):
buf = r.read(1024)
expected = 1024*chr(i)
self.assertEquals(buf, expected,
"expected=%r..%r, found=%r..%r iter=%d"
% (expected[:4], expected[-4:], buf[:4], buf[-4:], i))
gt.wait()
def __init__(self, nthreads=2):
self.nthreads = nthreads
self._run_queue = Queue()
self._result_queue = Queue()
self._threads = []
self._alive = True
if nthreads <= 0:
return
# We spawn a greenthread whose job it is to pull results from the
# worker threads via a real Queue and send them to eventlet Events so
# that the calling greenthreads can be awoken.
#
# Since each OS thread has its own collection of greenthreads, it
# doesn't work to have the worker thread send stuff to the event, as
# it then notifies its own thread-local eventlet hub to wake up, which
# doesn't do anything to help out the actual calling greenthread over
# in the main thread.
#
# Thus, each worker sticks its results into a result queue and then
# writes a byte to a pipe, signaling the result-consuming greenlet (in
# the main thread) to wake up and consume results.
#
# This is all stuff that eventlet.tpool does, but that code can't have
# multiple instances instantiated. Since the object server uses one
# pool per disk, we have to reimplement this stuff.
_raw_rpipe, self.wpipe = os.pipe()
self.rpipe = greenio.GreenPipe(_raw_rpipe, 'rb', bufsize=0)
for _junk in xrange(nthreads):
thr = stdlib_threading.Thread(
target=self._worker,
args=(self._run_queue, self._result_queue))
thr.daemon = True
thr.start()
self._threads.append(thr)
# This is the result-consuming greenthread that runs in the main OS
# thread, as described above.
self._consumer_coro = greenthread.spawn_n(self._consume_results,
self._result_queue)
def test_seek_on_buffered_pipe(self):
f = greenio.GreenPipe(self.tempdir+"/TestFile", 'w+', 1024)
self.assertEquals(f.tell(),0)
f.seek(0,2)
self.assertEquals(f.tell(),0)
f.write('1234567890')
f.seek(0,2)
self.assertEquals(f.tell(),10)
f.seek(0)
value = f.read(1)
self.assertEqual(value, '1')
self.assertEquals(f.tell(),1)
value = f.read(1)
self.assertEqual(value, '2')
self.assertEquals(f.tell(),2)
f.seek(0, 1)
self.assertEqual(f.readline(), '34567890')
f.seek(0)
self.assertEqual(f.readline(), '1234567890')
f.seek(0, 2)
self.assertEqual(f.readline(), '')
def __init__(self, args, bufsize=0, *argss, **kwds):
self.args = args
# Forward the call to base-class constructor
subprocess_orig.Popen.__init__(self, args, 0, *argss, **kwds)
# Now wrap the pipes, if any. This logic is loosely borrowed from
# eventlet.processes.Process.run() method.
for attr in "stdin", "stdout", "stderr":
pipe = getattr(self, attr)
if pipe is not None and type(pipe) != greenio.GreenPipe:
# https://github.com/eventlet/eventlet/issues/243
# AttributeError: '_io.TextIOWrapper' object has no attribute 'mode'
mode = getattr(pipe, 'mode', '')
if not mode:
if pipe.readable():
mode += 'r'
if pipe.writable():
mode += 'w'
# ValueError: can't have unbuffered text I/O
if bufsize == 0:
bufsize = -1
wrapped_pipe = greenio.GreenPipe(pipe, mode, bufsize)
setattr(self, attr, wrapped_pipe)
def create(self):
"""
Context manager to create a data file and metadata file .
We create a temporary file first, and
then return a DiskFileWriter object to encapsulate the state.
.. note::
An implementation is not required to perform on-disk
preallocations even if the parameter is specified. But if it does
and it fails, it must raise a `DiskFileNoSpace` exception.
"""
#if not exists(self._tmpdir):
# mkdirs(self._tmpdir)
tmp_data_file = '_'.join([hash_path(self.account), \
hash_path(self.account, self.container), \
self._name_hash]) + self._data_ext
tmp_meta_file = '_'.join([hash_path(self.account), \
hash_path(self.account, self.container), \
self._name_hash]) + self._meta_ext
#need to check if we need to generate the error
def _create_tmp_file(tmpdir, tmp_data_file, tmp_meta_file):
if not exists(tmpdir):
mkdirs(tmpdir)
fd_data, tmppath_data = create_tmp_file(tmpdir, tmp_data_file)
fd_meta, tmppath_meta = create_tmp_file(tmpdir, tmp_meta_file)
return (fd_data, tmppath_data, fd_meta, tmppath_meta)
(fd_data, tmppath_data, fd_meta, tmppath_meta) = \
self._threadpool_write.force_run_in_thread( \
_create_tmp_file, self._tmpdir, tmp_data_file, tmp_meta_file)
_raw_rpipe, wpipe = os.pipe()
rpipe = greenio.GreenPipe(_raw_rpipe, 'rb', bufsize=0)
#fd_data, tmppath_data = create_tmp_file(self._tmpdir, tmp_data_file)
#rfd_meta, tmppath_meta = create_tmp_file(self._tmpdir, tmp_meta_file)
try:
yield DiskFileWriter(self._name, self._name_hash, self._datadir,
self._metadir, fd_data, fd_meta, tmppath_data,
tmppath_meta, rpipe, wpipe,
self._threadpool_write, self._object_lib,
self._logger)
finally:
def _finally(fd_data, fd_meta, tmppath_data, tmppath_meta):
try:
os.close(fd_data)
os.close(fd_meta)
except OSError:
pass
try:
os.unlink(tmppath_data)
os.unlink(tmppath_meta)
except OSError:
pass
self._threadpool_write.force_run_in_thread(_finally, fd_data,
fd_meta, tmppath_data,
tmppath_meta)
rpipe.close()
os.close(wpipe)
def test_truncate(self):
f = greenio.GreenPipe(self.tempdir + "/TestFile", 'wb+', 1024)
f.write(b'1234567890')
f.truncate(9)
self.assertEqual(f.tell(), 9)
def _create_pipe(self):
rpipe, wpipe = os.pipe()
rfile = greenio.GreenPipe(rpipe, 'rb', 0)
wfile = greenio.GreenPipe(wpipe, 'wb', 0)
return rfile, wfile
def __iter__(self):
"""Returns an iterator over the data file."""
_raw_rpipe, self.wpipe = os.pipe()
self.rpipe = greenio.GreenPipe(_raw_rpipe, 'rb', bufsize=0)
try:
dropped_cache = 0
self._bytes_read = 0
self._started_at_0 = False
self._read_to_eof = False
use_hash = False
if self._fp.tell() == 0:
self._started_at_0 = True
use_hash = True
self._object_lib.init_md5_hash(self._fp.fileno())
chunk = "".zfill(self._disk_chunk_size)
#Performance Start time
start_time = time.time()
while True:
self._object_lib.read_chunk(self._fp.fileno(), chunk,
self._disk_chunk_size, self.wpipe)
ret = self.rpipe.read(4)
readBytes, = struct.unpack("i", ret)
if readBytes == -1:
raise IOError("read data file error")
if readBytes != 0:
self._bytes_read += readBytes
chunk = chunk[:readBytes]
if self._bytes_read - dropped_cache > (1024 * 1024):
self._drop_cache(self._fp.fileno(), dropped_cache,
self._bytes_read - dropped_cache)
dropped_cache = self._bytes_read
yield chunk
if readBytes != self._disk_chunk_size:
self._read_to_eof = True
self._drop_cache(self._fp.fileno(), dropped_cache,
self._bytes_read - dropped_cache)
break
#Performance Stat for GET Request
size_in_mb = self._bytes_read / (1024 * 1024)
elapsed_time = time.time() - start_time
self.get_object_size += size_in_mb
self.get_time += elapsed_time
if self.statInstance:
if elapsed_time:
band = self._bytes_read / elapsed_time
self.statInstance.avgBandForDownloadObject = self.get_object_size / self.get_time
if self.statInstance.maxBandForDownloadObject < band:
self.statInstance.maxBandForDownloadObject = band
if self.statInstance.minBandForDownloadObject == 0:
self.statInstance.minBandForDownloadObject = band
else:
if self.statInstance.minBandForDownloadObject > band:
self.statInstance.minBandForDownloadObject = band
except (Exception, Timeout) as e:
self._logger.error(
_('ERROR DiskFile %(data_file)s'
' read failure: %(exc)s : %(stack)s'), {
'exc': e,
'stack': ''.join(traceback.format_stack()),
'data_file': self._data_file
})
raise
finally:
if use_hash:
self._iter_etag = self._object_lib.get_md5_hash(
self._fp.fileno())
if not self._suppress_file_closing:
self.close()
self.rpipe.close()
os.close(self.wpipe)
