def rebuild_ctype(type_, wrapper, length): if length is not None: type_ = type_ * length ForkingPickler.register(type_, reduce_ctype) obj = type_.from_address(wrapper.get_address()) obj._wrapper = wrapper return obj
def rebuild_ctype(type_, wrapper, length): if length is not None: type_ = type_ * length ForkingPickler.register(type_, reduce_ctype) buf = wrapper.create_memoryview() obj = type_.from_buffer(buf) obj._wrapper = wrapper return obj
def dumps(obj, protocol=2): """ Serialize obj to string using the enhancments made in multiprocessing """ output = StringIO() ForkingPickler(output, protocol).dump(obj) return output.getvalue()
def send(self, obj): """Send a (picklable) object""" self._check_closed() self._check_writable() buf = io.BytesIO() ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(obj) self._send_bytes(buf.getbuffer())
def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe): debug('starting thread to feed data to pipe') from .util import is_exiting nacquire = notempty.acquire nrelease = notempty.release nwait = notempty.wait bpopleft = buffer.popleft sentinel = _sentinel if sys.platform != 'win32': wacquire = writelock.acquire wrelease = writelock.release else: wacquire = None try: while 1: nacquire() try: if not buffer: nwait() finally: nrelease() try: while 1: obj = bpopleft() if obj is sentinel: debug('feeder thread got sentinel -- exiting') close() return # serialize the data before acquiring the lock obj = ForkingPickler.dumps(obj) if wacquire is None: send_bytes(obj) else: wacquire() try: send_bytes(obj) finally: wrelease() except IndexError: pass except Exception as e: if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: return # Since this runs in a daemon thread the resources it uses # may be become unusable while the process is cleaning up. # We ignore errors which happen after the process has # started to cleanup. try: if is_exiting(): info('error in queue thread: %s', e) else: import traceback traceback.print_exc() except Exception: pass
def put(self, obj): # serialize the data before acquiring the lock obj = ForkingPickler.dumps(obj) if self._wlock is None: # writes to a message oriented win32 pipe are atomic self._writer.send_bytes(obj) else: with self._wlock: self._writer.send_bytes(obj)
def get(self, block=True, timeout=None): if block and timeout is None: with self._rlock: res = self._recv_bytes() self._sem.release() else: if block: deadline = time.time() + timeout if not self._rlock.acquire(block, timeout): raise Empty try: if block: timeout = deadline - time.time() if timeout < 0 or not self._poll(timeout): raise Empty elif not self._poll(): raise Empty res = self._recv_bytes() self._sem.release() finally: self._rlock.release() # unserialize the data after having released the lock return ForkingPickler.loads(res)
if timeout <= 0: return select.select(object_list, [], [], 0)[0] else: deadline = time.time() + timeout while True: try: return select.select(object_list, [], [], timeout)[0] except OSError as e: if e.errno != errno.EINTR: raise if timeout is not None: timeout = deadline - time.time() # # Make connection and socket objects sharable if possible # if sys.platform == 'win32': from . import reduction ForkingPickler.register(socket.socket, reduction.reduce_socket) ForkingPickler.register(Connection, reduction.reduce_connection) ForkingPickler.register(PipeConnection, reduction.reduce_pipe_connection) else: try: from . import reduction except ImportError: pass else: ForkingPickler.register(socket.socket, reduction.reduce_socket) ForkingPickler.register(Connection, reduction.reduce_connection)
return new_handle def reduce_connection(conn): rh = reduce_handle(conn.fileno()) return (rebuild_connection, (rh, conn.readable, conn.writable)) def rebuild_connection(reduced_handle, readable, writable): handle = rebuild_handle(reduced_handle) return _multiprocessing.Connection(handle, readable=readable, writable=writable) ForkingPickler.register(_multiprocessing.Connection, reduce_connection) def fromfd(fd, family, type_, proto=0): s = socket.fromfd(fd, family, type_, proto) if s.__class__ is not socket.socket: s = socket.socket(_sock=s) return s def reduce_socket(s): reduced_handle = reduce_handle(s.fileno()) return (rebuild_socket, (reduced_handle, s.family, s.type, s.proto)) def rebuild_socket(reduced_handle, family, type_, proto):
def get(self): with self._rlock: res = self._reader.recv_bytes() # unserialize the data after having released the lock return ForkingPickler.loads(res)
import array import queue from traceback import format_exc from multiprocessing import Process, current_process, active_children, Pool, util, connection from multiprocessing.process import AuthenticationString from multiprocessing.forking import Popen, ForkingPickler from time import time as _time # # Register some things for pickling # def reduce_array(a): return array.array, (a.typecode, a.tobytes()) ForkingPickler.register(array.array, reduce_array) view_types = [type(getattr({}, name)()) for name in ('items','keys','values')] if view_types[0] is not list: # only needed in Py3.0 def rebuild_as_list(obj): return list, (list(obj),) for view_type in view_types: ForkingPickler.register(view_type, rebuild_as_list) import copyreg copyreg.pickle(view_type, rebuild_as_list) # # Type for identifying shared objects # class Token(object):
from traceback import format_exc from multiprocessing import Process, current_process, active_children, Pool, util, connection from multiprocessing.process import AuthenticationString from multiprocessing.forking import exit, Popen, assert_spawning, ForkingPickler from multiprocessing.util import Finalize, info try: from cPickle import PicklingError except ImportError: from pickle import PicklingError def reduce_array(a): return (array.array, (a.typecode, a.tostring())) ForkingPickler.register(array.array, reduce_array) view_types = [ type(getattr({}, name)()) for name in ('items', 'keys', 'values') ] class Token(object): __slots__ = ('typeid', 'address', 'id') def __init__(self, typeid, address, id): self.typeid, self.address, self.id = typeid, address, id def __getstate__(self): return (self.typeid, self.address, self.id) def __setstate__(self, state):
def send(self, obj): """Send a (picklable) object""" self._check_closed() self._check_writable() self._send_bytes(ForkingPickler.dumps(obj))
from pickle import PicklingError from multiprocessing import Process, current_process, active_children, Pool, util, connection from multiprocessing.process import AuthenticationString from multiprocessing.forking import exit, Popen, assert_spawning, ForkingPickler from multiprocessing.util import Finalize, info # # Register some things for pickling # def reduce_array(a): return array.array, (a.typecode, a.tobytes()) ForkingPickler.register(array.array, reduce_array) view_types = [ type(getattr({}, name)()) for name in ('items', 'keys', 'values') ] if view_types[0] is not list: # only needed in Py3.0 def rebuild_as_list(obj): return list, (list(obj), ) for view_type in view_types: ForkingPickler.register(view_type, rebuild_as_list) import copyreg copyreg.pickle(view_type, rebuild_as_list) #
def recv(self): """Receive a (picklable) object""" self._check_closed() self._check_readable() buf = self._recv_bytes() return ForkingPickler.loads(buf.getbuffer())
new_handle = recv_handle(conn) conn.close() return new_handle def reduce_connection(conn): rh = reduce_handle(conn.fileno()) return (rebuild_connection, (rh, conn.readable, conn.writable)) def rebuild_connection(reduced_handle, readable, writable): handle = rebuild_handle(reduced_handle) return _multiprocessing.Connection(handle, readable=readable, writable=writable) ForkingPickler.register(_multiprocessing.Connection, reduce_connection) def fromfd(fd, family, type_, proto = 0): s = socket.fromfd(fd, family, type_, proto) if s.__class__ is not socket.socket: s = socket.socket(_sock=s) return s def reduce_socket(s): reduced_handle = reduce_handle(s.fileno()) return (rebuild_socket, (reduced_handle, s.family, s.type, s.proto))
def dump(obj, file, protocol=2): """ Serialize obj to file using the enhancments made in multiprocessing """ return ForkingPickler(file, protocol).dump(obj)
import Queue from traceback import format_exc from multiprocessing import Process, current_process, active_children, Pool, util, connection from multiprocessing.process import AuthenticationString from multiprocessing.forking import exit, Popen, assert_spawning, ForkingPickler from multiprocessing.util import Finalize, info try: from cPickle import PicklingError except ImportError: from pickle import PicklingError def reduce_array(a): return (array.array, (a.typecode, a.tostring())) ForkingPickler.register(array.array, reduce_array) view_types = [ type(getattr({}, name)()) for name in ('items', 'keys', 'values') ] class Token(object): __slots__ = ('typeid', 'address', 'id') def __init__(self, typeid, address, id): self.typeid, self.address, self.id = typeid, address, id def __getstate__(self): return (self.typeid, self.address, self.id) def __setstate__(self, state): self.typeid, self.address, self.id = state def __repr__(self):
return _poll(object_list, 0) else: deadline = time.time() + timeout while True: try: return _poll(object_list, timeout) except OSError as e: if e.errno != errno.EINTR: raise if timeout is not None: timeout = deadline - time.time() # # Make connection and socket objects sharable if possible # if sys.platform == 'win32': from . import reduction ForkingPickler.register(socket.socket, reduction.reduce_socket) ForkingPickler.register(Connection, reduction.reduce_connection) ForkingPickler.register(PipeConnection, reduction.reduce_pipe_connection) else: try: from . import reduction except ImportError: pass else: ForkingPickler.register(socket.socket, reduction.reduce_socket) ForkingPickler.register(Connection, reduction.reduce_connection)