def rebuild_ctype(type_, wrapper, length): if length is not None: type_ = type_ * length ForkingPickler.register(type_, reduce_ctype) obj = type_.from_address(wrapper.get_address()) obj._wrapper = wrapper return obj
def rebuild_ctype(type_, wrapper, length): if length is not None: type_ = type_ * length ForkingPickler.register(type_, reduce_ctype) buf = wrapper.create_memoryview() obj = type_.from_buffer(buf) obj._wrapper = wrapper return obj
import Queue from traceback import format_exc from multiprocessing import Process, current_process, active_children, Pool, util, connection from multiprocessing.process import AuthenticationString from multiprocessing.forking import exit, Popen, assert_spawning, ForkingPickler from multiprocessing.util import Finalize, info try: from cPickle import PicklingError except ImportError: from pickle import PicklingError def reduce_array(a): return (array.array, (a.typecode, a.tostring())) ForkingPickler.register(array.array, reduce_array) view_types = [ type(getattr({}, name)()) for name in ('items', 'keys', 'values') ] class Token(object): __slots__ = ('typeid', 'address', 'id') def __init__(self, typeid, address, id): self.typeid, self.address, self.id = typeid, address, id def __getstate__(self): return (self.typeid, self.address, self.id) def __setstate__(self, state): self.typeid, self.address, self.id = state def __repr__(self):
import array import queue from traceback import format_exc from multiprocessing import Process, current_process, active_children, Pool, util, connection from multiprocessing.process import AuthenticationString from multiprocessing.forking import Popen, ForkingPickler from time import time as _time # # Register some things for pickling # def reduce_array(a): return array.array, (a.typecode, a.tobytes()) ForkingPickler.register(array.array, reduce_array) view_types = [type(getattr({}, name)()) for name in ('items','keys','values')] if view_types[0] is not list: # only needed in Py3.0 def rebuild_as_list(obj): return list, (list(obj),) for view_type in view_types: ForkingPickler.register(view_type, rebuild_as_list) import copyreg copyreg.pickle(view_type, rebuild_as_list) # # Type for identifying shared objects # class Token(object):
new_handle = recv_handle(conn) conn.close() return new_handle def reduce_connection(conn): rh = reduce_handle(conn.fileno()) return (rebuild_connection, (rh, conn.readable, conn.writable)) def rebuild_connection(reduced_handle, readable, writable): handle = rebuild_handle(reduced_handle) return _multiprocessing.Connection(handle, readable=readable, writable=writable) ForkingPickler.register(_multiprocessing.Connection, reduce_connection) def fromfd(fd, family, type_, proto = 0): s = socket.fromfd(fd, family, type_, proto) if s.__class__ is not socket.socket: s = socket.socket(_sock=s) return s def reduce_socket(s): reduced_handle = reduce_handle(s.fileno()) return (rebuild_socket, (reduced_handle, s.family, s.type, s.proto))
if timeout <= 0: return select.select(object_list, [], [], 0)[0] else: deadline = time.time() + timeout while True: try: return select.select(object_list, [], [], timeout)[0] except OSError as e: if e.errno != errno.EINTR: raise if timeout is not None: timeout = deadline - time.time() # # Make connection and socket objects sharable if possible # if sys.platform == 'win32': from . import reduction ForkingPickler.register(socket.socket, reduction.reduce_socket) ForkingPickler.register(Connection, reduction.reduce_connection) ForkingPickler.register(PipeConnection, reduction.reduce_pipe_connection) else: try: from . import reduction except ImportError: pass else: ForkingPickler.register(socket.socket, reduction.reduce_socket) ForkingPickler.register(Connection, reduction.reduce_connection)
return _poll(object_list, 0) else: deadline = time.time() + timeout while True: try: return _poll(object_list, timeout) except OSError as e: if e.errno != errno.EINTR: raise if timeout is not None: timeout = deadline - time.time() # # Make connection and socket objects sharable if possible # if sys.platform == 'win32': from . import reduction ForkingPickler.register(socket.socket, reduction.reduce_socket) ForkingPickler.register(Connection, reduction.reduce_connection) ForkingPickler.register(PipeConnection, reduction.reduce_pipe_connection) else: try: from . import reduction except ImportError: pass else: ForkingPickler.register(socket.socket, reduction.reduce_socket) ForkingPickler.register(Connection, reduction.reduce_connection)
try: from cPickle import PicklingError except ImportError: from pickle import PicklingError # # Register some things for pickling # def reduce_array(a): return array.array, (a.typecode, a.tostring()) ForkingPickler.register(array.array, reduce_array) view_types = [ type(getattr({}, name)()) for name in ('items', 'keys', 'values') ] # # Type for identifying shared objects # class Token(object): ''' Type to uniquely indentify a shared object ''' __slots__ = ('typeid', 'address', 'id')
from pickle import PicklingError from multiprocessing import Process, current_process, active_children, Pool, util, connection from multiprocessing.process import AuthenticationString from multiprocessing.forking import exit, Popen, assert_spawning, ForkingPickler from multiprocessing.util import Finalize, info # # Register some things for pickling # def reduce_array(a): return array.array, (a.typecode, a.tobytes()) ForkingPickler.register(array.array, reduce_array) view_types = [ type(getattr({}, name)()) for name in ('items', 'keys', 'values') ] if view_types[0] is not list: # only needed in Py3.0 def rebuild_as_list(obj): return list, (list(obj), ) for view_type in view_types: ForkingPickler.register(view_type, rebuild_as_list) import copyreg copyreg.pickle(view_type, rebuild_as_list) #
# def reduce_connection(conn): rh = reduce_handle(conn.fileno()) return rebuild_connection, (rh, conn.readable, conn.writable) def rebuild_connection(reduced_handle, readable, writable): handle = rebuild_handle(reduced_handle) return _multiprocessing.Connection(handle, readable=readable, writable=writable) ForkingPickler.register(_multiprocessing.Connection, reduce_connection) # # Register `socket.socket` with `ForkingPickler` # def fromfd(fd, family, type_, proto=0): s = socket.fromfd(fd, family, type_, proto) if s.__class__ is not socket.socket: s = socket.socket(_sock=s) return s def reduce_socket(s): reduced_handle = reduce_handle(s.fileno())