def __init__(self, floating=None, shared_memory=False, numpy_dtype=None): if numpy_dtype: if numpy: log.info('Using numpy') if numpy_dtype in NUMPY_DEFAULTS: numpy_dtype = 'float32' if numpy_dtype not in numpy.sctypeDict: raise ValueError(BAD_NUMPY_TYPE_ERROR % numpy_dtype) else: log.error('The numpy module is not available.') log.error(importer.MISSING_MESSAGE % ('numpy', 'numpy')) if shared_memory and numpy_dtype: log.error('Shared memory for numpy arrays is not yet supported.') numpy_dtype = None if floating is None: floating = not shared_memory c_type = c_float if floating else c_uint8 if shared_memory: self.bytes = lambda size: RawArray(c_uint8, size) self.color_list = lambda size: RawArray(3 * c_type, size) # Note https://stackoverflow.com/questions/37705974/ elif numpy_dtype: self.bytes = bytearray self.color_list = lambda size: numpy.zeros((size, 3), numpy_dtype) else: self.bytes = bytearray self.color_list = lambda size: [(0, 0, 0)] * size
def _allocate_memory(wavelengths, item_count, shared_memory): # Allocate numpy array to store this SpectrumArray into if not shared_memory: # If we're not using shared memory (which the multiprocessing module can share between threads), # we allocate a simple numpy array values = np.empty([item_count, len(wavelengths)]) value_errors = np.empty([item_count, len(wavelengths)]) else: # If we need to shared this array between threads (read only!), then we allocate the memory as a # multiprocessing RawArray wavelengths_shared_base = RawArray(c_double, wavelengths.size) wavelengths_shared = np.frombuffer(wavelengths_shared_base) wavelengths_shared[:] = wavelengths[:] wavelengths = wavelengths_shared values_shared_base = RawArray(c_double, wavelengths.size * item_count) values = np.frombuffer(values_shared_base) values = values.reshape([item_count, len(wavelengths)]) value_errors_shared_base = RawArray(c_double, wavelengths.size * item_count) value_errors = np.frombuffer(value_errors_shared_base) value_errors = value_errors.reshape([item_count, len(wavelengths)]) return wavelengths, values, value_errors
def __init__(self, dataname_tuples, pdgIDs, nWorkers, num_loaders, filters=[]): self.dataname_tuples = sorted(dataname_tuples) self.nClasses = len(dataname_tuples[0]) self.total_files = len(dataname_tuples) # per class self.num_per_file = len(dataname_tuples) * [0] self.num_loaders = num_loaders self.lock = RLock() self.fileInMemory = Value('i', 0, lock=self.lock) self.fileInMemoryFirstIndex = Value('i', 0, lock=self.lock) self.fileInMemoryLastIndex = Value('i', -1, lock=self.lock) self.mem_index = Value('i', 1) # either 0 or 1. used for mem management. self.loadNext = Event() self.loadFile = Event() self.load_barrier = Barrier(self.num_loaders + 1) self.batch_barrier = Barrier(nWorkers - (self.num_loaders + 1)) self.worker_files = [ RawArray(ctypes.c_char, len(dataname_tuples[0][0]) + 50) for _ in range(self.num_loaders) ] self.data = {} ########################################### # prepare memory to share with workers # # take a sample file and get keys and over allocate # what if we overallocate for both classes? # we should overallocate for both classes # if user runs into memory problems, use fewer num_loaders. with h5py.File(dataname_tuples[0][0]) as sample: for key in sample.keys(): # print(key) old_shape = sample[key].shape size = self.nClasses * self.num_loaders self.new_shape = list(old_shape) for dim in old_shape: size *= dim self.new_shape[ 0] = self.nClasses * self.num_loaders * old_shape[0] buff = RawArray(ctypes.c_float, size) # prepare mem for num_loaders self.data[key] = np.frombuffer(buff, dtype=np.float32).reshape( self.new_shape) # map numpy array on buffer classID_buff = RawArray( ctypes.c_int, (2 * self.nClasses * self.num_loaders * 200)) # print(classID_buff) self.data['classID'] = np.frombuffer( classID_buff, dtype=np.int) #.reshape(self.nClasses*self.num_loaders*200) # print(self.data['classID'].shape) ########################################### self.pdgIDs = {} self.filters = filters for i, ID in enumerate(pdgIDs): self.pdgIDs[ID] = i self.countEvents()
def __init__(self, capacity, data_size, data_tree = None, policy = 'random', passes_before_random = 0.): if data_tree == None: self.capacity = capacity # for all priority values # calculate singular layers where it cannot be fully devided by 2: width = 1; self.num_of_nodes = 0 while width < capacity: self.num_of_nodes += width width *= 2 self.tree_buffer = RawArray('d', self.num_of_nodes + capacity) self.tree = np.frombuffer(self.tree_buffer,dtype='float64') # [--------------Parent nodes-------------][-------leaves to recode priority-------] # size: self.num_of_nodes size: capacity self.data_buffer = RawArray('f', capacity*data_size) self.data = np.frombuffer(self.data_buffer,dtype='float32').reshape((capacity,data_size)) # for all transitions self.data_size = data_size # [--------------data frame-------------] # size: capacity self.len = RawValue('i',0) self.passes = - passes_before_random assert self.passes <= 0 #self.childrens = [] if policy == 'sequential': self.sequential = True elif policy == 'random': self.sequential = False else: self.capacity, self.len, self.passes, self.sequential, self.data_size, self.num_of_nodes = capacity, data_tree.len, data_tree.passes, data_tree.sequential, data_tree.data_size, data_tree.num_of_nodes self.data = np.frombuffer(data_tree.data_buffer,dtype='float32').reshape((self.capacity,self.data_size)) self.tree = np.frombuffer(data_tree.tree_buffer,dtype='float64')
def __init__(self): params = { 'chords_amp': RawValue(ctypes.c_float), 'chords_chroma': RawArray(ctypes.c_float, 12 * [0.]), 'chords_mfcc': RawArray(ctypes.c_float, 64 * [0.]), 'chords_dissonance': RawValue(ctypes.c_float), 'bass_amp': RawValue(ctypes.c_float), 'bass_pitch': RawValue(ctypes.c_float), 'bass_has_pitch': RawValue(ctypes.c_float), 'drums_amp': RawValue(ctypes.c_float), 'drums_onset': RawValue(ctypes.c_float), 'drums_centroid': RawValue(ctypes.c_float), } params['chords_amp'].value = 0. params['chords_dissonance'].value = 0. params['bass_amp'].value = 0. params['bass_pitch'].value = 0. params['bass_has_pitch'].value = 0. params['drums_amp'].value = 0. params['drums_onset'].value = 0. params['drums_centroid'].value = 0. self.generator = Generator(params) self.osc = OSCServer(params) self.exit = Event()
def populate_expression_queue(self): for gene_id in self.gene_fname_mapping: n_trans = self.gene_ntranscripts_mapping[gene_id] self.mle_estimates[gene_id] = RawArray( 'd', [-1]*(n_trans+1)) self.ubs[gene_id] = RawArray( 'd', [-1]*n_trans) self.lbs[gene_id] = RawArray( 'd', [-1]*n_trans)
def __init__(self, shape, dtype, parity_obj): type_id = np_type_id_to_ctypes(dtype) self.__shared1 = RawArray(type_id, np.product(shape)) self.__np_array1 = np.frombuffer(self.__shared1, dtype=dtype).reshape(shape) self.__shared2 = RawArray(type_id, np.product(shape)) self.__np_array2 = np.frombuffer(self.__shared2, dtype=dtype).reshape(shape) self.__parity = parity_obj
def __init__(self, array_len, array_type, np_array_type, buffer_len=DEFAULT_BUFFER_LEN, array_lock=True): """Inits the SharedBuffer object with size and data type. Args: buffer_len: An integer size of the buffer array_len: An integer size of each buffer element (usually numpy array) array_type: A ctypes data type of buffer elements, e.g. 'd' np_array_type: A numpy data type of buffer elements, e.g. 'float64' array_lock: A bool specifying whether the buffer will be used with Lock """ self.array_len = array_len self.np_array_type = np_array_type self._buffer_len = buffer_len self._array_type = array_type # Data is stored in a circular buffer of shared arrays self._data_buffer = [] if array_lock: for _ in range(self._buffer_len): self._data_buffer.append( np.frombuffer(Array(self._array_type, self.array_len).get_obj(), dtype=self.np_array_type)) # We also store time stamps corresponding to each array record self._timestamp_buffer = Array('d', self._buffer_len) # We also store the index corresponding to each array record self._index_buffer = Array('l', self._buffer_len) else: # use RawArray without internal lock if needed for _ in range(self._buffer_len): self._data_buffer.append( np.frombuffer(RawArray(self._array_type, self.array_len), dtype=self.np_array_type)) self._timestamp_buffer = RawArray('d', self._buffer_len) self._index_buffer = RawArray('l', self._buffer_len) # Value of `index_buffer` is always set to `self._counter`, which is then increased self._counter = 0 # buffer_p is a pointer which always points to the next available slot in `data_buffer` # where the newest data array can be stored self._buffer_p = Value('i', 0) # This variable is set to 1 when a new array is stored and # set to 0 when a new array is read self._data_updated = Value('i', 0) # Lock to ensure that changing the `data_updated` as well as the data itself is atomic self._access_lock = Lock()
def make_pos_uniform(self, dt, p_scale, b_scale, figname, ncore = 0, ndt_decay = 0, roi_ratio = 2.0, k1 = 1.0, k2 = 0.5, chop_ratio = 1, spercent = 0.01, seed = -1, bfile = None, vpfile = None, local_plot = False, check = True): if not self.adjusted: self.adjust_pos(bfile = bfile, vpfile = vpfile) if ncore == 0: ncore = mp.cpu_count() print(f'{ncore} cores found') nx = int(np.ceil(self.nx*chop_ratio)) ny = int(np.ceil(self.ny*chop_ratio)) if nx*ny < ncore: nx = int(np.ceil(np.sqrt(ncore) * self.nx/self.ny)) ny = int(np.ceil(np.sqrt(ncore) * self.ny/self.nx)) ## prepare shared memory raw_shared_cmem = RawArray(c_int32, np.zeros(ncore, dtype = 'i4')) shared_cmem = np.frombuffer(raw_shared_cmem, dtype = 'i4') raw_shared_dmem = RawArray(c_double, np.zeros(self.nLGN*8, dtype = 'f8')) raw_shared_imem = RawArray(c_int32, np.zeros(self.nLGN, dtype = 'i4')) shared_dmem = np.frombuffer(raw_shared_dmem, dtype = 'f8') shared_imem = np.frombuffer(raw_shared_imem, dtype = 'i4') # populate position in shared array and assign chop references pos = np.empty(1, dtype = object) shared_dmem[:2*self.nLGN] = self.pos.flatten() pos[0] = shared_dmem[:2*self.nLGN].view().reshape((2,self.nLGN)) raw_shared_bmem = RawArray(c_double, np.zeros(nx*ny*6, dtype = 'f8')) shared_bmem = np.frombuffer(raw_shared_bmem, dtype = 'f8') raw_shared_nmem = RawArray(c_int32, np.zeros(nx*ny*6, dtype = 'i4')) # for neighbor_list and id shared_nmem = np.frombuffer(raw_shared_nmem, dtype = 'i4') subarea = self.subgrid[0] * self.subgrid[1] area = subarea * np.sum(self.Pi > 0) print(f'grid area: {area}, used in simulation') A = self.Pi.copy() A[self.Pi <= 0] = 0 A[self.Pi > 0] = 1 bound, btype = self.define_bound(A) self.pos = parallel_repel(area, self.subgrid, pos, shared_dmem, shared_imem, shared_bmem, shared_nmem, shared_cmem, p_scale, bound, btype, b_scale, nx, ny, ncore, dt, spercent, figname, ndt_decay, 1.0, roi_ratio, k1, k2, seed, 1.0, local_plot) if vpfile is not None: with open(vpfile+'-us.bin','wb') as f: np.array([1, self.nLGN]).astype('i4').tofile(f) self.pos.tofile(f) if check: self.check_pos() # check outer boundary only self.pos_uniform = True
def drawacc(components, osr2mp4, skin, n=0): from osr2mp4.ImageProcess.Objects.Scores.Accuracy import Accuracy from osr2mp4.ImageProcess.Objects.Components.TimePie import TimePie from osr2mp4.ImageProcess.PrepareFrames.Scores.Accuracy import prepare_accuracy from osr2mp4.ImageProcess.Objects.Scores.ScoreNumbers import ScoreNumbers from osr2mp4.ImageProcess.Objects.Components.ScorebarBG import ScorebarBG from osr2mp4.ImageProcess.PrepareFrames.Components.ScorebarBG import prepare_scorebarbg from PIL import Image from osr2mp4.VideoProcess.Setup import get_buffer from multiprocessing import Process, Pipe from multiprocessing.sharedctypes import RawArray import ctypes shared = RawArray(ctypes.c_uint8, osr2mp4.settings.height * osr2mp4.settings.width * 4) np_img, background = get_buffer(shared, osr2mp4.settings) scorenumbers = ScoreNumbers(osr2mp4.settings.scale, osr2mp4.settings) frames = prepare_accuracy(scorenumbers) components.accuracy = Accuracy(frames, skin.fonts["ScoreOverlap"], osr2mp4.settings) scorebarbg = prepare_scorebarbg(osr2mp4.settings.scale, ["", background], osr2mp4.settings) components.timepie = TimePie(components.accuracy, 0, 100, scorebarbg, osr2mp4.settings) components.scorebarbg = ScorebarBG(scorebarbg, 100, osr2mp4.settings, False) components.scorebarbg.add_to_frame(background, 100, False) components.timepie.add_to_frame(np_img, background, 30, components.scorebarbg.h, 100, False) components.accuracy.add_to_frame(background, False) background.save(f"test{n}.png")
def test_split_share_data(cooc_rows, cooc_cols, cooc_data): raw_array_int_10 = RawArray(typecode_or_type='i', size_or_initializer=10) raw_array_float_10 = RawArray(typecode_or_type='f', size_or_initializer=10) raws_rows_list, raws_cols_list, raws_coocs_list = split_share_data( cooc_rows, cooc_cols, cooc_data, 4) # Check Length assert len(raws_rows_list) == len(raws_cols_list) == len(raws_coocs_list) assert len(raws_rows_list[0]) == len(raws_cols_list[0]) == len( raws_coocs_list[0]) # Check Type assert type(raws_rows_list[0]) == type( raws_cols_list[0]) == type(raw_array_int_10) assert type(raws_coocs_list[0]) == type(raw_array_float_10)
def _create_data(self, shapes, ctype, buffers): """ Create data """ buffer_size = int(sum(np.prod(x) for x in shapes)) dtype = np.dtype(ctype) data = tuple(RawArray(ctype, buffer_size) for _ in range(buffers)) np_data = tuple(self._np_from_shared(arr, shapes, dtype) for arr in data) return data, np_data
def _create_mp_stream(self): if isinstance(self._stream, CupidMPInputStream): return self._stream if self._mp_stream is not None: return self._mp_stream from multiprocessing.sharedctypes import RawArray req_queue = multiprocessing.Queue() rep_queue = multiprocessing.Queue() buf = RawArray(ctypes.c_char, options.cupid.mp_buffer_size) def _mp_thread(): try: while True: read_size = req_queue.get() if read_size < 0: return try: read_size = self._stream.readinto(buf) except SubprocessStreamEOFError: rep_queue.put(-1) return rep_queue.put(read_size) finally: self.close() stream_thread = threading.Thread(target=_mp_thread) stream_thread.daemon = True stream_thread.start() self._mp_stream = CupidMPInputStream(buf, req_queue, rep_queue) return self._mp_stream
def __init__(self, struct, size=20): """ struct - a ctypes object (value, array or struct) size - number of slots in the buffer. If the number of slots exceeds the number of python ints which will fit in an os.pipe buffer, this will block indefinitely. I don't see a way round this. On the plus side, at least it fails early, rather than on a call to the put method. """ buf = RawArray(struct, int(size)) self.buffer = buf #self.buffer = numpy.frombuffer(buf, dtype=numpy.dtype(buf._type_)) stock_out, stock_in = Pipe(duplex=False) queue_out, queue_in = Pipe(duplex=False) stock_out_lock = Lock() stock_in_lock = Lock() queue_out_lock = Lock() queue_in_lock = Lock() self.stock_closed = RawValue('h', 0) self.queue_closed = RawValue('h', 0) for i in xrange(size): stock_in.send(i) self.map = {} self._put_obj = (stock_out, stock_out_lock, queue_in, queue_in_lock) self._ret_obj = (stock_in, stock_in_lock) self._get_obj = (queue_out, queue_out_lock)
def _build_refill_data(self): if self._input_stream is None: return self._refill_data if self._refill_data is not None: return self._refill_data from multiprocessing.sharedctypes import RawArray req_queue = multiprocessing.Queue() rep_queue = multiprocessing.Queue() buf = RawArray(ctypes.c_char, options.cupid.mp_buffer_size) def _mp_thread(): try: while True: req_body = req_queue.get(timeout=60) if req_body is None: return left_size, bound = req_body try: buf[:left_size] = buf[bound - left_size:bound] read_size = self._input_stream.readinto(buf, left_size) except SubprocessStreamEOFError: return rep_queue.put(read_size) finally: rep_queue.put(-1) self.close() stream_thread = threading.Thread(target=_mp_thread) stream_thread.daemon = True stream_thread.start() self._refill_data = (buf, req_queue, rep_queue) return self._refill_data
def _create_mp_stream(self): if isinstance(self._stream, CupidMPInputStream): return self._mp_stream if self._mp_stream is not None: return self._mp_stream from multiprocessing.sharedctypes import RawArray req_queue = multiprocessing.Queue() rep_queue = multiprocessing.Queue() buf = RawArray(ctypes.c_char, options.cupid.mp_buffer_size) def _mp_thread(): try: while True: size = req_queue.get() if size < 0: break self.write(buf[:size]) rep_queue.put(size) finally: self.close() stream_thread = threading.Thread(target=_mp_thread) stream_thread.daemon = True stream_thread.start() self._mp_stream = CupidMPOutputStream(buf, req_queue, rep_queue) return self._mp_stream
def __init__(self, params, y=-1, a0=0): self.beta, self.sigma = params.beta, params.sigma self.R, self.W, self.y = params.R, params.W, y # self.mls = mls = (y+1 if (y >= 0) and (y <= W+R-2) else W+R) # mls is maximum life span self.aN = aN = params.aN self.aa = aa = params.aa self.tol, self.neg = params.tol, params.neg """ SURVIVAL PROB., PRODUCTIVITY TRANSITION PROB. AND ... """ self.sp = sp = params.sp self.pi = pi = params.pi # muz[y] : distribution of productivity of y-yrs agents self.muz = muz = params.muz self.ef = ef = params.ef self.zN = zN = params.zN self.mls = mls = params.mls """ container for value function and expected value function """ # v[y,j,i] is the value of an y-yrs-old agent with asset i and productity j self.v = zeros((mls, zN, aN)) # ev[y,j,ni] is the expected value when the agent's next period asset is ni self.ev = zeros((mls, zN, aN)) """ container for policy functions """ self.a = zeros((mls, zN, aN)) self.c = zeros((mls, zN, aN)) """ distribution of agents w.r.t. age, productivity and asset for each age, distribution over all productivities and assets add up to 1 """ # self.mu = zeros(mls*zN*aN).reshape(mls,zN,aN) self.vmu = RawArray(c_double, mls * zN * aN)
def update_serial(self): self.new_mesh = RawArray(np.ctypeslib.ctypes.c_int8, self.rows * self.columns) # np.zeros(self.n *self.n, dtype=np.int8) for i in range(self.rows): for j in range(self.columns): self.update_cell(i, j) self.mesh = self.new_mesh
def create_shared_array(data: np.ndarray, return_shared_data=False): data = np.asarray(data) if data.dtype == np.complex: data_type = "complex" data_buffer = RawArray("d", int(np.prod(data.shape)) * 2) else: data_type = np.ctypeslib.as_ctypes_type(data.dtype) data_buffer = RawArray(data_type, int(np.prod(data.shape))) buffer = (data_buffer, data.shape, data_type) data_shared = array_from_buffer(buffer) data_shared[:] = data[:] if return_shared_data: return buffer, data_shared else: return buffer
def testInitLockFalse(self): buffer = SharedBuffer(array_len=self.array_len, array_type=self.array_type, np_array_type=self.np_array_type, array_lock=False) # Test array types are correct self.assertEqual(len(buffer._data_buffer), self.buffer_len) self.assertIsInstance(buffer._data_buffer[0], np.ndarray) self.assertIs(buffer._data_buffer[0].dtype, np.dtype(self.np_array_type)) self.assertIsInstance( buffer._data_buffer[0].base, type(Array(self.array_type, self.array_len).get_obj())) self.assertIsInstance(buffer._timestamp_buffer, type(RawArray("d", self.buffer_len))) self.assertIsInstance(buffer._index_buffer, type(RawArray("l", self.buffer_len)))
def __setstate__(self, state): """ Method overloaded for support of pickling. """ shape = state['_DoubleBufferedSharedNumpyArray__np_array1'].shape dtype = state['_DoubleBufferedSharedNumpyArray__np_array1'].dtype type_id = np_type_id_to_ctypes(dtype) self.__shared1 = RawArray(type_id, np.product(shape)) self.__np_array1 = np.frombuffer(self.__shared1, dtype=dtype).reshape(shape) np.copyto(self.__np_array1, state['_DoubleBufferedSharedNumpyArray__np_array1']) self.__shared2 = RawArray(type_id, np.product(shape)) self.__np_array2 = np.frombuffer(self.__shared2, dtype=dtype).reshape(shape) np.copyto(self.__np_array2, state['_DoubleBufferedSharedNumpyArray__np_array2']) self.__parity = state['_DoubleBufferedSharedNumpyArray__parity']
def __init__(self, net, memory, batch_size, gamma=0.99, backup_period=200, args={}): torch.backends.cudnn.benchmark = True # take the input arguments if net.__class__ == direct_DQN: self.net = net.__class__(**net.inputs) self.target_net = net.__class__(**net.inputs) self.measurement = False self.input_length = net.inputs['data_length'] else: self.net = net.__class__(net.inputs) self.target_net = net.__class__(net.inputs) self.measurement = True self.input_length = net.inputs self.net.load_state_dict(net.state_dict()) self.gamma = gamma self.memory = memory self.batch_size = batch_size self.backup_counter = 0 self.backup_period = backup_period # set report print() self.clear_report() # prepare to train self.net, self.target_net = self.net.cuda(), self.target_net.cuda() # the optimizer *** self.optim = LaProp(self.net.parameters(), lr=args.lr, centered=True, betas=(0.9, 0.9995)) #,amsgrad=True , centered=True self.net.train() self.target_net.train() # prepare the shared memory to do sampling self.transitions_storage = RawArray( 'f', self.batch_size * self.memory.tree.data_size) self.transitions = torch.from_numpy( np.frombuffer(self.transitions_storage, dtype='float32').reshape( (self.batch_size, self.memory.tree.data_size))) if self.measurement: # avoid reallocating memory self.next_states_storage = torch.empty(self.batch_size, 2, self.input_length, device='cuda') self.previous_states_storage = torch.empty(self.batch_size, 2, self.input_length, device='cuda')
def load_matrix_from_file(self, fname): with open(self.fmt_path(fname), 'r') as f: tmpmatrix = json.load(f) global g_matrix g_matrix = RawArray('d', len(tmpmatrix)) for i, z in enumerate(tmpmatrix): g_matrix[i] = z self.matrix = g_matrix del tmpmatrix
def setup_workers(y, x, trf_length, delta, mindelta, nsegs, error): n_y, n_times = y.shape n_x, _ = x.shape y_buffer = RawArray('d', n_y * n_times) y_buffer[:] = y.ravel() x_buffer = RawArray('d', n_x * n_times) x_buffer[:] = x.ravel() job_queue = Queue(200) result_queue = Queue(200) args = (y_buffer, x_buffer, n_y, n_times, n_x, trf_length, delta, mindelta, nsegs, error, job_queue, result_queue) for _ in range(N_WORKERS): Process(target=boosting_worker, args=args).start() return job_queue, result_queue
def split_share_data(rows, cols, coocs, split_n): """ This method takes the rows, cols and cooc(currence) from the glove co-occurrence sparse matrix and splits it in sub-lists, formatted in RawArray to be accessed by multiple processes in parallel. This allows keeps the GIL from replicating the memory space when multiprocessing :param rows: indexes of the non-empty rows of the co-occurrence matrix :param cols: indexes of the non-empty cols of the co-occurrence matrix :param coocs: non-empty values of the co-occurrence matrix :param split_n: number in which split the arrays :return: 3 lists of RawArrays """ total_length = len(rows) raws_rows_list = list() raws_cols_list = list() raws_coocs_list = list() for ix in range(split_n): min_ix = ix * total_length // split_n max_ix = min((ix + 1) * total_length // split_n, total_length - 1) split_len = max_ix - min_ix # Create the empty RawArrays rows_raw = RawArray(typecode_or_type='i', size_or_initializer=split_len) cols_raw = RawArray(typecode_or_type='i', size_or_initializer=split_len) coocs_raw = RawArray(typecode_or_type='f', size_or_initializer=split_len) # Cast the c-types to numpy types, and reshape rows_np = np.frombuffer(rows_raw, dtype=np.int32).reshape(split_len) cols_np = np.frombuffer(cols_raw, dtype=np.int32).reshape(split_len) coocs_np = np.frombuffer(coocs_raw, dtype=np.float32).reshape(split_len) # Copy data to our shared array np.copyto(rows_np, rows[min_ix: max_ix]) np.copyto(cols_np, cols[min_ix: max_ix]) np.copyto(coocs_np, coocs[min_ix: max_ix]) # Add data to the lists raws_rows_list.append(rows_raw) raws_cols_list.append(cols_raw) raws_coocs_list.append(coocs_raw) return raws_rows_list, raws_cols_list, raws_coocs_list
def aucell4r(df_rnk: pd.DataFrame, signatures: Sequence[Type[GeneSignature]], auc_threshold: float = 0.05, noweights: bool = False, normalize: bool = False, num_workers: int = cpu_count()) -> pd.DataFrame: """ Calculate enrichment of gene signatures for single cells. :param df_rnk: The rank matrix (n_cells x n_genes). :param signatures: The gene signatures or regulons. :param auc_threshold: The fraction of the ranked genome to take into account for the calculation of the Area Under the recovery Curve. :param noweights: Should the weights of the genes part of a signature be used in calculation of enrichment? :param normalize: Normalize the AUC values to a maximum of 1.0 per regulon. :param num_workers: The number of cores to use. :return: A dataframe with the AUCs (n_cells x n_modules). """ if num_workers == 1: # Show progress bar ... aucs = pd.concat([enrichment4cells(df_rnk, module.noweights() if noweights else module, auc_threshold=auc_threshold) for module in tqdm(signatures)]).unstack("Regulon") aucs.columns = aucs.columns.droplevel(0) else: # Decompose the rankings dataframe: the index and columns are shared with the child processes via pickling. genes = df_rnk.columns.values cells = df_rnk.index.values # The actual rankings are shared directly. This is possible because during a fork from a parent process the child # process inherits the memory of the parent process. A RawArray is used instead of a synchronize Array because # these rankings are read-only. shared_ro_memory_array = RawArray(DTYPE_C, mul(*df_rnk.shape)) array = np.frombuffer(shared_ro_memory_array, dtype=DTYPE) # Copy the contents of df_rank into this shared memory block using row-major ordering. array[:] = df_rnk.values.flatten(order='C') # The resulting AUCs are returned via a synchronize array. auc_mtx = Array('d', len(cells) * len(signatures)) # Double precision floats. # Convert the modules to modules with uniform weights if necessary. if noweights: signatures = list(map(lambda m: m.noweights(), signatures)) # Do the analysis in separate child processes. chunk_size = ceil(float(len(signatures)) / num_workers) processes = [Process(target=_enrichment, args=(shared_ro_memory_array, chunk, genes, cells, auc_threshold, auc_mtx, (chunk_size*len(cells))*idx)) for idx, chunk in enumerate(chunked(signatures, chunk_size))] for p in processes: p.start() for p in processes: p.join() # Reconstitute the results array. Using C or row-major ordering. aucs = pd.DataFrame(data=np.ctypeslib.as_array(auc_mtx.get_obj()).reshape(len(signatures), len(cells)), columns=pd.Index(data=cells, name='Cell'), index=pd.Index(data=list(map(attrgetter("name"), signatures)), name='Regulon')).T return aucs/aucs.max(axis=0) if normalize else aucs
def __get_shared_numpy( self, numpy_shape ): # The fixed evidence array is shared between processes shared_array = RawArray(c_uint32, (numpy_shape[0] * numpy_shape[1])) dt = np.dtype("uint32") flat_np = np.frombuffer(shared_array, dt) shared_np = np.reshape(flat_np, numpy_shape) return shared_np
def create_shared_array(data, return_buffer=False): data = np.asarray(data) shared_data = RawArray("d", int(np.prod(data.shape))) buffered_data = np.frombuffer(shared_data).reshape(data.shape) buffered_data[:] = data[:] if return_buffer: return shared_data, buffered_data else: return shared_data
def _get_shared(self, array, dtype=c_float): """ Returns a RawArray backed numpy array that can be shared between processes. :param array: the array to be shared :param dtype: the RawArray dtype to use :return: the RawArray backed numpy array """ shape = array.shape shared = RawArray(dtype, array.reshape(-1)) return np.frombuffer(shared, dtype).reshape(shape)
def tran(par, kt, c0, c1, N=5): T = par.T vl = par.yN * par.hN * par.zN * par.aN """Generate mu of T cohorts who die in t = 0,...,T-1 with initial asset g0.apath[-t-1]""" VM = [RawArray(c_double, vl) for t in range(T)] VC = [RawArray(c_double, vl) for t in range(T)] VRT = [RawArray(c_double, vl) for t in range(T)] VIN = [RawArray(c_double, vl) for t in range(T)] for n in range(N): start_time = datetime.now() print(str(n + 1) + 'th loop started at {}'.format(start_time)) jobs = [] # for t, vmu in enumerate(VM): for t in range(T): p = Process(target=sub1, args=(t, VM[t], VC[t], VRT[t], VIN[t], kt.ps, c0, c1, par)) p.start() jobs.append(p) # if t % 40 == 0: # print 'processing another 40 cohorts...' if len(jobs) % 8 == 0: for p in jobs: p.join() jobs = [] if len(jobs) > 0: for p in jobs: p.join() kt.aggregate(par, VM, VC, VRT, VIN) for t in linspace(1, T - 1, 20).astype(int): kt.prices(n=n + 1, t=t) kt.update(par, n=n + 1) end_time = datetime.now() print('this loop finished in {}\n'.format(end_time - start_time)) if kt.converged(): print('Transition Path Converged! in', n + 1, 'iterations.') break if n >= N - 1: print('Transition Path Not Converged! in', n + 1, 'iterations.') break return kt, VM, VC, VRT, VIN