def __init__(self, floating=None, shared_memory=False, numpy_dtype=None):
if numpy_dtype:
if numpy:
log.info('Using numpy')
if numpy_dtype in NUMPY_DEFAULTS:
numpy_dtype = 'float32'
if numpy_dtype not in numpy.sctypeDict:
raise ValueError(BAD_NUMPY_TYPE_ERROR % numpy_dtype)
else:
log.error('The numpy module is not available.')
log.error(importer.MISSING_MESSAGE % ('numpy', 'numpy'))
if shared_memory and numpy_dtype:
log.error('Shared memory for numpy arrays is not yet supported.')
numpy_dtype = None
if floating is None:
floating = not shared_memory
c_type = c_float if floating else c_uint8
if shared_memory:
self.bytes = lambda size: RawArray(c_uint8, size)
self.color_list = lambda size: RawArray(3 * c_type, size)
# Note https://stackoverflow.com/questions/37705974/
elif numpy_dtype:
self.bytes = bytearray
self.color_list = lambda size: numpy.zeros((size, 3), numpy_dtype)
else:
self.bytes = bytearray
self.color_list = lambda size: [(0, 0, 0)] * size
def _allocate_memory(wavelengths, item_count, shared_memory):
# Allocate numpy array to store this SpectrumArray into
if not shared_memory:
# If we're not using shared memory (which the multiprocessing module can share between threads),
# we allocate a simple numpy array
values = np.empty([item_count, len(wavelengths)])
value_errors = np.empty([item_count, len(wavelengths)])
else:
# If we need to shared this array between threads (read only!), then we allocate the memory as a
# multiprocessing RawArray
wavelengths_shared_base = RawArray(c_double, wavelengths.size)
wavelengths_shared = np.frombuffer(wavelengths_shared_base)
wavelengths_shared[:] = wavelengths[:]
wavelengths = wavelengths_shared
values_shared_base = RawArray(c_double, wavelengths.size * item_count)
values = np.frombuffer(values_shared_base)
values = values.reshape([item_count, len(wavelengths)])
value_errors_shared_base = RawArray(c_double, wavelengths.size * item_count)
value_errors = np.frombuffer(value_errors_shared_base)
value_errors = value_errors.reshape([item_count, len(wavelengths)])
return wavelengths, values, value_errors
def __init__(self,
dataname_tuples,
pdgIDs,
nWorkers,
num_loaders,
filters=[]):
self.dataname_tuples = sorted(dataname_tuples)
self.nClasses = len(dataname_tuples[0])
self.total_files = len(dataname_tuples) # per class
self.num_per_file = len(dataname_tuples) * [0]
self.num_loaders = num_loaders
self.lock = RLock()
self.fileInMemory = Value('i', 0, lock=self.lock)
self.fileInMemoryFirstIndex = Value('i', 0, lock=self.lock)
self.fileInMemoryLastIndex = Value('i', -1, lock=self.lock)
self.mem_index = Value('i',
1) # either 0 or 1. used for mem management.
self.loadNext = Event()
self.loadFile = Event()
self.load_barrier = Barrier(self.num_loaders + 1)
self.batch_barrier = Barrier(nWorkers - (self.num_loaders + 1))
self.worker_files = [
RawArray(ctypes.c_char,
len(dataname_tuples[0][0]) + 50)
for _ in range(self.num_loaders)
]
self.data = {}
###########################################
# prepare memory to share with workers #
# take a sample file and get keys and over allocate
# what if we overallocate for both classes?
# we should overallocate for both classes
# if user runs into memory problems, use fewer num_loaders.
with h5py.File(dataname_tuples[0][0]) as sample:
for key in sample.keys():
# print(key)
old_shape = sample[key].shape
size = self.nClasses * self.num_loaders
self.new_shape = list(old_shape)
for dim in old_shape:
size *= dim
self.new_shape[
0] = self.nClasses * self.num_loaders * old_shape[0]
buff = RawArray(ctypes.c_float,
size) # prepare mem for num_loaders
self.data[key] = np.frombuffer(buff, dtype=np.float32).reshape(
self.new_shape) # map numpy array on buffer
classID_buff = RawArray(
ctypes.c_int, (2 * self.nClasses * self.num_loaders * 200))
# print(classID_buff)
self.data['classID'] = np.frombuffer(
classID_buff,
dtype=np.int) #.reshape(self.nClasses*self.num_loaders*200)
# print(self.data['classID'].shape)
###########################################
self.pdgIDs = {}
self.filters = filters
for i, ID in enumerate(pdgIDs):
self.pdgIDs[ID] = i
self.countEvents()
def __init__(self, capacity, data_size, data_tree = None, policy = 'random', passes_before_random = 0.):
if data_tree == None:
self.capacity = capacity # for all priority values
# calculate singular layers where it cannot be fully devided by 2:
width = 1; self.num_of_nodes = 0
while width < capacity:
self.num_of_nodes += width
width *= 2
self.tree_buffer = RawArray('d', self.num_of_nodes + capacity)
self.tree = np.frombuffer(self.tree_buffer,dtype='float64')
# [--------------Parent nodes-------------][-------leaves to recode priority-------]
# size: self.num_of_nodes size: capacity
self.data_buffer = RawArray('f', capacity*data_size)
self.data = np.frombuffer(self.data_buffer,dtype='float32').reshape((capacity,data_size)) # for all transitions
self.data_size = data_size
# [--------------data frame-------------]
# size: capacity
self.len = RawValue('i',0)
self.passes = - passes_before_random
assert self.passes <= 0
#self.childrens = []
if policy == 'sequential': self.sequential = True
elif policy == 'random': self.sequential = False
else:
self.capacity, self.len, self.passes, self.sequential, self.data_size, self.num_of_nodes = capacity, data_tree.len, data_tree.passes, data_tree.sequential, data_tree.data_size, data_tree.num_of_nodes
self.data = np.frombuffer(data_tree.data_buffer,dtype='float32').reshape((self.capacity,self.data_size))
self.tree = np.frombuffer(data_tree.tree_buffer,dtype='float64')
def __init__(self):
params = {
'chords_amp': RawValue(ctypes.c_float),
'chords_chroma': RawArray(ctypes.c_float, 12 * [0.]),
'chords_mfcc': RawArray(ctypes.c_float, 64 * [0.]),
'chords_dissonance': RawValue(ctypes.c_float),
'bass_amp': RawValue(ctypes.c_float),
'bass_pitch': RawValue(ctypes.c_float),
'bass_has_pitch': RawValue(ctypes.c_float),
'drums_amp': RawValue(ctypes.c_float),
'drums_onset': RawValue(ctypes.c_float),
'drums_centroid': RawValue(ctypes.c_float),
}
params['chords_amp'].value = 0.
params['chords_dissonance'].value = 0.
params['bass_amp'].value = 0.
params['bass_pitch'].value = 0.
params['bass_has_pitch'].value = 0.
params['drums_amp'].value = 0.
params['drums_onset'].value = 0.
params['drums_centroid'].value = 0.
self.generator = Generator(params)
self.osc = OSCServer(params)
self.exit = Event()
def populate_expression_queue(self):
for gene_id in self.gene_fname_mapping:
n_trans = self.gene_ntranscripts_mapping[gene_id]
self.mle_estimates[gene_id] = RawArray(
'd', [-1]*(n_trans+1))
self.ubs[gene_id] = RawArray(
'd', [-1]*n_trans)
self.lbs[gene_id] = RawArray(
'd', [-1]*n_trans)
def __init__(self, shape, dtype, parity_obj):
type_id = np_type_id_to_ctypes(dtype)
self.__shared1 = RawArray(type_id, np.product(shape))
self.__np_array1 = np.frombuffer(self.__shared1,
dtype=dtype).reshape(shape)
self.__shared2 = RawArray(type_id, np.product(shape))
self.__np_array2 = np.frombuffer(self.__shared2,
dtype=dtype).reshape(shape)
self.__parity = parity_obj
def __init__(self,
array_len,
array_type,
np_array_type,
buffer_len=DEFAULT_BUFFER_LEN,
array_lock=True):
"""Inits the SharedBuffer object with size and data type.
Args:
buffer_len: An integer size of the buffer
array_len: An integer size of each buffer element (usually numpy array)
array_type: A ctypes data type of buffer elements, e.g. 'd'
np_array_type: A numpy data type of buffer elements, e.g. 'float64'
array_lock: A bool specifying whether the buffer will be used with Lock
"""
self.array_len = array_len
self.np_array_type = np_array_type
self._buffer_len = buffer_len
self._array_type = array_type
# Data is stored in a circular buffer of shared arrays
self._data_buffer = []
if array_lock:
for _ in range(self._buffer_len):
self._data_buffer.append(
np.frombuffer(Array(self._array_type,
self.array_len).get_obj(),
dtype=self.np_array_type))
# We also store time stamps corresponding to each array record
self._timestamp_buffer = Array('d', self._buffer_len)
# We also store the index corresponding to each array record
self._index_buffer = Array('l', self._buffer_len)
else:
# use RawArray without internal lock if needed
for _ in range(self._buffer_len):
self._data_buffer.append(
np.frombuffer(RawArray(self._array_type, self.array_len),
dtype=self.np_array_type))
self._timestamp_buffer = RawArray('d', self._buffer_len)
self._index_buffer = RawArray('l', self._buffer_len)
# Value of `index_buffer` is always set to `self._counter`, which is then increased
self._counter = 0
# buffer_p is a pointer which always points to the next available slot in `data_buffer`
# where the newest data array can be stored
self._buffer_p = Value('i', 0)
# This variable is set to 1 when a new array is stored and
# set to 0 when a new array is read
self._data_updated = Value('i', 0)
# Lock to ensure that changing the `data_updated` as well as the data itself is atomic
self._access_lock = Lock()
def make_pos_uniform(self, dt, p_scale, b_scale, figname, ncore = 0, ndt_decay = 0, roi_ratio = 2.0, k1 = 1.0, k2 = 0.5, chop_ratio = 1, spercent = 0.01, seed = -1, bfile = None, vpfile = None, local_plot = False, check = True):
if not self.adjusted:
self.adjust_pos(bfile = bfile, vpfile = vpfile)
if ncore == 0:
ncore = mp.cpu_count()
print(f'{ncore} cores found')
nx = int(np.ceil(self.nx*chop_ratio))
ny = int(np.ceil(self.ny*chop_ratio))
if nx*ny < ncore:
nx = int(np.ceil(np.sqrt(ncore) * self.nx/self.ny))
ny = int(np.ceil(np.sqrt(ncore) * self.ny/self.nx))
## prepare shared memory
raw_shared_cmem = RawArray(c_int32, np.zeros(ncore, dtype = 'i4'))
shared_cmem = np.frombuffer(raw_shared_cmem, dtype = 'i4')
raw_shared_dmem = RawArray(c_double, np.zeros(self.nLGN*8, dtype = 'f8'))
raw_shared_imem = RawArray(c_int32, np.zeros(self.nLGN, dtype = 'i4'))
shared_dmem = np.frombuffer(raw_shared_dmem, dtype = 'f8')
shared_imem = np.frombuffer(raw_shared_imem, dtype = 'i4')
# populate position in shared array and assign chop references
pos = np.empty(1, dtype = object)
shared_dmem[:2*self.nLGN] = self.pos.flatten()
pos[0] = shared_dmem[:2*self.nLGN].view().reshape((2,self.nLGN))
raw_shared_bmem = RawArray(c_double, np.zeros(nx*ny*6, dtype = 'f8'))
shared_bmem = np.frombuffer(raw_shared_bmem, dtype = 'f8')
raw_shared_nmem = RawArray(c_int32, np.zeros(nx*ny*6, dtype = 'i4')) # for neighbor_list and id
shared_nmem = np.frombuffer(raw_shared_nmem, dtype = 'i4')
subarea = self.subgrid[0] * self.subgrid[1]
area = subarea * np.sum(self.Pi > 0)
print(f'grid area: {area}, used in simulation')
A = self.Pi.copy()
A[self.Pi <= 0] = 0
A[self.Pi > 0] = 1
bound, btype = self.define_bound(A)
self.pos = parallel_repel(area, self.subgrid, pos, shared_dmem, shared_imem, shared_bmem, shared_nmem, shared_cmem, p_scale, bound, btype, b_scale, nx, ny, ncore, dt, spercent, figname, ndt_decay, 1.0, roi_ratio, k1, k2, seed, 1.0, local_plot)
if vpfile is not None:
with open(vpfile+'-us.bin','wb') as f:
np.array([1, self.nLGN]).astype('i4').tofile(f)
self.pos.tofile(f)
if check:
self.check_pos() # check outer boundary only
self.pos_uniform = True
def drawacc(components, osr2mp4, skin, n=0):
from osr2mp4.ImageProcess.Objects.Scores.Accuracy import Accuracy
from osr2mp4.ImageProcess.Objects.Components.TimePie import TimePie
from osr2mp4.ImageProcess.PrepareFrames.Scores.Accuracy import prepare_accuracy
from osr2mp4.ImageProcess.Objects.Scores.ScoreNumbers import ScoreNumbers
from osr2mp4.ImageProcess.Objects.Components.ScorebarBG import ScorebarBG
from osr2mp4.ImageProcess.PrepareFrames.Components.ScorebarBG import prepare_scorebarbg
from PIL import Image
from osr2mp4.VideoProcess.Setup import get_buffer
from multiprocessing import Process, Pipe
from multiprocessing.sharedctypes import RawArray
import ctypes
shared = RawArray(ctypes.c_uint8,
osr2mp4.settings.height * osr2mp4.settings.width * 4)
np_img, background = get_buffer(shared, osr2mp4.settings)
scorenumbers = ScoreNumbers(osr2mp4.settings.scale, osr2mp4.settings)
frames = prepare_accuracy(scorenumbers)
components.accuracy = Accuracy(frames, skin.fonts["ScoreOverlap"],
osr2mp4.settings)
scorebarbg = prepare_scorebarbg(osr2mp4.settings.scale, ["", background],
osr2mp4.settings)
components.timepie = TimePie(components.accuracy, 0, 100, scorebarbg,
osr2mp4.settings)
components.scorebarbg = ScorebarBG(scorebarbg, 100, osr2mp4.settings,
False)
components.scorebarbg.add_to_frame(background, 100, False)
components.timepie.add_to_frame(np_img, background, 30,
components.scorebarbg.h, 100, False)
components.accuracy.add_to_frame(background, False)
background.save(f"test{n}.png")
def test_split_share_data(cooc_rows, cooc_cols, cooc_data):
raw_array_int_10 = RawArray(typecode_or_type='i', size_or_initializer=10)
raw_array_float_10 = RawArray(typecode_or_type='f', size_or_initializer=10)
raws_rows_list, raws_cols_list, raws_coocs_list = split_share_data(
cooc_rows, cooc_cols, cooc_data, 4)
# Check Length
assert len(raws_rows_list) == len(raws_cols_list) == len(raws_coocs_list)
assert len(raws_rows_list[0]) == len(raws_cols_list[0]) == len(
raws_coocs_list[0])
# Check Type
assert type(raws_rows_list[0]) == type(
raws_cols_list[0]) == type(raw_array_int_10)
assert type(raws_coocs_list[0]) == type(raw_array_float_10)
def _create_data(self, shapes, ctype, buffers):
""" Create data """
buffer_size = int(sum(np.prod(x) for x in shapes))
dtype = np.dtype(ctype)
data = tuple(RawArray(ctype, buffer_size) for _ in range(buffers))
np_data = tuple(self._np_from_shared(arr, shapes, dtype) for arr in data)
return data, np_data
def _create_mp_stream(self):
if isinstance(self._stream, CupidMPInputStream):
return self._stream
if self._mp_stream is not None:
return self._mp_stream
from multiprocessing.sharedctypes import RawArray
req_queue = multiprocessing.Queue()
rep_queue = multiprocessing.Queue()
buf = RawArray(ctypes.c_char, options.cupid.mp_buffer_size)
def _mp_thread():
try:
while True:
read_size = req_queue.get()
if read_size < 0:
return
try:
read_size = self._stream.readinto(buf)
except SubprocessStreamEOFError:
rep_queue.put(-1)
return
rep_queue.put(read_size)
finally:
self.close()
stream_thread = threading.Thread(target=_mp_thread)
stream_thread.daemon = True
stream_thread.start()
self._mp_stream = CupidMPInputStream(buf, req_queue, rep_queue)
return self._mp_stream
def __init__(self, struct, size=20):
"""
struct - a ctypes object (value, array or struct)
size - number of slots in the buffer.
If the number of slots exceeds the number of python ints which
will fit in an os.pipe buffer, this will block indefinitely.
I don't see a way round this. On the plus side, at least it fails
early, rather than on a call to the put method.
"""
buf = RawArray(struct, int(size))
self.buffer = buf
#self.buffer = numpy.frombuffer(buf, dtype=numpy.dtype(buf._type_))
stock_out, stock_in = Pipe(duplex=False)
queue_out, queue_in = Pipe(duplex=False)
stock_out_lock = Lock()
stock_in_lock = Lock()
queue_out_lock = Lock()
queue_in_lock = Lock()
self.stock_closed = RawValue('h', 0)
self.queue_closed = RawValue('h', 0)
for i in xrange(size):
stock_in.send(i)
self.map = {}
self._put_obj = (stock_out, stock_out_lock, queue_in, queue_in_lock)
self._ret_obj = (stock_in, stock_in_lock)
self._get_obj = (queue_out, queue_out_lock)
def _build_refill_data(self):
if self._input_stream is None:
return self._refill_data
if self._refill_data is not None:
return self._refill_data
from multiprocessing.sharedctypes import RawArray
req_queue = multiprocessing.Queue()
rep_queue = multiprocessing.Queue()
buf = RawArray(ctypes.c_char, options.cupid.mp_buffer_size)
def _mp_thread():
try:
while True:
req_body = req_queue.get(timeout=60)
if req_body is None:
return
left_size, bound = req_body
try:
buf[:left_size] = buf[bound - left_size:bound]
read_size = self._input_stream.readinto(buf, left_size)
except SubprocessStreamEOFError:
return
rep_queue.put(read_size)
finally:
rep_queue.put(-1)
self.close()
stream_thread = threading.Thread(target=_mp_thread)
stream_thread.daemon = True
stream_thread.start()
self._refill_data = (buf, req_queue, rep_queue)
return self._refill_data
def _create_mp_stream(self):
if isinstance(self._stream, CupidMPInputStream):
return self._mp_stream
if self._mp_stream is not None:
return self._mp_stream
from multiprocessing.sharedctypes import RawArray
req_queue = multiprocessing.Queue()
rep_queue = multiprocessing.Queue()
buf = RawArray(ctypes.c_char, options.cupid.mp_buffer_size)
def _mp_thread():
try:
while True:
size = req_queue.get()
if size < 0:
break
self.write(buf[:size])
rep_queue.put(size)
finally:
self.close()
stream_thread = threading.Thread(target=_mp_thread)
stream_thread.daemon = True
stream_thread.start()
self._mp_stream = CupidMPOutputStream(buf, req_queue, rep_queue)
return self._mp_stream
def __init__(self, params, y=-1, a0=0):
self.beta, self.sigma = params.beta, params.sigma
self.R, self.W, self.y = params.R, params.W, y
# self.mls = mls = (y+1 if (y >= 0) and (y <= W+R-2) else W+R) # mls is maximum life span
self.aN = aN = params.aN
self.aa = aa = params.aa
self.tol, self.neg = params.tol, params.neg
""" SURVIVAL PROB., PRODUCTIVITY TRANSITION PROB. AND ... """
self.sp = sp = params.sp
self.pi = pi = params.pi
# muz[y] : distribution of productivity of y-yrs agents
self.muz = muz = params.muz
self.ef = ef = params.ef
self.zN = zN = params.zN
self.mls = mls = params.mls
""" container for value function and expected value function """
# v[y,j,i] is the value of an y-yrs-old agent with asset i and productity j
self.v = zeros((mls, zN, aN))
# ev[y,j,ni] is the expected value when the agent's next period asset is ni
self.ev = zeros((mls, zN, aN))
""" container for policy functions """
self.a = zeros((mls, zN, aN))
self.c = zeros((mls, zN, aN))
""" distribution of agents w.r.t. age, productivity and asset
for each age, distribution over all productivities and assets add up to 1 """
# self.mu = zeros(mls*zN*aN).reshape(mls,zN,aN)
self.vmu = RawArray(c_double, mls * zN * aN)
def update_serial(self):
self.new_mesh = RawArray(np.ctypeslib.ctypes.c_int8,
self.rows * self.columns) # np.zeros(self.n *self.n, dtype=np.int8)
for i in range(self.rows):
for j in range(self.columns):
self.update_cell(i, j)
self.mesh = self.new_mesh
def create_shared_array(data: np.ndarray, return_shared_data=False):
data = np.asarray(data)
if data.dtype == np.complex:
data_type = "complex"
data_buffer = RawArray("d", int(np.prod(data.shape)) * 2)
else:
data_type = np.ctypeslib.as_ctypes_type(data.dtype)
data_buffer = RawArray(data_type, int(np.prod(data.shape)))
buffer = (data_buffer, data.shape, data_type)
data_shared = array_from_buffer(buffer)
data_shared[:] = data[:]
if return_shared_data:
return buffer, data_shared
else:
return buffer
def testInitLockFalse(self):
buffer = SharedBuffer(array_len=self.array_len,
array_type=self.array_type,
np_array_type=self.np_array_type,
array_lock=False)
# Test array types are correct
self.assertEqual(len(buffer._data_buffer), self.buffer_len)
self.assertIsInstance(buffer._data_buffer[0], np.ndarray)
self.assertIs(buffer._data_buffer[0].dtype,
np.dtype(self.np_array_type))
self.assertIsInstance(
buffer._data_buffer[0].base,
type(Array(self.array_type, self.array_len).get_obj()))
self.assertIsInstance(buffer._timestamp_buffer,
type(RawArray("d", self.buffer_len)))
self.assertIsInstance(buffer._index_buffer,
type(RawArray("l", self.buffer_len)))
def __setstate__(self, state):
"""
Method overloaded for support of pickling.
"""
shape = state['_DoubleBufferedSharedNumpyArray__np_array1'].shape
dtype = state['_DoubleBufferedSharedNumpyArray__np_array1'].dtype
type_id = np_type_id_to_ctypes(dtype)
self.__shared1 = RawArray(type_id, np.product(shape))
self.__np_array1 = np.frombuffer(self.__shared1,
dtype=dtype).reshape(shape)
np.copyto(self.__np_array1,
state['_DoubleBufferedSharedNumpyArray__np_array1'])
self.__shared2 = RawArray(type_id, np.product(shape))
self.__np_array2 = np.frombuffer(self.__shared2,
dtype=dtype).reshape(shape)
np.copyto(self.__np_array2,
state['_DoubleBufferedSharedNumpyArray__np_array2'])
self.__parity = state['_DoubleBufferedSharedNumpyArray__parity']
def __init__(self,
net,
memory,
batch_size,
gamma=0.99,
backup_period=200,
args={}):
torch.backends.cudnn.benchmark = True
# take the input arguments
if net.__class__ == direct_DQN:
self.net = net.__class__(**net.inputs)
self.target_net = net.__class__(**net.inputs)
self.measurement = False
self.input_length = net.inputs['data_length']
else:
self.net = net.__class__(net.inputs)
self.target_net = net.__class__(net.inputs)
self.measurement = True
self.input_length = net.inputs
self.net.load_state_dict(net.state_dict())
self.gamma = gamma
self.memory = memory
self.batch_size = batch_size
self.backup_counter = 0
self.backup_period = backup_period
# set report print()
self.clear_report()
# prepare to train
self.net, self.target_net = self.net.cuda(), self.target_net.cuda()
# the optimizer ***
self.optim = LaProp(self.net.parameters(),
lr=args.lr,
centered=True,
betas=(0.9,
0.9995)) #,amsgrad=True , centered=True
self.net.train()
self.target_net.train()
# prepare the shared memory to do sampling
self.transitions_storage = RawArray(
'f', self.batch_size * self.memory.tree.data_size)
self.transitions = torch.from_numpy(
np.frombuffer(self.transitions_storage, dtype='float32').reshape(
(self.batch_size, self.memory.tree.data_size)))
if self.measurement: # avoid reallocating memory
self.next_states_storage = torch.empty(self.batch_size,
2,
self.input_length,
device='cuda')
self.previous_states_storage = torch.empty(self.batch_size,
2,
self.input_length,
device='cuda')
def load_matrix_from_file(self, fname):
with open(self.fmt_path(fname), 'r') as f:
tmpmatrix = json.load(f)
global g_matrix
g_matrix = RawArray('d', len(tmpmatrix))
for i, z in enumerate(tmpmatrix):
g_matrix[i] = z
self.matrix = g_matrix
del tmpmatrix
def setup_workers(y, x, trf_length, delta, mindelta, nsegs, error):
n_y, n_times = y.shape
n_x, _ = x.shape
y_buffer = RawArray('d', n_y * n_times)
y_buffer[:] = y.ravel()
x_buffer = RawArray('d', n_x * n_times)
x_buffer[:] = x.ravel()
job_queue = Queue(200)
result_queue = Queue(200)
args = (y_buffer, x_buffer, n_y, n_times, n_x, trf_length, delta, mindelta,
nsegs, error, job_queue, result_queue)
for _ in range(N_WORKERS):
Process(target=boosting_worker, args=args).start()
return job_queue, result_queue
def split_share_data(rows, cols, coocs, split_n):
"""
This method takes the rows, cols and cooc(currence) from the glove co-occurrence sparse matrix and splits it
in sub-lists, formatted in RawArray to be accessed by multiple processes in parallel.
This allows keeps the GIL from replicating the memory space when multiprocessing
:param rows: indexes of the non-empty rows of the co-occurrence matrix
:param cols: indexes of the non-empty cols of the co-occurrence matrix
:param coocs: non-empty values of the co-occurrence matrix
:param split_n: number in which split the arrays
:return: 3 lists of RawArrays
"""
total_length = len(rows)
raws_rows_list = list()
raws_cols_list = list()
raws_coocs_list = list()
for ix in range(split_n):
min_ix = ix * total_length // split_n
max_ix = min((ix + 1) * total_length // split_n, total_length - 1)
split_len = max_ix - min_ix
# Create the empty RawArrays
rows_raw = RawArray(typecode_or_type='i', size_or_initializer=split_len)
cols_raw = RawArray(typecode_or_type='i', size_or_initializer=split_len)
coocs_raw = RawArray(typecode_or_type='f', size_or_initializer=split_len)
# Cast the c-types to numpy types, and reshape
rows_np = np.frombuffer(rows_raw, dtype=np.int32).reshape(split_len)
cols_np = np.frombuffer(cols_raw, dtype=np.int32).reshape(split_len)
coocs_np = np.frombuffer(coocs_raw, dtype=np.float32).reshape(split_len)
# Copy data to our shared array
np.copyto(rows_np, rows[min_ix: max_ix])
np.copyto(cols_np, cols[min_ix: max_ix])
np.copyto(coocs_np, coocs[min_ix: max_ix])
# Add data to the lists
raws_rows_list.append(rows_raw)
raws_cols_list.append(cols_raw)
raws_coocs_list.append(coocs_raw)
return raws_rows_list, raws_cols_list, raws_coocs_list
def aucell4r(df_rnk: pd.DataFrame, signatures: Sequence[Type[GeneSignature]],
auc_threshold: float = 0.05, noweights: bool = False, normalize: bool = False,
num_workers: int = cpu_count()) -> pd.DataFrame:
"""
Calculate enrichment of gene signatures for single cells.
:param df_rnk: The rank matrix (n_cells x n_genes).
:param signatures: The gene signatures or regulons.
:param auc_threshold: The fraction of the ranked genome to take into account for the calculation of the
Area Under the recovery Curve.
:param noweights: Should the weights of the genes part of a signature be used in calculation of enrichment?
:param normalize: Normalize the AUC values to a maximum of 1.0 per regulon.
:param num_workers: The number of cores to use.
:return: A dataframe with the AUCs (n_cells x n_modules).
"""
if num_workers == 1:
# Show progress bar ...
aucs = pd.concat([enrichment4cells(df_rnk,
module.noweights() if noweights else module,
auc_threshold=auc_threshold) for module in tqdm(signatures)]).unstack("Regulon")
aucs.columns = aucs.columns.droplevel(0)
else:
# Decompose the rankings dataframe: the index and columns are shared with the child processes via pickling.
genes = df_rnk.columns.values
cells = df_rnk.index.values
# The actual rankings are shared directly. This is possible because during a fork from a parent process the child
# process inherits the memory of the parent process. A RawArray is used instead of a synchronize Array because
# these rankings are read-only.
shared_ro_memory_array = RawArray(DTYPE_C, mul(*df_rnk.shape))
array = np.frombuffer(shared_ro_memory_array, dtype=DTYPE)
# Copy the contents of df_rank into this shared memory block using row-major ordering.
array[:] = df_rnk.values.flatten(order='C')
# The resulting AUCs are returned via a synchronize array.
auc_mtx = Array('d', len(cells) * len(signatures)) # Double precision floats.
# Convert the modules to modules with uniform weights if necessary.
if noweights:
signatures = list(map(lambda m: m.noweights(), signatures))
# Do the analysis in separate child processes.
chunk_size = ceil(float(len(signatures)) / num_workers)
processes = [Process(target=_enrichment, args=(shared_ro_memory_array, chunk,
genes, cells, auc_threshold,
auc_mtx, (chunk_size*len(cells))*idx))
for idx, chunk in enumerate(chunked(signatures, chunk_size))]
for p in processes:
p.start()
for p in processes:
p.join()
# Reconstitute the results array. Using C or row-major ordering.
aucs = pd.DataFrame(data=np.ctypeslib.as_array(auc_mtx.get_obj()).reshape(len(signatures), len(cells)),
columns=pd.Index(data=cells, name='Cell'),
index=pd.Index(data=list(map(attrgetter("name"), signatures)), name='Regulon')).T
return aucs/aucs.max(axis=0) if normalize else aucs
def __get_shared_numpy(
self, numpy_shape
): # The fixed evidence array is shared between processes
shared_array = RawArray(c_uint32, (numpy_shape[0] * numpy_shape[1]))
dt = np.dtype("uint32")
flat_np = np.frombuffer(shared_array, dt)
shared_np = np.reshape(flat_np, numpy_shape)
return shared_np
def create_shared_array(data, return_buffer=False):
data = np.asarray(data)
shared_data = RawArray("d", int(np.prod(data.shape)))
buffered_data = np.frombuffer(shared_data).reshape(data.shape)
buffered_data[:] = data[:]
if return_buffer:
return shared_data, buffered_data
else:
return shared_data
def _get_shared(self, array, dtype=c_float):
"""
Returns a RawArray backed numpy array that can be shared between processes.
:param array: the array to be shared
:param dtype: the RawArray dtype to use
:return: the RawArray backed numpy array
"""
shape = array.shape
shared = RawArray(dtype, array.reshape(-1))
return np.frombuffer(shared, dtype).reshape(shape)
def tran(par, kt, c0, c1, N=5):
T = par.T
vl = par.yN * par.hN * par.zN * par.aN
"""Generate mu of T cohorts who die in t = 0,...,T-1
with initial asset g0.apath[-t-1]"""
VM = [RawArray(c_double, vl) for t in range(T)]
VC = [RawArray(c_double, vl) for t in range(T)]
VRT = [RawArray(c_double, vl) for t in range(T)]
VIN = [RawArray(c_double, vl) for t in range(T)]
for n in range(N):
start_time = datetime.now()
print(str(n + 1) + 'th loop started at {}'.format(start_time))
jobs = []
# for t, vmu in enumerate(VM):
for t in range(T):
p = Process(target=sub1,
args=(t, VM[t], VC[t], VRT[t], VIN[t], kt.ps, c0, c1,
par))
p.start()
jobs.append(p)
# if t % 40 == 0:
# print 'processing another 40 cohorts...'
if len(jobs) % 8 == 0:
for p in jobs:
p.join()
jobs = []
if len(jobs) > 0:
for p in jobs:
p.join()
kt.aggregate(par, VM, VC, VRT, VIN)
for t in linspace(1, T - 1, 20).astype(int):
kt.prices(n=n + 1, t=t)
kt.update(par, n=n + 1)
end_time = datetime.now()
print('this loop finished in {}\n'.format(end_time - start_time))
if kt.converged():
print('Transition Path Converged! in', n + 1, 'iterations.')
break
if n >= N - 1:
print('Transition Path Not Converged! in', n + 1, 'iterations.')
break
return kt, VM, VC, VRT, VIN
