def _assemble_kernel_mat(
self,
R_desc,
R_d_desc,
tril_perms_lin,
sig,
desc, # TODO: document me
use_E_cstr=False,
col_idxs=np.s_[:], # TODO: document me
callback=None,
):
r"""
Compute force field kernel matrix.
The Hessian of the Matern kernel is used with n = 2 (twice
differentiable). Each row and column consists of matrix-valued blocks,
which encode the interaction of one training point with all others. The
result is stored in shared memory (a global variable).
Parameters
----------
R_desc : :obj:`numpy.ndarray`
Array containing the descriptor for each training point.
R_d_desc : :obj:`numpy.ndarray`
Array containing the gradient of the descriptor for
each training point.
tril_perms_lin : :obj:`numpy.ndarray`
1D array containing all recovered permutations
expanded as one large permutation to be applied to a
tiled copy of the object to be permuted.
sig : int
Hyper-parameter :math:`\sigma`(kernel length scale).
use_E_cstr : bool, optional
True: include energy constraints in the kernel,
False: default (s)GDML kernel.
callback : callable, optional
Kernel assembly progress function that takes three
arguments:
current : int
Current progress (number of completed entries).
total : int
Task size (total number of entries to create).
done_str : :obj:`str`, optional
Once complete, this string contains the
time it took to assemble the kernel (seconds).
cols_m_limit : int, optional
Only generate the columns up to index 'cols_m_limit'. This creates
a M*3N x cols_m_limit*3N kernel matrix, instead of M*3N x M*3N.
cols_3n_keep_idxs : :obj:`numpy.ndarray`, optional
Only generate columns with the given indices in the 3N x 3N
kernel function. The resulting kernel matrix will have dimension
M*3N x M*len(cols_3n_keep_idxs).
Returns
-------
:obj:`numpy.ndarray`
Force field kernel matrix.
"""
global glob
# Note: This function does not support unsorted (ascending) index arrays.
# if not isinstance(col_idxs, slice):
# assert np.array_equal(col_idxs, np.sort(col_idxs))
n_train, dim_d = R_d_desc.shape[:2]
dim_i = 3 * int((1 + np.sqrt(8 * dim_d + 1)) / 2)
# Determine size of kernel matrix.
K_n_rows = n_train * dim_i
if isinstance(col_idxs, slice): # indexed by slice
K_n_cols = len(range(*col_idxs.indices(K_n_rows)))
else: # indexed by list
# TODO: throw exeption with description
assert len(col_idxs) == len(
set(col_idxs)) # assume no dublicate indices
# TODO: throw exeption with description
# Note: This function does not support unsorted (ascending) index arrays.
assert np.array_equal(col_idxs, np.sort(col_idxs))
K_n_cols = len(col_idxs)
# Account for additional rows and columns due to energy constraints in the kernel matrix.
if use_E_cstr:
K_n_rows += n_train
K_n_cols += n_train
# Make sure no indices are outside of the valid range.
if K_n_cols > K_n_rows:
raise ValueError('Columns indexed beyond range.')
exploit_sym = False
cols_m_limit = None
# Check if range is a subset of training points (as opposed to a subset of partials of multiple points).
is_M_subset = (isinstance(col_idxs, slice) and
(col_idxs.start is None or col_idxs.start % dim_i == 0)
and
(col_idxs.stop is None or col_idxs.stop % dim_i == 0)
and col_idxs.step is None)
if is_M_subset:
M_slice_start = (None if col_idxs.start is None else int(
col_idxs.start / dim_i))
M_slice_stop = None if col_idxs.stop is None else int(
col_idxs.stop / dim_i)
M_slice = slice(M_slice_start, M_slice_stop)
J = range(*M_slice.indices(n_train))
if M_slice_start is None:
exploit_sym = True
cols_m_limit = M_slice_stop
else:
if isinstance(col_idxs, slice):
random = list(range(*col_idxs.indices(n_train * dim_i)))
else:
random = col_idxs
# M - number training
# N - number atoms
n_idxs = np.mod(random, dim_i)
m_idxs = (np.array(random) / dim_i).astype(int)
m_idxs_uniq = np.unique(m_idxs) # which points to include?
m_n_idxs = [
list(n_idxs[np.where(m_idxs == m_idx)])
for m_idx in m_idxs_uniq
]
m_n_idxs_lens = [len(m_n_idx) for m_n_idx in m_n_idxs]
m_n_idxs_lens.insert(0, 0)
blk_start_idxs = list(
np.cumsum(m_n_idxs_lens[:-1]
)) # index within K at which each block starts
# tupels: (block index in final K, block index global, indices of partials within block)
J = list(zip(blk_start_idxs, m_idxs_uniq, m_n_idxs))
callback(0, 100) # 0%
K = mp.RawArray('d', K_n_rows * K_n_cols)
glob['K'], glob['K_shape'] = K, (K_n_rows, K_n_cols)
glob['R_desc'], glob['R_desc_shape'] = _share_array(R_desc, 'd')
glob['R_d_desc'], glob['R_d_desc_shape'] = _share_array(R_d_desc, 'd')
glob['desc_func'] = desc
start = timeit.default_timer()
pool = Pool(self._max_processes)
todo, done = K_n_cols, 0
for done_wkr in pool.imap_unordered(
partial(
_assemble_kernel_mat_wkr,
tril_perms_lin=tril_perms_lin,
sig=sig,
use_E_cstr=use_E_cstr,
exploit_sym=exploit_sym,
cols_m_limit=cols_m_limit,
),
J,
):
done += done_wkr
if callback is not None:
callback(done, todo, newline_when_done=False)
pool.close()
pool.join(
) # Wait for the worker processes to terminate (to measure total runtime correctly).
stop = timeit.default_timer()
if callback is not None:
dur_s = (stop - start) / 2
sec_disp_str = 'took {:.1f} s'.format(
dur_s) if dur_s >= 0.1 else ''
callback(DONE, sec_disp_str=sec_disp_str)
# Release some memory.
glob.pop('K', None)
glob.pop('R_desc', None)
glob.pop('R_d_desc', None)
return np.frombuffer(K).reshape(glob['K_shape'])
bbi = shp.bounding_box
left = np.nonzero((bounds > bbi.left))[0][0]
right = np.nonzero((bounds > bbi.right))[0][0]
bids = range(left, right + 1)
for bid in bids:
bins[bid].append(shp)
ids[bid].append(i)
sf.close()
# vectorized parallel
GRID_DIM = 80
t1 = time.time()
dims = grid(bb, GRID_DIM)
#bboxes is an n x 4 array of type double
c_bboxes = mp.RawArray(ctypes.c_double, len(shps) * 4)
bboxes = np.frombuffer(c_bboxes)
bboxes.shape = (len(shps), 4)
for i, shp in enumerate(shps):
bboxes[i][:] = shp.bounding_box[:]
origin = np.array([bb[0], bb[1], bb[0], bb[1]])
grid_cells = ((bboxes - origin) / dims).astype(int) # [r0,c0, r1,c1]
grid_cells[:, [2, 3]] += 1
max_g = grid_cells.max(axis=1)
n = len(shps)
#global rows_2_polys
#global cols_2_polys
#allocate ctype arrays that we can manually pass memory pointers
#this should be what ipyhton does natiely with view['variable'] = variable
#1. create the memory space, 2. get a numpy view, 3. populate, 4. set the shape
def allocate_data_buffer(self, shared=False):
"""
allocate fully_populated_shape data buffer for cached data
if shared, return a multiprocessing.Array that can be shared across subprocesses
if not shared, return a numpy ndarrray
Parameters
----------
shared: boolean
Returns
-------
multiprocessing.Array or numpy ndarray sized to hole fully_populated utility array
"""
assert not self.is_open
assert shared == self.network_los.multiprocess()
dtype_name = DTYPE_NAME
dtype = np.dtype(DTYPE_NAME)
# multiprocessing.Array argument buffer_size must be int, not np.int64
shape = self.uid_calculator.fully_populated_shape
buffer_size = util.iprod(self.uid_calculator.fully_populated_shape)
csz = buffer_size * dtype.itemsize
logger.info(
f"TVPBCache.allocate_data_buffer allocating data buffer "
f"shape {shape} buffer_size {buffer_size} total size: {csz} ({tracing.si_units(csz)})"
)
if shared:
if dtype_name == 'float64':
typecode = 'd'
elif dtype_name == 'float32':
typecode = 'f'
else:
raise RuntimeError(
"allocate_data_buffer unrecognized dtype %s" % dtype_name)
if RAWARRAY:
with memo("TVPBCache.allocate_data_buffer allocate RawArray"):
buffer = multiprocessing.RawArray(typecode, buffer_size)
logger.info(
f"TVPBCache.allocate_data_buffer allocated shared multiprocessing.RawArray as buffer"
)
else:
with memo("TVPBCache.allocate_data_buffer allocate Array"):
buffer = multiprocessing.Array(typecode, buffer_size)
logger.info(
f"TVPBCache.allocate_data_buffer allocated shared multiprocessing.Array as buffer"
)
else:
buffer = np.empty(buffer_size, dtype=dtype)
np.copyto(buffer, np.nan) # fill with np.nan
logger.info(
f"TVPBCache.allocate_data_buffer allocating non-shared numpy array as buffer"
)
return buffer
def create_core_catalog_mevolved(virialFlag,
writeOutputFlag=True,
useLocalHost=False):
"""
Appends mevolved to core catalog and saves output in HDF5.
Works by computing mevolved for step+1 at each step and saving that in memory.
"""
if writeOutputFlag:
print 'Reading data from {} and writing output to {}'.format(
cc_data_dir, cc_output_dir)
global cc, coresMaskedArray, distance_upper_bound, pool_mergedCoreTag, coresKDTree, KDTree_idx
cc = {}
cc_prev = {}
for step in tqdm(steps):
# Read in cc for step
cc = {v: gio.gio_read(fname_cc(step, 'input'), v)[0] for v in vars_cc}
if virialFlag:
# Convert all mass to virial
cc['infall_mass'] = SHMLM.m_vir(cc['infall_mass'], step)
satellites_mask = cc['central'] == 0
centrals_mask = cc['central'] == 1
# assert np.sum(satellites_mask) + np.sum(centrals_mask) == len(cc['infall_mass']), 'central flag not 0 or 1.'
numSatellites = np.sum(satellites_mask)
# Verify there are no satellites at first step
if step == steps[0]:
assert numSatellites == 0, 'Satellites found at first step.'
# Add column for m_evolved and initialize to 0
for A in A_arr:
for zeta in zeta_arr:
cc[m_evolved_col(A, zeta)] = np.zeros_like(cc['infall_mass'])
cc['mergedCoreTag'] = np.zeros_like(cc['core_tag'])
cc['mergedCoreStep'] = np.zeros_like(cc['infall_step'])
# wasInFragment: persistent flag that is False by default and becomes True when core is inside a fragment halo
cc['wasInFragment'] = cc['fof_halo_tag'] < 0
if step != steps[0]:
_, i1, i2 = np.intersect1d(cc['core_tag'],
cc_prev['core_tag'],
return_indices=True)
cc['wasInFragment'][i1] = np.logical_or(
cc['wasInFragment'][i1], cc_prev['wasInFragment'][i2])
cc['mergedCoreTag'][i1] = cc_prev['mergedCoreTag'][i2]
cc['mergedCoreStep'][i1] = cc_prev['mergedCoreStep'][i2]
# If there are satellites (not applicable for first step)
if numSatellites != 0:
# Match satellites to prev step cores using core tag.
vals, idx1, idx2 = np.intersect1d(cc['core_tag'][satellites_mask],
cc_prev['core_tag'],
return_indices=True)
# assert len(vals) == len(cc['core_tag'][satellites_mask]), 'All cores from prev step did not carry over.'
# assert len(cc['core_tag'][satellites_mask]) == len(np.unique(cc['core_tag'][satellites_mask])), 'Satellite core tags not unique for this time step.'
for A in A_arr:
for zeta in zeta_arr:
# Set m_evolved of all satellites that have core tag match on prev step to next_m_evolved of prev step.
# DOUBLE MASK ASSIGNMENT: cc['m_evolved'][satellites_mask][idx1] = cc_prev['next_m_evolved'][idx2]
cc[m_evolved_col(A, zeta)][np.flatnonzero(satellites_mask)
[idx1]] = cc_prev[m_evolved_col(
A, zeta, next=True)][idx2]
# Initialize m array (corresponds with cc[satellites_mask]) to be either infall_mass (step after infall) or m_evolved (subsequent steps).
m = (cc[m_evolved_col(A, zeta)][satellites_mask]
== 0) * cc['infall_mass'][satellites_mask] + (
cc[m_evolved_col(A, zeta)][satellites_mask] !=
0) * cc[m_evolved_col(A, zeta)][satellites_mask]
# Set m_evolved of satellites with m_evolved=0 to infall mass.
cc[m_evolved_col(A, zeta)][satellites_mask] = m
# Initialize M array (corresponds with cc[satellites_mask]) to be host tree node mass of each satellite
idx_m21 = many_to_one(cc['tree_node_index'][satellites_mask],
cc['tree_node_index'][centrals_mask])
M, X, Y, Z = (cc[k][centrals_mask][idx_m21]
for k in ['infall_mass', 'x', 'y', 'z'])
KDTreemask = cc['mergedCoreTag'][satellites_mask] == 0
x = SHMLM.periodic_bcs(cc['x'][satellites_mask][KDTreemask],
X[KDTreemask], SHMLM.BOXSIZE)
y = SHMLM.periodic_bcs(cc['y'][satellites_mask][KDTreemask],
Y[KDTreemask], SHMLM.BOXSIZE)
z = SHMLM.periodic_bcs(cc['z'][satellites_mask][KDTreemask],
Z[KDTreemask], SHMLM.BOXSIZE)
# coresMaskedArray = np.vstack((x, y, z)).T
coresMaskedArray_Tree = np.vstack(
(np.r_[x,
cc['x'][centrals_mask]], np.r_[y,
cc['y'][centrals_mask]],
np.r_[z, cc['z'][centrals_mask]])).T
coresMaskedArray = coresMaskedArray_Tree
KDTree_idx = np.r_[np.flatnonzero(satellites_mask)[KDTreemask],
np.flatnonzero(centrals_mask)]
coresKDTree = spatial.cKDTree(coresMaskedArray_Tree)
# Set mergedCoreTag as central core of KDTreemask cores with no mergedCoreTag that are `merged`
'''
disruptMask = cc['merged'][satellites_mask][KDTreemask]|(cc['radius'][satellites_mask][KDTreemask] >= SHMLM.CORERADIUSCUT)
cc['mergedCoreTag'][ np.flatnonzero(satellites_mask)[KDTreemask] ] += disruptMask*cc['core_tag'][centrals_mask][idx_m21][KDTreemask]
cc['mergedCoreStep'][ np.flatnonzero(satellites_mask)[KDTreemask] ] += disruptMask*step
'''
### core merging detection
# rvir_KDTreemask = SHMLM.getRvir(cc[m_evolved_col(1.1, 0.068)][satellites_mask][KDTreemask], SHMLM.step2z[step]) #infall should be exact m_evolved in production
# rvir_KDTreemask = SHMLM.getRvir(cc['infall_mass'][satellites_mask][KDTreemask], SHMLM.step2z[step]) #infall should be exact m_evolved in production
# distance_upper_bound = SHMLM.COREVIRIALRADIUSFACTOR * rvir_KDTreemask
distance_upper_bound = SHMLM.COREVIRIALRADIUSFACTOR * SHMLM.getRvir(
cc['infall_mass'][KDTree_idx], SHMLM.step2z[step])
import ctypes
# pool_mergedCoreTag = { i:np.full(len(KDTree_idx), 0, dtype=np.int64) for i in range(1,pool_size+1) }
pool_mergedCoreTag = {
i: multiprocessing.RawArray(
ctypes.c_int64, np.zeros(len(KDTree_idx), dtype=np.int64))
for i in range(1, pool_size + 1)
}
# Search radius of 2*rvir around each core
from datetime import datetime
tstart = datetime.now()
work = np.arange(len(distance_upper_bound))
t1 = datetime.now() - tstart
print "work=", t1
tstart = datetime.now()
p = Pool(pool_size)
t2 = datetime.now() - tstart
print "Pool=", t2
tstart = datetime.now()
p.map(query, np.array_split(work, pool_size))
t3 = datetime.now() - tstart
print "mapp=", t3
tstart = datetime.now()
p.close()
t4 = datetime.now() - tstart
print "close=", t4
tstart = datetime.now()
p.join()
t5 = datetime.now() - tstart
print "join=", t5
tstart = datetime.now()
# mergedCoreTag = np.full_like(pool_mergedCoreTag[1], 0)
mergedCoreTag = np.zeros(len(KDTree_idx), dtype=np.int64)
# pool_mergedCoreTag = {k:np.array(pool_mergedCoreTag[k]) for k in pool_mergedCoreTag.keys()}
for i in range(1, pool_size + 1):
mCT = np.array(pool_mergedCoreTag[i])
newMergedMask = mCT != 0
# print np.sum(newMergedMask)
mergedCoreTag[newMergedMask] = mCT[newMergedMask]
t6 = datetime.now() - tstart
newMergedMask = mergedCoreTag != 0
cc['mergedCoreTag'][
KDTree_idx[newMergedMask]] = mergedCoreTag[newMergedMask]
cc['mergedCoreStep'][KDTree_idx[newMergedMask]] = step
print np.sum(cc['mergedCoreTag'] != 0)
t7 = datetime.now() - tstart
print t1, t2, t3, t4, t5, t6, t7
###
print 'Merged centrals: ', np.sum(
cc['mergedCoreTag'][centrals_mask] != 0)
cc['mergedCoreTag'][centrals_mask] = 0
cc['mergedCoreStep'][centrals_mask] = 0
if writeOutputFlag:
# Write output to disk
h5_write_dict(fname_cc(step, 'output'), cc, 'coredata')
# Compute m_evolved of satellites according to SHMLModel for NEXT time step and save as cc_prev['next_m_evolved'] in memory.
# Mass loss assumed to begin at step AFTER infall detected.
if step != steps[-1]:
# cc_prev = { 'core_tag':cc['core_tag'].copy(), 'wasInFragment':cc['wasInFragment'].copy() }
cc_prev = {
k: cc[k].copy()
for k in [
'core_tag', 'wasInFragment', 'mergedCoreTag',
'mergedCoreStep'
]
}
if numSatellites != 0:
localhost_m21 = many_to_one(cc['host_core'][satellites_mask],
cc['core_tag'])
for A in A_arr:
for zeta in zeta_arr:
cc_prev[m_evolved_col(A, zeta, next=True)] = np.zeros_like(
cc['infall_mass'])
if numSatellites != 0: # If there are satellites (not applicable for first step)
m = cc[m_evolved_col(A, zeta)][satellites_mask]
if useLocalHost:
Mlocal = cc[m_evolved_col(A, zeta)][localhost_m21]
M_A_zeta = (Mlocal
== 0) * M + (Mlocal != 0) * Mlocal
cc_prev[m_evolved_col(
A, zeta,
next=True)][satellites_mask] = SHMLM.m_evolved(
m0=m,
M0=M_A_zeta,
step=steps[steps.index(step) + 1],
step_prev=step,
A=A,
zeta=zeta)
else:
cc_prev[m_evolved_col(
A, zeta,
next=True)][satellites_mask] = SHMLM.m_evolved(
m0=m,
M0=M,
step=steps[steps.index(step) + 1],
step_prev=step,
A=A,
zeta=zeta)
return cc
def np_mp_array(shape, dtype):
"""Allocate a numpy array on OS shared memory."""
size = int(np.prod(shape))
nbytes = size * np.dtype(dtype).itemsize
mp_array = mp.RawArray(ctypes.c_char, nbytes)
return np.frombuffer(mp_array, dtype=dtype, count=size).reshape(shape)
def __init__(self,
data,
ncomp=256,
gamma0=10,
m0=None,
nu0=None,
Phi0=None,
e0=5,
f0=0.1,
g0=0.1,
h0=0.1,
mu0=None,
Sigma0=None,
weights0=None,
alpha0=1,
gpu=None,
parallel=False,
verbose=False):
if not issubclass(type(data), HDPNormalMixture):
# check for functioning gpu
if _has_gpu:
self.dev_list = np.asarray((0), dtype=np.int)
self.dev_list.shape = 1
if gpu is not None:
if type(gpu) is bool:
self.gpu = gpu
else:
self.gpu = True
self.dev_list = np.asarray(np.abs(gpu), dtype=np.int)
if self.dev_list.shape == ():
self.dev_list.shape = 1
self.dev_list = np.unique(self.dev_list)
else:
self.gpu = True
else:
self.gpu = False
self.parallel = parallel
# get the data .. should add checks here later
self.data = [np.asarray(d) for d in data]
self.ngroups = len(self.data)
self.ndim = self.data[0].shape[1]
self.nobs = tuple([d.shape[0] for d in self.data])
# need for ident code
self.cumobs = np.zeros(self.ngroups + 1)
self.cumobs[1:] = np.asarray(self.nobs).cumsum()
self.ncomp = ncomp
if m0 is not None:
if len(m0) == self.ndim:
self.mu_prior_mean = m0.copy()
elif len(m0) == 1:
self.mu_prior_mean = m0 * np.ones(self.ndim)
else:
self.mu_prior_mean = np.zeros(self.ndim)
self.gamma = gamma0 * np.ones(ncomp)
self._set_initial_values(alpha0, nu0, Phi0, mu0, Sigma0, weights0,
e0, f0)
# initialize hdp specific vars
if weights0 is None:
self._weights0 = np.zeros((self.ngroups, self.ncomp),
dtype=np.float)
self._weights0.fill(1 / self.ncomp)
else:
self._weights0 = weights0.copy()
self._stick_beta0 = stats.beta.rvs(1,
self._alpha0,
size=self.ncomp - 1)
self._beta0 = break_sticks(self._stick_beta0)
self._alpha00 = 1.0
self.e0, self.f0 = g0, h0
self.prop_scale = 0.05 * np.ones(self.ncomp)
self.prop_scale[-1] = 1.
else:
# get all important vars from input class
self.data = data.data
self.ngroups, self.nobs, self.ndim, self.ncomp = data.ngroups, data.nobs, data.ndim, data.ncomp
self.cumobs = data.cumobs.copy()
self._weights0 = data.weights[-1].copy()
self._stick_beta0 = data.stick_beta.copy()
self._beta0 = break_sticks(self._stick_beta0)
self.e0, self.f0 = data.e0, data.f0
self.e, self.f = data.e, data.f
self._nu0 = data._nu0
self._Phi0 = data._Phi0
self.mu_prior_mean = data.mu_prior_mean.copy()
self.gamma = data.gamma.copy()
self._alpha0 = data.alpha[-1].copy()
self._alpha00 = data.alpha0[-1].copy()
self._weights0 = data.weights[-1].copy()
self._mu0 = data.mu[-1].copy()
self._Sigma0 = data.Sigma[-1].copy()
self.prop_scale = data.prop_scale.copy()
self.gpu = data.gpu
if self.gpu:
self.dev_list = np.unique(data.dev_list)
self.parallel = data.parallel
self.AR = np.zeros(self.ncomp)
# verbosity
self.verbose = verbose
# data working var
self.data_shared_mem = multiprocessing.RawArray(
'd',
sum(self.nobs) * self.ndim)
self.alldata = np.frombuffer(self.data_shared_mem).reshape(
sum(self.nobs), self.ndim)
for i in xrange(self.ngroups):
self.alldata[
self.cumobs[i]:self.cumobs[i + 1], :] = self.data[i].copy()
if self.parallel:
self.num_cores = min(min(multiprocessing.cpu_count(), self.ncomp),
16)
compsperdev = self.ncomp / self.num_cores
self.work_queue = [
multiprocessing.Queue() for i in xrange(self.num_cores)
]
self.result_queue = [
multiprocessing.Queue() for i in xrange(self.num_cores)
]
self.workers = [
CPUWorker(np.zeros(
(1, 1)), self.gamma, self.mu_prior_mean, self._Phi0,
self._nu0, self.work_queue[i], self.result_queue[i])
for i in xrange(self.num_cores)
]
self.compsdevmap = {}
cumcomps = 0
for i in xrange(self.num_cores):
self.compsdevmap[i] = [
int(cumcomps),
int(min(cumcomps + compsperdev, self.ncomp))
]
cumcomps += compsperdev
self.compsdevmap[self.num_cores - 1][1] = self.ncomp
for thd in self.workers:
thd.set_data(self.data_shared_mem, sum(self.nobs), self.ndim)
def mapper(i):
offset = i * block_size
block = bytes(shared_data[offset:offset + block_size])
for j in range(1000):
decrypted = des.decrypt(block)
block = decrypted
if i == 0:
intermediate = xor64d(block, iv_global)
else:
offset_prev = (i - 1) * block_size
intermediate = xor64d(block,
shit[offset_prev:offset_prev + block_size])
output_data[offset:offset + block_size] = intermediate
shit = multiprocessing.RawArray(ctypes.c_ubyte, encryptedCBC)
shared_data = multiprocessing.RawArray(ctypes.c_ubyte, encryptedCBC)
output_data = multiprocessing.RawArray(ctypes.c_ubyte, encryptedCBC)
pool = multiprocessing.Pool(8)
print(no_blocks)
start = time.time()
pool.map(mapper, range(no_blocks))
end = time.time()
print(end - start)
print(bytes(output_data))
def calculate_spectrum(
x,
line_function,
*line_args,
ncpus=1,
# multiprocess_divide='lines', # 'lines' or 'x','xshared'
multiprocess_divide='lines_shared', # 'lines' or 'x','xshared'
# multiprocess_divide='x', # 'lines' or 'x','xshared'
# multiprocess_divide='xshared', # 'lines' or 'x','xshared'
yin=None,
**line_kwargs,
):
"""Compute a spectrum of lines using line_function which is applied to
positional arguments line_args and with common kwargs.
line_function must add to y in place with a yin argument."""
ncpus = min(ncpus, multiprocessing.cpu_count())
## run as a single process
if ncpus == 1:
if yin is None:
yin = np.zeros(x.shape, dtype=float)
for args in zip(*line_args):
line_function(x, *args, **line_kwargs, yin=yin)
return yin
## multiprocessing version -- divide lines between processes
elif multiprocess_divide == 'lines':
## get indices to divide lines
with multiprocessing.Pool(ncpus, ) as p:
y = []
n = len(line_args[0])
jstep = int(n / ncpus) + 1
jbeg = 0
while jbeg < n:
## start async processes
jend = min(jbeg + jstep, n)
y.append(
p.apply_async(
calculate_spectrum,
args=(x, line_function,
*[t[jbeg:jend] for t in line_args]),
kwds=line_kwargs,
))
jbeg = jend
## run proceses, tidy keyboard interrupt, sum to give full spectrum
try:
p.close()
p.join()
except KeyboardInterrupt as err:
p.terminate()
p.join()
raise err
y = np.sum([t.get() for t in y], axis=0)
return y
## multiprocessing version -- divide x-range between processes
elif multiprocess_divide == 'x':
## get indices to divide lines
with multiprocessing.Pool(ncpus) as p:
y = []
n = len(x)
jstep = int(n / ncpus) + 1
jbeg = 0
while jbeg < n:
## start async processes
jend = min(jbeg + jstep, n)
y.append(
p.apply_async(
calculate_spectrum,
args=(x[jbeg:jend], line_function, *line_args),
kwds=line_kwargs,
))
jbeg = jend
## run proceses, tidy keyboard interrupt, sum to give full spectrum
try:
p.close()
p.join()
except KeyboardInterrupt as err:
p.terminate()
p.join()
raise err
y = np.concatenate([t.get() for t in y])
return y
elif multiprocess_divide == 'xshared':
## get indices to divide lines
n = len(x)
jstep = int(n / ncpus) + 1
jbeg = 0
## a writable shared memory array -- no lock
yraw = multiprocessing.RawArray('d', n)
y = np.frombuffer(yraw, dtype=np.float64).reshape((n, ))
y[:] = 0.0
with multiprocessing.Pool(
ncpus,
initializer=_calculate_spectrum_init_worker,
initargs=(x, y, line_function, line_args, line_kwargs),
) as p:
while jbeg < n:
## start async processes
jend = min(jbeg + jstep, n)
p.apply_async(_calculate_spectrum_worker, args=(jbeg, jend))
jbeg = jend
## run proceses, tidy keyboard interrupt, sum to give full spectrum
try:
p.close()
p.join()
except KeyboardInterrupt as err:
p.terminate()
p.join()
raise err
return y
elif multiprocess_divide == 'lines_shared':
## get indices to divide lines
n = len(line_args[0])
jstep = int(n / ncpus) + 1
jbeg = 0
## a writable shared memory array -- no lock
yraw = multiprocessing.Array('d', len(x))
y = np.frombuffer(yraw.get_obj(), dtype=np.float64).reshape((len(x), ))
y[:] = 0.0
with multiprocessing.Pool(
ncpus,
initializer=_calculate_spectrum_init_worker,
initargs=(x, y, line_function, line_args, line_kwargs),
) as p:
while jbeg < n:
## start async processes
jend = min(jbeg + jstep, n)
p.apply_async(_calculate_spectrum_worker_lines,
args=(jbeg, jend))
jbeg = jend
## run proceses, tidy keyboard interrupt, sum to give full spectrum
try:
p.close()
p.join()
except KeyboardInterrupt as err:
p.terminate()
p.join()
raise err
return y
else:
raise Exception('Unknown {multiprocess_divide=}')
def mp_empty_int(shape):
shared_array_base = mp.RawArray(ct.c_int, int(np.prod(shape)))
shared_array = np.ctypeslib.as_array(shared_array_base).reshape(*shape)
return shared_array
def np_mp_array(shape, dtype):
size = int(np.prod(shape))
nbytes = size * np.dtype(dtype).itemsize
mp_array = mp.RawArray(ctypes.c_char, nbytes)
return np.frombuffer(mp_array, dtype=dtype, count=size).reshape(shape)
def main():
logger = mp.log_to_stderr()
logger.setLevel(logging.CRITICAL)
#logger.setLevel(logging.FATAL)
# create shared array
mgr = mp.Manager()
d = mgr.dict()
d = {}
N, M = int(1e7), 11
base_ptr = mp.RawArray(ctypes.c_double, N)
arr = tonumpyarray(base_ptr)
arr[:] = np.array(np.random.uniform(size=N) * 1000, np.int)
arr_orig = arr.copy()
d["array"] = np.array(np.random.uniform(size=N) * 1000, np.int)
base_ptr2 = mp.RawArray(ctypes.c_double, N)
arr2 = tonumpyarray(base_ptr2)
arr2[:] = np.array(np.random.uniform(size=N), np.int)
arr_orig2 = arr2.copy()
base_ptr3 = mp.RawArray(ctypes.c_double, N)
arr3 = tonumpyarray(base_ptr3)
arr3[:] = np.random.uniform(size=N)
arr_orig3 = arr3.copy()
lst_event = []
for i in range(5):
lst_event.append(mp.Event())
lst_event[0].set()
flag_ptr = mp.RawArray(ctypes.c_bool, 5)
flag = [0, 0, 0, 0, 0]
flag[0] = True
flag[1] = False
flag[2] = False
flag[3] = False
flag[4] = False
#d["flag"] = lst_event
flag_ptr2 = mp.RawArray(ctypes.c_bool, 5)
flag2 = flag_ptr2
flag2[0] = True
flag2[1] = False
flag2[2] = False
flag2[3] = False
flag2[4] = False
flag_ptr3 = mp.RawArray(ctypes.c_bool, 5)
flag3 = flag_ptr3
flag3[0] = True
flag3[1] = False
flag3[2] = False
flag3[3] = False
flag3[4] = False
value_ptr = mp.RawArray(ctypes.c_int, 2)
value_ptr[0] = 0
value_ptr[1] = 0
d["value"] = [0, 0]
ptr_lst = []
ptr_lst.append(base_ptr)
ptr_lst.append(lst_event)
ptr_lst.append(value_ptr)
ptr_lst2 = []
ptr_lst2.append(base_ptr2)
ptr_lst2.append(flag_ptr2)
ptr_lst2.append(value_ptr)
ptr_lst3 = []
ptr_lst3.append(base_ptr3)
ptr_lst3.append(flag_ptr3)
ptr_lst3.append(value_ptr)
#p1 = mp.Process(target=exec1, args=(d, lst_event,))
#p2 = mp.Process(target=exec2, args=(d, lst_event,))
#p3 = mp.Process(target=exec1, args=(ptr_lst2,))
#p4 = mp.Process(target=exec2, args=(ptr_lst2,))
#p5 = mp.Process(target=exec1, args=(ptr_lst3,))
#p6 = mp.Process(target=exec2, args=(ptr_lst3,))
t = time.time()
exec1(d, lst_event)
exec2(d, lst_event)
#p3.start()
#p4.start()
#p5.start()
#p6.start()
#p1.join()
#p2.join()
#info(arr)
arr = d["array"]
crit(arr[0])
crit(arr[arr.size // 2 - 1])
crit(arr[arr.size // 2])
crit(arr[arr.size - 1])
# write to arr from different processes
# with closing(mp.Pool(initializer=init, initargs=(shared_arr,))) as p:
# # many processes access the same slice
# stop_f = N // 10
# p.map_async(f, [slice(stop_f)]*M)
# # many processes access different slices of the same array
# assert M % 2 # odd
# step = N // 10
# p.map_async(g, [slice(i, i + step) for i in range(stop_f, N, step)])
# p.join()
# assert np.allclose(((-1)**M)*tonumpyarray(shared_arr), arr_orig)
return t
def parallel_hungarian(G, p):
start_time = time.time()
# Randomly generate some data
data = np.array(G)
X_shape = (len(G), len(G))
X = mp.RawArray('d', X_shape[0] * X_shape[1])
X_np = np.frombuffer(X).reshape(X_shape)
# Copy data to our shared array.
np.copyto(X_np, data)
performance = {}
#G1 = len(G)*[]
manager = mp.Manager()
zeros_of_columns = manager.list()
zeros_of_rows = manager.list()
que = queue.Queue()
for i in range(len(G)):
zeros_of_rows.append(manager.dict())
zeros_of_columns.append(manager.dict())
#G1.append(G[i].copy())
#print(G1)
threads_of_rows = []
threads_of_columns = []
if (p > len(G)):
p = len(G)
d = int(len(G) / p)
if len(G) % 2 != 0:
#print("new thread was added because the dimention of the adjacency matrix is an odd number")
p = p + 1
#print(d)
performance["init"] = (time.time() - start_time)
start_time = time.time()
for i in range(p): # O(p)
start = i * d
#Gk = G1[start : start + d]
n = d
if i + 1 == p:
n = len(G) - (i * d)
#Gk = G1[start : ]
#print(type(G))
t = mp.Process(target=subtract_min_from_rows,
args=(X, X_shape, start, n, zeros_of_rows,
zeros_of_columns))
threads_of_rows.append(t)
for i in range(len(threads_of_rows)): # O(p)
#print("start threads_of_rows ",i)
threads_of_rows[i].start()
#threads_of_rows[0].start()
#threads_of_rows[0].join()
for i in range(len(threads_of_rows)):
# print("join")
threads_of_rows[i].join()
performance["subtract_min_from_rows"] = (time.time() - start_time)
start_time = time.time()
for i in range(p):
n = d
s = i * d
if i + 1 == p:
n = len(G) - s
t = mp.Process(target=subtract_min_from_columns,
args=(X, X_shape, s, n, zeros_of_rows,
zeros_of_columns))
threads_of_columns.append(t)
for i in range(len(threads_of_columns)):
#print("start threads_of_columns ",i)
threads_of_columns[i].start()
#for i in range(len(threads_of_columns)):
#print("join")
threads_of_columns[i].join()
performance["subtract_min_from_columns"] = (time.time() - start_time)
#print("result ")
#for start in range(len(G)):
# print(zeros_of_rows[start],type(zeros_of_rows[start]))
r = []
c = []
#print(G1,zeros_of_rows,zeros_of_columns)
ri = 0
for i in zeros_of_rows:
#print(type(i))
r.append({})
for (j, k) in i.items():
#print("g",j)
r[ri][j] = k
ri = ri + 1
ci = 0
for i in zeros_of_columns:
c.append({})
for (j, k) in i.items():
c[ci][j] = k
ci = ci + 1
#print(r,c)
#print(zeros_of_rows,zeros_of_columns)
number_of_lines, horizental_lines, vertical_lines = cover_zeros(r, c)
#print("number of lines",number_of_lines, horizental_lines ,vertical_lines )
#print("let's loop")
while (number_of_lines != len(G)):
min, row_index, column_index = min_in_graph(G, horizental_lines,
vertical_lines)
#print("G[ ",row_index," ][ ",column_index," ] = ",min)
match, zeros_of_rows, zeros_of_columns = subtract_min_from_graph(
G, min, horizental_lines, vertical_lines, zeros_of_rows,
zeros_of_columns)
#print(match, zeros_of_rows,zeros_of_columns)
r = []
c = []
ri = 0
for i in zeros_of_rows:
#print(type(i))
r.append({})
for (j, k) in i.items():
#print("g",j)
r[ri][j] = k
ri = ri + 1
ci = 0
for i in zeros_of_columns:
c.append({})
for (j, k) in i.items():
c[ci][j] = k
ci = ci + 1
#print(G1)
number_of_lines, horizental_lines, vertical_lines = cover_zeros(r, c)
#print(G1)
#print("number of lines",number_of_lines, horizental_lines ,vertical_lines )
if (number_of_lines == len(G)):
assignments = assign_tasks_to_workers(zeros_of_rows, zeros_of_columns)
#print("assignemt",assignments)
performance["rest"] = (time.time() - start_time)
return assignments, performance
thread of execution.
"""
X_shape = (10, 10)
G = [[5, 8, 47, 49, 33, 34, 45, 34, 54, 59],
[88, 79, 46, 23, 4, 54, 321, 54, 85, 43],
[75, 17, 1, 34, 55, 234, 2, 58, 654, 59],
[98, 74, 90, 41, 68, 12, 1, 13, 466, 52],
[12, 67, 43, 32, 51, 890, 59, 85, 76, 37],
[890, 90, 89, 89, 808, 9, 34, 674, 56, 52],
[89, 8, 3, 890, 34, 8, 378, 65, 85, 36],
[323, 123, 49, 959, 49, 3, 8, 95, 36, 74],
[3, 2, 23, 54, 59, 76, 65, 54, 559, 5 ],
[6, 54, 45, 95, 59, 87, 90, 237, 23, 232]]
# Randomly generate some data
data = np.array(G)
X = multiprocessing.RawArray('d', X_shape[0] * X_shape[1])
X_np = np.frombuffer(X).reshape(X_shape)
# Copy data to our shared array.
np.copyto(X_np, data)
print(type(X),type(X_np))
p = []
n = 2
for i in range(n):
p.append(multiprocessing.Process(target=parallel_function, args=(i,i,X_shape, X)))
for i in range(n):
p[i].start()
for i in range(n):
p[i].join()
def __init__(self, fun, dim):
self.func = fun
self.statMutex = mp.Lock()
self.best_x = mp.RawArray(ct.c_double, dim)
self.best_y = mp.RawValue(ct.c_double, sys.float_info.max)
self.count = mp.RawValue(ct.c_int, 0)
def run_l2d(p, die_on_error=False):
config = p.config
mod_tools.initialize_parameters(p)
resols = p.resol_spatial_l2d # km
resolt = p.resol_temporal_l2d # days
grd = build_grid(p)
lon0, lon1, lat0, lat1 = grd['modelbox']
# Filtering as a function of latitude (10d at Equator, 14 d at 60º)
resolt = (-4 * numpy.cos(numpy.deg2rad(grd['lat'])) + 9) * resolt
# #####################################
# READ L2B OBS
# #####################################
tcycle = p.satcycle
# Time domain
time0, time1, dt = p.time_domain
# TODO change
resoltmax = numpy.max(resolt)
c0 = int((max(time0 - resoltmax, 0)) / tcycle) + 1
c1 = int((time1 + resoltmax) / tcycle) + 1
obs = {}
# c0 = 1 ; c1 = 2
p2 = mod_tools.todict(p)
jobs = []
print(datetime.datetime.now())
for cycle in range(c0, c1 + 1, 1):
_pat = '{}_c{:02d}_p*'.format(p.config, cycle)
pat = os.path.join(p.outdatadir, _pat)
listfile = glob.glob(pat)
# Create specific funtion to retrieve time_unit
if not listfile:
continue
filev = os.path.join(p.outdatadir, listfile[0])
_, _, time_unit = read_l2b(filev)
for pattern in listfile:
jobs.append([p2, pattern, lon0, lon1, lat0, lat1, time0, time1,
resoltmax, grd['dlonlr'], grd['dlatlr']])
ok = False
task = 0
results = []
try:
ok, results = build_obs(p.proc_count, jobs, die_on_error,
p.progress_bar)
except skimulator.mod_parallel.MultiprocessingError:
logger.error('An error occurred with the multiprocessing framework')
traceback.print_exception(*sys.exc_info())
sys.exit(1)
except skimulator.mod_parallel.DyingOnError:
logger.error('An error occurred and all errors are fatal')
sys.exit(1)
print()
print('Aggregating results...', datetime.datetime.now())
for i in range(len(results)):
obs_rjobs = results[i]
#key, ind_key, obs_r = results[i]
for j in range(len(obs_rjobs)):
try:
obs_r = obs_rjobs[j]
except:
obs_r = obs_rjobs
obs_r_keys = list(obs_r.keys())
for key in obs_r_keys:
if key not in obs.keys():
obs[key] = {}
for ind_key in obs_r[key].keys():
if ind_key not in obs[key].keys():
obs[key][ind_key] = []
obs[key][ind_key] = numpy.concatenate((obs[key][ind_key],
obs_r[key][ind_key]))
del(obs_r[key])
del(results)
print('Building shared obs vector...', datetime.datetime.now())
# Populate the shared observations vector
global obs_vector
obs_vector = {}
for key in obs:
obs_vector[key] = {}
obs_ind_keys = list(obs[key].keys())
for ind_key in obs_ind_keys:
_buffer = multiprocessing.RawArray(ctypes.c_float,
numpy.size(obs[key][ind_key]))
narray = numpy.frombuffer(_buffer, dtype='float32')
numpy.copyto(narray, obs[key][ind_key])
obs_vector[key][ind_key] = _buffer
# Release memory from original array to keep the total amount of
# RAM as low as possible
if 'time' != key:
del(obs[key][ind_key]) # release memory as soon as possible
print(datetime.datetime.now())
print('Memory housekeeping...', datetime.datetime.now())
keys = list(obs.keys())
for key in keys:
if 'time' != key:
del(obs[key])
print(datetime.datetime.now())
#import pdb ; pdb.set_trace()
# #####################################
# Independant time loop for each analysis
# #####################################
enstime = numpy.arange(time0, time1 + dt, dt)
time_model = numpy.arange(time0, time1, p.timestep)
# Center at noon
window = dt / 2.
enstime = enstime + window
indice_mask = make_mask(p, 'ucur', grd)
list_key = {'ucur':'ux_true', 'vcur':'uy_true'} #, 'uwnd':'uwnd',
#'vwnd': 'vwnd'}
list_input = {}
for key in list_key:
list_input[key] = p.list_input_var_l2d[key]
for it, timeref in enumerate(enstime):
print(it, timeref)
iobs = {}
for ind_key in obs['time'].keys():
iobs[ind_key] = numpy.where((obs['time'][ind_key] > (timeref
- numpy.max(resoltmax)))
& (obs['time'][ind_key] < (timeref
+ numpy.max(resoltmax))))[0]
make_oi(p, p2, grd, iobs, resols, timeref, resolt, indice_mask,
die_on_error=die_on_error)
# Interpolate model data to have a true reference
#indice_model = it
#print('read model')
#model_data, out_var, list_file = read_model(p, it, p.dim_time,
# list_input)
#print('interpolate model')
#if global_domain is False:
# interpolate_model(p, model_data, out_var, grd, list_key)
pattern = os.path.join(p.outdatadir, '{}_{}_l2d_'.format(config,
p.config_l2d))
save_l2d(pattern, timeref, window, time_unit, grd)
print(datetime.datetime.now())
def mp_filled_bool(input_array):
shape = input_array.shape
shared_array_base = mp.RawArray(ct.c_bool, input_array.flatten())
shared_array = np.ctypeslib.as_array(shared_array_base).reshape(*shape)
return shared_array
def share_array(arr_np, typecode):
arr = mp.RawArray(typecode, arr_np.ravel())
return arr, arr_np.shape
def get_all_DRs_parallel(time_start,
time_end,
probe,
pad,
cmp,
kmin=0.0,
kmax=1.0,
Nk=1000,
knorm=True,
nsec=None,
HM_filter_mhz=50,
complex_out=True):
if nsec is None:
DR_path = save_dir + 'DISP_{}_cc_{:03}_{:03}_{:03}.npz'.format(
save_string, int(cmp[0]), int(cmp[1]), int(cmp[2]))
else:
DR_path = save_dir + 'DISP_{}_cc_{:03}_{:03}_{:03}_{}sec.npz'.format(
save_string, int(cmp[0]), int(cmp[1]), int(cmp[2]), nsec)
if os.path.exists(DR_path) == False:
# Load data
times, B0, name, mass, charge, density, tper, ani, cold_dens = \
extract_species_arrays(time_start, time_end, probe, pad, cmp=np.asarray(cmp),
return_raw_ne=True, nsec=nsec, HM_filter_mhz=HM_filter_mhz)
Nt = times.shape[0]
N_procs = 7
procs = []
# Create raw shared memory arrays with shapes. Store in dict to send with each worker
k_shape = (Nt, Nk)
k_shm = multiprocessing.RawArray('d', Nt * Nk)
k_dict = {'arr': k_shm, 'shape': k_shape}
CPDR_shape = (Nt, Nk, 3, 2)
CPDR_shm = multiprocessing.RawArray('d', Nt * Nk * 3 * 2)
CPDR_dict = {'arr': CPDR_shm, 'shape': CPDR_shape}
WPDR_shape = (Nt, Nk, 3, 2)
WPDR_shm = multiprocessing.RawArray('d', Nt * Nk * 3 * 2)
WPDR_dict = {'arr': WPDR_shm, 'shape': WPDR_shape}
HPDR_shape = (Nt, Nk, 3, 2)
HPDR_shm = multiprocessing.RawArray('d', Nt * Nk * 3 * 2)
HPDR_dict = {'arr': HPDR_shm, 'shape': HPDR_shape}
k_np = np.frombuffer(k_shm).reshape(k_shape)
CPDR_np = np.frombuffer(CPDR_shm).reshape(CPDR_shape)
WPDR_np = np.frombuffer(WPDR_shm).reshape(WPDR_shape)
HPDR_np = np.frombuffer(HPDR_shm).reshape(HPDR_shape)
# Split input data into a list of chunks
time_chunks = np.array_split(times, N_procs)
field_chunks = np.array_split(B0, N_procs)
density_chunks = np.array_split(density, N_procs, axis=1)
tper_chunks = np.array_split(tper, N_procs, axis=1)
ani_chunks = np.array_split(ani, N_procs, axis=1)
# Instatiate each process with a different chunk
acc = 0
start = time.time()
for xx in range(N_procs):
print('Starting process', xx)
proc = multiprocessing.Process(
target=get_DRs_chunked,
args=(Nk, kmin, kmax, knorm, time_chunks[xx], field_chunks[xx],
name, mass, charge, density_chunks[xx], tper_chunks[xx],
ani_chunks[xx], k_dict, CPDR_dict, WPDR_dict, HPDR_dict),
kwargs={
'st': acc,
'worker': xx
})
procs.append(proc)
proc.start()
acc += time_chunks[xx].shape[0]
# Complete processes
for proc in procs:
proc.join()
print('All processes complete')
end = time.time()
print('Total parallel time = {}s'.format(str(end - start)))
# Make output complex or keep as real in [X, 2] array
if complex_out == True:
CPDR_out = np.zeros((Nt, Nk, 3), dtype=np.complex128)
WPDR_out = np.zeros((Nt, Nk, 3), dtype=np.complex128)
HPDR_out = np.zeros((Nt, Nk, 3), dtype=np.complex128)
for ii in range(Nt):
for jj in range(Nk):
for kk in range(3):
CPDR_out[ii, jj,
kk] = CPDR_np[ii, jj, kk,
0] + 1j * CPDR_np[ii, jj, kk, 1]
WPDR_out[ii, jj,
kk] = WPDR_np[ii, jj, kk,
0] + 1j * WPDR_np[ii, jj, kk, 1]
HPDR_out[ii, jj,
kk] = HPDR_np[ii, jj, kk,
0] + 1j * HPDR_np[ii, jj, kk, 1]
else:
CPDR_out = CPDR_np
WPDR_out = WPDR_np
HPDR_out = HPDR_np
# Saves data used for DR calculation as well, for future reference (and plotting)
if os.path.exists(save_dir) == False:
os.makedirs(save_dir)
print('Saving dispersion history...')
np.savez(DR_path,
all_CPDR=CPDR_out,
all_WPDR=WPDR_out,
all_HPDR=HPDR_out,
all_k=k_np,
comp=np.asarray(cmp),
times=times,
B0=B0,
name=name,
mass=mass,
charge=charge,
density=density,
tper=tper,
ani=ani,
cold_dens=cold_dens,
HM_filter_mhz=np.array([HM_filter_mhz]))
else:
print('Dispersion results already exist, loading from file...')
DR_file = np.load(DR_path)
k_np = DR_file['all_k']
CPDR_out = DR_file['all_CPDR']
WPDR_out = DR_file['all_WPDR']
HPDR_out = DR_file['all_HPDR']
times = DR_file['times']
B0 = DR_file['B0']
name = DR_file['name']
mass = DR_file['mass']
charge = DR_file['charge']
density = DR_file['density']
tper = DR_file['tper']
ani = DR_file['ani']
cold_dens = DR_file['cold_dens']
return k_np, CPDR_out, WPDR_out, HPDR_out, \
times, B0, name, mass, charge, density, tper, ani, cold_dens
# Yikang Liao <*****@*****.**>
# Multi-processing Video Files Reader
import logging
import multiprocessing as mp
import ctypes
import numpy as np
import cv2
import random
logger = logging.getLogger(__name__)
PROCESSES = 64
MAX_FLOAT_NUM = 1024 * 3 * 32 * 224 * 224
ret_base = mp.RawArray(ctypes.c_float, MAX_FLOAT_NUM)
counter = mp.RawValue(ctypes.c_int, 0)
def sample_clip_func((filename, p, batch_size, n_frame, crop_size, scale_w,
scale_h, is_train, temporal_center)):
ret = np.frombuffer(ret_base,
dtype=np.float32,
count=batch_size * 3 * n_frame * crop_size *
crop_size).reshape(
(batch_size, 3, n_frame, crop_size, crop_size))
tmp = None
while tmp is None:
v = cv2.VideoCapture(filename)
width, height, length = v.get(cv2.CAP_PROP_FRAME_WIDTH), v.get(cv2.CAP_PROP_FRAME_HEIGHT), \
v.get(cv2.CAP_PROP_FRAME_COUNT)
def mcrun():
'''
Monte Carlo simulation for electrons with ADP and intervalley scattering.
Set parameters between line 155 and 190.
Returns
-------
None.
'''
# create bins
bins_z = sv.bins_z
bins_xy = sv.bins_xy
bins_t = sv.bins_t #number of temporal bins to record current data in # NEEDS FIXING, NOW I HAVE TO SPECIFY BINS AT 3 DIFFERENT PLACES
#make a place to store the electric field vector V2.0
e_field = np.zeros(bins_t * bins_z * bins_xy**2 * 3)
raw_array = mp.RawArray('d', int(bins_t * bins_z * bins_xy**2 * 3))
raw_array_np = np.frombuffer(raw_array,
dtype=np.float64) #.reshape(X_shape)
np.copyto(raw_array_np, e_field)
#define one parallel process for each cpu
workers = mp.cpu_count()
poolen = mp.Pool(workers, initializer=initProcess,
initargs=(raw_array, )) # V2.0
# calculates gmax for each vally
diffrent_cycles = 6
valleys = [1 + (x % 6)
for x in range(diffrent_cycles)] # repeating 0,1,2,3,4,5 list
gmax_list = poolen.map(gmax, list(zip(*[valleys])))
###################################################
start_time = tm.time() #start counting cpu time
print('TL= ', sv.T_L, ' B= ', sv.B, ' U= ', sv.U, ' Xid= ',
sv.Xid / sv.q, ' Xiu= ', sv.Xiu /
sv.q) #to be able to follow the progress in the command window
jz2 = np.zeros(bins_t) # zero the current vector etc.
xtotinval = np.zeros(6)
ttotinval = np.zeros(6)
Ettot = np.zeros(6)
xinttot = np.zeros(6)
xvtot = np.zeros(6)
############################################################################################################
E_field = np.zeros([100, 100, 29, 29, 3])
number_e_matrix = np.zeros([100, 100, 29, 29])
for run in range(2): #V2.0
print(run)
run_start_time = tm.time()
# Parallel call via poolen.map, first all the indata is merged into an array for all electrons (Nr=incycles)
valleys = [1 + (x % 6)
for x in range(sv.incycles)] # repeating 0,1,2,3,4,5 list
Gammamax = [gmax_list[int(x % 6)] for x in range(sv.incycles)]
mapper = poolen.map(multi_run_wrap, list(zip(*[valleys, Gammamax])))
resultlist = list(mapper) #this is the outdata from inloop6 (fortran)
summed = [sum(x) for x in zip(*resultlist)
] #outdata summed over the electrons
xtotinval = summed[
0] #xtotinval(n)= total distance travelled in valley n (n=1..6)
ttotinval = summed[
1] #ttotinval(n)= total time spent in valley n (n=1..6)
Ettot = summed[
2] #Ettot(n)= energy integrated over time in valley n (n=1..6) ???
xinttot = summed[
3] #xinttot(n)= distance integrated over time in valley n (n=1..6)
xvtot = summed[
4] #xvtot(n)= distance times velocity integrated over time in valley n (n=1..6) ???
jz2 = summed[5] #jz2(n)= current in valley n (n=1..6) ???
run_end_time = tm.time()
print('Elapsed time scatt: ', run_end_time - run_start_time)
#V2.0 sorterar ut alla positionenr för elektronerna
pos_i = np.zeros(
bins_t * 4
) # zero the position vector etc. make sure that the format of the vectors is correct and all zero values are gone
for i in range(sv.incycles): # allt under är V2.0
pos_i = np.array(resultlist[i][6]).reshape((bins_t, 4), order='F')
resultlist[i][6] = pos_i[np.nonzero(pos_i[:, 3]), :]
pos_raw = np.array(
np.hstack((np.reshape(resultlist, sv.incycles * 7)[6::7])))
pos_bins = []
pos, number_e = np.unique(
np.floor(
np.divide(pos_raw, [
sv.length / bins_z, sv.length / bins_xy,
sv.length / bins_xy, 22e-9 / bins_t
])),
return_index=False,
return_inverse=False,
return_counts=True,
axis=1) # kan gå snabbare om jag gör den för varje tidsteg
pos = np.reshape(pos, (number_e.size, 4))
pos_raw = np.reshape(pos_raw, (-1, 4))
ind_greater_zero = (number_e > 0) # ta bort alla små väden
pos = pos[ind_greater_zero, :]
number_e = number_e[ind_greater_zero]
pos_end = pos[(pos[:, 0] > bins_z - 1), :]
number_e_end = number_e[(pos[:, 0] > bins_z - 1)]
##simulats that the electrons get stuck in Nv centers
part_e_Nv = 0.1 # part of electrons that hits Nv center
count = 0
k = 0
for i in range(pos_end[:, 0].size):
for j in range(int(pos_end[i, 3]), int(bins_t)):
count += 1
## lägger till electroner till slutet the end
pos_e_Nv = np.zeros([count, 4])
number_e_Nv = np.zeros(count)
for i in range(pos_end[:, 0].size):
for j in range(int(pos_end[i, 3]), int(bins_t)):
pos_e_Nv[k, :] = [bins_z - 1, pos_end[i, 1], pos_end[i, 2], j]
number_e_Nv[k] = part_e_Nv * number_e_end[i]
k += 1
pos = np.vstack((pos, pos_e_Nv))
number_e = np.append(number_e, number_e_Nv)
###############################################################
ind_inside = ((pos[:, 0] < bins_z) & (0 < pos[:, 0]) &
(pos[:, 1] <= int(
(bins_xy - 1) / 2)) & (pos[:, 1] >= -int(
(bins_xy - 1) / 2)) & (pos[:, 2] <= int(
(bins_xy - 1) / 2)) & (pos[:, 2] >= -int(
(bins_xy - 1) / 2)))
pos = pos[ind_inside, :]
number_e = number_e[ind_inside]
ind_sort = np.lexsort((pos[:, 0], pos[:, 3]), axis=0)
pos = pos[ind_sort, :]
number_e = number_e[ind_sort]
b_tid = tm.time()
ratio_add = 0.02
number_e_matrix = (
(1 - ratio_add) * number_e_matrix +
ratio_add * electron_smoothing(pos[:, 0].size, pos, number_e))
run_start_time = tm.time()
print('Elapsed time number_e_matrix: ', run_start_time - b_tid)
####################################################################
e_pos_matrix_i = [number_e_matrix[x, :, :, :] for x in range(bins_t)]
mapper = poolen.map(Efind, list(zip(*[e_pos_matrix_i])))
result_efield_pool = list(
mapper) #this is the outdata from inloop6 (fortran)
E_field = np.stack(result_efield_pool, axis=0)
#print(np.mean(E_field))
b_tid = tm.time()
print('Elapsed time E_field: ', b_tid - run_start_time)
########################################################################
raw_array_np = np.frombuffer(raw_array,
dtype=np.float64) #.reshape(X_shape)
np.copyto(raw_array_np,
np.reshape(E_field * 1e8 / sv.incycles, (1, -1), order='F'))
np.savetxt('Pos_end' + str(run) + '.txt', pos_raw)
############################################################################
np.savetxt('Pos_end.txt', pos_raw)
#np.savetxt('number_e_matrix.txt',np.reshape(number_e_matrix, (-1,1)))
#np.savetxt('pos_all.txt',np.reshape(pos, (-1,1)))
#np.savetxt('Pos_end'+'.txt',pos_raw)
plt.figure(20)
plt.imshow(number_e_matrix[40, :, :, :].sum(axis=1))
plt.figure(21)
plt.imshow(E_field[40, :, :, :, 2].sum(axis=1))
########################################################################################################################
end_time = tm.time()
print('Elapsed time: ', end_time - start_time)
plt.show()
poolen.close() #close the paralell workers
poolen.join()
def grid_visibilities_parallel(self,
visibilities,
min_attenuation=1e-10,
N=120):
"""
Grid a set of visibilities from baselines onto a UV grid.
Uses Fourier (Gaussian) beam weighting to perform the gridding.
Parameters
----------
visibilities : complex (n_baselines, n_freq)-array
The visibilities at each basline and frequency.
Returns
-------
visgrid : (ngrid, ngrid, n_freq)-array
The visibility grid, in Jy.
"""
#Find out the number of frequencies to process per thread
nfreq = len(self.frequencies)
numperthread = int(np.ceil(nfreq / self.n_obs))
offset = 0
nfreqstart = np.zeros(self.n_obs, dtype=int)
nfreqend = np.zeros(self.n_obs, dtype=int)
infreq = np.zeros(self.n_obs, dtype=int)
for i in range(self.n_obs):
nfreqstart[i] = offset
nfreqend[i] = offset + numperthread
if (i == self.n_obs - 1):
infreq[i] = nfreq - offset
else:
infreq[i] = numperthread
offset += numperthread
# Set the last process to the number of frequencies
nfreqend[-1] = nfreq
processes = []
ugrid = np.linspace(-self.uv_max, self.uv_max,
self.n_uv + 1) # +1 because these are bin edges.
centres = (ugrid[1:] + ugrid[:-1]) / 2
visgrid = np.zeros((self.n_uv, self.n_uv, len(self.frequencies)),
dtype=np.complex128)
if (os.path.exists(self.datafile[0][:-4] + ".kernel_weights.npy")):
kernel_weights = np.load(self.datafile[0][:-4] +
".kernel_weights.npy")
else:
kernel_weights = None
if kernel_weights is None:
weights = np.zeros((self.n_uv, self.n_uv, len(self.frequencies)))
visgrid_buff_real = []
visgrid_buff_imag = []
weights_buff = []
#Lets split this array up into chunks
for i in range(self.n_obs):
visgrid_buff_real.append(
multiprocessing.RawArray(
np.sctype2char(visgrid.real),
visgrid[:, :, nfreqstart[i]:nfreqend[i]].size))
visgrid_buff_imag.append(
multiprocessing.RawArray(
np.sctype2char(visgrid.imag),
visgrid[:, :, nfreqstart[i]:nfreqend[i]].size))
visgrid_tmp_real = np.frombuffer(visgrid_buff_real[i])
visgrid_tmp_imag = np.frombuffer(visgrid_buff_imag[i])
visgrid_tmp_real = visgrid[:, :,
nfreqstart[i]:nfreqend[i]].real.flatten(
)
visgrid_tmp_imag = visgrid[:, :,
nfreqstart[i]:nfreqend[i]].imag.flatten(
)
if (kernel_weights is None):
weights_buff.append(
multiprocessing.RawArray(
np.sctype2char(weights),
weights[:, :, nfreqstart[i]:nfreqend[i]].size))
weights_tmp = np.frombuffer(weights_buff[i])
weights_tmp = weights[:, :, nfreqstart[i]:nfreqend[i]]
else:
weights_buff.append(None)
processes.append(
multiprocessing.Process(
target=self._grid_visibilities_buff,
args=(self.n_uv, visgrid_buff_real[i],
visgrid_buff_imag[i], weights_buff[i],
visibilities[:, nfreqstart[i]:nfreqend[i]],
self.frequencies[nfreqstart[i]:nfreqend[i]],
self.baselines, centres,
self._instr_core.sigma(
self.frequencies[nfreqstart[i]:nfreqend[i]]),
min_attenuation, N)))
for p in processes:
p.start()
for p in processes:
p.join()
for i in range(self.n_obs):
visgrid[:, :, nfreqstart[i]:nfreqend[i]].real = np.frombuffer(
visgrid_buff_real[i]).reshape(self.n_uv, self.n_uv,
nfreqend[i] - nfreqstart[i])
visgrid[:, :, nfreqstart[i]:nfreqend[i]].imag = np.frombuffer(
visgrid_buff_imag[i]).reshape(self.n_uv, self.n_uv,
nfreqend[i] - nfreqstart[i])
if (kernel_weights is None):
weights[:, :, nfreqstart[i]:nfreqend[i]] = np.frombuffer(
weights_buff[i]).reshape(self.n_uv, self.n_uv,
nfreqend[i] - nfreqstart[i])
if kernel_weights is None:
kernel_weights = weights
visgrid[kernel_weights != 0] /= kernel_weights[kernel_weights != 0]
return visgrid, kernel_weights
if __name__ == '__main__':
coeffs = np.array([[random.randint(0, MAXHASH) for j in range(NF)]
for i in range(2)])
files = [
os.path.join(root, f) for root, _, fnames in os.walk('.')
for f in fnames
]
if len(sys.argv) > 1:
files = files[:int(sys.argv[1])]
filenum = len(files)
signatures = np.ctypeslib.as_array(
mp.RawArray(ctypes.c_ulong, filenum * NF)).reshape(filenum, NF)
aux_data = (signatures, files, coeffs)
with mp.Pool(mp.cpu_count(), initializer, (aux_data, )) as p:
# shingle files and create signatures
for i in tqdm(p.imap(shingle_wrapper, range(filenum), chunksize=100),
total=filenum,
desc="shingling"):
pass
# compare signatures
results = []
for s in tqdm(p.imap_unordered(hashcount_wrapper,
range(1, filenum),
chunksize=100),
def _assemble_kernel_mat(
self,
R_desc,
R_d_desc,
n_perms,
tril_perms_lin,
sig,
use_E_cstr=False,
progr_callback=None,
j_end=None,
):
r"""
Compute force field kernel matrix.
The Hessian of the Matern kernel is used with n = 2 (twice
differentiable). Each row and column consists of matrix-valued blocks,
which encode the interaction of one training point with all others. The
result is stored in shared memory (a global variable).
Parameters
----------
R_desc : :obj:`numpy.ndarray`
Array containing the descriptor for each training point.
R_d_desc : :obj:`numpy.ndarray`
Array containing the gradient of the descriptor for
each training point.
n_perms : int
Number of individual permutations encoded in
`tril_perms_lin`.
tril_perms_lin : :obj:`numpy.ndarray`
1D array containing all recovered permutations
expanded as one large permutation to be applied to a
tiled copy of the object to be permuted.
sig : int
Hyper-parameter :math:`\sigma`(kernel length scale).
use_E_cstr : bool, optional
True: include energy constraints in the kernel,
False: default (s)GDML kernel.
progress_callback : callable, optional
Kernel assembly progress function that takes three
arguments:
current : int
Current progress (number of completed entries).
total : int
Task size (total number of entries to create).
done_str : :obj:`str`, optional
Once complete, this string contains the
time it took to assemble the kernel (seconds).
j_end : int
Only generate the columns up to 'j_end'. This creates a
M*3N x j_end*3N kernel matrix, instead of M*3N x M*3N.
Returns
-------
:obj:`numpy.ndarray`
Force field kernel matrix.
"""
global glob
n_train, dim_d, dim_i = R_d_desc.shape
if j_end == None:
j_end = n_train
dim_K = n_train * dim_i
dim_K_ny = j_end * dim_i
dim_K += n_train if use_E_cstr else 0
dim_K_ny += j_end if use_E_cstr else 0
K = mp.RawArray('d', dim_K_ny * dim_K)
glob['K'], glob['K_shape'] = K, (dim_K, dim_K_ny)
glob['R_desc'], glob['R_desc_shape'] = _share_array(R_desc, 'd')
glob['R_d_desc'], glob['R_d_desc_shape'] = _share_array(R_d_desc, 'd')
start = timeit.default_timer()
pool = mp.Pool(self._max_processes)
todo = (j_end**2 - j_end) // 2 + j_end
if j_end is not n_train:
todo += (n_train - j_end) * j_end
done_total = 0
for done in pool.imap_unordered(
partial(
_assemble_kernel_mat_wkr,
n_perms=n_perms,
tril_perms_lin=tril_perms_lin,
sig=sig,
use_E_cstr=use_E_cstr,
j_end=j_end,
),
list(range(j_end)),
):
done_total += done
if progr_callback is not None:
if done_total == todo:
stop = timeit.default_timer()
progr_callback(done_total,
todo,
sec_disp_str='(%.1f s)' %
((stop - start) / 2))
else:
progr_callback(done_total, todo)
pool.close()
# Release some memory.
glob.pop('K', None)
glob.pop('R_desc', None)
glob.pop('R_d_desc', None)
return np.frombuffer(K).reshape(glob['K_shape'])
def get_rule_score_and_indices(rule_bundle, training_examples,
testing_examples, weights_i, rule_weights, tree,
y, x1, x2, holdout, rule_train_index,
rule_test_index):
### ADD IN
if rule_bundle.size == 1:
rule_score = util.calc_score(tree, rule_weights, rule_train_index)
motif_bundle = []
regulator_bundle = []
return rule_score, rule_train_index, rule_test_index
# Get a lock
lock_stable = multiprocessing.Lock()
# Initialize shared data objects
theta_alphas = multiprocessing.RawArray(ctypes.c_double, rule_bundle.size)
bundle_train_pred = multiprocessing.RawArray(ctypes.c_double,
y.num_row * y.num_col)
bundle_test_pred = multiprocessing.RawArray(ctypes.c_double,
y.num_row * y.num_col)
# Store the value of the next rule that needs to be worked on
rule_index_cntr = multiprocessing.Value('i', 0)
# Pack arguments
stable_args = [y, x1, x2, rule_index_cntr, rule_bundle, \
training_examples, testing_examples,
rule_weights.w_pos, rule_weights.w_neg,
(lock_stable, theta_alphas, bundle_train_pred, bundle_test_pred)]
# Fork worker processes, and wait for them to return
fork_and_wait(config.NCPU, return_rule_index, stable_args)
### Get results back into the right format ()
theta_alphas = np.array(theta_alphas)
if y.sparse:
bundle_train_pred = csr_matrix(
np.reshape(np.array(bundle_train_pred), (y.data.shape)))
bundle_test_pred = csr_matrix(
np.reshape(np.array(bundle_test_pred), (y.data.shape)))
else:
bundle_train_pred = np.reshape(np.array(bundle_train_pred),
(y.data.shape))
bundle_test_pred = np.reshape(np.array(bundle_test_pred),
(y.data.shape))
# Calculate theta
min_val = min([abs(a) for a in theta_alphas])
theta = sum([abs(alph) - min_val for alph in theta_alphas]) / 2
# new index is where absolute value greater than theta
new_train_rule_ind = (abs(bundle_train_pred) > theta)
new_test_rule_ind = (abs(bundle_test_pred) > theta)
# calculate W+ and W- for new rule
w_pos = util.element_mult(weights_i, holdout.ind_train_up)
w_neg = util.element_mult(weights_i, holdout.ind_train_down)
w_bundle_pos = util.element_mult(w_pos, new_train_rule_ind)
w_bundle_neg = util.element_mult(w_neg, new_train_rule_ind)
# get score of new rule
rule_bundle_score = 0.5 * np.log(
(w_bundle_pos.sum() + config.TUNING_PARAMS.epsilon) /
(w_bundle_neg.sum() + config.TUNING_PARAMS.epsilon))
return rule_bundle_score, new_train_rule_ind, new_test_rule_ind
def extract_params_as_shared_arrays(link):
assert isinstance(link, chainer.Link)
shared_arrays = {}
for param_name, param in link.namedparams():
shared_arrays[param_name] = mp.RawArray('f', param.array.ravel())
return shared_arrays
def create_sim_data(self):
# Keypress data
keyPressed = 0
self.simData["/control/outputs/keypress/shape"] = []
self.simData["/control/outputs/keypress/dtype"] = np.int
self.simData["/control/outputs/keypress"] = []
for i in range(2):
base_ptr = mp.Value(ctypes.c_uint8, keyPressed)
self.simData["/control/outputs/keypress"].append(base_ptr)
for name in self.script_info:
self.simData["/" + name[0] + "/inputs/keypress"] = 0
self.connectivity_matrix["/" + name[0] + "/inputs/keypress"] = \
"/control/outputs/keypress"
self.simData["/" + name[0] + "/outputs/keypress"] = []
for i in range(2):
base_ptr = mp.RawValue(ctypes.c_uint8, 0)
self.simData["/" + name[0] +
"/outputs/keypress"].append(base_ptr)
# Create simulation data
iteration = 0
self.simData["/simulation/iteration"] = mp.RawValue(
ctypes.c_int, iteration)
longestDelay = 0.0
self.simData["/simulation/longestDelay"] = mp.Value(
ctypes.c_float, longestDelay)
buffer_current = 0
self.simData["/simulation/buffer_current"] = mp.RawValue(
ctypes.c_int, buffer_current)
iterations_per_second = 0.0
self.simData["/simulation/iterations_per_second"] = mp.Value(
ctypes.c_float, iterations_per_second)
if "/simulation/time_step" in self.simData:
self.simData["/simulation/time_step"] = \
mp.RawValue(ctypes.c_double, self.simData["/simulation/time_step"])
else:
time_step = 1.0
self.simData["/simulation/time_step"] = mp.RawValue(
ctypes.c_double, time_step)
self.simData["/adder/outputs/signal"] = []
for i in range(2):
base_ptr = mp.RawValue(ctypes.c_double)
self.simData["/adder/outputs/signal"].append(base_ptr)
self.simData["/low_pass/outputs/signal"] = []
for i in range(2):
base_ptr = mp.RawValue(ctypes.c_double)
self.simData["/low_pass/outputs/signal"].append(base_ptr)
# Create simulation data for every module
for idx, script in enumerate(self.scripts):
name = self.script_info[idx][0]
if "module_configuration" in script.__dict__:
config = script.module_configuration()
if len(self.script_info[idx]) == 3:
#Load scenario file if specified.
scenario_file_name = self.script_info[idx][2]
try:
with open(scenario_file_name, "r") as fp:
scenario = json.load(fp)
for key in scenario:
config[key] = scenario[key]
except IOError:
crit("{} defaults loaded. Unable to load scenario "
"file, {}".format(name, scenario_file_name))
except KeyError as e:
crit("Invalid entry {} in scenario "
"file {}".format(e, scenario_file_name))
exit()
info("{} loaded with {}.".format(name, scenario_file_name))
else:
info("{} defaults loaded.".format(name))
# update all variables from user defined inputs
for key in config:
while (isinstance(config[key], str)) and \
(len(config[key][0]) > 0) and \
(config[key][0] == "/"):
try:
config[key] = self.simData[config[key]]
except KeyError as e:
crit("{}, variable {} does not "
"exist.".format(name, e))
exit()
# Add all entries into simulation dictionary
for key in config:
self.simData["/" + name + key] = config[key]
# Remove all user defined constants from connectivity matrix and update
# modules simData with new value from user defined constants
to_be_remove = []
for key in self.connectivity_matrix:
if (("/" + name + "/")
in key) and ("/user/"
in self.connectivity_matrix[key]):
to_be_remove.append(key)
for user in to_be_remove:
key = self.connectivity_matrix[user]
try:
self.simData[user] = self.simData[key]
config[
"/" +
"/".join(user.split("/")[2:])] = self.simData[key]
key = user
except KeyError as e:
crit("{}, variable {} does not "
"exist.".format("Connectivity Matrix", e))
exit()
while (isinstance(self.simData[key], str)) and \
(len(self.simData[key][0]) > 0) and \
(self.simData[key][0] == "/"):
try:
self.simData[key] = self.simData[self.simData[key]]
config["/" + "/".join(key.split("/")
[2:])] = self.simData[key]
except KeyError as e:
crit("{}, variable {} does not "
"exist.".format(name, e))
exit()
del self.connectivity_matrix[key]
# Assign memory to all outputs
# Currently not support matrix assigments as defaults for inputs. - TODO
outputs_added = []
for key in config:
if "outputs" in key:
output = key.split('/')[2] # ["", "outputs", "output"]
if output not in outputs_added:
outputs_added.append(output)
test = "/outputs/" + output
try:
data_type = config[test + "/dtype"]
shape = config[test + "/shape"]
default = config[test + "/default"]
except KeyError as e:
crit("{:15}, Incomplete output, '{}' "
"does not exist.".format(name, e))
exit()
# Convert data types
try:
dtype, ctype = self.select_data_type(data_type)
except ValueError:
crit("{} invalid data type '{}'".format(
name + test, data_type))
exit()
# Check shape data
if not isinstance(shape, list):
crit("{} invalid shape '{}'".format(
name + test, data_type))
exit()
else:
for d in shape:
if not isinstance(d, int):
crit(
"{} invalid dimension '{}'".format(
name + test, d))
exit()
# Check default value is acceptable, TODO
# Create memory allocation and buffers
self.simData["/" + name + test +
"/shape"] = tuple(shape)
self.simData["/" + name + test + "/dtype"] = dtype
self.simData["/" + name + test] = []
elements = int(np.prod(shape))
if elements > 1:
if data_type == "str":
if not isinstance(defaults, str):
defaults = ""
empty = np.zeros(elements, dtype)
empty[:len(defaults)] = defaults[:]
defaults = empty
elif isinstance(default, str):
try:
defaults = np.loadtxt(defaults,
dtype=dtype)
except:
crit("{} default loading, problem "
"with {}.".format(name, default))
else:
defaults = np.ones(elements,
dtype=dtype) * default
for i in range(2):
base_ptr = mp.RawArray(ctype, elements)
arr = np.frombuffer(base_ptr, dtype)
arr[:] = defaults
self.simData["/" + name +
test].append(base_ptr)
if data_type == "str":
self.simData["/" + name + test +
"/dtype"] = "str"
else:
defaults = default
for i in range(2):
base_ptr = mp.RawValue(ctype, defaults)
self.simData["/" + name +
test].append(base_ptr)
else:
crit("{:20} has no configuration data.".format(name))
# Check connectivity matrix assignments are valid
for key in self.connectivity_matrix:
val = self.connectivity_matrix[key]
if isinstance(
val,
str) and val[0] == '/' and val.split("/")[1] != "control":
test = val.split("/")[1]
valid = False
for idx, script in enumerate(self.scripts):
name = self.script_info[idx][0]
if test == name:
valid = True
break
if valid == False:
crit(
"Connectivity Matrix error: For input {}, {} not a valid input."
.format(key, val))
exit()
def __init__(
self,
width,
height,
pixelFormat=PySpin.PixelFormat_BGR8, #PySpin.PixelFormat_BayerRG8,
offsetX=0,
offsetY=0,
verbose=False,
channels=3, # Number of color channels in images (for example, 1 for grayscale, 3 for RGB, 4 for RGBA)
imageDataType=ctypes.
c_uint8, # ctype of data of each pixel's channel value
imageBitsPerPixel=8, # Number of bits per pixel
outputType='PySpin', # Specify how to return data at receiver endpoint. Options are PySpin, numpy, PIL, rawBuffer
outputCopy=True, # If true, the output data type references a copied buffer, rather than the original. Caution - the original buffer is not synchronized, and may be overwritten at any time!
fileWriter=None, # fileWriter must be either None or a function that takes the specified output type and writes it to a file.
lockForOutput=True, # Should the shared memory buffer be locked during fileWriter call? If outputCopy=False and fileWriter is not None, it is recommended that lockForOutput=True
maxBufferSize=1 # Maximum number of images to allocate. Attempting to allocate more than that will raise an index error
):
self.verbose = verbose
self.maxBufferSize = maxBufferSize
if self.maxBufferSize < 1:
raise ValueError("maxBufferSize must be greater than 1")
self.width = width
self.height = height
self.channels = channels
self.metadataQueue = mp.Queue(maxsize=maxBufferSize)
imageDataSize = self.width * self.height * imageBitsPerPixel * self.channels // 8
if imageDataSize > 8000000000:
print(
"Warning, super big buffer! Do you have enough RAM? Buffer will be {f} frames and {b} GB"
.format(f=maxBufferSize,
b=round(maxBufferSize * imageDataSize / 8000000000,
2)))
self.readLag = mp.Value(ctypes.c_uint32, 0)
self.nextID = 0
self.buffersReady = False
self.buffers = []
self.bufferLocks = []
self.npBuffers = []
for k in range(self.maxBufferSize):
# Create a series of shared memory locations, one for each image buffer slot
self.buffers.append(mp.RawArray(imageDataType, imageDataSize))
# Create synchronization locks to ensure that any given buffer element
# is not being read from and written to at the same time
self.bufferLocks.append(mp.Lock())
self.receiver = SharedImageReceiver(width=self.width,
height=self.height,
channels=self.channels,
pixelFormat=pixelFormat,
verbose=self.verbose,
offsetX=offsetX,
offsetY=offsetY,
buffers=self.buffers,
bufferLocks=self.bufferLocks,
outputType=outputType,
outputCopy=outputCopy,
fileWriter=fileWriter,
imageDataType=imageDataType,
lockForOutput=lockForOutput,
metadataQueue=self.metadataQueue,
readLag=self.readLag)
def _init_par_objs_batchpolopt(self):
"""
Any init_par_objs() method in a derived class must call this method,
and, following that, may append() the SimpleContainer objects as needed.
"""
n = self.n_parallel
self.rank = None
shareds = SimpleContainer(
sum_discounted_return=mp.RawArray('d', n),
num_traj=mp.RawArray('i', n),
sum_return=mp.RawArray('d', n),
max_return=mp.RawArray('d', n),
min_return=mp.RawArray('d', n),
sum_raw_return=mp.RawArray('d', n),
max_raw_return=mp.RawArray('d', n),
min_raw_return=mp.RawArray('d', n),
max_bonus=mp.RawArray('d', n),
min_bonus=mp.RawArray('d', n),
sum_bonus=mp.RawArray('d', n),
sum_path_len=mp.RawArray('i', n),
max_path_len=mp.RawArray('i', n),
min_path_len=mp.RawArray('i', n),
num_steps=mp.RawArray('i', n),
num_valids=mp.RawArray('d', n),
sum_ent=mp.RawArray('d', n),
)
##HT: for explained variance (yeah I know it's clumsy)
shareds.append(baseline_stats=SimpleContainer(
y_sum_vec=mp.RawArray('d', n),
y_square_sum_vec=mp.RawArray('d', n),
y_pred_error_sum_vec=mp.RawArray('d', n),
y_pred_error_square_sum_vec=mp.RawArray('d', n),
))
barriers = SimpleContainer(dgnstc=mp.Barrier(n), )
self._par_objs = (shareds, barriers)
self.baseline.init_par_objs(n_parallel=n)
if self.exemplar is not None:
self.exemplar.init_par_objs(n_parallel=n)
for model in model_list:
model_inst = model()
model_process[i].append(
mp.Process(target=model_inst.run,
args=(model_output_list[i], live_data_list[i],
current_idx_list[i])))
model_process[i][-1].start()
# Run interface with web
api.run()
if __name__ == '__main__':
mp.freeze_support()
repo = DAL._Repository()
# shared objects
manager = mp.Manager()
live_data_list = [mp.RawArray('b', N_max) for _ in range(5)]
current_idx_list = [mp.RawValue('i', 0) for _ in range(5)]
model_output_list = [manager.dict(output_dic_template) for _ in range(5)]
is_recording = [False for _ in range(NUM_CHANNELS)]
#_Listen to the stream of data
data_feed_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM,
socket.IPPROTO_UDP)
data_feed_socket.bind((data_feed_HOST, data_feed_PORT))
# Run
main()
def _makeIndex(self):
'''
Reads the methratio file and creates an hdf5 file of the following structure
file[chrom_dataset][position] = [context_index, strand_index, c, ct, g, ga]
# Attributes
self.methFile (str): reads Methylation input file
self.methBin (str): writes the structured binary output
self.sorted_chroms (dict): reads the chomosomes to initializing h5 file
self.chrom_dict (dict): reads chromosome lengths for initializing h5 file
'''
if not self.n_cores or self.n_cores > 1:
# Open file handles
H5 = h5py.File(self.methBin, 'w')
#IM = open(self.methFile, 'r')
# Create datasets
for chrom in self.sorted_chroms:
chrom_len = self.chrom_dict[chrom]
# [context, strand, c, ct, g, ga]
H5.create_dataset(chrom, (chrom_len, 6), compression='lzf', chunks=True, fillvalue=-1, dtype='i')
H5.close()
NP = min(self.n_cores, mp.cpu_count()) if self.n_cores else mp.cpu_count()
logger.debug("Creating index using %i cores"%(NP))
# Create global variables
global syncArray, chromArray, currentChrom
syncArray = mp.RawArray('B', [0]*NP)
chromArray = mp.RawArray('i', max(self.chrom_dict.values())*6)
np_chromArray = np.frombuffer(chromArray, dtype='i')
np_chromArray[:] = -1
currentChrom = mp.RawArray('c', 30)
currentChrom.value = get_first_chrom(self.methFile).encode('ascii')
# Launch workers
pool = mp.Pool(NP)
pool.map(index_worker, [(self.methFile, self.methBin, self.chrom_dict, i, NP) for i in range(NP)])
pool.close()
pool.join()
# Get an error if the shared mem is not deleted
del syncArray, chromArray, currentChrom
else:
logger.debug("Creating index using 1 core")
# Open file handles
H5 = h5py.File(self.methBin, 'w')
IM = open(self.methFile, 'r')
# Create datasets
for chrom in self.sorted_chroms:
chrom_len = self.chrom_dict[chrom]
# [context, strand, c, ct, g, ga]
H5.create_dataset(chrom, (chrom_len, 6), compression='gzip', compression_opts=6, chunks=True, fillvalue=-1, dtype='i')
# Skip the first line
firstLine = IM.readline()
#chr pos strand context ratio eff_CT_count C_count CT_count rev_G_count rev_GA_count CI_lower CI_upper
#test1 14 - CHG 0.000 1.00 0 1 0 0 0.000 0.793
index_buffer = []
value_buffer = []
start_time, end_time = time.time(), time.time()
current_chrom = False
buffer_size = 1000
for i, line in enumerate(ifilter(lambda x: x[0] != "#", IM)):
tmp = line.split('\t')
chrom, pos, strandIndex = (tmp[0], int(tmp[1])-1, self.strands.index(tmp[2]))
if not current_chrom: current_chrom = chrom
cIndex = self.contexts.index(tmp[3])
c, ct, g, ga = map(int, tmp[6:10])
# Write if chrom changes
if current_chrom != chrom:
H5[current_chrom][index_buffer] = value_buffer
end_time = time.time()
elapsed_time = end_time - start_time
#print("Processed %i records per second"%(len(index_buffer)/elapsed_time))
#print("Finished %s"%(current_chrom))
index_buffer, value_buffer = ([], [])
current_chrom = chrom
index_buffer.append(pos)
value_buffer.append((cIndex, strandIndex, c, ct, g, ga))
start_time = time.time()
elif (i+1)%buffer_size == 0:
index_buffer.append(pos)
value_buffer.append((cIndex, strandIndex, c, ct, g, ga))
H5[current_chrom][index_buffer] = np.array(value_buffer)
#end_time = time.time()
#elapsed_time = end_time - start_time
#print("Processed %i records per second"%(len(index_buffer)/elapsed_time))
index_buffer, value_buffer = ([], [])
start_time = time.time()
else:
index_buffer.append(pos)
value_buffer.append((cIndex, strandIndex, c, ct, g, ga))
if index_buffer:
H5[current_chrom][index_buffer] = value_buffer
index_buffer, value_buffer = ([], [])
IM.close()
H5.close()