def create_shm_dict(keys, shape, dtype=np.float64):
'''
Create dict of shm buffers
'''
buffers = o()
shapes = o()
for k in keys:
buffers[k] = init_shm_ndarray(shape, dtype)
shapes[k] = shape
return o(buffers=buffers, shape=shapes)
def create_shm_dict_inputs(shape_map, dtype=np.float64):
'''
Create dict of shm buffers
'''
buffers = o()
shapes = o()
for k in shape_map:
if shape_map[k] is None or len(shape_map[k]) == 0:
buffers[k] = init_shm_ndarray((1, ), dtype)
else:
buffers[k] = init_shm_ndarray(shape_map[k], dtype)
shapes[k] = shape_map[k]
return o(buffers=buffers, shapes=shapes)
def _update_dicts(self):
self.variables = o()
self.groups = o()
self.attrs = o()
for v in self.ncd_group.variables:
self.variables[v] = self.ncd_group.variables[v]
if hasattr(self.ncd_group, 'var_name'):
self.awra_var = self.ncd_group.variables[self.ncd_group.var_name]
else:
self.awra_var = self._imply_variable()
for a in self.ncd_group.ncattrs():
self.attrs[a] = self.ncd_group.getncattr(a)
for g in self.ncd_group.groups:
self.groups[g] = DatasetManager(self.ncd_group.groups[g])
def __init__(self, ps):
self.key = ps[0]
self.function = o(fn=ps[0][0], line=ps[0][1], func_name=ps[0][2])
stats = ps[1]
self.ncalls = stats[0]
self.tot_time = stats[2]
self.cum_time = stats[3]
self.callers = stats[4]
def shm_to_nd_dict(buffers, shape=None, order='C'):
'''
Convert a dictionary of shared memory arrays into process-local numpy arrays
Shape can be a single common shape, or a dictionary of shapes
'''
nd_dict = o()
to_shape = shape
for k, v in list(buffers.items()):
if hasattr(shape, 'items'):
to_shape = shape[k]
nd_dict[k] = shm_as_ndarray(v, to_shape, order)
return nd_dict
def shm_to_nd_dict_inputs(buffers, shapes, order='C'):
'''
Convert a dictionary of shared memory arrays into process-local numpy arrays
Shape can be a single common shape, or a dictionary of shapes
'''
nd_dict = o()
for k, v in list(buffers.items()):
if shapes[k] is None or len(shapes[k]) == 0:
nd_dict[k] = shm_as_ndarray(v, (1, ), order) #np.asscalar(a)
else:
nd_dict[k] = shm_as_ndarray(v, shapes[k], order)
return nd_dict
def split_data(self, data, indices):
'''
For a given block of data whose shape matches the supplied
indices, return a list of (segment,index,data) dicts, where index
is localised to the segment
'''
if isinstance(indices, Coordinates):
indices = self.coordinates.get_index(indices)
seg_indices = self.split_indices(indices)
out_data = []
for seg_i in seg_indices:
seg_data = data[simplify_indices(seg_i.global_index)]
out_split = o(segment=seg_i.segment,
index=seg_i.local_index,
data=seg_data)
out_data.append(out_split)
return out_data
def split_indices(self, indices):
'''
For a given set of (global) indices, return a list of
(segment,local_index,global_index) dicts, where local index
maps to the segment, and global_index to a block of data
the shape of indices, but normalised to start at 0
'''
if type(indices) in [slice, int]:
indices = [indices]
dim_idx = indices[self.dim_pos]
if type(dim_idx) == int:
_, seg = self._locate_segment(dim_idx)
out_index = self._substitute_index(indices, dim_idx - seg.start)
return [
o(segment=seg, local_index=out_index, global_index=indices)
]
if type(dim_idx) == slice:
if dim_idx.step != None:
raise Exception("Stepped slices unsupported")
out_indices = []
idx_start = dim_idx.start if dim_idx.start != None else 0
idx_stop = (dim_idx.stop -
1) if dim_idx.stop not in [PY_INTMAX, None
] else self.dim_len - 1
cur_i, cur_seg = self._locate_segment(idx_start)
cur_start = idx_start
cur_idx = cur_start + 1
while cur_idx <= idx_stop:
if cur_idx > cur_seg.end:
local_slice = self._substitute_index(
indices,
slice(cur_start - cur_seg.start,
cur_idx - cur_seg.start))
g_i = self._normalize_index(indices)
global_slice = self._substitute_index(
g_i, slice(cur_start - idx_start, cur_idx - idx_start))
out_indices.append(
o(segment=cur_seg,
local_index=local_slice,
global_index=global_slice))
cur_i += 1
cur_seg = self.segments[cur_i]
cur_start = cur_idx
cur_idx += 1
local_slice = self._substitute_index(
indices,
slice(cur_start - cur_seg.start,
cur_idx - cur_seg.start)) #-1 ?
g_i = self._normalize_index(indices)
global_slice = self._substitute_index(
g_i, slice(cur_start - idx_start, cur_idx - idx_start)) # -1?
out_indices.append(
o(segment=cur_seg,
local_index=local_slice,
global_index=global_slice))
return out_indices