def create_shm_dict(keys, shape, dtype=np.float64): ''' Create dict of shm buffers ''' buffers = o() shapes = o() for k in keys: buffers[k] = init_shm_ndarray(shape, dtype) shapes[k] = shape return o(buffers=buffers, shape=shapes)
def create_shm_dict_inputs(shape_map, dtype=np.float64): ''' Create dict of shm buffers ''' buffers = o() shapes = o() for k in shape_map: if shape_map[k] is None or len(shape_map[k]) == 0: buffers[k] = init_shm_ndarray((1, ), dtype) else: buffers[k] = init_shm_ndarray(shape_map[k], dtype) shapes[k] = shape_map[k] return o(buffers=buffers, shapes=shapes)
def _update_dicts(self): self.variables = o() self.groups = o() self.attrs = o() for v in self.ncd_group.variables: self.variables[v] = self.ncd_group.variables[v] if hasattr(self.ncd_group, 'var_name'): self.awra_var = self.ncd_group.variables[self.ncd_group.var_name] else: self.awra_var = self._imply_variable() for a in self.ncd_group.ncattrs(): self.attrs[a] = self.ncd_group.getncattr(a) for g in self.ncd_group.groups: self.groups[g] = DatasetManager(self.ncd_group.groups[g])
def __init__(self, ps): self.key = ps[0] self.function = o(fn=ps[0][0], line=ps[0][1], func_name=ps[0][2]) stats = ps[1] self.ncalls = stats[0] self.tot_time = stats[2] self.cum_time = stats[3] self.callers = stats[4]
def shm_to_nd_dict(buffers, shape=None, order='C'): ''' Convert a dictionary of shared memory arrays into process-local numpy arrays Shape can be a single common shape, or a dictionary of shapes ''' nd_dict = o() to_shape = shape for k, v in list(buffers.items()): if hasattr(shape, 'items'): to_shape = shape[k] nd_dict[k] = shm_as_ndarray(v, to_shape, order) return nd_dict
def shm_to_nd_dict_inputs(buffers, shapes, order='C'): ''' Convert a dictionary of shared memory arrays into process-local numpy arrays Shape can be a single common shape, or a dictionary of shapes ''' nd_dict = o() for k, v in list(buffers.items()): if shapes[k] is None or len(shapes[k]) == 0: nd_dict[k] = shm_as_ndarray(v, (1, ), order) #np.asscalar(a) else: nd_dict[k] = shm_as_ndarray(v, shapes[k], order) return nd_dict
def split_data(self, data, indices): ''' For a given block of data whose shape matches the supplied indices, return a list of (segment,index,data) dicts, where index is localised to the segment ''' if isinstance(indices, Coordinates): indices = self.coordinates.get_index(indices) seg_indices = self.split_indices(indices) out_data = [] for seg_i in seg_indices: seg_data = data[simplify_indices(seg_i.global_index)] out_split = o(segment=seg_i.segment, index=seg_i.local_index, data=seg_data) out_data.append(out_split) return out_data
def split_indices(self, indices): ''' For a given set of (global) indices, return a list of (segment,local_index,global_index) dicts, where local index maps to the segment, and global_index to a block of data the shape of indices, but normalised to start at 0 ''' if type(indices) in [slice, int]: indices = [indices] dim_idx = indices[self.dim_pos] if type(dim_idx) == int: _, seg = self._locate_segment(dim_idx) out_index = self._substitute_index(indices, dim_idx - seg.start) return [ o(segment=seg, local_index=out_index, global_index=indices) ] if type(dim_idx) == slice: if dim_idx.step != None: raise Exception("Stepped slices unsupported") out_indices = [] idx_start = dim_idx.start if dim_idx.start != None else 0 idx_stop = (dim_idx.stop - 1) if dim_idx.stop not in [PY_INTMAX, None ] else self.dim_len - 1 cur_i, cur_seg = self._locate_segment(idx_start) cur_start = idx_start cur_idx = cur_start + 1 while cur_idx <= idx_stop: if cur_idx > cur_seg.end: local_slice = self._substitute_index( indices, slice(cur_start - cur_seg.start, cur_idx - cur_seg.start)) g_i = self._normalize_index(indices) global_slice = self._substitute_index( g_i, slice(cur_start - idx_start, cur_idx - idx_start)) out_indices.append( o(segment=cur_seg, local_index=local_slice, global_index=global_slice)) cur_i += 1 cur_seg = self.segments[cur_i] cur_start = cur_idx cur_idx += 1 local_slice = self._substitute_index( indices, slice(cur_start - cur_seg.start, cur_idx - cur_seg.start)) #-1 ? g_i = self._normalize_index(indices) global_slice = self._substitute_index( g_i, slice(cur_start - idx_start, cur_idx - idx_start)) # -1? out_indices.append( o(segment=cur_seg, local_index=local_slice, global_index=global_slice)) return out_indices