def declare(self, name, dtype, shape, unlim=None, addr=-1):
current = self.items.get(name)
if dtype == dict:
if current is not None:
if current.isgroup():
return current
raise KeyError("already a non-group item {}".format(name))
item = PDBGroup(self)
elif dtype == list:
if current is not None:
if current.islist() == 2:
return current
raise KeyError("already a non-list item {}".format(name))
item = QnDList(PDBGroup(self), 1)
else:
if current is not None:
raise KeyError("attempt to redeclare {}".format(name))
if dtype is None and name == '_':
# Assume we are creating a QList.
dtype = npdtype('u1')
elif isinstance(dtype, npdtype) and dtype.kind == 'S':
# frontend never passes 'U' dtype
shape = shape + (dtype.itemsize,)
dtype = npdtype('S1')
item = PDBLeaf(self, addr, dtype, shape, unlim)
if unlim:
item = QnDList(item, None if hasattr(addr, '__iter__') or
addr != -1 else 1)
self.items[name] = item
return item
def make_nd_array(c_pointer,
shape,
dtype=float64,
order='C',
own_data=True):
"""
replacement for: np.ctypeslib.as_array(x)
that doesn't work with python3
http://stackoverflow.com/questions/4355524/getting-data-from-ctypes-array-into-numpy
"""
from numpy import prod, float64, ndarray
from numpy import dtype as npdtype
arr_size = prod(shape[:]) * npdtype(dtype).itemsize
if sys.version_info.major >= 3:
buf_from_mem = ctypes.pythonapi.PyMemoryView_FromMemory
buf_from_mem.restype = ctypes.py_object
buf_from_mem.argtypes = (ctypes.c_void_p, ctypes.c_int,
ctypes.c_int)
buffer = buf_from_mem(c_pointer, arr_size, 0x100)
else:
buf_from_mem = ctypes.pythonapi.PyBuffer_FromMemory
buf_from_mem.restype = ctypes.py_object
buffer = buf_from_mem(c_pointer, arr_size)
arr = ndarray(tuple(shape[:]), dtype, buffer, order=order)
if own_data and not arr.flags.owndata:
return arr.copy()
else:
return arr
def __decode_array(fp_read, no_bytes, object_hook, object_pairs_hook, intern_object_keys, islittle):
marker, counting, count, type_, dims = __get_container_params(fp_read, False, no_bytes, object_hook, object_pairs_hook, intern_object_keys, islittle)
# special case - no data (None or bool)
if type_ in __TYPES_NO_DATA:
return [__METHOD_MAP[type_](fp_read, type_, islittle)] * count
# special case - bytes array
if type_ == TYPE_UINT8 and not no_bytes and len(dims)==0:
container = fp_read(count)
if len(container) < count:
raise DecoderException('Container bytes array too short')
return container
container = []
while count > 0 and (counting or marker != ARRAY_END):
if marker == TYPE_NOOP:
marker = fp_read(1)
continue
# decode value
try:
value = __METHOD_MAP[marker](fp_read, marker, islittle)
except KeyError:
handled = False
else:
handled = True
# handle outside above except (on KeyError) so do not have unfriendly "exception within except" backtrace
if not handled:
if marker == ARRAY_START:
value = __decode_array(fp_read, no_bytes, object_hook, object_pairs_hook, intern_object_keys, islittle)
elif marker == OBJECT_START:
value = __decode_object(fp_read, no_bytes, object_hook, object_pairs_hook, intern_object_keys, islittle)
else:
raise DecoderException('Invalid marker within array')
container.append(value)
if counting:
count -= 1
if count and type_ == TYPE_NONE:
marker = fp_read(1)
if len(dims)>0:
container=list(reduce(lambda x, y: map(list, zip(*y*(x,))), (iter(container), ) +tuple(dims[:0:-1])))
container=ndarray(container, dtype=npdtype(__DTYPE_MAP[type_]))
return container
def declare(self, name, dtype, shape, unlim=None):
h5item = self.h5item
if dtype == dict:
return H5Group(h5item.create_group(name))
if dtype == list:
return QnDList(H5Group(h5item.create_group(name)), 1)
if dtype is None or (shape and not all(shape)):
h5item[name] = npdtype('u1') if dtype is None else dtype
item = h5item[name]
if dtype is None:
item.attrs['__none__'] = True
else:
item.attrs['__shape__'] = shape
elif unlim:
item = h5item.create_dataset(name, (1,)+shape, dtype=dtype,
maxshape=(None,)+shape)
else:
item = h5item.create_dataset(name, shape, dtype=dtype)
item = H5Leaf(item, h5item)
return QnDList(item, 1) if unlim else item
def make_nd_array(c_pointer, shape, dtype=float64, order='C', own_data=True):
"""
replacement for: np.ctypeslib.as_array(x)
that doesn't work with python3
http://stackoverflow.com/questions/4355524/getting-data-from-ctypes-array-into-numpy
"""
from numpy import prod,float64,ndarray
from numpy import dtype as npdtype
arr_size = prod(shape[:]) * npdtype(dtype).itemsize
if sys.version_info.major >= 3:
buf_from_mem = ctypes.pythonapi.PyMemoryView_FromMemory
buf_from_mem.restype = ctypes.py_object
buf_from_mem.argtypes = (ctypes.c_void_p, ctypes.c_int, ctypes.c_int)
buffer = buf_from_mem(c_pointer, arr_size, 0x100)
else:
buf_from_mem = ctypes.pythonapi.PyBuffer_FromMemory
buf_from_mem.restype = ctypes.py_object
buffer = buf_from_mem(c_pointer, arr_size)
arr = ndarray(tuple(shape[:]), dtype, buffer, order=order)
if own_data and not arr.flags.owndata:
return arr.copy()
else:
return arr
'0')) # allows config info to be debugged
read_config([DEFAULT_CONFIG_FILE, USER_CONFIG_FILE, USER_ENV_CONFIG_FILE])
# verbosity level from environment has priority, otherwise use config, which
# defaults to 0. Note that we looked at the env var previously to allow debug of config
VERBOSE = int(
os.getenv('PYFUSION_VERBOSE',
config.get('global', 'VERBOSE', vars={'VERBOSE': '0'})))
## Variable precision - allows data sets to be much bigger - up to 4x bdb Dec-2012
root_dir = os.path.split(os.path.abspath(__file__))[0]
try:
from numpy import dtype as npdtype
# Beware! vars takes precedence over others!
prec_med = npdtype(config.get(
'global', 'precision_medium')) #,vars={'precision_medium':'float32'}))
except:
print('defaulting medium precision value, either because of error or'\
' missing value precision_medium in globals')
from numpy import dtype as npdtype
prec_med = npdtype('float32')
# warning - may need to move this earlier - for some reason the fftw3 import
# doesn't work if it is ahead of this one.
try:
COLORS = config.get('global', 'colors')
except:
if VERBOSE > 0: print('colors not in config file - no color!')
COLORS = None
try:
NSAMPLES = int(os.getenv('PYFUSION_NSAMPLES',
config.get('global','NSAMPLES',vars={'NSAMPLES':'0'})))
root_dir = os.path.split(os.path.abspath( __file__ ))[0]
# cache control is likely to be computer dependent
try:
TMP_FREE_BYTES = float(config.get('global','tmp_free_bytes'))
except:
TMP_FREE_BYTES = 1e9
print('defaulting [global]temp_free_bytes to {t:,}'.format(t=TMP_FREE_BYTES))
try:
from numpy import dtype as npdtype
# Beware! vars takes precedence over others!
prec_med=npdtype(config.get('global','precision_medium')) #,vars={'precision_medium':'float32'}))
except:
print('defaulting medium precision value, either because of error or'\
' missing value precision_medium in globals')
from numpy import dtype as npdtype
prec_med=npdtype('float32')
# warning - may need to move this earlier - for some reason the fftw3 import
# doesn't work if it is ahead of this one.
try:
COLORS=config.get('global','colors')
except:
if VERBOSE>0: print('colors not in config file - no color!')
COLORS=None
#