class Operation(object):
default_backends = ["numexpr", "cython", "numpy"]
_imps = None # implementations
opname = None
short_name = None
def __init__(self, name, short_name, implementations=(), doc=""):
self.opname = name
self.short_name = short_name
self._imps = OrderedDict()
self.add_implementations(implementations)
setattr(self, "__doc__", doc)
@property
def __name__(self):
return self.opname
def add_implementation(self, name, func):
self._imps[name] = func
def add_implementations(self, implementations):
for name, func in implementations:
self.add_implementation(name, func)
def _get_imp(self, preferred, only=False):
if not isinstance(preferred, (list, tuple)):
if preferred is None:
preferred = list(self._imps.keys())
else:
preferred = [preferred]
for name in preferred:
if name in self._imps:
return self._imps[name]
msg = "{0} :: {1}".format(self.opname, preferred)
if only:
raise verror.BackendNotFound(msg)
logger.info("No preferred backends available: " + msg)
for name in self.default_backends:
if name in self._imps:
return self._imps[name]
if len(self._imps) == 0:
raise verror.BackendNotFound("No backends available")
return list(self._imps.values())[0]
def __call__(self, *args, **kwargs):
preferred = kwargs.pop("preferred", None)
only = kwargs.pop("only", False)
func = self._get_imp(preferred, only)
return func(*args, **kwargs)
class Operation(object):
default_backends = ["numexpr", "cython", "numpy"]
_imps = None # implementations
opname = None
short_name = None
def __init__(self, name, short_name, implementations=(), doc=""):
self.opname = name
self.short_name = short_name
self._imps = OrderedDict()
self.add_implementations(implementations)
setattr(self, "__doc__", doc)
@property
def __name__(self):
return self.opname
def add_implementation(self, name, func):
self._imps[name] = func
def add_implementations(self, implementations):
for name, func in implementations:
self.add_implementation(name, func)
def _get_imp(self, preferred, only=False):
if not isinstance(preferred, (list, tuple)):
if preferred is None:
preferred = list(self._imps.keys())
else:
preferred = [preferred]
for name in preferred:
if name in self._imps:
return self._imps[name]
msg = "{0} :: {1}".format(self.opname, preferred)
if only:
raise verror.BackendNotFound(msg)
logger.info("No preferred backends available: " + msg)
for name in self.default_backends:
if name in self._imps:
return self._imps[name]
if len(self._imps) == 0:
raise verror.BackendNotFound("No backends available")
return list(self._imps.values())[0]
def __call__(self, *args, **kwargs):
preferred = kwargs.pop("preferred", None)
only = kwargs.pop("only", False)
func = self._get_imp(preferred, only)
return func(*args, **kwargs)
def save_fields(cls, fname, flds, **kwargs):
assert len(flds) > 0
fname = os.path.expanduser(os.path.expandvars(fname))
if isinstance(flds, list):
if isinstance(flds[0], (list, tuple)):
flds = OrderedDict(flds)
else:
flds = OrderedDict([(fld.name, fld) for fld in flds])
fld_dict = {}
# setup crds
# FIXME: all coordinates are saved as non-uniform, the proper
# way to do this is to have let coordinate format its own
# hdf5 / xdmf / numpy binary output
fld0 = next(iter(flds.values()))
clist = fld0.crds.get_clist(full_arrays=True)
axis_names = []
for axis_name, crdarr in clist:
fld_dict[axis_name] = crdarr
axis_names.append(axis_name)
fld_dict[cls._KEY_CRDS] = np.array(axis_names)
# setup fields
# dict comprehension invalid in Python 2.6
# fld_names = {key.lower(): [] for key in cls._KEY_FLDS.keys()}
fld_names = {}
for key in cls._KEY_FLDS.keys():
fld_names[key.lower()] = []
for name, fld in flds.items():
fld_names[fld.center.lower()].append(name)
fld_dict[name] = fld.data
for center, names_lst in fld_names.items():
fld_dict[cls._KEY_FLDS[center.lower()]] = np.array(names_lst)
if fname.endswith(".npz"):
fname = fname[:-4]
np.savez(fname, **fld_dict)
def save_fields(cls, fname, flds, complevel=0, compression='gzip',
compression_opts=None, **kwargs):
""" save some fields using the format given by the class """
# FIXME: this is only good for writing cartesian rectilnear flds
# FIXME: axes are renamed if flds[0] is 1D or 2D
assert len(flds) > 0
fname = os.path.expanduser(os.path.expandvars(fname))
if complevel and compression == 'gzip' and compression_opts is None:
compression_opts = complevel
# TODO: what if compression != 'gzip'
do_compression = compression_opts is not None
if isinstance(flds, list):
if isinstance(flds[0], (list, tuple)):
flds = OrderedDict(flds)
else:
flds = OrderedDict([(fld.name, fld) for fld in flds])
# FIXME: all coordinates are saved as non-uniform, the proper
# way to do this is to have let coordinate format its own
# hdf5 / xdmf / numpy binary output
fld0 = next(iter(flds.values()))
clist = fld0.crds.get_clist(full_arrays=True)
crd_arrs = [np.array([0.0])] * 3
crd_names = ["x", "y", "z"]
for i, c in enumerate(clist):
crd_arrs[i] = c[1]
crd_shape = [len(arr) for arr in crd_arrs]
time = fld0.time
# write arrays to the hdf5 file
with h5py.File(fname, 'w') as f:
for axis_name, arr in zip(crd_names, crd_arrs):
loc = cls._CRDS_GROUP + '/' + axis_name
if do_compression:
f.create_dataset(loc, data=arr, compression=compression,
compression_opts=compression_opts)
else:
f[loc] = arr
for name, fld in flds.items():
loc = cls._FLD_GROUPS[fld.center.lower()] + '/' + name
# xdmf files use kji ordering
if do_compression:
f.create_dataset(loc, data=fld.data.T, compression=compression,
compression_opts=compression_opts)
else:
f[loc] = fld.data.T
# big bad openggcm time_str hack to put basetime into hdf5 file
for fld in flds.values():
try:
tfmt = "%Y:%m:%d:%H:%M:%S.%f"
sec_td = viscid.as_timedelta64(fld.time, 's')
dtime = viscid.as_datetime(fld.basetime + sec_td).strftime(tfmt)
epoch = viscid.readers.openggcm.GGCM_EPOCH
ts = viscid.as_timedelta(fld.basetime - epoch).total_seconds()
ts += fld.time
timestr = "time= {0} {1:.16e} {2} 300c".format(fld.time, ts, dtime)
f.create_group('openggcm')
f['openggcm'].attrs['time_str'] = np.string_(timestr)
break
except viscid.NoBasetimeError:
pass
# now write an xdmf file
xdmf_fname = os.path.splitext(fname)[0] + ".xdmf"
relh5fname = "./" + os.path.basename(fname)
with open(xdmf_fname, 'w') as f:
xloc = cls._CRDS_GROUP + '/' + crd_names[0]
yloc = cls._CRDS_GROUP + '/' + crd_names[1]
zloc = cls._CRDS_GROUP + '/' + crd_names[2]
dim_str = " ".join([str(l) for l in crd_shape][::-1])
f.write(cls._XDMF_TEMPLATE_BEGIN.format(time=time))
s = cls._XDMF_TEMPLATE_RECTILINEAR_GRID_BEGIN.format(
grid_name="vgrid", crd_dims=dim_str, h5fname=relh5fname,
xdim=crd_shape[0], ydim=crd_shape[1], zdim=crd_shape[2],
xloc=xloc, yloc=yloc, zloc=zloc)
f.write(s)
for fld in flds.values():
_crd_system = viscid.as_crd_system(fld, None)
if _crd_system:
f.write(cls._XDMF_INFO_TEMPLATE.format(name="crd_system",
value=_crd_system))
break
for name, fld in flds.items():
fld = fld.as_flat().T
dt = fld.dtype.name.rstrip("0123456789").title()
precision = fld.dtype.itemsize
fld_dim_str = " ".join([str(l) for l in fld.shape])
loc = cls._FLD_GROUPS[fld.center.lower()] + '/' + name
s = cls._XDMF_TEMPLATE_ATTRIBUTE.format(
fld_name=name,
fld_type=fld.fldtype, center=fld.center.title(),
dtype=dt, precision=precision, fld_dims=fld_dim_str,
h5fname=relh5fname, fld_loc=loc)
f.write(s)
f.write(cls._XDMF_TEMPLATE_GRID_END)
f.write(cls._XDMF_TEMPLATE_END)
class Bucket(object):
""" This is basically a glorified dict
It's a convenient dict-like object if you want lots of
keys for a given value.
NOTE:
You can add non-hashable items, but this is poorly tested.
When adding / removing non-hashable items (items, not handles)
the comparison is done using the object's id. This is
fundamentally different than using an object's __hash__, but
it should be fairly transparent.
"""
_ordered = False
_ref_count = None # keys are hashable items, values are # of times item was added
_hash_lookup = None # keys are hashable items, values are actual items
_handles = None # keys are hashable items, values are list of handles
_items = None # keys are handles, values are actual items
# if index handle, set_item adds this number as a handle and increments it
# this is useful for hiding loads that are not user initiated, such as
# an xdmf file loading an h5 file under the covers
_int_counter = None
def __init__(self, ordered=False):
self._ordered = ordered
self._set_empty_dicts()
self._int_counter = 0
def _set_empty_dicts(self):
if self._ordered:
self._ref_count = OrderedDict()
self._hash_lookup = OrderedDict()
self._handles = OrderedDict()
self._items = OrderedDict()
else:
self._ref_count = {}
self._hash_lookup = {}
self._handles = {}
self._items = {}
@staticmethod
def _make_hashable(item):
try:
hash(item)
return item
except TypeError:
return "<{0} @ {1}>".format(type(item), hex(id(item)))
def items(self):
for hashable_item, item in self._hash_lookup.items():
yield self._handles[hashable_item], item
def keys(self):
return self._handles.values()
def values(self):
return self._hash_lookup.values()
def set_item(self, handles, item, index_handle=True, _add_ref=False):
""" if index_handle is true then the index of item will be included as
a handle making the bucket indexable like a list """
# found = False
if handles is None:
handles = []
if not isinstance(handles, list):
raise TypeError("handle must by of list type")
# make sure we have a hashable "item" for doing reverse
# lookups of handles using an item
hashable_item = self._make_hashable(item)
if hashable_item not in self._hash_lookup:
if index_handle:
handles += [self._int_counter]
self._int_counter += 1
handles_added = []
for h in handles:
# check if we're stealing a handle from another item
try:
hash(h)
except TypeError:
logger.error("A bucket says handle '{0}' is not hashable, "
"ignoring it".format(h))
continue
if (h in self._items) and (item is self._items[h]):
continue
elif h in self._items:
logger.error("The handle '{0}' is being hijacked! Memory leak "
"could ensue.".format(h))
# romove handle from old item, since this check is here,
# there sholdn't be 2 items with the same handle in the
# items dict
old_item = self._items[h]
old_hashable_item = self._make_hashable(old_item)
self._handles[old_hashable_item].remove(h)
if len(self._handles[old_hashable_item]) == 0:
self.remove_item(old_item)
self._items[h] = item
handles_added.append(h)
try:
self._handles[hashable_item] += handles_added
if _add_ref:
self._ref_count[hashable_item] += 1
except KeyError:
if len(handles_added) == 0:
logger.error("No valid handles given, item '{0}' not added to "
"bucket".format(hashable_item))
else:
self._handles[hashable_item] = handles_added
self._hash_lookup[hashable_item] = item
self._ref_count[hashable_item] = 1
return None
def _remove_item(self, item):
"""remove item no matter what
You may want to use remove_
, raises ValueError if item is not found """
hashable_item = self._make_hashable(item)
handles = self._handles[hashable_item]
for h in handles:
del self._items[h]
del self._hash_lookup[hashable_item]
del self._handles[hashable_item]
del self._ref_count[hashable_item]
def _remove_item_by_handle(self, handle):
self._remove_item(self._items[handle])
def remove_item(self, item):
self._remove_item(item)
def remove_item_by_handle(self, handle):
""" remove item by handle, raises KeyError if handle is not found """
self.remove_item(self._items[handle])
def remove_reference(self, item, _ref_count=1):
hashable_item = self._make_hashable(item)
try:
self._ref_count[hashable_item] -= _ref_count
except KeyError:
item = self[item]
hashable_item = self._make_hashable(item)
if _ref_count:
self._ref_count[hashable_item] -= _ref_count
else:
self._ref_count[hashable_item] = 0
# FIXME: unload_all_files breaks this assert check... probably a bug
# assert self._ref_count[hashable_item] >= 0, \
# "problem with bucket ref counting {0}".format(hashable_item)
if self._ref_count[hashable_item] <= 0:
self._remove_item(item)
def remove_all_items(self):
""" unload all items """
self._set_empty_dicts()
def items_as_list(self):
return list(self._hash_lookup.values())
def get_primary_handles(self):
"""Return a list of the first handles for all items"""
return [handles[0] for handles in self._handles.values()]
def handle_string(self, prefix=""):
""" return string representation of handles and items """
# this is inefficient, but probably doesn't matter
s = ""
for item, handles in self._handles.items():
hands = [repr(h) for h in handles]
s += "{0}handles: {1}\n".format(prefix, ", ".join(hands))
s += "{0} item: {1}\n".format(prefix, str(item))
return s
def print_tree(self, prefix=""):
print(self.handle_string(prefix=prefix), end='')
def __getitem__(self, handle):
return self._items[handle]
def __setitem__(self, key, value):
if isinstance(key, (list, tuple)):
key = list(key)
elif key is not None:
key = [key]
self.set_item(key, value)
def __delitem__(self, handle):
try:
self.remove_item_by_handle(handle)
except (KeyError, TypeError):
# maybe we are asking to remove an item explicitly
self.remove_item(handle)
def __iter__(self):
return self.values().__iter__()
def contains_item(self, item):
hashable_item = self._make_hashable(item)
return hashable_item in self._handles
def contains_handle(self, handle):
try:
return handle in self._items
except TypeError:
return False
def __contains__(self, handle):
return self.contains_handle(handle) or self.contains_item(handle)
def __len__(self):
return len(self._hash_lookup)
def __str__(self):
return self.handle_string()
class Bucket(object):
""" This is basically a glorified dict
It's a convenient dict-like object if you want lots of
keys for a given value.
NOTE:
You can add non-hashable items, but this is poorly tested.
When adding / removing non-hashable items (items, not handles)
the comparison is done using the object's id. This is
fundamentally different than using an object's __hash__, but
it should be fairly transparent.
"""
_ordered = False
_ref_count = None # keys are hashable items, values are # of times item was added
_hash_lookup = None # keys are hashable items, values are actual items
_handles = None # keys are hashable items, values are list of handles
_items = None # keys are handles, values are actual items
# if index handle, set_item adds this number as a handle and increments it
# this is useful for hiding loads that are not user initiated, such as
# an xdmf file loading an h5 file under the covers
_int_counter = None
def __init__(self, ordered=False):
self._ordered = ordered
self._set_empty_dicts()
self._int_counter = 0
def _set_empty_dicts(self):
if self._ordered:
self._ref_count = OrderedDict()
self._hash_lookup = OrderedDict()
self._handles = OrderedDict()
self._items = OrderedDict()
else:
self._ref_count = {}
self._hash_lookup = {}
self._handles = {}
self._items = {}
@staticmethod
def _make_hashable(item):
try:
hash(item)
return item
except TypeError:
return "<{0} @ {1}>".format(type(item), hex(id(item)))
def items(self):
for hashable_item, item in self._hash_lookup.items():
yield self._handles[hashable_item], item
def keys(self):
return self._handles.values()
def values(self):
return self._hash_lookup.values()
def set_item(self, handles, item, index_handle=True, _add_ref=False):
""" if index_handle is true then the index of item will be included as
a handle making the bucket indexable like a list """
# found = False
if handles is None:
handles = []
if not isinstance(handles, list):
raise TypeError("handle must by of list type")
# make sure we have a hashable "item" for doing reverse
# lookups of handles using an item
hashable_item = self._make_hashable(item)
if hashable_item not in self._hash_lookup:
if index_handle:
handles += [self._int_counter]
self._int_counter += 1
handles_added = []
for h in handles:
# check if we're stealing a handle from another item
try:
hash(h)
except TypeError:
logger.error("A bucket says handle '{0}' is not hashable, "
"ignoring it".format(h))
continue
if (h in self._items) and (item is self._items[h]):
continue
elif h in self._items:
logger.error("The handle '{0}' is being hijacked! Memory leak "
"could ensue.".format(h))
# romove handle from old item, since this check is here,
# there sholdn't be 2 items with the same handle in the
# items dict
old_item = self._items[h]
old_hashable_item = self._make_hashable(old_item)
self._handles[old_hashable_item].remove(h)
if len(self._handles[old_hashable_item]) == 0:
self.remove_item(old_item)
self._items[h] = item
handles_added.append(h)
try:
self._handles[hashable_item] += handles_added
if _add_ref:
self._ref_count[hashable_item] += 1
except KeyError:
if len(handles_added) == 0:
logger.error("No valid handles given, item '{0}' not added to "
"bucket".format(hashable_item))
else:
self._handles[hashable_item] = handles_added
self._hash_lookup[hashable_item] = item
self._ref_count[hashable_item] = 1
return None
def _remove_item(self, item):
"""remove item no matter what
You may want to use remove_
, raises ValueError if item is not found """
hashable_item = self._make_hashable(item)
handles = self._handles[hashable_item]
for h in handles:
del self._items[h]
del self._hash_lookup[hashable_item]
del self._handles[hashable_item]
del self._ref_count[hashable_item]
def _remove_item_by_handle(self, handle):
self._remove_item(self._items[handle])
def remove_item(self, item):
self._remove_item(item)
def remove_item_by_handle(self, handle):
""" remove item by handle, raises KeyError if handle is not found """
self.remove_item(self._items[handle])
def remove_reference(self, item, _ref_count=1):
hashable_item = self._make_hashable(item)
try:
self._ref_count[hashable_item] -= _ref_count
except KeyError:
item = self[item]
hashable_item = self._make_hashable(item)
if _ref_count:
self._ref_count[hashable_item] -= _ref_count
else:
self._ref_count[hashable_item] = 0
# FIXME: unload_all_files breaks this assert check... probably a bug
# assert self._ref_count[hashable_item] >= 0, \
# "problem with bucket ref counting {0}".format(hashable_item)
if self._ref_count[hashable_item] <= 0:
self._remove_item(item)
def remove_all_items(self):
""" unload all items """
self._set_empty_dicts()
def items_as_list(self):
return list(self._hash_lookup.values())
def get_primary_handles(self):
"""Return a list of the first handles for all items"""
return [handles[0] for handles in self._handles.values()]
def handle_string(self, prefix=""):
""" return string representation of handles and items """
# this is inefficient, but probably doesn't matter
s = ""
for item, handles in self._handles.items():
hands = [repr(h) for h in handles]
s += "{0}handles: {1}\n".format(prefix, ", ".join(hands))
s += "{0} item: {1}\n".format(prefix, str(item))
return s
def print_tree(self, prefix=""):
print(self.handle_string(prefix=prefix), end='')
def __getitem__(self, handle):
return self._items[handle]
def __setitem__(self, key, value):
if isinstance(key, (list, tuple)):
key = list(key)
elif key is not None:
key = [key]
self.set_item(key, value)
def __delitem__(self, handle):
try:
self.remove_item_by_handle(handle)
except (KeyError, TypeError):
# maybe we are asking to remove an item explicitly
self.remove_item(handle)
def __iter__(self):
return self.values().__iter__()
def contains_item(self, item):
hashable_item = self._make_hashable(item)
return hashable_item in self._handles
def contains_handle(self, handle):
try:
return handle in self._items
except TypeError:
return False
def __contains__(self, handle):
return self.contains_handle(handle) or self.contains_item(handle)
def __len__(self):
return len(self._hash_lookup)
def __str__(self):
return self.handle_string()
def save_fields(cls, fname, flds, **kwargs):
""" save some fields using the format given by the class """
# FIXME: this is only good for writing cartesian rectilnear flds
# FIXME: axes are renamed if flds[0] is 1D or 2D
assert len(flds) > 0
fname = os.path.expanduser(os.path.expandvars(fname))
if isinstance(flds, list):
if isinstance(flds[0], (list, tuple)):
flds = OrderedDict(flds)
else:
flds = OrderedDict([(fld.name, fld) for fld in flds])
# FIXME: all coordinates are saved as non-uniform, the proper
# way to do this is to have let coordinate format its own
# hdf5 / xdmf / numpy binary output
fld0 = next(iter(flds.values()))
clist = fld0.crds.get_clist(full_arrays=True)
crd_arrs = [np.array([0.0])] * 3
crd_names = ["x", "y", "z"]
for i, c in enumerate(clist):
crd_arrs[i] = c[1]
crd_shape = [len(arr) for arr in crd_arrs]
time = fld0.time
# write arrays to the hdf5 file
with h5py.File(fname, 'w') as f:
for axis_name, arr in zip(crd_names, crd_arrs):
loc = cls._CRDS_GROUP + '/' + axis_name
f[loc] = arr
for name, fld in flds.items():
loc = cls._FLD_GROUPS[fld.center.lower()] + '/' + name
# xdmf files use kji ordering
f[loc] = fld.data.T
# big bad openggcm time_str hack to put basetime into hdf5 file
for fld in flds.values():
try:
tfmt = "%Y:%m:%d:%H:%M:%S.%f"
sec_td = viscid.as_timedelta64(fld.time, 's')
dtime = viscid.as_datetime(fld.basetime + sec_td).strftime(tfmt)
epoch = viscid.readers.openggcm.GGCM_EPOCH
ts = viscid.as_timedelta(fld.basetime - epoch).total_seconds()
ts += fld.time
timestr = "time= {0} {1:.16e} {2} 300c".format(fld.time, ts, dtime)
f.create_group('openggcm')
f['openggcm'].attrs['time_str'] = np.string_(timestr)
break
except viscid.NoBasetimeError:
pass
# now write an xdmf file
xdmf_fname = os.path.splitext(fname)[0] + ".xdmf"
relh5fname = "./" + os.path.basename(fname)
with open(xdmf_fname, 'w') as f:
xloc = cls._CRDS_GROUP + '/' + crd_names[0]
yloc = cls._CRDS_GROUP + '/' + crd_names[1]
zloc = cls._CRDS_GROUP + '/' + crd_names[2]
dim_str = " ".join([str(l) for l in crd_shape][::-1])
f.write(cls._XDMF_TEMPLATE_BEGIN.format(time=time))
s = cls._XDMF_TEMPLATE_RECTILINEAR_GRID_BEGIN.format(
grid_name="vgrid", crd_dims=dim_str, h5fname=relh5fname,
xdim=crd_shape[0], ydim=crd_shape[1], zdim=crd_shape[2],
xloc=xloc, yloc=yloc, zloc=zloc)
f.write(s)
for fld in flds.values():
_crd_system = viscid.as_crd_system(fld, None)
if _crd_system:
f.write(cls._XDMF_INFO_TEMPLATE.format(name="crd_system",
value=_crd_system))
break
for name, fld in flds.items():
fld = fld.as_flat().T
dt = fld.dtype.name.rstrip("0123456789").title()
precision = fld.dtype.itemsize
fld_dim_str = " ".join([str(l) for l in fld.shape])
loc = cls._FLD_GROUPS[fld.center.lower()] + '/' + name
s = cls._XDMF_TEMPLATE_ATTRIBUTE.format(
fld_name=name,
fld_type=fld.fldtype, center=fld.center.title(),
dtype=dt, precision=precision, fld_dims=fld_dim_str,
h5fname=relh5fname, fld_loc=loc)
f.write(s)
f.write(cls._XDMF_TEMPLATE_GRID_END)
f.write(cls._XDMF_TEMPLATE_END)
