def __init__(self, filename):
self.kw = dict()
with open(filename, "r") as f:
for line in f:
line = line.strip()
if line.startswith('#') or not line:
continue
words = line.split()
funcname = "parse_" + words[0]
if hasattr(self, funcname):
parse = getattr(self, funcname)
parse(words[1:], f)
elif words[0] in self._keywords:
descr = self._keywords[words[0]]
if descr[0] == 'i':
values = [int(w) for w in words[1:]]
elif descr[0] == 'f':
values = [float(w) for w in words[1:]]
elif descr[0] == 's':
values = words[1:]
else:
raise NotImplementedError("unhandled descr {0}"
"".format(descr))
if len(descr) > 1:
n_values = int(descr[1:])
assert len(words[1:]) == n_values
else:
values, = values
self.kw[words[0]] = values
else:
logger.error("Parsing problem: {0}".format(words))
self._setup()
def field_dict(self, time=None, fld_names=None, **kwargs):
""" fields as dict of {name: field} """
child = self.active_child
if child is None:
logger.error("Could not get appropriate child...")
return None
else:
return child.field_dict(time=time, fld_names=fld_names)
def get_field(self, fldname, time=None, slc=Ellipsis):
""" recurse down active children to get a field """
child = self.active_child
if child is None:
logger.error("Could not get appropriate child...")
return None
else:
return child.get_field(fldname, time=time, slc=slc)
def get_grid(self, time=None):
""" recurse down active children to get a field """
child = self.active_child.resolve()
if child is None:
logger.error("Could not get appropriate child...")
return None
else:
return child.get_grid(time=time)
def get_grid(self, time=None):
""" recurse down active children to get a field """
child = self.active_child.resolve()
if child is None:
logger.error("Could not get appropriate child...")
return None
else:
return child.get_grid(time=time)
def get_field(self, fldname, time=None, slc=Ellipsis):
""" recurse down active children to get a field """
child = self.active_child
if child is None:
logger.error("Could not get appropriate child...")
return None
else:
return child.get_field(fldname, time=time, slc=slc)
def field_dict(self, time=None, fld_names=None, **kwargs):
""" fields as dict of {name: field} """
child = self.active_child
if child is None:
logger.error("Could not get appropriate child...")
return None
else:
return child.field_dict(time=time, fld_names=fld_names)
def iter_field_items(self, time=None, fld_names=None):
""" generator for (name, field) in the active dataset,
this will recurse down to a grid """
child = self.active_child
if child is None:
logger.error("Could not get appropriate child...")
return None
else:
return child.iter_field_items(time=time, fld_names=fld_names)
def iter_field_items(self, time=None, fld_names=None):
""" generator for (name, field) in the active dataset,
this will recurse down to a grid """
child = self.active_child
if child is None:
logger.error("Could not get appropriate child...")
return None
else:
return child.iter_field_items(time=time, fld_names=fld_names)
def _read_info(self):
# this takes super long when reading 3 hrs worth of ggcm data
# over sshfs
# import pdb; pdb.set_trace()
try:
with np.load(self.fname) as f:
dset = f[self.loc]
self._shape = dset.shape
self._dtype = dset.dtype
except IOError:
logger.error("Problem opening npz file, '%s'", self.fname)
raise
def field_dict(self, time=None, fld_names=None):
""" fields as dict of {name: field} """
if time is not None:
child = self.get_child(time)
else:
child = self.active_child
if child is None:
logger.error("Could not get appropriate child...")
return None
else:
return child.field_dict(fld_names=fld_names)
def _read_info(self):
# this takes super long when reading 3 hrs worth of ggcm data
# over sshfs
# import pdb; pdb.set_trace()
try:
with np.load(self.fname) as f:
dset = f[self.loc]
self._shape = dset.shape
self._dtype = dset.dtype
except IOError:
logger.error("Problem opening npz file, '%s'", self.fname)
raise
def _read_info(self):
try:
with h5py.File(self.fname, 'r') as f:
dset = self._resolve_loc(f)
self._shape = list(dset.shape)
if self.comp_dim is not None:
self._shape.pop(self.comp_dim)
self._shape = tuple(self._shape)
self._dtype = dset.dtype
except IOError:
logger.error("Problem opening hdf5 file, '%s'", self.fname)
raise
def _read_info(self):
try:
with h5py.File(self.fname, 'r') as f:
dset = self._resolve_loc(f)
self._shape = list(dset.shape)
if self.comp_dim is not None:
self._shape.pop(self.comp_dim)
self._shape = tuple(self._shape)
self._dtype = dset.dtype
except IOError:
logger.error("Problem opening hdf5 file, '%s'", self.fname)
raise
def field_dict(self, time=None, fld_names=None):
""" fields as dict of {name: field} """
if time is not None:
child = self.get_child(time)
else:
child = self.active_child
if child is None:
logger.error("Could not get appropriate child...")
return None
else:
return child.field_dict(fld_names=fld_names)
def iter_fields(self, time=None, named=None):
""" generator for fields in the active dataset,
this will recurse down to a grid """
if time is not None:
child = self.get_child(time)
else:
child = self.active_child
if child is None:
logger.error("Could not get appropriate child...")
return None
else:
return child.iter_fields(time=time, named=named)
def iter_fields(self, time=None, named=None):
""" generator for fields in the active dataset,
this will recurse down to a grid """
if time is not None:
child = self.get_child(time)
else:
child = self.active_child
if child is None:
logger.error("Could not get appropriate child...")
return None
else:
return child.iter_fields(time=time, named=named)
def add(self, child, set_active=True):
if child is None:
raise RuntimeError()
if child.time is None:
child.time = 0.0
logger.error("A child with no time? Something is strange...")
# this keeps the children in time order
self.prepare_child(child)
self.children.append((child.time, child))
self.children.sort(key=itemgetter(0))
# binary in sorting... maybe more efficient?
# bisect.insort(self.children, (child.time, child))
if set_active:
self.active_child = child
def add(self, child, set_active=True):
if child is None:
raise RuntimeError()
if child.time is None:
child.time = 0.0
logger.error("A child with no time? Something is strange...")
# this keeps the children in time order
self.prepare_child(child)
self.children.append((child.time, child))
self.children.sort(key=itemgetter(0))
# binary in sorting... maybe more efficient?
# bisect.insort(self.children, (child.time, child))
if set_active:
self.active_child = child
def set_item(self, handles, item, index_handle=True, _add_ref=False):
""" if index_handle is true then the index of item will be included as
a handle making the bucket indexable like a list """
# found = False
if handles is None:
handles = []
if not isinstance(handles, list):
raise TypeError("handle must by of list type")
# make sure we have a hashable "item" for doing reverse
# lookups of handles using an item
hashable_item = self._make_hashable(item)
if hashable_item not in self._hash_lookup:
if index_handle:
handles += [self._int_counter]
self._int_counter += 1
handles_added = []
for h in handles:
# check if we're stealing a handle from another item
try:
hash(h)
except TypeError:
logger.error("A bucket says handle '{0}' is not hashable, "
"ignoring it".format(h))
continue
if (h in self._items) and (item is self._items[h]):
continue
elif h in self._items:
logger.error("The handle '{0}' is being hijacked! Memory leak "
"could ensue.".format(h))
# romove handle from old item, since this check is here,
# there sholdn't be 2 items with the same handle in the
# items dict
old_item = self._items[h]
old_hashable_item = self._make_hashable(old_item)
self._handles[old_hashable_item].remove(h)
if len(self._handles[old_hashable_item]) == 0:
self.remove_item(old_item)
self._items[h] = item
handles_added.append(h)
try:
self._handles[hashable_item] += handles_added
if _add_ref:
self._ref_count[hashable_item] += 1
except KeyError:
if len(handles_added) == 0:
logger.error("No valid handles given, item '{0}' not added to "
"bucket".format(hashable_item))
else:
self._handles[hashable_item] = handles_added
self._hash_lookup[hashable_item] = item
self._ref_count[hashable_item] = 1
return None
def set_item(self, handles, item, index_handle=True, _add_ref=False):
""" if index_handle is true then the index of item will be included as
a handle making the bucket indexable like a list """
# found = False
if handles is None:
handles = []
if not isinstance(handles, list):
raise TypeError("handle must by of list type")
# make sure we have a hashable "item" for doing reverse
# lookups of handles using an item
hashable_item = self._make_hashable(item)
if hashable_item not in self._hash_lookup:
if index_handle:
handles += [self._int_counter]
self._int_counter += 1
handles_added = []
for h in handles:
# check if we're stealing a handle from another item
try:
hash(h)
except TypeError:
logger.error("A bucket says handle '{0}' is not hashable, "
"ignoring it".format(h))
continue
if (h in self._items) and (item is self._items[h]):
continue
elif h in self._items:
logger.error("The handle '{0}' is being hijacked! Memory leak "
"could ensue.".format(h))
# romove handle from old item, since this check is here,
# there sholdn't be 2 items with the same handle in the
# items dict
old_item = self._items[h]
old_hashable_item = self._make_hashable(old_item)
self._handles[old_hashable_item].remove(h)
if len(self._handles[old_hashable_item]) == 0:
self.remove_item(old_item)
self._items[h] = item
handles_added.append(h)
try:
self._handles[hashable_item] += handles_added
if _add_ref:
self._ref_count[hashable_item] += 1
except KeyError:
if len(handles_added) == 0:
logger.error("No valid handles given, item '{0}' not added to "
"bucket".format(hashable_item))
else:
self._handles[hashable_item] = handles_added
self._hash_lookup[hashable_item] = item
self._ref_count[hashable_item] = 1
return None
def _do_mhd_to_gse_on_read(self):
"""Return True if we """
# we already know what this data file needs
if self.has_info("_viscid_do_mhd_to_gse_on_read"):
return self.find_info("_viscid_do_mhd_to_gse_on_read")
# do we already know the crd system of this grid?
crd_system = self.find_info("crd_system", None)
freshly_determined_crd_system = crd_system is None
# I guess not, can we figure out the crd system of this grid?
if crd_system is None and self.find_info('assume_mhd_crds', False):
crd_system = "mhd"
if crd_system is None and self.find_info("_viscid_log_fname"):
# try to intuit the _crd system based on the log file and grid
try:
# if we're using a mirdip IC, and low edge is at least
# twice smaller than the high edge, then assume
# it's a magnetosphere box with xl < 0.0 is the sunward
# edge in "MHD" coordinates
is_openggcm = self.find_info('ggcm_mhd_type') == "ggcm"
# this 2nd check is in case the ggcm_mhd view in the
# log file is mangled... this happens sometimes
ic_type = self.find_info('ggcm_mhd_ic_type', '')
is_openggcm |= ic_type.startswith("mirdip")
# note that these default values are total hacks for fortran
# runs which don't spew mrc information @ the beginning
xl = float(self.find_info('mrc_crds_l')[0])
xh = float(self.find_info('mrc_crds_h')[0])
if is_openggcm and xl < 0.0 and xh > 0.0 and -2 * xl < xh:
crd_system = "mhd"
elif is_openggcm and xl < 0.0 and xh > 0.0 and -2 * xh > xl:
crd_system = "gse"
else:
crd_system = "other"
except KeyError as e:
logger.warning("Could not determine coordiname system; "
"either the logfile is mangled, or "
"the libmrc options I'm using in infer "
"crd system have changed (%s)", e.args[0])
if crd_system is None:
crd_system = "unknown"
if freshly_determined_crd_system:
self.set_info("crd_system", crd_system)
# now that we have an idea what the crd_system is, determine
# whether or not to do a mhd -> gse translation
request = str(self.mhd_to_gse_on_read).strip().lower()
if request == 'true':
viscid.logger.warning("'mhd_to_gse_on_read = true' is deprecated due "
"to lack of clarity. Please use 'auto', or if "
"you really want to always flip the axes, use "
"'force'. Only use 'force' if you are certain, "
"since even non-magnetosphere OpenGGCM grids "
"will be flipped, and you will be confused "
"some day when you open an MHD-in-a-box run, "
"and you have forgetten about this message.")
ret = True
elif request == 'force':
ret = True
elif request == 'false':
ret = False
elif request.startswith("auto"):
default = True if request.endswith('true') else False
if crd_system == "mhd":
ret = True
elif crd_system == "gse":
ret = False
else:
log_fname = self.find_info("_viscid_log_fname")
# which error / warning to print depends on why crd_system
# neither mhd | gse; was logfile reading turned off, was
# the logfile not found, or was the logfile simply mangled?
if default:
default_action = "flipping axes since default is True"
else:
default_action = "not flipping axes since default is False"
if log_fname is False:
logger.error("If you're using 'auto' for mhd->gse "
"conversion, reading the logfile MUST be "
"turned on. ({0})".format(default_action))
elif log_fname is None:
logger.warning("Tried to determine coordinate system using "
"logfile parameters, but no logfile found. "
"Copy over the log file to use auto mhd->gse "
"conversion. ({0})".format(default_action))
else:
logger.warning("Could not determine crd_system used for this "
"grid on disk ({0})".format(default_action))
# crd_system is either 'other' or 'unknown'
ret = default
else:
raise ValueError("Invalid value for mhd_to_gse_on_read: "
"'{0}'; valid choices: (True, False, auto, "
"auto_true, force)".format(request))
self.set_info("_viscid_do_mhd_to_gse_on_read", ret)
return ret