def convert_timelike(value):
if value in (None, ''):
return None
elif viscid.is_datetime_like(value, conservative=True):
return viscid.as_datetime64(value)
elif viscid.is_timedelta_like(value, conservative=True):
return viscid.as_timedelta64(value)
else:
return value
def parse_timestring(timestr):
prefix = 'time='
timestr.strip()
if not timestr.startswith(prefix):
raise ValueError("Time string '{0}' is malformed".format(timestr))
timestr = timestr[len(prefix):].split()
t = float(timestr[0])
uttime = viscid.as_datetime64(timestr[2])
basetime = uttime - viscid.as_timedelta64(t, 's')
basetime = viscid.time_as_seconds(basetime, 1)
return basetime, uttime
def from_dataframe(frame, crd_cols=None, time_col='time', datetime_col='datetime'):
"""Make either a DatasetTemporal or Grid from pandas dataframe
Args:
frame (pandas.DataFrame): frame to parse
crd_cols (List[Str], None): list of column names for coordinates
time_col (str): column name of times
datetime_col (str): column name of datetimes
Returns:
DatasetTemporal or Grid
Raises:
ValueError: if only 1 row given and crd_cols is None
"""
import pandas
# discover times and possible basetime
try:
unique_times = frame[time_col].drop_duplicates()
if 'datetime' in frame:
unique_datetimes = frame[datetime_col].drop_duplicates()
if len(unique_times) > 1:
dt_datetime = unique_datetimes.iloc[1] - unique_datetimes.iloc[0]
dt_time = unique_times.iloc[1] - unique_times.iloc[0]
t0_timedelta = unique_times.iloc[0] * (dt_datetime / dt_time)
else:
t0_timedelta = viscid.as_timedelta64(1e6 * unique_times.iloc[0], 'us')
basetime = unique_datetimes.iloc[0] - t0_timedelta
else:
basetime = None
frame0 = frame[frame[time_col] == unique_times[0]]
except KeyError:
unique_times = np.array([0.0])
basetime = None
frame0 = frame
# discover crd_cols if not given
if crd_cols is None:
frame1 = frame0.drop([time_col, datetime_col], axis=1, errors='ignore')
if len(frame1) <= 1:
raise ValueError("With only 1 row, crd_cols must be specified.")
for icol in range(frame1.shape[1]):
diff = frame1.iloc[1, icol] - frame1.iloc[0, icol]
if diff != np.zeros((1,), dtype=diff.dtype):
break
crd_cols = frame1.columns[:icol + 1]
# discover field shape and make coordinates
crd_arrs = [frame[col].drop_duplicates() for col in crd_cols]
shape = [len(arr) for arr in crd_arrs]
crds = viscid.arrays2crds(crd_arrs, crd_names=crd_cols)
fld_names = list(frame.columns)
for _col in [time_col, datetime_col] + list(crd_cols):
if _col in fld_names:
fld_names.remove(_col)
# wrap everything up into grids
grids = []
for time in unique_times:
grid = Grid()
grid.time = time
grid.basetime = basetime
try:
frame1 = frame[frame[time_col] == time]
except KeyError:
frame1 = frame
for name in fld_names:
arr = frame1[name].values.reshape(shape)
fld = viscid.wrap_field(arr, crds, name=name, center='node')
grid.add_field(fld)
grids.append(grid)
if len(grids) > 1:
ret = DatasetTemporal()
for grid in grids:
ret.add(grid)
ret.basetime = basetime
else:
ret = grids[0]
return ret
def _main():
fld = viscid.mfield[-4:4:24j, -5:5:16j, -6:6:20j]
fld_f = viscid.scalar_fields_to_vector([fld, fld, fld], layout='flat')
fld_i = fld_f.as_layout('interlaced')
check_nd_sel(fld_f,
np.s_[...],
good_sel_list=np.s_[:, :, :],
good_sel_names=['x', 'y', 'z'],
good_sel_newmask=[False, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_[0],
good_sel_list=np.s_[0, :, :],
good_sel_names=['x', 'y', 'z'],
good_sel_newmask=[False, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_[..., :3j],
good_sel_list=np.s_[:, :, :3j],
good_sel_names=['x', 'y', 'z'],
good_sel_newmask=[False, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_[..., None, :3j],
good_sel_list=np.s_[:, :, np.newaxis, :3j],
good_sel_names=['x', 'y', 'new-x0', 'z'],
good_sel_newmask=[False, False, True, False],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_[..., :3j, None],
good_sel_list=np.s_[:, :, :3j, np.newaxis],
good_sel_names=['x', 'y', 'z', 'new-x0'],
good_sel_newmask=[False, False, False, True],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_[..., None, :3j, None],
good_sel_list=np.s_[:, :, np.newaxis, :3j, np.newaxis],
good_sel_names=['x', 'y', 'new-x0', 'z', 'new-x1'],
good_sel_newmask=[False, False, True, False, True],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_[None, 'u=None', ..., None, :3j, None],
good_sel_list=np.s_[np.newaxis, np.newaxis, :, :, np.newaxis,
:3j, np.newaxis],
good_sel_names=['new-x0', 'u', 'x', 'y', 'new-x1', 'z', 'new-x2'],
good_sel_newmask=[True, True, False, False, True, False, True],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_[:3j, ...],
good_sel_list=np.s_[:3j, :, :],
good_sel_names=['x', 'y', 'z'],
good_sel_newmask=[False, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_[0, ..., :3j],
good_sel_list=np.s_[0, :, :3j],
good_sel_names=['x', 'y', 'z'],
good_sel_newmask=[False, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_[0, 'u=newaxis', ..., :3j],
good_sel_list=np.s_[0, np.newaxis, :, :3j],
good_sel_names=['x', 'u', 'y', 'z'],
good_sel_newmask=[False, True, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_[0, np.newaxis, ..., :3j],
good_sel_list=np.s_[0, np.newaxis, :, :3j],
good_sel_names=['x', 'new-x0', 'y', 'z'],
good_sel_newmask=[False, True, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_[0, ..., 'u=newaxis', :3j],
good_sel_list=np.s_[0, :, np.newaxis, :3j],
good_sel_names=['x', 'y', 'u', 'z'],
good_sel_newmask=[False, False, True, False],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_[0, ..., np.newaxis, :3j],
good_sel_list=np.s_[0, :, np.newaxis, :3j],
good_sel_names=['x', 'y', 'new-x0', 'z'],
good_sel_newmask=[False, False, True, False],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_['z=4:10, x=:3'],
good_sel_list=np.s_[':3', :, '4:10'],
good_sel_names=['x', 'y', 'z'],
good_sel_newmask=[False, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_['z', None, 'u=None', :, :, None, 'z=:3j', None],
good_sel_list=np.s_[np.newaxis, np.newaxis, :, :, np.newaxis,
':3j', np.newaxis],
good_sel_names=['new-x0', 'u', 'x', 'y', 'new-x1', 'z', 'new-x2'],
good_sel_newmask=[True, True, False, False, True, False, True],
good_comp_sel=2,
)
check_nd_sel(fld_f,
np.s_['z=4:10, newaxis, x=:3'],
good_sel_list=np.s_[':3', :, '4:10', np.newaxis],
good_sel_names=['x', 'y', 'z', 'new-x0'],
good_sel_newmask=[False, False, False, True],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_['z=4:10, u=newaxis, x=:3'],
good_sel_list=np.s_[':3', :, '4:10', np.newaxis],
good_sel_names=['x', 'y', 'z', 'u'],
good_sel_newmask=[False, False, False, True],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_['u=newaxis, z=4:10, x=:3'],
good_sel_list=np.s_[np.newaxis, ':3', :, '4:10'],
good_sel_names=['u', 'x', 'y', 'z'],
good_sel_newmask=[True, False, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(fld_f,
np.s_['z=4:10, x=:3, u=newaxis'],
good_sel_list=np.s_[':3', :, '4:10', np.newaxis],
good_sel_names=['x', 'y', 'z', 'u'],
good_sel_newmask=[False, False, False, True],
good_comp_sel=slice(None),
)
check_slc_val('UT2010-01-01', np.datetime64('2010-01-01'))
check_slc_val('T2010-01-01:T60:12:01.0', slice(viscid.as_datetime64('2010-01-01'),
viscid.as_timedelta64('60:12:01.0')))
check_slc_val('[2010-01-01, 2011-02-02]', viscid.as_datetime64(['2010-01-01',
'2011-02-02']))
check_slc_val('[T31, 12:20.1]', viscid.as_timedelta64(['31', '12:20.1']))
check_slc_val('[T31, T12]', viscid.as_timedelta64(['31', '12']))
check_slc_val('13j:12:3', np.s_[13j:12:3])
check_slc_val(np.s_[13j:12:3], np.s_[13j:12:3])
check_slc_val(np.s_[13j:'T12':3], np.s_[13j:viscid.as_timedelta64(12):3])
check_slc_val(np.s_[13j:'2010-01-01':3],
np.s_[13j:viscid.as_datetime64('2010-01-01'):3])
check_slc_val(np.s_[13j:'None':3], np.s_[13j:None:3])
# ret = viscid.std_sel2index(np.s_[:3], x, np.s_[:3])
x = np.array([-1, -0.5, 0, 0.5, 1])
# start, step > 0
check_sbv(np.s_[-1.0j:], x, np.s_[0:])
check_sbv(np.s_[-0.9999999j:], x, np.s_[0:])
check_sbv(np.s_[-0.9999j:], x, np.s_[1:])
check_sbv(np.s_[-0.8j:], x, np.s_[1:])
check_sbv(np.s_[-0.6j:], x, np.s_[1:])
check_sbv(np.s_[0.5j:], x, np.s_[3:])
# start, step < 0
check_sbv(np.s_[-1.0j::-1], x, np.s_[0::-1])
check_sbv(np.s_[-0.51j::-1], x, np.s_[0::-1])
check_sbv(np.s_[-0.50000001j::-1], x, np.s_[1::-1])
check_sbv(np.s_[-0.5j::-1], x, np.s_[1::-1])
check_sbv(np.s_[-0.4j::-1], x, np.s_[1::-1])
check_sbv(np.s_[-0.6j::-1], x, np.s_[0::-1])
check_sbv(np.s_[0.5j::-1], x, np.s_[3::-1])
# stop, step > 0
check_sbv(np.s_[:-1.1j], x, np.s_[:0])
check_sbv(np.s_[:-1j], x, np.s_[:1])
check_sbv(np.s_[:-0.51j], x, np.s_[:1])
check_sbv(np.s_[:-0.5000001j], x, np.s_[:2])
check_sbv(np.s_[:-0.5j], x, np.s_[:2])
check_sbv(np.s_[:-0.48j], x, np.s_[:2])
# stop, step < 0
check_sbv(np.s_[:-1.0j:-1], x, np.s_[::-1])
check_sbv(np.s_[:-0.51j:-1], x, np.s_[:0:-1])
check_sbv(np.s_[:-0.5j:-1], x, np.s_[:0:-1])
check_sbv(np.s_[:-0.499999j:-1], x, np.s_[:0:-1])
check_sbv(np.s_[:-0.49j:-1], x, np.s_[:1:-1])
# length-zero slice
check_sbv(np.s_[:-10j:1], x, np.s_[:0:1])
check_sbv(np.s_[:10j:-1], x, np.s_[:5:-1])
crds = fld.crds
check_crd_slice(crds, np.s_[-2j:2j, 0j:, :0j],
(-1.91304348, 0.33333, -6.0),
(1.91304348, 5.0, -0.31578947),
(12, 8, 10),
('x', 'y', 'z')
)
check_crd_slice(crds, np.s_[-2j:2j, 0j, :0j],
(-1.91304348, -6.0),
(1.91304348, -0.31578947),
(12, 10),
('x', 'z')
)
check_crd_slice(crds, np.s_[-2j:2j, 0j, :0j],
(-1.91304348, 0.33333, -6.0),
(1.91304348, 0.33333, -0.31578947),
(12, 1, 10),
('x', 'y', 'z'),
variant='keep'
)
check_crd_slice(crds, np.s_[-2j:2j, 0j, 'w=newaxis', :0j],
(-1.91304348, 0.33333, 0.0, -6.0),
(1.91304348, 0.33333, 0.0, -0.31578947),
(12, 1, 1, 10),
('x', 'y', 'w', 'z'),
variant='keep'
)
check_crd_slice(crds.slice_and_keep(np.s_[0j]), np.s_[..., 0j],
(-5.0,), (5.0,), (16,), ('y'),
variant='reduce'
)
check_fld_slice(fld, np.s_[-2j:2j, 0j:, :0j],
(-1.91304348, 0.33333, -6.0),
(1.91304348, 5.0, -0.31578947),
(12, 8, 10),
('x', 'y', 'z')
)
check_fld_slice(fld, np.s_[-2j:2j, 0j:, 0j],
(-1.91304348, 0.33333, -0.3157894),
(1.91304348, 5.0, -0.31578947),
(12, 8, 1),
('x', 'y', 'z'),
variant='keep'
)
check_fld_slice(fld.slice_and_keep(np.s_[:, 5j]),
np.s_[-2j:2j, :, 0j],
(-1.91304348,),
(1.91304348,),
(12,),
('x',),
variant='reduce'
)
# fld = viscid.mfield[-4:4:24j, -5:5:16j, -6:6:20j]
x = fld.get_crd('x')
# check slice-by-in-array
xslc = [1, 3, 5, 7]
assert np.allclose(fld[xslc].get_crd('x'), x[xslc])
assert np.allclose(fld[np.array(xslc)].get_crd('x'), x[xslc])
assert np.allclose(fld[str(xslc)].get_crd('x'), x[xslc])
bool_arr = np.zeros((fld.shape[0]), dtype=np.bool_)
bool_arr[xslc] = True
assert np.allclose(fld[list(bool_arr)].get_crd('x'), x[xslc])
assert np.allclose(fld[bool_arr].get_crd('x'), x[xslc])
assert np.allclose(fld[str(bool_arr)].get_crd('x'), x[xslc])
val_arr = 1j * x[xslc]
assert np.allclose(fld[list(val_arr)].get_crd('x'), x[xslc])
assert np.allclose(fld[val_arr].get_crd('x'), x[xslc])
assert np.allclose(fld[str(val_arr)].get_crd('x'), x[xslc])
assert isinstance(fld[list(val_arr)].crds, viscid.coordinate.UniformCrds)
assert isinstance(fld[val_arr].crds, viscid.coordinate.UniformCrds)
assert isinstance(fld[str(val_arr)].crds, viscid.coordinate.UniformCrds)
# will turn uniform into non-uniform
xslc = [1, 3, 5, 8]
assert np.allclose(fld[xslc].get_crd('x'), x[xslc])
assert np.allclose(fld[np.array(xslc)].get_crd('x'), x[xslc])
assert np.allclose(fld[str(xslc)].get_crd('x'), x[xslc])
bool_arr = np.zeros((fld.shape[0]), dtype=np.bool_)
bool_arr[xslc] = True
assert np.allclose(fld[list(bool_arr)].get_crd('x'), x[xslc])
assert np.allclose(fld[bool_arr].get_crd('x'), x[xslc])
assert np.allclose(fld[str(bool_arr)].get_crd('x'), x[xslc])
val_arr = 1j * x[xslc]
assert np.allclose(fld[list(val_arr)].get_crd('x'), x[xslc])
assert np.allclose(fld[val_arr].get_crd('x'), x[xslc])
assert np.allclose(fld[str(val_arr)].get_crd('x'), x[xslc])
assert isinstance(fld[list(val_arr)].crds, viscid.coordinate.NonuniformCrds)
assert isinstance(fld[val_arr].crds, viscid.coordinate.NonuniformCrds)
assert isinstance(fld[str(val_arr)].crds, viscid.coordinate.NonuniformCrds)
########### cell centered....
fld_cc = fld.as_centered('cell')
x_cc = fld_cc.get_crd('x')
# check slice-by-in-array
xslc = [1, 3, 5, 7]
assert np.allclose(fld_cc[xslc].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[np.array(xslc)].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[str(xslc)].get_crd('x'), x_cc[xslc])
bool_arr = np.zeros((fld_cc.shape[0]), dtype=np.bool_)
bool_arr[xslc] = True
assert np.allclose(fld_cc[list(bool_arr)].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[bool_arr].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[str(bool_arr)].get_crd('x'), x_cc[xslc])
val_arr = 1j * x_cc[xslc]
assert np.allclose(fld_cc[list(val_arr)].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[val_arr].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[str(val_arr)].get_crd('x'), x_cc[xslc])
assert isinstance(fld_cc[list(val_arr)].crds, viscid.coordinate.UniformCrds)
assert isinstance(fld_cc[val_arr].crds, viscid.coordinate.UniformCrds)
assert isinstance(fld_cc[str(val_arr)].crds, viscid.coordinate.UniformCrds)
# will turn uniform into non-uniform
xslc = [1, 3, 5, 8]
assert np.allclose(fld_cc[xslc].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[np.array(xslc)].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[str(xslc)].get_crd('x'), x_cc[xslc])
bool_arr = np.zeros((fld_cc.shape[0]), dtype=np.bool_)
bool_arr[xslc] = True
assert np.allclose(fld_cc[list(bool_arr)].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[bool_arr].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[str(bool_arr)].get_crd('x'), x_cc[xslc])
val_arr = 1j * x_cc[xslc]
assert np.allclose(fld_cc[list(val_arr)].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[val_arr].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[str(val_arr)].get_crd('x'), x_cc[xslc])
assert isinstance(fld_cc[list(val_arr)].crds, viscid.coordinate.NonuniformCrds)
assert isinstance(fld_cc[val_arr].crds, viscid.coordinate.NonuniformCrds)
assert isinstance(fld_cc[str(val_arr)].crds, viscid.coordinate.NonuniformCrds)
return 0
def time_as_timedelta64(self):
return viscid.as_timedelta64(self.time, unit='s')
def save_fields(cls, fname, flds, complevel=0, compression='gzip',
compression_opts=None, **kwargs):
""" save some fields using the format given by the class """
# FIXME: this is only good for writing cartesian rectilnear flds
# FIXME: axes are renamed if flds[0] is 1D or 2D
assert len(flds) > 0
fname = os.path.expanduser(os.path.expandvars(fname))
if complevel and compression == 'gzip' and compression_opts is None:
compression_opts = complevel
# TODO: what if compression != 'gzip'
do_compression = compression_opts is not None
if isinstance(flds, list):
if isinstance(flds[0], (list, tuple)):
flds = OrderedDict(flds)
else:
flds = OrderedDict([(fld.name, fld) for fld in flds])
# FIXME: all coordinates are saved as non-uniform, the proper
# way to do this is to have let coordinate format its own
# hdf5 / xdmf / numpy binary output
fld0 = next(iter(flds.values()))
clist = fld0.crds.get_clist(full_arrays=True)
crd_arrs = [np.array([0.0])] * 3
crd_names = ["x", "y", "z"]
for i, c in enumerate(clist):
crd_arrs[i] = c[1]
crd_shape = [len(arr) for arr in crd_arrs]
time = fld0.time
# write arrays to the hdf5 file
with h5py.File(fname, 'w') as f:
for axis_name, arr in zip(crd_names, crd_arrs):
loc = cls._CRDS_GROUP + '/' + axis_name
if do_compression:
f.create_dataset(loc, data=arr, compression=compression,
compression_opts=compression_opts)
else:
f[loc] = arr
for name, fld in flds.items():
loc = cls._FLD_GROUPS[fld.center.lower()] + '/' + name
# xdmf files use kji ordering
if do_compression:
f.create_dataset(loc, data=fld.data.T, compression=compression,
compression_opts=compression_opts)
else:
f[loc] = fld.data.T
# big bad openggcm time_str hack to put basetime into hdf5 file
for fld in flds.values():
try:
tfmt = "%Y:%m:%d:%H:%M:%S.%f"
sec_td = viscid.as_timedelta64(fld.time, 's')
dtime = viscid.as_datetime(fld.basetime + sec_td).strftime(tfmt)
epoch = viscid.readers.openggcm.GGCM_EPOCH
ts = viscid.as_timedelta(fld.basetime - epoch).total_seconds()
ts += fld.time
timestr = "time= {0} {1:.16e} {2} 300c".format(fld.time, ts, dtime)
f.create_group('openggcm')
f['openggcm'].attrs['time_str'] = np.string_(timestr)
break
except viscid.NoBasetimeError:
pass
# now write an xdmf file
xdmf_fname = os.path.splitext(fname)[0] + ".xdmf"
relh5fname = "./" + os.path.basename(fname)
with open(xdmf_fname, 'w') as f:
xloc = cls._CRDS_GROUP + '/' + crd_names[0]
yloc = cls._CRDS_GROUP + '/' + crd_names[1]
zloc = cls._CRDS_GROUP + '/' + crd_names[2]
dim_str = " ".join([str(l) for l in crd_shape][::-1])
f.write(cls._XDMF_TEMPLATE_BEGIN.format(time=time))
s = cls._XDMF_TEMPLATE_RECTILINEAR_GRID_BEGIN.format(
grid_name="vgrid", crd_dims=dim_str, h5fname=relh5fname,
xdim=crd_shape[0], ydim=crd_shape[1], zdim=crd_shape[2],
xloc=xloc, yloc=yloc, zloc=zloc)
f.write(s)
for fld in flds.values():
_crd_system = viscid.as_crd_system(fld, None)
if _crd_system:
f.write(cls._XDMF_INFO_TEMPLATE.format(name="crd_system",
value=_crd_system))
break
for name, fld in flds.items():
fld = fld.as_flat().T
dt = fld.dtype.name.rstrip("0123456789").title()
precision = fld.dtype.itemsize
fld_dim_str = " ".join([str(l) for l in fld.shape])
loc = cls._FLD_GROUPS[fld.center.lower()] + '/' + name
s = cls._XDMF_TEMPLATE_ATTRIBUTE.format(
fld_name=name,
fld_type=fld.fldtype, center=fld.center.title(),
dtype=dt, precision=precision, fld_dims=fld_dim_str,
h5fname=relh5fname, fld_loc=loc)
f.write(s)
f.write(cls._XDMF_TEMPLATE_GRID_END)
f.write(cls._XDMF_TEMPLATE_END)
def save_fields(cls, fname, flds, **kwargs):
""" save some fields using the format given by the class """
# FIXME: this is only good for writing cartesian rectilnear flds
# FIXME: axes are renamed if flds[0] is 1D or 2D
assert len(flds) > 0
fname = os.path.expanduser(os.path.expandvars(fname))
# FIXME: all coordinates are saved as non-uniform, the proper
# way to do this is to have let coordinate format its own
# hdf5 / xdmf / numpy binary output
clist = flds[0].crds.get_clist(full_arrays=True)
crd_arrs = [np.array([0.0])] * 3
crd_names = ["x", "y", "z"]
for i, c in enumerate(clist):
crd_arrs[i] = c[1]
crd_shape = [len(arr) for arr in crd_arrs]
time = flds[0].time
# write arrays to the hdf5 file
with h5py.File(fname, 'w') as f:
for axis_name, arr in zip(crd_names, crd_arrs):
loc = cls._CRDS_GROUP + '/' + axis_name
f[loc] = arr
for fld in flds:
loc = cls._FLD_GROUPS[fld.center.lower()] + '/' + fld.name
# xdmf files use kji ordering
f[loc] = fld.data.T
# big bad openggcm time_str hack to put basetime into hdf5 file
for fld in flds:
try:
tfmt = "%Y:%m:%d:%H:%M:%S.%f"
sec_td = viscid.as_timedelta64(fld.time, 's')
dtime = viscid.as_datetime(fld.basetime + sec_td).strftime(tfmt)
epoch = viscid.readers.openggcm.GGCM_EPOCH
ts = viscid.as_timedelta(fld.basetime - epoch).total_seconds()
ts += fld.time
timestr = "time= {0} {1:.16e} {2} 300c".format(fld.time, ts, dtime)
f.create_group('openggcm')
f['openggcm'].attrs['time_str'] = np.string_(timestr)
break
except viscid.NoBasetimeError:
pass
# now write an xdmf file
xdmf_fname = os.path.splitext(fname)[0] + ".xdmf"
relh5fname = "./" + os.path.basename(fname)
with open(xdmf_fname, 'w') as f:
xloc = cls._CRDS_GROUP + '/' + crd_names[0]
yloc = cls._CRDS_GROUP + '/' + crd_names[1]
zloc = cls._CRDS_GROUP + '/' + crd_names[2]
dim_str = " ".join([str(l) for l in crd_shape][::-1])
f.write(cls._XDMF_TEMPLATE_BEGIN.format(time=time))
s = cls._XDMF_TEMPLATE_RECTILINEAR_GRID_BEGIN.format(
grid_name="vgrid", crd_dims=dim_str, h5fname=relh5fname,
xdim=crd_shape[0], ydim=crd_shape[1], zdim=crd_shape[2],
xloc=xloc, yloc=yloc, zloc=zloc)
f.write(s)
for fld in flds:
_crd_system = viscid.get_crd_system(fld, None)
if _crd_system:
f.write(cls._XDMF_INFO_TEMPLATE.format(name="crd_system",
value=_crd_system))
break
for fld in flds:
fld = fld.as_flat().T
dt = fld.dtype.name.rstrip("0123456789").title()
precision = fld.dtype.itemsize
fld_dim_str = " ".join([str(l) for l in fld.shape])
loc = cls._FLD_GROUPS[fld.center.lower()] + '/' + fld.name
s = cls._XDMF_TEMPLATE_ATTRIBUTE.format(
fld_name=fld.name,
fld_type=fld.fldtype, center=fld.center.title(),
dtype=dt, precision=precision, fld_dims=fld_dim_str,
h5fname=relh5fname, fld_loc=loc)
f.write(s)
f.write(cls._XDMF_TEMPLATE_GRID_END)
f.write(cls._XDMF_TEMPLATE_END)
def from_dataframe(frame,
crd_cols=None,
time_col='time',
datetime_col='datetime'):
"""Make either a DatasetTemporal or Grid from pandas dataframe
Args:
frame (pandas.DataFrame): frame to parse
crd_cols (List[Str], None): list of column names for coordinates
time_col (str): column name of times
datetime_col (str): column name of datetimes
Returns:
DatasetTemporal or Grid
Raises:
ValueError: if only 1 row given and crd_cols is None
"""
import pandas
# discover times and possible basetime
try:
unique_times = frame[time_col].drop_duplicates()
if 'datetime' in frame:
unique_datetimes = frame[datetime_col].drop_duplicates()
if len(unique_times) > 1:
dt_datetime = unique_datetimes.iloc[1] - unique_datetimes.iloc[
0]
dt_time = unique_times.iloc[1] - unique_times.iloc[0]
t0_timedelta = unique_times.iloc[0] * (dt_datetime / dt_time)
else:
t0_timedelta = viscid.as_timedelta64(
1e6 * unique_times.iloc[0], 'us')
basetime = unique_datetimes.iloc[0] - t0_timedelta
else:
basetime = None
frame0 = frame[frame[time_col] == unique_times[0]]
except KeyError:
unique_times = np.array([0.0])
basetime = None
frame0 = frame
# discover crd_cols if not given
if crd_cols is None:
frame1 = frame0.drop([time_col, datetime_col], axis=1, errors='ignore')
if len(frame1) <= 1:
raise ValueError("With only 1 row, crd_cols must be specified.")
for icol in range(frame1.shape[1]):
diff = frame1.iloc[1, icol] - frame1.iloc[0, icol]
if diff != np.zeros((1, ), dtype=diff.dtype):
break
crd_cols = frame1.columns[:icol + 1]
# discover field shape and make coordinates
crd_arrs = [frame[col].drop_duplicates() for col in crd_cols]
shape = [len(arr) for arr in crd_arrs]
crds = viscid.arrays2crds(crd_arrs, crd_names=crd_cols)
fld_names = list(frame.columns)
for _col in [time_col, datetime_col] + list(crd_cols):
if _col in fld_names:
fld_names.remove(_col)
# wrap everything up into grids
grids = []
for time in unique_times:
grid = Grid()
grid.time = time
grid.basetime = basetime
try:
frame1 = frame[frame[time_col] == time]
except KeyError:
frame1 = frame
for name in fld_names:
arr = frame1[name].values.reshape(shape)
fld = viscid.wrap_field(arr, crds, name=name, center='node')
grid.add_field(fld)
grids.append(grid)
if len(grids) > 1:
ret = DatasetTemporal()
for grid in grids:
ret.add(grid)
ret.basetime = basetime
else:
ret = grids[0]
return ret
def time_as_timedelta64(self):
return viscid.as_timedelta64(self.time, unit='s')
def time_as_timedelta64(self):
return viscid.as_timedelta64(1e6 * self.time, 'us')
def _main():
fld = viscid.mfield[-4:4:24j, -5:5:16j, -6:6:20j]
fld_f = viscid.scalar_fields_to_vector([fld, fld, fld], layout='flat')
fld_i = fld_f.as_layout('interlaced')
check_nd_sel(
fld_f,
np.s_[...],
good_sel_list=np.s_[:, :, :],
good_sel_names=['x', 'y', 'z'],
good_sel_newmask=[False, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_[0],
good_sel_list=np.s_[0, :, :],
good_sel_names=['x', 'y', 'z'],
good_sel_newmask=[False, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_[..., :3j],
good_sel_list=np.s_[:, :, :3j],
good_sel_names=['x', 'y', 'z'],
good_sel_newmask=[False, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_[..., None, :3j],
good_sel_list=np.s_[:, :, np.newaxis, :3j],
good_sel_names=['x', 'y', 'new-x0', 'z'],
good_sel_newmask=[False, False, True, False],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_[..., :3j, None],
good_sel_list=np.s_[:, :, :3j, np.newaxis],
good_sel_names=['x', 'y', 'z', 'new-x0'],
good_sel_newmask=[False, False, False, True],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_[..., None, :3j, None],
good_sel_list=np.s_[:, :, np.newaxis, :3j, np.newaxis],
good_sel_names=['x', 'y', 'new-x0', 'z', 'new-x1'],
good_sel_newmask=[False, False, True, False, True],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_[None, 'u=None', ..., None, :3j, None],
good_sel_list=np.s_[np.newaxis, np.newaxis, :, :, np.newaxis, :3j,
np.newaxis],
good_sel_names=['new-x0', 'u', 'x', 'y', 'new-x1', 'z', 'new-x2'],
good_sel_newmask=[True, True, False, False, True, False, True],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_[:3j, ...],
good_sel_list=np.s_[:3j, :, :],
good_sel_names=['x', 'y', 'z'],
good_sel_newmask=[False, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_[0, ..., :3j],
good_sel_list=np.s_[0, :, :3j],
good_sel_names=['x', 'y', 'z'],
good_sel_newmask=[False, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_[0, 'u=newaxis', ..., :3j],
good_sel_list=np.s_[0, np.newaxis, :, :3j],
good_sel_names=['x', 'u', 'y', 'z'],
good_sel_newmask=[False, True, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_[0, np.newaxis, ..., :3j],
good_sel_list=np.s_[0, np.newaxis, :, :3j],
good_sel_names=['x', 'new-x0', 'y', 'z'],
good_sel_newmask=[False, True, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_[0, ..., 'u=newaxis', :3j],
good_sel_list=np.s_[0, :, np.newaxis, :3j],
good_sel_names=['x', 'y', 'u', 'z'],
good_sel_newmask=[False, False, True, False],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_[0, ..., np.newaxis, :3j],
good_sel_list=np.s_[0, :, np.newaxis, :3j],
good_sel_names=['x', 'y', 'new-x0', 'z'],
good_sel_newmask=[False, False, True, False],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_['z=4:10, x=:3'],
good_sel_list=np.s_[':3', :, '4:10'],
good_sel_names=['x', 'y', 'z'],
good_sel_newmask=[False, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_['z', None, 'u=None', :, :, None, 'z=:3j', None],
good_sel_list=np.s_[np.newaxis, np.newaxis, :, :, np.newaxis, ':3j',
np.newaxis],
good_sel_names=['new-x0', 'u', 'x', 'y', 'new-x1', 'z', 'new-x2'],
good_sel_newmask=[True, True, False, False, True, False, True],
good_comp_sel=2,
)
check_nd_sel(
fld_f,
np.s_['z=4:10, newaxis, x=:3'],
good_sel_list=np.s_[':3', :, '4:10', np.newaxis],
good_sel_names=['x', 'y', 'z', 'new-x0'],
good_sel_newmask=[False, False, False, True],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_['z=4:10, u=newaxis, x=:3'],
good_sel_list=np.s_[':3', :, '4:10', np.newaxis],
good_sel_names=['x', 'y', 'z', 'u'],
good_sel_newmask=[False, False, False, True],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_['u=newaxis, z=4:10, x=:3'],
good_sel_list=np.s_[np.newaxis, ':3', :, '4:10'],
good_sel_names=['u', 'x', 'y', 'z'],
good_sel_newmask=[True, False, False, False],
good_comp_sel=slice(None),
)
check_nd_sel(
fld_f,
np.s_['z=4:10, x=:3, u=newaxis'],
good_sel_list=np.s_[':3', :, '4:10', np.newaxis],
good_sel_names=['x', 'y', 'z', 'u'],
good_sel_newmask=[False, False, False, True],
good_comp_sel=slice(None),
)
check_slc_val('UT2010-01-01', np.datetime64('2010-01-01'))
check_slc_val(
'T2010-01-01:T60:12:01.0',
slice(viscid.as_datetime64('2010-01-01'),
viscid.as_timedelta64('60:12:01.0')))
check_slc_val('[2010-01-01, 2011-02-02]',
viscid.as_datetime64(['2010-01-01', '2011-02-02']))
check_slc_val('[T31, 12:20.1]', viscid.as_timedelta64(['31', '12:20.1']))
check_slc_val('[T31, T12]', viscid.as_timedelta64(['31', '12']))
check_slc_val('13j:12:3', np.s_[13j:12:3])
check_slc_val(np.s_[13j:12:3], np.s_[13j:12:3])
check_slc_val(np.s_[13j:'T12':3], np.s_[13j:viscid.as_timedelta64(12):3])
check_slc_val(np.s_[13j:'2010-01-01':3],
np.s_[13j:viscid.as_datetime64('2010-01-01'):3])
check_slc_val(np.s_[13j:'None':3], np.s_[13j:None:3])
# ret = viscid.std_sel2index(np.s_[:3], x, np.s_[:3])
x = np.array([-1, -0.5, 0, 0.5, 1])
# start, step > 0
check_sbv(np.s_[-1.0j:], x, np.s_[0:])
check_sbv(np.s_[-0.9999999j:], x, np.s_[0:])
check_sbv(np.s_[-0.9999j:], x, np.s_[1:])
check_sbv(np.s_[-0.8j:], x, np.s_[1:])
check_sbv(np.s_[-0.6j:], x, np.s_[1:])
check_sbv(np.s_[0.5j:], x, np.s_[3:])
# start, step < 0
check_sbv(np.s_[-1.0j::-1], x, np.s_[0::-1])
check_sbv(np.s_[-0.51j::-1], x, np.s_[0::-1])
check_sbv(np.s_[-0.50000001j::-1], x, np.s_[1::-1])
check_sbv(np.s_[-0.5j::-1], x, np.s_[1::-1])
check_sbv(np.s_[-0.4j::-1], x, np.s_[1::-1])
check_sbv(np.s_[-0.6j::-1], x, np.s_[0::-1])
check_sbv(np.s_[0.5j::-1], x, np.s_[3::-1])
# stop, step > 0
check_sbv(np.s_[:-1.1j], x, np.s_[:0])
check_sbv(np.s_[:-1j], x, np.s_[:1])
check_sbv(np.s_[:-0.51j], x, np.s_[:1])
check_sbv(np.s_[:-0.5000001j], x, np.s_[:2])
check_sbv(np.s_[:-0.5j], x, np.s_[:2])
check_sbv(np.s_[:-0.48j], x, np.s_[:2])
# stop, step < 0
check_sbv(np.s_[:-1.0j:-1], x, np.s_[::-1])
check_sbv(np.s_[:-0.51j:-1], x, np.s_[:0:-1])
check_sbv(np.s_[:-0.5j:-1], x, np.s_[:0:-1])
check_sbv(np.s_[:-0.499999j:-1], x, np.s_[:0:-1])
check_sbv(np.s_[:-0.49j:-1], x, np.s_[:1:-1])
# length-zero slice
check_sbv(np.s_[:-10j:1], x, np.s_[:0:1])
check_sbv(np.s_[:10j:-1], x, np.s_[:5:-1])
crds = fld.crds
check_crd_slice(crds, np.s_[-2j:2j,
0j:, :0j], (-1.91304348, 0.33333, -6.0),
(1.91304348, 5.0, -0.31578947), (12, 8, 10),
('x', 'y', 'z'))
check_crd_slice(crds, np.s_[-2j:2j, 0j, :0j], (-1.91304348, -6.0),
(1.91304348, -0.31578947), (12, 10), ('x', 'z'))
check_crd_slice(crds,
np.s_[-2j:2j, 0j, :0j], (-1.91304348, 0.33333, -6.0),
(1.91304348, 0.33333, -0.31578947), (12, 1, 10),
('x', 'y', 'z'),
variant='keep')
check_crd_slice(crds,
np.s_[-2j:2j, 0j,
'w=newaxis', :0j], (-1.91304348, 0.33333, 0.0, -6.0),
(1.91304348, 0.33333, 0.0, -0.31578947), (12, 1, 1, 10),
('x', 'y', 'w', 'z'),
variant='keep')
check_crd_slice(crds.slice_and_keep(np.s_[0j]),
np.s_[..., 0j], (-5.0, ), (5.0, ), (16, ), ('y'),
variant='reduce')
check_fld_slice(fld, np.s_[-2j:2j, 0j:, :0j], (-1.91304348, 0.33333, -6.0),
(1.91304348, 5.0, -0.31578947), (12, 8, 10),
('x', 'y', 'z'))
check_fld_slice(fld,
np.s_[-2j:2j, 0j:, 0j], (-1.91304348, 0.33333, -0.3157894),
(1.91304348, 5.0, -0.31578947), (12, 8, 1),
('x', 'y', 'z'),
variant='keep')
check_fld_slice(fld.slice_and_keep(np.s_[:, 5j]),
np.s_[-2j:2j, :, 0j], (-1.91304348, ), (1.91304348, ),
(12, ), ('x', ),
variant='reduce')
# fld = viscid.mfield[-4:4:24j, -5:5:16j, -6:6:20j]
x = fld.get_crd('x')
# check slice-by-in-array
xslc = [1, 3, 5, 7]
assert np.allclose(fld[xslc].get_crd('x'), x[xslc])
assert np.allclose(fld[np.array(xslc)].get_crd('x'), x[xslc])
assert np.allclose(fld[str(xslc)].get_crd('x'), x[xslc])
bool_arr = np.zeros((fld.shape[0]), dtype=np.bool_)
bool_arr[xslc] = True
assert np.allclose(fld[list(bool_arr)].get_crd('x'), x[xslc])
assert np.allclose(fld[bool_arr].get_crd('x'), x[xslc])
assert np.allclose(fld[str(bool_arr)].get_crd('x'), x[xslc])
val_arr = 1j * x[xslc]
assert np.allclose(fld[list(val_arr)].get_crd('x'), x[xslc])
assert np.allclose(fld[val_arr].get_crd('x'), x[xslc])
assert np.allclose(fld[str(val_arr)].get_crd('x'), x[xslc])
assert isinstance(fld[list(val_arr)].crds, viscid.coordinate.UniformCrds)
assert isinstance(fld[val_arr].crds, viscid.coordinate.UniformCrds)
assert isinstance(fld[str(val_arr)].crds, viscid.coordinate.UniformCrds)
# will turn uniform into non-uniform
xslc = [1, 3, 5, 8]
assert np.allclose(fld[xslc].get_crd('x'), x[xslc])
assert np.allclose(fld[np.array(xslc)].get_crd('x'), x[xslc])
assert np.allclose(fld[str(xslc)].get_crd('x'), x[xslc])
bool_arr = np.zeros((fld.shape[0]), dtype=np.bool_)
bool_arr[xslc] = True
assert np.allclose(fld[list(bool_arr)].get_crd('x'), x[xslc])
assert np.allclose(fld[bool_arr].get_crd('x'), x[xslc])
assert np.allclose(fld[str(bool_arr)].get_crd('x'), x[xslc])
val_arr = 1j * x[xslc]
assert np.allclose(fld[list(val_arr)].get_crd('x'), x[xslc])
assert np.allclose(fld[val_arr].get_crd('x'), x[xslc])
assert np.allclose(fld[str(val_arr)].get_crd('x'), x[xslc])
assert isinstance(fld[list(val_arr)].crds,
viscid.coordinate.NonuniformCrds)
assert isinstance(fld[val_arr].crds, viscid.coordinate.NonuniformCrds)
assert isinstance(fld[str(val_arr)].crds, viscid.coordinate.NonuniformCrds)
########### cell centered....
fld_cc = fld.as_centered('cell')
x_cc = fld_cc.get_crd('x')
# check slice-by-in-array
xslc = [1, 3, 5, 7]
assert np.allclose(fld_cc[xslc].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[np.array(xslc)].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[str(xslc)].get_crd('x'), x_cc[xslc])
bool_arr = np.zeros((fld_cc.shape[0]), dtype=np.bool_)
bool_arr[xslc] = True
assert np.allclose(fld_cc[list(bool_arr)].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[bool_arr].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[str(bool_arr)].get_crd('x'), x_cc[xslc])
val_arr = 1j * x_cc[xslc]
assert np.allclose(fld_cc[list(val_arr)].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[val_arr].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[str(val_arr)].get_crd('x'), x_cc[xslc])
assert isinstance(fld_cc[list(val_arr)].crds,
viscid.coordinate.UniformCrds)
assert isinstance(fld_cc[val_arr].crds, viscid.coordinate.UniformCrds)
assert isinstance(fld_cc[str(val_arr)].crds, viscid.coordinate.UniformCrds)
# will turn uniform into non-uniform
xslc = [1, 3, 5, 8]
assert np.allclose(fld_cc[xslc].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[np.array(xslc)].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[str(xslc)].get_crd('x'), x_cc[xslc])
bool_arr = np.zeros((fld_cc.shape[0]), dtype=np.bool_)
bool_arr[xslc] = True
assert np.allclose(fld_cc[list(bool_arr)].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[bool_arr].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[str(bool_arr)].get_crd('x'), x_cc[xslc])
val_arr = 1j * x_cc[xslc]
assert np.allclose(fld_cc[list(val_arr)].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[val_arr].get_crd('x'), x_cc[xslc])
assert np.allclose(fld_cc[str(val_arr)].get_crd('x'), x_cc[xslc])
assert isinstance(fld_cc[list(val_arr)].crds,
viscid.coordinate.NonuniformCrds)
assert isinstance(fld_cc[val_arr].crds, viscid.coordinate.NonuniformCrds)
assert isinstance(fld_cc[str(val_arr)].crds,
viscid.coordinate.NonuniformCrds)
return 0
