def convert_timelike(value): if value in (None, ''): return None elif viscid.is_datetime_like(value, conservative=True): return viscid.as_datetime64(value) elif viscid.is_timedelta_like(value, conservative=True): return viscid.as_timedelta64(value) else: return value
def parse_timestring(timestr): prefix = 'time=' timestr.strip() if not timestr.startswith(prefix): raise ValueError("Time string '{0}' is malformed".format(timestr)) timestr = timestr[len(prefix):].split() t = float(timestr[0]) uttime = viscid.as_datetime64(timestr[2]) basetime = uttime - viscid.as_timedelta64(t, 's') basetime = viscid.time_as_seconds(basetime, 1) return basetime, uttime
def from_dataframe(frame, crd_cols=None, time_col='time', datetime_col='datetime'): """Make either a DatasetTemporal or Grid from pandas dataframe Args: frame (pandas.DataFrame): frame to parse crd_cols (List[Str], None): list of column names for coordinates time_col (str): column name of times datetime_col (str): column name of datetimes Returns: DatasetTemporal or Grid Raises: ValueError: if only 1 row given and crd_cols is None """ import pandas # discover times and possible basetime try: unique_times = frame[time_col].drop_duplicates() if 'datetime' in frame: unique_datetimes = frame[datetime_col].drop_duplicates() if len(unique_times) > 1: dt_datetime = unique_datetimes.iloc[1] - unique_datetimes.iloc[0] dt_time = unique_times.iloc[1] - unique_times.iloc[0] t0_timedelta = unique_times.iloc[0] * (dt_datetime / dt_time) else: t0_timedelta = viscid.as_timedelta64(1e6 * unique_times.iloc[0], 'us') basetime = unique_datetimes.iloc[0] - t0_timedelta else: basetime = None frame0 = frame[frame[time_col] == unique_times[0]] except KeyError: unique_times = np.array([0.0]) basetime = None frame0 = frame # discover crd_cols if not given if crd_cols is None: frame1 = frame0.drop([time_col, datetime_col], axis=1, errors='ignore') if len(frame1) <= 1: raise ValueError("With only 1 row, crd_cols must be specified.") for icol in range(frame1.shape[1]): diff = frame1.iloc[1, icol] - frame1.iloc[0, icol] if diff != np.zeros((1,), dtype=diff.dtype): break crd_cols = frame1.columns[:icol + 1] # discover field shape and make coordinates crd_arrs = [frame[col].drop_duplicates() for col in crd_cols] shape = [len(arr) for arr in crd_arrs] crds = viscid.arrays2crds(crd_arrs, crd_names=crd_cols) fld_names = list(frame.columns) for _col in [time_col, datetime_col] + list(crd_cols): if _col in fld_names: fld_names.remove(_col) # wrap everything up into grids grids = [] for time in unique_times: grid = Grid() grid.time = time grid.basetime = basetime try: frame1 = frame[frame[time_col] == time] except KeyError: frame1 = frame for name in fld_names: arr = frame1[name].values.reshape(shape) fld = viscid.wrap_field(arr, crds, name=name, center='node') grid.add_field(fld) grids.append(grid) if len(grids) > 1: ret = DatasetTemporal() for grid in grids: ret.add(grid) ret.basetime = basetime else: ret = grids[0] return ret
def _main(): fld = viscid.mfield[-4:4:24j, -5:5:16j, -6:6:20j] fld_f = viscid.scalar_fields_to_vector([fld, fld, fld], layout='flat') fld_i = fld_f.as_layout('interlaced') check_nd_sel(fld_f, np.s_[...], good_sel_list=np.s_[:, :, :], good_sel_names=['x', 'y', 'z'], good_sel_newmask=[False, False, False], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_[0], good_sel_list=np.s_[0, :, :], good_sel_names=['x', 'y', 'z'], good_sel_newmask=[False, False, False], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_[..., :3j], good_sel_list=np.s_[:, :, :3j], good_sel_names=['x', 'y', 'z'], good_sel_newmask=[False, False, False], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_[..., None, :3j], good_sel_list=np.s_[:, :, np.newaxis, :3j], good_sel_names=['x', 'y', 'new-x0', 'z'], good_sel_newmask=[False, False, True, False], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_[..., :3j, None], good_sel_list=np.s_[:, :, :3j, np.newaxis], good_sel_names=['x', 'y', 'z', 'new-x0'], good_sel_newmask=[False, False, False, True], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_[..., None, :3j, None], good_sel_list=np.s_[:, :, np.newaxis, :3j, np.newaxis], good_sel_names=['x', 'y', 'new-x0', 'z', 'new-x1'], good_sel_newmask=[False, False, True, False, True], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_[None, 'u=None', ..., None, :3j, None], good_sel_list=np.s_[np.newaxis, np.newaxis, :, :, np.newaxis, :3j, np.newaxis], good_sel_names=['new-x0', 'u', 'x', 'y', 'new-x1', 'z', 'new-x2'], good_sel_newmask=[True, True, False, False, True, False, True], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_[:3j, ...], good_sel_list=np.s_[:3j, :, :], good_sel_names=['x', 'y', 'z'], good_sel_newmask=[False, False, False], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_[0, ..., :3j], good_sel_list=np.s_[0, :, :3j], good_sel_names=['x', 'y', 'z'], good_sel_newmask=[False, False, False], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_[0, 'u=newaxis', ..., :3j], good_sel_list=np.s_[0, np.newaxis, :, :3j], good_sel_names=['x', 'u', 'y', 'z'], good_sel_newmask=[False, True, False, False], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_[0, np.newaxis, ..., :3j], good_sel_list=np.s_[0, np.newaxis, :, :3j], good_sel_names=['x', 'new-x0', 'y', 'z'], good_sel_newmask=[False, True, False, False], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_[0, ..., 'u=newaxis', :3j], good_sel_list=np.s_[0, :, np.newaxis, :3j], good_sel_names=['x', 'y', 'u', 'z'], good_sel_newmask=[False, False, True, False], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_[0, ..., np.newaxis, :3j], good_sel_list=np.s_[0, :, np.newaxis, :3j], good_sel_names=['x', 'y', 'new-x0', 'z'], good_sel_newmask=[False, False, True, False], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_['z=4:10, x=:3'], good_sel_list=np.s_[':3', :, '4:10'], good_sel_names=['x', 'y', 'z'], good_sel_newmask=[False, False, False], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_['z', None, 'u=None', :, :, None, 'z=:3j', None], good_sel_list=np.s_[np.newaxis, np.newaxis, :, :, np.newaxis, ':3j', np.newaxis], good_sel_names=['new-x0', 'u', 'x', 'y', 'new-x1', 'z', 'new-x2'], good_sel_newmask=[True, True, False, False, True, False, True], good_comp_sel=2, ) check_nd_sel(fld_f, np.s_['z=4:10, newaxis, x=:3'], good_sel_list=np.s_[':3', :, '4:10', np.newaxis], good_sel_names=['x', 'y', 'z', 'new-x0'], good_sel_newmask=[False, False, False, True], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_['z=4:10, u=newaxis, x=:3'], good_sel_list=np.s_[':3', :, '4:10', np.newaxis], good_sel_names=['x', 'y', 'z', 'u'], good_sel_newmask=[False, False, False, True], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_['u=newaxis, z=4:10, x=:3'], good_sel_list=np.s_[np.newaxis, ':3', :, '4:10'], good_sel_names=['u', 'x', 'y', 'z'], good_sel_newmask=[True, False, False, False], good_comp_sel=slice(None), ) check_nd_sel(fld_f, np.s_['z=4:10, x=:3, u=newaxis'], good_sel_list=np.s_[':3', :, '4:10', np.newaxis], good_sel_names=['x', 'y', 'z', 'u'], good_sel_newmask=[False, False, False, True], good_comp_sel=slice(None), ) check_slc_val('UT2010-01-01', np.datetime64('2010-01-01')) check_slc_val('T2010-01-01:T60:12:01.0', slice(viscid.as_datetime64('2010-01-01'), viscid.as_timedelta64('60:12:01.0'))) check_slc_val('[2010-01-01, 2011-02-02]', viscid.as_datetime64(['2010-01-01', '2011-02-02'])) check_slc_val('[T31, 12:20.1]', viscid.as_timedelta64(['31', '12:20.1'])) check_slc_val('[T31, T12]', viscid.as_timedelta64(['31', '12'])) check_slc_val('13j:12:3', np.s_[13j:12:3]) check_slc_val(np.s_[13j:12:3], np.s_[13j:12:3]) check_slc_val(np.s_[13j:'T12':3], np.s_[13j:viscid.as_timedelta64(12):3]) check_slc_val(np.s_[13j:'2010-01-01':3], np.s_[13j:viscid.as_datetime64('2010-01-01'):3]) check_slc_val(np.s_[13j:'None':3], np.s_[13j:None:3]) # ret = viscid.std_sel2index(np.s_[:3], x, np.s_[:3]) x = np.array([-1, -0.5, 0, 0.5, 1]) # start, step > 0 check_sbv(np.s_[-1.0j:], x, np.s_[0:]) check_sbv(np.s_[-0.9999999j:], x, np.s_[0:]) check_sbv(np.s_[-0.9999j:], x, np.s_[1:]) check_sbv(np.s_[-0.8j:], x, np.s_[1:]) check_sbv(np.s_[-0.6j:], x, np.s_[1:]) check_sbv(np.s_[0.5j:], x, np.s_[3:]) # start, step < 0 check_sbv(np.s_[-1.0j::-1], x, np.s_[0::-1]) check_sbv(np.s_[-0.51j::-1], x, np.s_[0::-1]) check_sbv(np.s_[-0.50000001j::-1], x, np.s_[1::-1]) check_sbv(np.s_[-0.5j::-1], x, np.s_[1::-1]) check_sbv(np.s_[-0.4j::-1], x, np.s_[1::-1]) check_sbv(np.s_[-0.6j::-1], x, np.s_[0::-1]) check_sbv(np.s_[0.5j::-1], x, np.s_[3::-1]) # stop, step > 0 check_sbv(np.s_[:-1.1j], x, np.s_[:0]) check_sbv(np.s_[:-1j], x, np.s_[:1]) check_sbv(np.s_[:-0.51j], x, np.s_[:1]) check_sbv(np.s_[:-0.5000001j], x, np.s_[:2]) check_sbv(np.s_[:-0.5j], x, np.s_[:2]) check_sbv(np.s_[:-0.48j], x, np.s_[:2]) # stop, step < 0 check_sbv(np.s_[:-1.0j:-1], x, np.s_[::-1]) check_sbv(np.s_[:-0.51j:-1], x, np.s_[:0:-1]) check_sbv(np.s_[:-0.5j:-1], x, np.s_[:0:-1]) check_sbv(np.s_[:-0.499999j:-1], x, np.s_[:0:-1]) check_sbv(np.s_[:-0.49j:-1], x, np.s_[:1:-1]) # length-zero slice check_sbv(np.s_[:-10j:1], x, np.s_[:0:1]) check_sbv(np.s_[:10j:-1], x, np.s_[:5:-1]) crds = fld.crds check_crd_slice(crds, np.s_[-2j:2j, 0j:, :0j], (-1.91304348, 0.33333, -6.0), (1.91304348, 5.0, -0.31578947), (12, 8, 10), ('x', 'y', 'z') ) check_crd_slice(crds, np.s_[-2j:2j, 0j, :0j], (-1.91304348, -6.0), (1.91304348, -0.31578947), (12, 10), ('x', 'z') ) check_crd_slice(crds, np.s_[-2j:2j, 0j, :0j], (-1.91304348, 0.33333, -6.0), (1.91304348, 0.33333, -0.31578947), (12, 1, 10), ('x', 'y', 'z'), variant='keep' ) check_crd_slice(crds, np.s_[-2j:2j, 0j, 'w=newaxis', :0j], (-1.91304348, 0.33333, 0.0, -6.0), (1.91304348, 0.33333, 0.0, -0.31578947), (12, 1, 1, 10), ('x', 'y', 'w', 'z'), variant='keep' ) check_crd_slice(crds.slice_and_keep(np.s_[0j]), np.s_[..., 0j], (-5.0,), (5.0,), (16,), ('y'), variant='reduce' ) check_fld_slice(fld, np.s_[-2j:2j, 0j:, :0j], (-1.91304348, 0.33333, -6.0), (1.91304348, 5.0, -0.31578947), (12, 8, 10), ('x', 'y', 'z') ) check_fld_slice(fld, np.s_[-2j:2j, 0j:, 0j], (-1.91304348, 0.33333, -0.3157894), (1.91304348, 5.0, -0.31578947), (12, 8, 1), ('x', 'y', 'z'), variant='keep' ) check_fld_slice(fld.slice_and_keep(np.s_[:, 5j]), np.s_[-2j:2j, :, 0j], (-1.91304348,), (1.91304348,), (12,), ('x',), variant='reduce' ) # fld = viscid.mfield[-4:4:24j, -5:5:16j, -6:6:20j] x = fld.get_crd('x') # check slice-by-in-array xslc = [1, 3, 5, 7] assert np.allclose(fld[xslc].get_crd('x'), x[xslc]) assert np.allclose(fld[np.array(xslc)].get_crd('x'), x[xslc]) assert np.allclose(fld[str(xslc)].get_crd('x'), x[xslc]) bool_arr = np.zeros((fld.shape[0]), dtype=np.bool_) bool_arr[xslc] = True assert np.allclose(fld[list(bool_arr)].get_crd('x'), x[xslc]) assert np.allclose(fld[bool_arr].get_crd('x'), x[xslc]) assert np.allclose(fld[str(bool_arr)].get_crd('x'), x[xslc]) val_arr = 1j * x[xslc] assert np.allclose(fld[list(val_arr)].get_crd('x'), x[xslc]) assert np.allclose(fld[val_arr].get_crd('x'), x[xslc]) assert np.allclose(fld[str(val_arr)].get_crd('x'), x[xslc]) assert isinstance(fld[list(val_arr)].crds, viscid.coordinate.UniformCrds) assert isinstance(fld[val_arr].crds, viscid.coordinate.UniformCrds) assert isinstance(fld[str(val_arr)].crds, viscid.coordinate.UniformCrds) # will turn uniform into non-uniform xslc = [1, 3, 5, 8] assert np.allclose(fld[xslc].get_crd('x'), x[xslc]) assert np.allclose(fld[np.array(xslc)].get_crd('x'), x[xslc]) assert np.allclose(fld[str(xslc)].get_crd('x'), x[xslc]) bool_arr = np.zeros((fld.shape[0]), dtype=np.bool_) bool_arr[xslc] = True assert np.allclose(fld[list(bool_arr)].get_crd('x'), x[xslc]) assert np.allclose(fld[bool_arr].get_crd('x'), x[xslc]) assert np.allclose(fld[str(bool_arr)].get_crd('x'), x[xslc]) val_arr = 1j * x[xslc] assert np.allclose(fld[list(val_arr)].get_crd('x'), x[xslc]) assert np.allclose(fld[val_arr].get_crd('x'), x[xslc]) assert np.allclose(fld[str(val_arr)].get_crd('x'), x[xslc]) assert isinstance(fld[list(val_arr)].crds, viscid.coordinate.NonuniformCrds) assert isinstance(fld[val_arr].crds, viscid.coordinate.NonuniformCrds) assert isinstance(fld[str(val_arr)].crds, viscid.coordinate.NonuniformCrds) ########### cell centered.... fld_cc = fld.as_centered('cell') x_cc = fld_cc.get_crd('x') # check slice-by-in-array xslc = [1, 3, 5, 7] assert np.allclose(fld_cc[xslc].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[np.array(xslc)].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[str(xslc)].get_crd('x'), x_cc[xslc]) bool_arr = np.zeros((fld_cc.shape[0]), dtype=np.bool_) bool_arr[xslc] = True assert np.allclose(fld_cc[list(bool_arr)].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[bool_arr].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[str(bool_arr)].get_crd('x'), x_cc[xslc]) val_arr = 1j * x_cc[xslc] assert np.allclose(fld_cc[list(val_arr)].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[val_arr].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[str(val_arr)].get_crd('x'), x_cc[xslc]) assert isinstance(fld_cc[list(val_arr)].crds, viscid.coordinate.UniformCrds) assert isinstance(fld_cc[val_arr].crds, viscid.coordinate.UniformCrds) assert isinstance(fld_cc[str(val_arr)].crds, viscid.coordinate.UniformCrds) # will turn uniform into non-uniform xslc = [1, 3, 5, 8] assert np.allclose(fld_cc[xslc].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[np.array(xslc)].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[str(xslc)].get_crd('x'), x_cc[xslc]) bool_arr = np.zeros((fld_cc.shape[0]), dtype=np.bool_) bool_arr[xslc] = True assert np.allclose(fld_cc[list(bool_arr)].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[bool_arr].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[str(bool_arr)].get_crd('x'), x_cc[xslc]) val_arr = 1j * x_cc[xslc] assert np.allclose(fld_cc[list(val_arr)].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[val_arr].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[str(val_arr)].get_crd('x'), x_cc[xslc]) assert isinstance(fld_cc[list(val_arr)].crds, viscid.coordinate.NonuniformCrds) assert isinstance(fld_cc[val_arr].crds, viscid.coordinate.NonuniformCrds) assert isinstance(fld_cc[str(val_arr)].crds, viscid.coordinate.NonuniformCrds) return 0
def time_as_timedelta64(self): return viscid.as_timedelta64(self.time, unit='s')
def save_fields(cls, fname, flds, complevel=0, compression='gzip', compression_opts=None, **kwargs): """ save some fields using the format given by the class """ # FIXME: this is only good for writing cartesian rectilnear flds # FIXME: axes are renamed if flds[0] is 1D or 2D assert len(flds) > 0 fname = os.path.expanduser(os.path.expandvars(fname)) if complevel and compression == 'gzip' and compression_opts is None: compression_opts = complevel # TODO: what if compression != 'gzip' do_compression = compression_opts is not None if isinstance(flds, list): if isinstance(flds[0], (list, tuple)): flds = OrderedDict(flds) else: flds = OrderedDict([(fld.name, fld) for fld in flds]) # FIXME: all coordinates are saved as non-uniform, the proper # way to do this is to have let coordinate format its own # hdf5 / xdmf / numpy binary output fld0 = next(iter(flds.values())) clist = fld0.crds.get_clist(full_arrays=True) crd_arrs = [np.array([0.0])] * 3 crd_names = ["x", "y", "z"] for i, c in enumerate(clist): crd_arrs[i] = c[1] crd_shape = [len(arr) for arr in crd_arrs] time = fld0.time # write arrays to the hdf5 file with h5py.File(fname, 'w') as f: for axis_name, arr in zip(crd_names, crd_arrs): loc = cls._CRDS_GROUP + '/' + axis_name if do_compression: f.create_dataset(loc, data=arr, compression=compression, compression_opts=compression_opts) else: f[loc] = arr for name, fld in flds.items(): loc = cls._FLD_GROUPS[fld.center.lower()] + '/' + name # xdmf files use kji ordering if do_compression: f.create_dataset(loc, data=fld.data.T, compression=compression, compression_opts=compression_opts) else: f[loc] = fld.data.T # big bad openggcm time_str hack to put basetime into hdf5 file for fld in flds.values(): try: tfmt = "%Y:%m:%d:%H:%M:%S.%f" sec_td = viscid.as_timedelta64(fld.time, 's') dtime = viscid.as_datetime(fld.basetime + sec_td).strftime(tfmt) epoch = viscid.readers.openggcm.GGCM_EPOCH ts = viscid.as_timedelta(fld.basetime - epoch).total_seconds() ts += fld.time timestr = "time= {0} {1:.16e} {2} 300c".format(fld.time, ts, dtime) f.create_group('openggcm') f['openggcm'].attrs['time_str'] = np.string_(timestr) break except viscid.NoBasetimeError: pass # now write an xdmf file xdmf_fname = os.path.splitext(fname)[0] + ".xdmf" relh5fname = "./" + os.path.basename(fname) with open(xdmf_fname, 'w') as f: xloc = cls._CRDS_GROUP + '/' + crd_names[0] yloc = cls._CRDS_GROUP + '/' + crd_names[1] zloc = cls._CRDS_GROUP + '/' + crd_names[2] dim_str = " ".join([str(l) for l in crd_shape][::-1]) f.write(cls._XDMF_TEMPLATE_BEGIN.format(time=time)) s = cls._XDMF_TEMPLATE_RECTILINEAR_GRID_BEGIN.format( grid_name="vgrid", crd_dims=dim_str, h5fname=relh5fname, xdim=crd_shape[0], ydim=crd_shape[1], zdim=crd_shape[2], xloc=xloc, yloc=yloc, zloc=zloc) f.write(s) for fld in flds.values(): _crd_system = viscid.as_crd_system(fld, None) if _crd_system: f.write(cls._XDMF_INFO_TEMPLATE.format(name="crd_system", value=_crd_system)) break for name, fld in flds.items(): fld = fld.as_flat().T dt = fld.dtype.name.rstrip("0123456789").title() precision = fld.dtype.itemsize fld_dim_str = " ".join([str(l) for l in fld.shape]) loc = cls._FLD_GROUPS[fld.center.lower()] + '/' + name s = cls._XDMF_TEMPLATE_ATTRIBUTE.format( fld_name=name, fld_type=fld.fldtype, center=fld.center.title(), dtype=dt, precision=precision, fld_dims=fld_dim_str, h5fname=relh5fname, fld_loc=loc) f.write(s) f.write(cls._XDMF_TEMPLATE_GRID_END) f.write(cls._XDMF_TEMPLATE_END)
def save_fields(cls, fname, flds, **kwargs): """ save some fields using the format given by the class """ # FIXME: this is only good for writing cartesian rectilnear flds # FIXME: axes are renamed if flds[0] is 1D or 2D assert len(flds) > 0 fname = os.path.expanduser(os.path.expandvars(fname)) # FIXME: all coordinates are saved as non-uniform, the proper # way to do this is to have let coordinate format its own # hdf5 / xdmf / numpy binary output clist = flds[0].crds.get_clist(full_arrays=True) crd_arrs = [np.array([0.0])] * 3 crd_names = ["x", "y", "z"] for i, c in enumerate(clist): crd_arrs[i] = c[1] crd_shape = [len(arr) for arr in crd_arrs] time = flds[0].time # write arrays to the hdf5 file with h5py.File(fname, 'w') as f: for axis_name, arr in zip(crd_names, crd_arrs): loc = cls._CRDS_GROUP + '/' + axis_name f[loc] = arr for fld in flds: loc = cls._FLD_GROUPS[fld.center.lower()] + '/' + fld.name # xdmf files use kji ordering f[loc] = fld.data.T # big bad openggcm time_str hack to put basetime into hdf5 file for fld in flds: try: tfmt = "%Y:%m:%d:%H:%M:%S.%f" sec_td = viscid.as_timedelta64(fld.time, 's') dtime = viscid.as_datetime(fld.basetime + sec_td).strftime(tfmt) epoch = viscid.readers.openggcm.GGCM_EPOCH ts = viscid.as_timedelta(fld.basetime - epoch).total_seconds() ts += fld.time timestr = "time= {0} {1:.16e} {2} 300c".format(fld.time, ts, dtime) f.create_group('openggcm') f['openggcm'].attrs['time_str'] = np.string_(timestr) break except viscid.NoBasetimeError: pass # now write an xdmf file xdmf_fname = os.path.splitext(fname)[0] + ".xdmf" relh5fname = "./" + os.path.basename(fname) with open(xdmf_fname, 'w') as f: xloc = cls._CRDS_GROUP + '/' + crd_names[0] yloc = cls._CRDS_GROUP + '/' + crd_names[1] zloc = cls._CRDS_GROUP + '/' + crd_names[2] dim_str = " ".join([str(l) for l in crd_shape][::-1]) f.write(cls._XDMF_TEMPLATE_BEGIN.format(time=time)) s = cls._XDMF_TEMPLATE_RECTILINEAR_GRID_BEGIN.format( grid_name="vgrid", crd_dims=dim_str, h5fname=relh5fname, xdim=crd_shape[0], ydim=crd_shape[1], zdim=crd_shape[2], xloc=xloc, yloc=yloc, zloc=zloc) f.write(s) for fld in flds: _crd_system = viscid.get_crd_system(fld, None) if _crd_system: f.write(cls._XDMF_INFO_TEMPLATE.format(name="crd_system", value=_crd_system)) break for fld in flds: fld = fld.as_flat().T dt = fld.dtype.name.rstrip("0123456789").title() precision = fld.dtype.itemsize fld_dim_str = " ".join([str(l) for l in fld.shape]) loc = cls._FLD_GROUPS[fld.center.lower()] + '/' + fld.name s = cls._XDMF_TEMPLATE_ATTRIBUTE.format( fld_name=fld.name, fld_type=fld.fldtype, center=fld.center.title(), dtype=dt, precision=precision, fld_dims=fld_dim_str, h5fname=relh5fname, fld_loc=loc) f.write(s) f.write(cls._XDMF_TEMPLATE_GRID_END) f.write(cls._XDMF_TEMPLATE_END)
def from_dataframe(frame, crd_cols=None, time_col='time', datetime_col='datetime'): """Make either a DatasetTemporal or Grid from pandas dataframe Args: frame (pandas.DataFrame): frame to parse crd_cols (List[Str], None): list of column names for coordinates time_col (str): column name of times datetime_col (str): column name of datetimes Returns: DatasetTemporal or Grid Raises: ValueError: if only 1 row given and crd_cols is None """ import pandas # discover times and possible basetime try: unique_times = frame[time_col].drop_duplicates() if 'datetime' in frame: unique_datetimes = frame[datetime_col].drop_duplicates() if len(unique_times) > 1: dt_datetime = unique_datetimes.iloc[1] - unique_datetimes.iloc[ 0] dt_time = unique_times.iloc[1] - unique_times.iloc[0] t0_timedelta = unique_times.iloc[0] * (dt_datetime / dt_time) else: t0_timedelta = viscid.as_timedelta64( 1e6 * unique_times.iloc[0], 'us') basetime = unique_datetimes.iloc[0] - t0_timedelta else: basetime = None frame0 = frame[frame[time_col] == unique_times[0]] except KeyError: unique_times = np.array([0.0]) basetime = None frame0 = frame # discover crd_cols if not given if crd_cols is None: frame1 = frame0.drop([time_col, datetime_col], axis=1, errors='ignore') if len(frame1) <= 1: raise ValueError("With only 1 row, crd_cols must be specified.") for icol in range(frame1.shape[1]): diff = frame1.iloc[1, icol] - frame1.iloc[0, icol] if diff != np.zeros((1, ), dtype=diff.dtype): break crd_cols = frame1.columns[:icol + 1] # discover field shape and make coordinates crd_arrs = [frame[col].drop_duplicates() for col in crd_cols] shape = [len(arr) for arr in crd_arrs] crds = viscid.arrays2crds(crd_arrs, crd_names=crd_cols) fld_names = list(frame.columns) for _col in [time_col, datetime_col] + list(crd_cols): if _col in fld_names: fld_names.remove(_col) # wrap everything up into grids grids = [] for time in unique_times: grid = Grid() grid.time = time grid.basetime = basetime try: frame1 = frame[frame[time_col] == time] except KeyError: frame1 = frame for name in fld_names: arr = frame1[name].values.reshape(shape) fld = viscid.wrap_field(arr, crds, name=name, center='node') grid.add_field(fld) grids.append(grid) if len(grids) > 1: ret = DatasetTemporal() for grid in grids: ret.add(grid) ret.basetime = basetime else: ret = grids[0] return ret
def time_as_timedelta64(self): return viscid.as_timedelta64(1e6 * self.time, 'us')
def _main(): fld = viscid.mfield[-4:4:24j, -5:5:16j, -6:6:20j] fld_f = viscid.scalar_fields_to_vector([fld, fld, fld], layout='flat') fld_i = fld_f.as_layout('interlaced') check_nd_sel( fld_f, np.s_[...], good_sel_list=np.s_[:, :, :], good_sel_names=['x', 'y', 'z'], good_sel_newmask=[False, False, False], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_[0], good_sel_list=np.s_[0, :, :], good_sel_names=['x', 'y', 'z'], good_sel_newmask=[False, False, False], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_[..., :3j], good_sel_list=np.s_[:, :, :3j], good_sel_names=['x', 'y', 'z'], good_sel_newmask=[False, False, False], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_[..., None, :3j], good_sel_list=np.s_[:, :, np.newaxis, :3j], good_sel_names=['x', 'y', 'new-x0', 'z'], good_sel_newmask=[False, False, True, False], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_[..., :3j, None], good_sel_list=np.s_[:, :, :3j, np.newaxis], good_sel_names=['x', 'y', 'z', 'new-x0'], good_sel_newmask=[False, False, False, True], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_[..., None, :3j, None], good_sel_list=np.s_[:, :, np.newaxis, :3j, np.newaxis], good_sel_names=['x', 'y', 'new-x0', 'z', 'new-x1'], good_sel_newmask=[False, False, True, False, True], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_[None, 'u=None', ..., None, :3j, None], good_sel_list=np.s_[np.newaxis, np.newaxis, :, :, np.newaxis, :3j, np.newaxis], good_sel_names=['new-x0', 'u', 'x', 'y', 'new-x1', 'z', 'new-x2'], good_sel_newmask=[True, True, False, False, True, False, True], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_[:3j, ...], good_sel_list=np.s_[:3j, :, :], good_sel_names=['x', 'y', 'z'], good_sel_newmask=[False, False, False], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_[0, ..., :3j], good_sel_list=np.s_[0, :, :3j], good_sel_names=['x', 'y', 'z'], good_sel_newmask=[False, False, False], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_[0, 'u=newaxis', ..., :3j], good_sel_list=np.s_[0, np.newaxis, :, :3j], good_sel_names=['x', 'u', 'y', 'z'], good_sel_newmask=[False, True, False, False], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_[0, np.newaxis, ..., :3j], good_sel_list=np.s_[0, np.newaxis, :, :3j], good_sel_names=['x', 'new-x0', 'y', 'z'], good_sel_newmask=[False, True, False, False], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_[0, ..., 'u=newaxis', :3j], good_sel_list=np.s_[0, :, np.newaxis, :3j], good_sel_names=['x', 'y', 'u', 'z'], good_sel_newmask=[False, False, True, False], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_[0, ..., np.newaxis, :3j], good_sel_list=np.s_[0, :, np.newaxis, :3j], good_sel_names=['x', 'y', 'new-x0', 'z'], good_sel_newmask=[False, False, True, False], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_['z=4:10, x=:3'], good_sel_list=np.s_[':3', :, '4:10'], good_sel_names=['x', 'y', 'z'], good_sel_newmask=[False, False, False], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_['z', None, 'u=None', :, :, None, 'z=:3j', None], good_sel_list=np.s_[np.newaxis, np.newaxis, :, :, np.newaxis, ':3j', np.newaxis], good_sel_names=['new-x0', 'u', 'x', 'y', 'new-x1', 'z', 'new-x2'], good_sel_newmask=[True, True, False, False, True, False, True], good_comp_sel=2, ) check_nd_sel( fld_f, np.s_['z=4:10, newaxis, x=:3'], good_sel_list=np.s_[':3', :, '4:10', np.newaxis], good_sel_names=['x', 'y', 'z', 'new-x0'], good_sel_newmask=[False, False, False, True], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_['z=4:10, u=newaxis, x=:3'], good_sel_list=np.s_[':3', :, '4:10', np.newaxis], good_sel_names=['x', 'y', 'z', 'u'], good_sel_newmask=[False, False, False, True], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_['u=newaxis, z=4:10, x=:3'], good_sel_list=np.s_[np.newaxis, ':3', :, '4:10'], good_sel_names=['u', 'x', 'y', 'z'], good_sel_newmask=[True, False, False, False], good_comp_sel=slice(None), ) check_nd_sel( fld_f, np.s_['z=4:10, x=:3, u=newaxis'], good_sel_list=np.s_[':3', :, '4:10', np.newaxis], good_sel_names=['x', 'y', 'z', 'u'], good_sel_newmask=[False, False, False, True], good_comp_sel=slice(None), ) check_slc_val('UT2010-01-01', np.datetime64('2010-01-01')) check_slc_val( 'T2010-01-01:T60:12:01.0', slice(viscid.as_datetime64('2010-01-01'), viscid.as_timedelta64('60:12:01.0'))) check_slc_val('[2010-01-01, 2011-02-02]', viscid.as_datetime64(['2010-01-01', '2011-02-02'])) check_slc_val('[T31, 12:20.1]', viscid.as_timedelta64(['31', '12:20.1'])) check_slc_val('[T31, T12]', viscid.as_timedelta64(['31', '12'])) check_slc_val('13j:12:3', np.s_[13j:12:3]) check_slc_val(np.s_[13j:12:3], np.s_[13j:12:3]) check_slc_val(np.s_[13j:'T12':3], np.s_[13j:viscid.as_timedelta64(12):3]) check_slc_val(np.s_[13j:'2010-01-01':3], np.s_[13j:viscid.as_datetime64('2010-01-01'):3]) check_slc_val(np.s_[13j:'None':3], np.s_[13j:None:3]) # ret = viscid.std_sel2index(np.s_[:3], x, np.s_[:3]) x = np.array([-1, -0.5, 0, 0.5, 1]) # start, step > 0 check_sbv(np.s_[-1.0j:], x, np.s_[0:]) check_sbv(np.s_[-0.9999999j:], x, np.s_[0:]) check_sbv(np.s_[-0.9999j:], x, np.s_[1:]) check_sbv(np.s_[-0.8j:], x, np.s_[1:]) check_sbv(np.s_[-0.6j:], x, np.s_[1:]) check_sbv(np.s_[0.5j:], x, np.s_[3:]) # start, step < 0 check_sbv(np.s_[-1.0j::-1], x, np.s_[0::-1]) check_sbv(np.s_[-0.51j::-1], x, np.s_[0::-1]) check_sbv(np.s_[-0.50000001j::-1], x, np.s_[1::-1]) check_sbv(np.s_[-0.5j::-1], x, np.s_[1::-1]) check_sbv(np.s_[-0.4j::-1], x, np.s_[1::-1]) check_sbv(np.s_[-0.6j::-1], x, np.s_[0::-1]) check_sbv(np.s_[0.5j::-1], x, np.s_[3::-1]) # stop, step > 0 check_sbv(np.s_[:-1.1j], x, np.s_[:0]) check_sbv(np.s_[:-1j], x, np.s_[:1]) check_sbv(np.s_[:-0.51j], x, np.s_[:1]) check_sbv(np.s_[:-0.5000001j], x, np.s_[:2]) check_sbv(np.s_[:-0.5j], x, np.s_[:2]) check_sbv(np.s_[:-0.48j], x, np.s_[:2]) # stop, step < 0 check_sbv(np.s_[:-1.0j:-1], x, np.s_[::-1]) check_sbv(np.s_[:-0.51j:-1], x, np.s_[:0:-1]) check_sbv(np.s_[:-0.5j:-1], x, np.s_[:0:-1]) check_sbv(np.s_[:-0.499999j:-1], x, np.s_[:0:-1]) check_sbv(np.s_[:-0.49j:-1], x, np.s_[:1:-1]) # length-zero slice check_sbv(np.s_[:-10j:1], x, np.s_[:0:1]) check_sbv(np.s_[:10j:-1], x, np.s_[:5:-1]) crds = fld.crds check_crd_slice(crds, np.s_[-2j:2j, 0j:, :0j], (-1.91304348, 0.33333, -6.0), (1.91304348, 5.0, -0.31578947), (12, 8, 10), ('x', 'y', 'z')) check_crd_slice(crds, np.s_[-2j:2j, 0j, :0j], (-1.91304348, -6.0), (1.91304348, -0.31578947), (12, 10), ('x', 'z')) check_crd_slice(crds, np.s_[-2j:2j, 0j, :0j], (-1.91304348, 0.33333, -6.0), (1.91304348, 0.33333, -0.31578947), (12, 1, 10), ('x', 'y', 'z'), variant='keep') check_crd_slice(crds, np.s_[-2j:2j, 0j, 'w=newaxis', :0j], (-1.91304348, 0.33333, 0.0, -6.0), (1.91304348, 0.33333, 0.0, -0.31578947), (12, 1, 1, 10), ('x', 'y', 'w', 'z'), variant='keep') check_crd_slice(crds.slice_and_keep(np.s_[0j]), np.s_[..., 0j], (-5.0, ), (5.0, ), (16, ), ('y'), variant='reduce') check_fld_slice(fld, np.s_[-2j:2j, 0j:, :0j], (-1.91304348, 0.33333, -6.0), (1.91304348, 5.0, -0.31578947), (12, 8, 10), ('x', 'y', 'z')) check_fld_slice(fld, np.s_[-2j:2j, 0j:, 0j], (-1.91304348, 0.33333, -0.3157894), (1.91304348, 5.0, -0.31578947), (12, 8, 1), ('x', 'y', 'z'), variant='keep') check_fld_slice(fld.slice_and_keep(np.s_[:, 5j]), np.s_[-2j:2j, :, 0j], (-1.91304348, ), (1.91304348, ), (12, ), ('x', ), variant='reduce') # fld = viscid.mfield[-4:4:24j, -5:5:16j, -6:6:20j] x = fld.get_crd('x') # check slice-by-in-array xslc = [1, 3, 5, 7] assert np.allclose(fld[xslc].get_crd('x'), x[xslc]) assert np.allclose(fld[np.array(xslc)].get_crd('x'), x[xslc]) assert np.allclose(fld[str(xslc)].get_crd('x'), x[xslc]) bool_arr = np.zeros((fld.shape[0]), dtype=np.bool_) bool_arr[xslc] = True assert np.allclose(fld[list(bool_arr)].get_crd('x'), x[xslc]) assert np.allclose(fld[bool_arr].get_crd('x'), x[xslc]) assert np.allclose(fld[str(bool_arr)].get_crd('x'), x[xslc]) val_arr = 1j * x[xslc] assert np.allclose(fld[list(val_arr)].get_crd('x'), x[xslc]) assert np.allclose(fld[val_arr].get_crd('x'), x[xslc]) assert np.allclose(fld[str(val_arr)].get_crd('x'), x[xslc]) assert isinstance(fld[list(val_arr)].crds, viscid.coordinate.UniformCrds) assert isinstance(fld[val_arr].crds, viscid.coordinate.UniformCrds) assert isinstance(fld[str(val_arr)].crds, viscid.coordinate.UniformCrds) # will turn uniform into non-uniform xslc = [1, 3, 5, 8] assert np.allclose(fld[xslc].get_crd('x'), x[xslc]) assert np.allclose(fld[np.array(xslc)].get_crd('x'), x[xslc]) assert np.allclose(fld[str(xslc)].get_crd('x'), x[xslc]) bool_arr = np.zeros((fld.shape[0]), dtype=np.bool_) bool_arr[xslc] = True assert np.allclose(fld[list(bool_arr)].get_crd('x'), x[xslc]) assert np.allclose(fld[bool_arr].get_crd('x'), x[xslc]) assert np.allclose(fld[str(bool_arr)].get_crd('x'), x[xslc]) val_arr = 1j * x[xslc] assert np.allclose(fld[list(val_arr)].get_crd('x'), x[xslc]) assert np.allclose(fld[val_arr].get_crd('x'), x[xslc]) assert np.allclose(fld[str(val_arr)].get_crd('x'), x[xslc]) assert isinstance(fld[list(val_arr)].crds, viscid.coordinate.NonuniformCrds) assert isinstance(fld[val_arr].crds, viscid.coordinate.NonuniformCrds) assert isinstance(fld[str(val_arr)].crds, viscid.coordinate.NonuniformCrds) ########### cell centered.... fld_cc = fld.as_centered('cell') x_cc = fld_cc.get_crd('x') # check slice-by-in-array xslc = [1, 3, 5, 7] assert np.allclose(fld_cc[xslc].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[np.array(xslc)].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[str(xslc)].get_crd('x'), x_cc[xslc]) bool_arr = np.zeros((fld_cc.shape[0]), dtype=np.bool_) bool_arr[xslc] = True assert np.allclose(fld_cc[list(bool_arr)].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[bool_arr].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[str(bool_arr)].get_crd('x'), x_cc[xslc]) val_arr = 1j * x_cc[xslc] assert np.allclose(fld_cc[list(val_arr)].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[val_arr].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[str(val_arr)].get_crd('x'), x_cc[xslc]) assert isinstance(fld_cc[list(val_arr)].crds, viscid.coordinate.UniformCrds) assert isinstance(fld_cc[val_arr].crds, viscid.coordinate.UniformCrds) assert isinstance(fld_cc[str(val_arr)].crds, viscid.coordinate.UniformCrds) # will turn uniform into non-uniform xslc = [1, 3, 5, 8] assert np.allclose(fld_cc[xslc].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[np.array(xslc)].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[str(xslc)].get_crd('x'), x_cc[xslc]) bool_arr = np.zeros((fld_cc.shape[0]), dtype=np.bool_) bool_arr[xslc] = True assert np.allclose(fld_cc[list(bool_arr)].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[bool_arr].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[str(bool_arr)].get_crd('x'), x_cc[xslc]) val_arr = 1j * x_cc[xslc] assert np.allclose(fld_cc[list(val_arr)].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[val_arr].get_crd('x'), x_cc[xslc]) assert np.allclose(fld_cc[str(val_arr)].get_crd('x'), x_cc[xslc]) assert isinstance(fld_cc[list(val_arr)].crds, viscid.coordinate.NonuniformCrds) assert isinstance(fld_cc[val_arr].crds, viscid.coordinate.NonuniformCrds) assert isinstance(fld_cc[str(val_arr)].crds, viscid.coordinate.NonuniformCrds) return 0