def field2d(src_lon, src_lat, src_field, dest_lon, dest_lat, dest_mask=None, nx=0, ny=0, weight=10, threads=2, pmap=None): """ Given a 2D field with time (dimensions [time, lat, lon]), interpolate onto a new grid and return the new field. This is a helper function when needing to interpolate data within files, etc. Parameters ---------- src_lon: numpy.ndarray longitude that field is on src_lat: numpy.ndarray latitude that field is on src_field: numpy.ndarray field to interpolate dest_lon: numpy.ndarray output longitudes to interpolate to dest_lat: numpy.ndarray output latitudes to interpolate to dest_mask: numpy.ndarray, optional mask to apply to interpolated data reftime: datetime, optional: Reference time as the epoch for z-grid file nx : float, optional: decorrelation length-scale for OA (same units as source data) ny : float, optional: decorrelation length-scale for OA (same units as source data) weight : int, optional: number of points to use in weighting matrix threads : int, optional: number of processing threads pmap : numpy.ndarray, optional: use the specified pmap rather than compute it Output ------ ndarray: interpolated field on the destination grid pmap: the pmap used in the inerpolation """ if pmap is None: tmp, pmap = seapy.oasurf(src_lon, src_lat, src_lat, dest_lon, dest_lat, weight=weight, nx=nx, ny=ny) if dest_mask is None: dest_mask = np.ones(dest_lat.shape) records = np.arange(0, src_field.shape[0]) maxrecs = np.maximum(1, np.minimum(records.size, np.int(_max_memory / (dest_lon.nbytes + src_lon.nbytes)))) for rn, recs in enumerate(seapy.chunker(records, maxrecs)): nfield = np.ma.array(Parallel(n_jobs=threads, verbose=2) (delayed(__interp2_thread)( src_lon, src_lat, src_field[i, :, :], dest_lon, dest_lat, pmap, weight, nx, ny, dest_mask) for i in recs), copy=False) return nfield, pmap
def __interp_grids(src_grid, child_grid, ncout, records=None, threads=2, nx=0, ny=0, weight=10, vmap=None, z_mask=False, pmap=None): """ internal method: Given a model file (average, history, etc.), interpolate the fields onto another gridded file. Parameters ---------- src_grid : seapy.model.grid data source (History, Average, etc. file) child_grid : seapy.model.grid output data grid ncout : netcdf output file [records] : array of the record indices to interpolate [threads] : number of processing threads [nx] : decorrelation length in grid-cells for x [ny] : decorrelation length in grid-cells for y [vmap] : variable name mapping [z_mask] : mask out depths in z-grids [pmap] : use the specified pmap rather than compute it Returns ------- None """ # If we don't have a variable map, then do a one-to-one mapping if vmap is None: vmap = dict() for k in seapy.roms.fields: vmap[k] = k # Generate a file to store the pmap information sname = getattr(src_grid, 'name', None) cname = getattr(child_grid, 'name', None) pmap_file = None if any(v is None for v in (sname, cname)) else \ sname + "_" + cname + "_pmap.npz" # Create or load the pmaps depending on if they exist if nx == 0: if hasattr(src_grid, "dm") and hasattr(child_grid, "dm"): nx = np.ceil(np.mean(src_grid.dm) / np.mean(child_grid.dm)) else: nx = 5 if ny == 0: if hasattr(src_grid, "dn") and hasattr(child_grid, "dn"): ny = np.ceil(np.mean(src_grid.dn) / np.mean(child_grid.dn)) else: ny = 5 if pmap is None: if pmap_file is not None and os.path.isfile(pmap_file): pmap = np.load(pmap_file) else: tmp = np.ma.masked_equal(src_grid.mask_rho, 0) tmp, pmaprho = seapy.oasurf(src_grid.lon_rho, src_grid.lat_rho, tmp, child_grid.lon_rho, child_grid.lat_rho, weight=weight, nx=nx, ny=ny) tmp = np.ma.masked_equal(src_grid.mask_u, 0) tmp, pmapu = seapy.oasurf(src_grid.lon_u, src_grid.lat_u, tmp, child_grid.lon_rho, child_grid.lat_rho, weight=weight, nx=nx, ny=ny) tmp = np.ma.masked_equal(src_grid.mask_v, 0) tmp, pmapv = seapy.oasurf(src_grid.lon_v, src_grid.lat_v, tmp, child_grid.lon_rho, child_grid.lat_rho, weight=weight, nx=nx, ny=ny) if pmap_file is not None: np.savez(pmap_file, pmaprho=pmaprho, pmapu=pmapu, pmapv=pmapv) pmap = {"pmaprho": pmaprho, "pmapu": pmapu, "pmapv": pmapv} # Get the time field ncsrc = seapy.netcdf(src_grid.filename) time = seapy.roms.get_timevar(ncsrc) # Interpolate the depths from the source to final grid src_depth = np.min(src_grid.depth_rho, 0) dst_depth = __interp2_thread(src_grid.lon_rho, src_grid.lat_rho, src_depth, child_grid.lon_rho, child_grid.lat_rho, pmap[ "pmaprho"], weight, nx, ny, child_grid.mask_rho) # Interpolate the scalar fields records = np.arange(0, ncsrc.variables[time].shape[0]) \ if records is None else np.atleast_1d(records) for src in vmap: dest = vmap[src] # Extra fields will probably be user tracers (biogeochemical) fld = seapy.roms.fields.get(dest, {"dims": 3}) # Only interpolate the fields we want in the destination if (dest not in ncout.variables) or ("rotate" in fld): continue if fld["dims"] == 2: # Compute the max number of hold in memory maxrecs = np.maximum(1, np.minimum(len(records), np.int(_max_memory / (child_grid.lon_rho.nbytes + src_grid.lon_rho.nbytes)))) for rn, recs in enumerate(seapy.chunker(records, maxrecs)): outr = np.s_[ rn * maxrecs:np.minimum((rn + 1) * maxrecs, len(records))] ndata = np.ma.array(Parallel(n_jobs=threads, verbose=2, max_nbytes=_max_memory) (delayed(__interp2_thread)( src_grid.lon_rho, src_grid.lat_rho, ncsrc.variables[src][i, :, :], child_grid.lon_rho, child_grid.lat_rho, pmap["pmaprho"], weight, nx, ny, child_grid.mask_rho) for i in recs), copy=False) ncout.variables[dest][outr, :, :] = ndata ncout.sync() else: maxrecs = np.maximum(1, np.minimum( len(records), np.int(_max_memory / (child_grid.lon_rho.nbytes * child_grid.n + src_grid.lon_rho.nbytes * src_grid.n)))) for rn, recs in enumerate(seapy.chunker(records, maxrecs)): outr = np.s_[ rn * maxrecs:np.minimum((rn + 1) * maxrecs, len(records))] ndata = np.ma.array(Parallel(n_jobs=threads, verbose=2, max_nbytes=_max_memory) (delayed(__interp3_thread)( src_grid.lon_rho, src_grid.lat_rho, src_grid.depth_rho, ncsrc.variables[src][i, :, :, :], child_grid.lon_rho, child_grid.lat_rho, child_grid.depth_rho, pmap["pmaprho"], weight, nx, ny, child_grid.mask_rho, up_factor=_up_scaling.get(dest, 1.0), down_factor=_down_scaling.get(dest, 1.0)) for i in recs), copy=False) if z_mask: __mask_z_grid(ndata, dst_depth, child_grid.depth_rho) ncout.variables[dest][outr, :, :, :] = ndata ncout.sync() # Rotate and Interpolate the vector fields. First, determine which # are the "u" and the "v" vmap fields try: velmap = { "u": list(vmap.keys())[list(vmap.values()).index("u")], "v": list(vmap.keys())[list(vmap.values()).index("v")]} except: warn("velocity not present in source file") return srcangle = src_grid.angle if src_grid.cgrid else None dstangle = child_grid.angle if child_grid.cgrid else None maxrecs = np.minimum(len(records), np.int(_max_memory / (2 * (child_grid.lon_rho.nbytes * child_grid.n + src_grid.lon_rho.nbytes * src_grid.n)))) for nr, recs in enumerate(seapy.chunker(records, maxrecs)): vel = Parallel(n_jobs=threads, verbose=2, max_nbytes=_max_memory)(delayed(__interp3_vel_thread)( src_grid.lon_rho, src_grid.lat_rho, src_grid.depth_rho, srcangle, ncsrc.variables[velmap["u"]][i, :, :, :], ncsrc.variables[velmap["v"]][i, :, :, :], child_grid.lon_rho, child_grid.lat_rho, child_grid.depth_rho, dstangle, pmap["pmaprho"], weight, nx, ny, child_grid.mask_rho) for i in recs) for j in range(len(vel)): vel_u = np.ma.array(vel[j][0], copy=False) vel_v = np.ma.array(vel[j][1], copy=False) if z_mask: __mask_z_grid(vel_u, dst_depth, child_grid.depth_rho) __mask_z_grid(vel_v, dst_depth, child_grid.depth_rho) if child_grid.cgrid: vel_u = seapy.model.rho2u(vel_u) vel_v = seapy.model.rho2v(vel_v) ncout.variables["u"][nr * maxrecs + j, :] = vel_u ncout.variables["v"][nr * maxrecs + j, :] = vel_v if "ubar" in ncout.variables: # Create ubar and vbar # depth = seapy.adddim(child_grid.depth_u, vel_u.shape[0]) ncout.variables["ubar"][nr * maxrecs + j, :] = \ np.sum(vel_u * child_grid.depth_u, axis=0) / \ np.sum(child_grid.depth_u, axis=0) if "vbar" in ncout.variables: # depth = seapy.adddim(child_grid.depth_v, vel_v.shape[0]) ncout.variables["vbar"][nr * maxrecs + j, :] = \ np.sum(vel_v * child_grid.depth_v, axis=0) / \ np.sum(child_grid.depth_v, axis=0) ncout.sync() # Return the pmap that was used return pmap
print("Usage: {:s} input_file output_file".format(sys.argv[0])) sys.exit() print("Convert {:s} to {:s}".format(infile, outfile)) maxrecs = 30 # Get the parameters inc = seapy.netcdf(infile) lat = len(inc.dimensions['lat']) lon = len(inc.dimensions['lon']) epoch, tvar = seapy.roms.get_reftime(inc) # Create the new file onc = seapy.roms.ncgen.create_frc_bulk( outfile, lat=lat, lon=lon, reftime=epoch, clobber=True) # Save the times onc.variables['time'][:] = inc.variables[tvar][:] ntimes = len(onc.dimensions['time']) onc.variables['lat'][:] = inc.variables['lat'][:] onc.variables['lon'][:] = inc.variables['lon'][:] # Copy the variables for v in seapy.roms.forcing.fields: print("{:s}, ".format(v), end='', flush=True) for l in seapy.chunker(np.arange(ntimes), maxrecs): onc.variables[v][l, :] = inc.variables[v][l, :] print('done.') onc.close() inc.close()
except: print("Usage: {:s} input_file output_file".format(sys.argv[0])) sys.exit() print("Convert {:s} to {:s}".format(infile, outfile)) maxrecs = 30 # Get the parameters inc = seapy.netcdf(infile) eta_rho = len(inc.dimensions['eta_rho']) xi_rho = len(inc.dimensions['xi_rho']) s_rho = len(inc.dimensions['s_rho']) epoch, tvar = seapy.roms.get_reftime(inc) # Create the new file onc = seapy.roms.ncgen.create_clim( outfile, eta_rho=eta_rho, xi_rho=xi_rho, s_rho=s_rho, reftime=epoch, clobber=True) # Save the times onc.variables['clim_time'][:] = inc.variables[tvar][:] ntimes = len(onc.dimensions['clim_time']) # Copy the variables for v in seapy.roms.fields: print("{:s}, ".format(v), end='', flush=True) for l in seapy.chunker(np.arange(ntimes), maxrecs): onc.variables[v][l, :] = inc.variables[v][l, :] print('done.') onc.close() inc.close()
def gen_direct_forcing(his_file, frc_file, cdl=None): """ Generate a direct forcing file from a history (or other ROMS output) file. It requires that sustr, svstr, shflux, and ssflux (or swflux) with salt be available. This will generate a forcing file that contains: sustr, svstr, swflux, and ssflux. Parameters ---------- his_file: string, The ROMS history (or other) file(s) (can use wildcards) that contains the fields to make forcing from frc_file: string, The output forcing file cdl: string, optional, Use the specified CDL file as the definition for the new netCDF file. Returns ------- None: Generates an output file of bulk forcings """ import os infile = seapy.netcdf(his_file) ref, _ = seapy.roms.get_reftime(infile) # Create the output file nc = seapy.roms.ncgen.create_frc_direct(frc_file, eta_rho=infile.dimensions[ 'eta_rho'].size, xi_rho=infile.dimensions[ 'xi_rho'].size, reftime=ref, clobber=True, title="Forcing from " + os.path.basename(his_file), cdl=cdl) # Copy the data over time = seapy.roms.num2date(infile, 'ocean_time') nc.variables['frc_time'][:] = seapy.roms.date2num(time, nc, 'frc_time') for x in seapy.progressbar.progress(seapy.chunker(range(len(time)), 1000)): nc.variables['SSS'][x, :, :] = seapy.convolve_mask( infile.variables['salt'][x, -1, :, :], copy=False) if 'EminusP' in infile.variables: nc.variables['swflux'][x, :, :] = seapy.convolve_mask( infile.variables['EminusP'][x, :, :], copy=False) * 86400 elif 'swflux' in infile.variables: nc.variables['swflux'][x, :, :] = seapy.convolve_mask( infile.variables['swflux'][x, :, :], copy=False) else: nc.variables['swflux'][x, :, :] = seapy.convolve_mask( infile.variables['ssflux'][x, :, :] / nc.variables['SSS'][x, :, :], copy=False) nc.sync() for f in ("sustr", "svstr", "shflux", "swrad"): if f in infile.variables: nc.variables[f][x, :, :] = seapy.convolve_mask( infile.variables[f][x, :, :], copy=False) nc.sync() for f in ("lat_rho", "lat_u", "lat_v", "lon_rho", "lon_u", "lon_v"): if f in infile.variables: nc.variables[f][:] = infile.variables[f][:] nc.close()
def gen_direct_forcing(his_file, frc_file, cdl=None): """ Generate a direct forcing file from a history (or other ROMS output) file. It requires that sustr, svstr, shflux, and ssflux (or swflux) with salt be available. This will generate a forcing file that contains: sustr, svstr, swflux, and ssflux. Parameters ---------- his_file: string, The ROMS history (or other) file(s) (can use wildcards) that contains the fields to make forcing from frc_file: string, The output forcing file cdl: string, optional, Use the specified CDL file as the definition for the new netCDF file. Returns ------- None: Generates an output file of bulk forcings """ import os infile = seapy.netcdf(his_file) ref, _ = seapy.roms.get_reftime(infile) # Create the output file nc = seapy.roms.ncgen.create_frc_direct( frc_file, eta_rho=infile.dimensions['eta_rho'].size, xi_rho=infile.dimensions['xi_rho'].size, reftime=ref, clobber=True, title="Forcing from " + os.path.basename(his_file), cdl=cdl) # Copy the data over time = seapy.roms.num2date(infile, 'ocean_time') nc.variables['frc_time'][:] = seapy.roms.date2num(time, nc, 'frc_time') for x in seapy.progressbar.progress(seapy.chunker(range(len(time)), 1000)): nc.variables['SSS'][x, :, :] = seapy.convolve_mask( infile.variables['salt'][x, -1, :, :], copy=False) if 'EminusP' in infile.variables: nc.variables['swflux'][x, :, :] = seapy.convolve_mask( infile.variables['EminusP'][x, :, :], copy=False) * 86400 elif 'swflux' in infile.variables: nc.variables['swflux'][x, :, :] = seapy.convolve_mask( infile.variables['swflux'][x, :, :], copy=False) else: nc.variables['swflux'][x, :, :] = seapy.convolve_mask( infile.variables['ssflux'][x, :, :] / nc.variables['SSS'][x, :, :], copy=False) nc.sync() for f in ("sustr", "svstr", "shflux", "swrad"): if f in infile.variables: nc.variables[f][x, :, :] = seapy.convolve_mask( infile.variables[f][x, :, :], copy=False) nc.sync() for f in ("lat_rho", "lat_u", "lat_v", "lon_rho", "lon_u", "lon_v"): if f in infile.variables: nc.variables[f][:] = infile.variables[f][:] nc.close()
def __interp_grids(src_grid, child_grid, ncsrc, ncout, records=None, threads=2, nx=0, ny=0, weight=10, vmap=None, z_mask=False, pmap=None): """ internal method: Given a model file (average, history, etc.), interpolate the fields onto another gridded file. Parameters ---------- src_grid : seapy.model.grid data of source child_grid : seapy.model.grid output data grid ncsrc : netcdf input file (History, Average, etc. file) ncout : netcdf output file [records] : array of the record indices to interpolate [threads] : number of processing threads [nx] : decorrelation length in grid-cells for x [ny] : decorrelation length in grid-cells for y [vmap] : variable name mapping [z_mask] : mask out depths in z-grids [pmap] : use the specified pmap rather than compute it Returns ------- None """ # If we don't have a variable map, then do a one-to-one mapping if vmap is None: vmap = dict() for k in seapy.roms.fields: vmap[k] = k # Generate a file to store the pmap information sname = getattr(src_grid, 'name', None) cname = getattr(child_grid, 'name', None) pmap_file = None if any(v is None for v in (sname, cname)) else \ sname + "_" + cname + "_pmap.npz" # Create or load the pmaps depending on if they exist if nx == 0: if hasattr(src_grid, "dm") and hasattr(child_grid, "dm"): nx = np.ceil(np.mean(src_grid.dm) / np.mean(child_grid.dm)) else: nx = 5 if ny == 0: if hasattr(src_grid, "dn") and hasattr(child_grid, "dn"): ny = np.ceil(np.mean(src_grid.dn) / np.mean(child_grid.dn)) else: ny = 5 if pmap is None: if pmap_file is not None and os.path.isfile(pmap_file): pmap = np.load(pmap_file) else: tmp = np.ma.masked_equal(src_grid.mask_rho, 0) tmp, pmaprho = seapy.oasurf(src_grid.lon_rho, src_grid.lat_rho, tmp, child_grid.lon_rho, child_grid.lat_rho, weight=weight, nx=nx, ny=ny) tmp = np.ma.masked_equal(src_grid.mask_u, 0) tmp, pmapu = seapy.oasurf(src_grid.lon_u, src_grid.lat_u, tmp, child_grid.lon_rho, child_grid.lat_rho, weight=weight, nx=nx, ny=ny) tmp = np.ma.masked_equal(src_grid.mask_v, 0) tmp, pmapv = seapy.oasurf(src_grid.lon_v, src_grid.lat_v, tmp, child_grid.lon_rho, child_grid.lat_rho, weight=weight, nx=nx, ny=ny) if pmap_file is not None: np.savez(pmap_file, pmaprho=pmaprho, pmapu=pmapu, pmapv=pmapv) pmap = {"pmaprho": pmaprho, "pmapu": pmapu, "pmapv": pmapv} # Get the time field time = seapy.roms.get_timevar(ncsrc) # Interpolate the depths from the source to final grid src_depth = np.min(src_grid.depth_rho, 0) dst_depth = __interp2_thread(src_grid.lon_rho, src_grid.lat_rho, src_depth, child_grid.lon_rho, child_grid.lat_rho, pmap[ "pmaprho"], weight, nx, ny, child_grid.mask_rho) # Interpolate the scalar fields records = np.arange(0, ncsrc.variables[time].shape[0]) \ if records is None else np.atleast_1d(records) for src in vmap: dest = vmap[src] # Extra fields will probably be user tracers (biogeochemical) fld = seapy.roms.fields.get(dest, {"dims": 3}) # Only interpolate the fields we want in the destination if (dest not in ncout.variables) or \ (src not in ncsrc.variables) or \ ("rotate" in fld): continue if fld["dims"] == 2: # Compute the max number of hold in memory maxrecs = np.maximum(1, np.minimum(len(records), np.int(_max_memory / (child_grid.lon_rho.nbytes + src_grid.lon_rho.nbytes)))) for rn, recs in enumerate(seapy.chunker(records, maxrecs)): outr = np.s_[ rn * maxrecs:np.minimum((rn + 1) * maxrecs, len(records))] ndata = np.ma.array(Parallel(n_jobs=threads, verbose=2, max_nbytes=_max_memory) (delayed(__interp2_thread)( src_grid.lon_rho, src_grid.lat_rho, ncsrc.variables[src][i, :, :], child_grid.lon_rho, child_grid.lat_rho, pmap["pmaprho"], weight, nx, ny, child_grid.mask_rho) for i in recs), copy=False) ncout.variables[dest][outr, :, :] = ndata ncout.sync() else: maxrecs = np.maximum(1, np.minimum( len(records), np.int(_max_memory / (child_grid.lon_rho.nbytes * child_grid.n + src_grid.lon_rho.nbytes * src_grid.n)))) for rn, recs in enumerate(seapy.chunker(records, maxrecs)): outr = np.s_[ rn * maxrecs:np.minimum((rn + 1) * maxrecs, len(records))] ndata = np.ma.array(Parallel(n_jobs=threads, verbose=2, max_nbytes=_max_memory) (delayed(__interp3_thread)( src_grid.lon_rho, src_grid.lat_rho, src_grid.depth_rho, ncsrc.variables[src][i, :, :, :], child_grid.lon_rho, child_grid.lat_rho, child_grid.depth_rho, pmap["pmaprho"], weight, nx, ny, child_grid.mask_rho, up_factor=_up_scaling.get(dest, 1.0), down_factor=_down_scaling.get(dest, 1.0)) for i in recs), copy=False) if z_mask: __mask_z_grid(ndata, dst_depth, child_grid.depth_rho) ncout.variables[dest][outr, :, :, :] = ndata ncout.sync() # Rotate and Interpolate the vector fields. First, determine which # are the "u" and the "v" vmap fields try: velmap = { "u": list(vmap.keys())[list(vmap.values()).index("u")], "v": list(vmap.keys())[list(vmap.values()).index("v")]} except: warn("velocity not present in source file") return srcangle = getattr(src_grid, 'angle', None) dstangle = getattr(child_grid, 'angle', None) maxrecs = np.minimum(len(records), np.int(_max_memory / (2 * (child_grid.lon_rho.nbytes * child_grid.n + src_grid.lon_rho.nbytes * src_grid.n)))) for nr, recs in enumerate(seapy.chunker(records, maxrecs)): vel = Parallel(n_jobs=threads, verbose=2, max_nbytes=_max_memory)(delayed(__interp3_vel_thread)( src_grid.lon_rho, src_grid.lat_rho, src_grid.depth_rho, srcangle, ncsrc.variables[velmap["u"]][i, :, :, :], ncsrc.variables[velmap["v"]][i, :, :, :], child_grid.lon_rho, child_grid.lat_rho, child_grid.depth_rho, dstangle, pmap["pmaprho"], weight, nx, ny, child_grid.mask_rho) for i in recs) for j in range(len(vel)): vel_u = np.ma.array(vel[j][0], copy=False) vel_v = np.ma.array(vel[j][1], copy=False) if z_mask: __mask_z_grid(vel_u, dst_depth, child_grid.depth_rho) __mask_z_grid(vel_v, dst_depth, child_grid.depth_rho) if child_grid.cgrid: vel_u = seapy.model.rho2u(vel_u) vel_v = seapy.model.rho2v(vel_v) ncout.variables["u"][nr * maxrecs + j, :] = vel_u ncout.variables["v"][nr * maxrecs + j, :] = vel_v if "ubar" in ncout.variables: # Create ubar and vbar # depth = seapy.adddim(child_grid.depth_u, vel_u.shape[0]) ncout.variables["ubar"][nr * maxrecs + j, :] = \ np.sum(vel_u * child_grid.depth_u, axis=0) / \ np.sum(child_grid.depth_u, axis=0) if "vbar" in ncout.variables: # depth = seapy.adddim(child_grid.depth_v, vel_v.shape[0]) ncout.variables["vbar"][nr * maxrecs + j, :] = \ np.sum(vel_v * child_grid.depth_v, axis=0) / \ np.sum(child_grid.depth_v, axis=0) ncout.sync() # Return the pmap that was used return pmap