# of CMEMS-Med-biogeochemistry-ScCP-1.0.pdf from commons.Timelist import TimeList from commons.time_interval import TimeInterval import commons.IOnames as IOnames import numpy as np import SatManager as Sat import matchup.matchup as matchup from commons.dataextractor import DataExtractor from layer_integral.mapbuilder import MapBuilder from commons.mask import Mask from commons.submask import SubMask from basins import OGS from commons.layer import Layer TheMask=Mask('/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc') MODEL_DIR="/pico/scratch/userexternal/gbolzon0/Carbonatic-17/wrkdir/POSTPROC/output/AVE_FREQ_1/TMP/" REF_DIR = "/pico/scratch/userexternal/gbolzon0/Carbonatic-01/SAT16/" Timestart="20140404" Time__end="20150701" TI = TimeInterval(Timestart,Time__end,"%Y%m%d") sat_TL = TimeList.fromfilenames(TI, REF_DIR,"*.nc",prefix='',dateformat='%Y%m%d') model_TL = TimeList.fromfilenames(TI, MODEL_DIR,"*.nc") IonamesFile = '../postproc/IOnames_sat.xml' IOname = IOnames.IOnames(IonamesFile) nFrames = model_TL.nTimes
from commons.mask import Mask import numpy as np import netCDF4 TheMask=Mask('/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc') CLIM_FILE="Climatology_KD490.nc" tmask = TheMask.mask_at_level(0) ncIN = netCDF4.Dataset(CLIM_FILE,'r') CLIM16 = np.array( ncIN.variables['Mean']) ncIN.close() jpk,jpj,jpi = TheMask.shape start_i=53 CLIM16[CLIM16==-999] = np.nan tmask=tmask[:, start_i:] VOIDS=np.zeros((365),np.int) for julian in range(365): clim16=CLIM16[julian,:,start_i:] ii = (np.isnan(clim16) & tmask) J,I = np.nonzero(ii) print I VOIDS[julian] = ii.sum() import sys sys.exit() import pylab as pl
def readMeshMask(self, fname): ''' Reads the meshmask.nc file and extracts the dimensions of the mesh as well as the longitude, latitude and nav_lev vectors. Requires NetCDF4 and bit.sea to work. Inputs: > fname: full path to the meshmask.nc file ''' # Load the meshmask (requires bit.sea) # This first mask is for loading the cell centered masks self._mask = Mask(fname, zlevelsvar="nav_lev", ylevelsmatvar="gphit", xlevelsmatvar="glamt") # This mask is for generating the mesh points mask = Mask(fname, zlevelsvar="gdepw", ylevelsmatvar="gphif", xlevelsmatvar="glamf") # Set variables dims = list(mask.shape) # We need to add the missing points dims[0] += 1 dims[1] += 1 dims[2] += 1 # Longitudinal coordinates, add one in the right Lon = np.insert(mask.xlevels, 0, mask.xlevels[0, :] - (mask.xlevels[0, :] - mask.xlevels[1, :]) / 2., axis=0) Lon = np.insert(Lon, 0, Lon[:, 0] - (Lon[:, 0] - Lon[:, 1]) / 2., axis=1) # Latitudinal coordinates, add one in the right Lat = np.insert(mask.ylevels, 0, mask.ylevels[0, :] + (mask.ylevels[0, :] - mask.ylevels[1, :]) / 2., axis=0) Lat = np.insert(Lat, 0, Lat[:, 0] - (Lat[:, 0] - Lat[:, 1]) / 2., axis=1) # Depth coordinates, add the one in the left nav_lev = np.append( mask.zlevels, mask.zlevels[-1] + (mask.zlevels[-1] - mask.zlevels[-2]) / 2.) # Return return dims, Lon, Lat, nav_lev
year = int(idate0[0:4]) month = int(idate0[4:6]) day = int(idate0[6:8]) idate1 = timerequestors.Weekly_req(year, month, day) print idate1 idate2 = timerequestors.Daily_req(year, month, day) print idate2 # Variable name VARLIST = ['P_l'] #'N3n','O2o'] read_adjusted = [True] #,False,False] # MASK of the domain TheMask = Mask( "/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc" ) nav_lev = TheMask.zlevels layer = Layer(0, 200) #layer of the Float profile???? # new depth from 0 to 200 meters with 1 meter of resolution NewPres = np.linspace(0, 200, 201) dimnewpress = len(NewPres) # data interpolated on the vertical Z f = open(idate0 + "." + VARLIST[0] + "_arg_mis.dat", "w") # OUTPUT x il 3DVAR iniz = " \n" f.writelines(iniz) # LIST of profiles for the selected variable in VARLIST # in the interval idate1.time_interval in the mediterranean area
import os from commons.time_interval import TimeInterval from commons.Timelist import TimeList from commons.mask import Mask from commons.layer import Layer from layer_integral.mapplot import mapplot import commons.timerequestors as requestors import Sat.SatManager as Sat import pylab as pl from layer_integral import coastline clon, clat = coastline.get() maskfile = os.getenv("MASKFILE") TheMask = Mask(maskfile) _, jpj, jpi = TheMask.shape Timestart = os.getenv("START_DATE") Time__end = os.getenv("END_DATE") INPUTDIR = args.inputdir OUTPUTDIR = args.outdir TI = TimeInterval(Timestart, Time__end, "%Y%m%d") TL = TimeList.fromfilenames(TI, INPUTDIR, "*.nc", prefix="", dateformat="%Y%m%d") # MY_YEAR = TimeInterval('20130101','20140101',"%Y%m%d")
import numpy as np from commons.time_interval import TimeInterval from commons.Timelist import TimeList from commons.mask import Mask from commons.layer import Layer from layer_integral.mapbuilder import MapBuilder from layer_integral.mapplot import mapplot, pl from commons.dataextractor import DataExtractor from commons.time_averagers import TimeAverager3D from layer_integral import coastline import commons.timerequestors as requestors clon,clat = coastline.get() TheMask=Mask('/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc') INPUTDIR = args.inputdir OUTPUTDIR = args.outdir var = args.varname LIMIT_PER_MASK=[5,5,5] LAYERLIST=[Layer(0,50)] #VARLIST=['ppn','N1p','N3n','PH_','pCO','P_l'] # saved as mg/m3/d --> * Heigh * 365/1000 #VARLIST=['DIC','AC_','PH_','pCO'] UNITS_DICT={ 'ppn' : 'gC/m^2/y', 'N1p' : 'mmol /m^3', 'N3n' : 'mmol /m^3', 'PH' : '',
class OGSmesh(object): ''' Python wrapper to the OGS C++ class that lets us interface with the OGS mesh (.ogsmsh) file. This class needs the libOGS.so that is generally deployed along with this class by the deployment scripts of the OGSParaView Suite. ''' def __init__(self, maskpath, maskname="meshmask.nc", maptype='merc', lib='libOGS'): ''' Class constructor for OGSmesh. Inputs: > maskpath: Full path to meshmask. > maptype: Kind of map projection (default: merc). > lib: Full path to the OGSmesh.so library. ''' # Class variables self._maskpath = maskpath self._maskname = maskname self._mask = None # meshmask will update after being read self._map = maptype # Interface with the C functions lib += '.dylib' if sys.platform == 'darwin' else '.so' self._OGSlib = ct.cdll.LoadLibrary(lib) @property def map(self): return self._map @map.setter def map(self, maptype): self._map = maptype def readMeshMask(self, fname): ''' Reads the meshmask.nc file and extracts the dimensions of the mesh as well as the longitude, latitude and nav_lev vectors. Requires NetCDF4 and bit.sea to work. Inputs: > fname: full path to the meshmask.nc file ''' # Load the meshmask (requires bit.sea) # This first mask is for loading the cell centered masks self._mask = Mask(fname, zlevelsvar="nav_lev", ylevelsmatvar="gphit", xlevelsmatvar="glamt") # This mask is for generating the mesh points mask = Mask(fname, zlevelsvar="gdepw", ylevelsmatvar="gphif", xlevelsmatvar="glamf") # Set variables dims = list(mask.shape) # We need to add the missing points dims[0] += 1 dims[1] += 1 dims[2] += 1 # Longitudinal coordinates, add one in the right Lon = np.insert(mask.xlevels, 0, mask.xlevels[0, :] - (mask.xlevels[0, :] - mask.xlevels[1, :]) / 2., axis=0) Lon = np.insert(Lon, 0, Lon[:, 0] - (Lon[:, 0] - Lon[:, 1]) / 2., axis=1) # Latitudinal coordinates, add one in the right Lat = np.insert(mask.ylevels, 0, mask.ylevels[0, :] + (mask.ylevels[0, :] - mask.ylevels[1, :]) / 2., axis=0) Lat = np.insert(Lat, 0, Lat[:, 0] - (Lat[:, 0] - Lat[:, 1]) / 2., axis=1) # Depth coordinates, add the one in the left nav_lev = np.append( mask.zlevels, mask.zlevels[-1] + (mask.zlevels[-1] - mask.zlevels[-2]) / 2.) # Return return dims, Lon, Lat, nav_lev def getMeshResolution(self, dims): ''' Returns the mesh resolution as a string (either "low", "mid" or "high") according to the dimensions of the mesh read. Inputs: > dims: dimensions. ''' if dims == (43, 160, 394): return "low" if dims == (72, 253, 722): return "mid" if dims == (125, 380, 1085): return "high" return None def generateBasinsMask(self): ''' Generate the basins_mask field where all basins are numbered from 1 to the number of basins. ''' return np.array([ SubMask(sub, maskobject=self._mask).mask.ravel() for sub in OGS.P.basin_list[:-1] ], dtype=c_uint8).T def generateCoastsMask(self): ''' Generate the continetnal shelf mask field where coastal areas are separated from the open sea (at depth 200m). FIX AMAL: the intersection with the med mask and mask_at_level returns zero for the last value of the mask, even if the depth is less than 200. To avoid that, we get the next depth value from 200. ''' dims = self._mask.shape # Extract mask at level 200 # This is all the places that have water at depth = 200 m jk_m = self._mask.getDepthIndex(200.) mask200_2D = self._mask.mask[jk_m + 1, :, :].copy() # FIX AMAL mask200_3D = np.array([mask200_2D for i in xrange(dims[0])]) # Extract mask for mediterranean sea # We want all the places that belong to the MED and that are water from 0 to 200 m s = SubMask(OGS.P.basin_list[-1], maskobject=self._mask) # Define coasts mask coasts_mask = np.zeros(dims, dtype=c_uint8) coasts_mask[~mask200_3D & s.mask] = 1 # Coast coasts_mask[mask200_3D & s.mask] = 2 # Open sea return coasts_mask def applyProjection(self, dims, Lon, Lat, Lon0=0., Lat0=0.): ''' Applies a map projection to the longitude and latitude vectors. The kind of map projection is defined in the class definition. Inputs: > dims: dimensions. > Lon: longitude vector. > Lat: latitude vector. ''' # Define map projection mproj = Basemap(projection = self._map, \ lat_0 = Lat0, \ lon_0 = Lon0, \ llcrnrlon = -5.3, \ llcrnrlat = 28.0, \ urcrnrlon = 37, \ urcrnrlat = 46.0, \ resolution = 'l' ) # Initialize arrays nLon = dims[2] nLat = dims[1] Lon2Meters = np.zeros((nLon, ), np.double) Lat2Meters = np.zeros((nLat, ), np.double) # Perform projection for ii in xrange(0, nLon): xpt, ypt = mproj(Lon[60, ii], Lat[60, 0]) # FIXED NEW Lon2Meters[ ii] = xpt if not self._map == 'cyl' else 6371e3 * np.deg2rad( xpt) for jj in xrange(0, nLat): xpt, ypt = mproj(Lon[0, nLon / 2], Lat[jj, nLon / 2]) Lat2Meters[ jj] = ypt if not self._map == 'cyl' else 6371e3 * np.deg2rad( ypt) # Return return np.sort(Lon2Meters), np.sort(Lat2Meters) def OGSwriteMesh(self, fname, wrkdir, Lon2Meters, Lat2Meters, nav_lev, basins_mask, coast_mask): ''' Wrapper for the C function writeOGSMesh inside the OGSmesh.so library. ''' writeMesh = self._OGSlib.OGSWriteMesh OGS.argtypes = [ c_char_p, c_char_p, c_int, c_int, c_int, c_double_p, c_double_p, c_uint8_p, c_uint8_p ] OGS.restype = c_int # Compute sizes of vectors nLon = Lon2Meters.shape[0] nLat = Lat2Meters.shape[0] nLev = nav_lev.shape[0] # Return class instance return writeMesh(fname,wrkdir,c_int(nLon),c_int(nLat),c_int(nLev),Lon2Meters.ctypes.data_as(c_double_p),\ Lat2Meters.ctypes.data_as(c_double_p), nav_lev.ctypes.data_as(c_double_p),\ basins_mask.ctypes.data_as(c_uint8_p),coast_mask.ctypes.data_as(c_uint8_p) ) def createOGSMesh(self, fname="mesh.ogsmsh", path="."): ''' creates a binary file containing all the mesh information for ParaView as well as the basins and coasts masks. Inputs: > fname: name of the output file to write. It will be written at the same path as the meshmask.nc file (Default: mesh.ogsmsh). ''' # Read the mesh mask dims, Lon, Lat, nav_lev = self.readMeshMask( os.path.join(self._maskpath, self._maskname)) # Obtain the coasts_mask and the basins_mask basins_mask = self.generateBasinsMask().ravel() coasts_mask = self.generateCoastsMask().ravel() # Project latitude and longitude according to map specifics Lon2Meters, Lat2Meters = self.applyProjection(dims, Lon, Lat) # Save into file self.OGSwriteMesh(fname, path, Lon2Meters, Lat2Meters, nav_lev, basins_mask, coasts_mask)
import numpy as np import scipy.io.netcdf as NC import glob,os import pickle import datetime from commons.mask import Mask from commons.submask import SubMask from basins import OGS maskfile = os.getenv('MASKFILE') TheMask = Mask(maskfile) tk_m = TheMask.getDepthIndex(200) mask200 = TheMask.mask_at_level(200) jpk,jpj,jpi =TheMask.shape dtype = [(sub.name, np.bool) for sub in OGS.P] SUB = np.zeros((jpj,jpi),dtype=dtype) for sub in OGS.P: sbmask = SubMask(sub,maskobject=TheMask).mask SUB[sub.name] = sbmask[0,:,:] SUB_LIST = ['alb', 'sww', 'swe', 'nwm', 'tyr', 'adn', 'ads', 'aeg', 'ion', 'lev', 'med'] numsub = len(SUB_LIST) def stats_open_coast(filelist,var): numd = len(filelist)
default=None, required=True, help=''' Path of maskfile''') return parser.parse_args() args = argument() nproc = args.mpiprocs max_proc_i = args.max_proc_i max_proc_j = args.max_proc_j from domdec import * from commons.mask import Mask TheMask = Mask(args.maskfile, dzvarname="e3t_0") tmask = TheMask.mask_at_level(0) jpjglo, jpiglo = tmask.shape USED_PROCS, COMMUNICATION = candidate_decompositions(tmask, max_proc_i, max_proc_j, nproc) choosen_procs, nproci, nprocj = get_best_decomposition(USED_PROCS, COMMUNICATION, nproc, jpiglo, jpjglo) dump_outfile(tmask, choosen_procs, nproci, nprocj) fig, ax = plot_decomposition(tmask, nproci, nprocj) fig.set_dpi(150) outfile = 'domdec_' + str(choosen_procs) + ".png"
# generates submask.nc for createGridDA # to be executed in MODEL/ or where meshmask.nc is from basins import V0 as OGS from commons.submask import SubMask from commons.mask import Mask TheMask = Mask('meshmask.nc', dzvarname="e3t_0") for sub in OGS.Pred: S = SubMask(sub, maskobject=TheMask) S.save_as_netcdf('submask.nc', maskvarname=sub.name)
annaCoast = False maskfile = os.getenv("MASKFILE"); if maskfile is None : print "Error: Environment variable MASKFILE must be defined " sys.exit(1) if annaCoast: kcoastfile = os.getenv( "KCOASTFILE"); if kcoastfile is None: print "Error: Environment variable KCOASTFILE must be defined " sys.exit(1) TheMask=Mask(maskfile,ylevelsmatvar="gphit", xlevelsmatvar="glamt") jpk,jpj,jpi = TheMask.shape tmask = TheMask.mask nav_lev = TheMask.zlevels Lon = TheMask.xlevels Lat = TheMask.ylevels area = TheMask.area e3t = TheMask.dz MEDmask = tmask.copy() MEDmask[:, Lon < -5.3] = False# Atlantic buffer mask200_2D = TheMask.mask_at_level(200.0)
# Author Giorgio Bolzon # useful to check differences between 2 directories # when md5sum cannot be used because of differences in NetCDF files import numpy as np from commons.dataextractor import DataExtractor from commons.mask import Mask import os, glob TheMask = Mask( '/gpfs/work/IscrC_MEDCOAST_0/test_swp/TEST01/wrkdir/MODEL/meshmask.nc') DIR1 = "/gpfs/work/IscrC_MEDCOAST_0/test_swp/test_FSA/ogstm/testcase/TEST01/wrkdir/MODEL/AVE_FREQ_2/" DIR2 = "/gpfs/work/IscrC_MEDCOAST_0/test_swp/TEST01/wrkdir/MODEL/AVE_FREQ_2/" searchstr = "ave.20000101-12:00:00*nc" filelist = glob.glob(DIR1 + searchstr) for file1 in filelist: file2 = DIR2 + os.path.basename(file1) prefix, datestr, varname, _ = os.path.basename(file2).rsplit(".") VAR1 = DataExtractor(TheMask, file1, varname).values VAR2 = DataExtractor(TheMask, file2, varname).values d = VAR2 - VAR1 print varname, (d**2).sum() if varname == 'O3h': break
from commons.time_interval import TimeInterval from commons.Timelist import TimeList from commons.mask import Mask from commons.layer import Layer from layer_integral.mapbuilder import MapBuilder from layer_integral.mapplot import mapplot, pl from commons.dataextractor import DataExtractor from commons.time_averagers import TimeAverager3D from layer_integral import coastline import commons.timerequestors as requestors clon,clat = coastline.get() maskfile = os.getenv('MASKFILE') TheMask = Mask(maskfile) Timestart=os.getenv("START_DATE") Time__end=os.getenv("END_DATE") INPUTDIR = args.inputdir OUTPUTDIR = args.outdir var = args.varname if args.optype=='mean': LAYERLIST=[Layer( 0, 10), \ Layer( 10, 50), \ Layer( 50, 100), \ Layer(100, 150), \ Layer(150, 300), \
import numpy as np from basins import V2 as OGS from commons.mask import Mask from commons.submask import SubMask import pylab as pl from layer_integral.mapplot import generic_mapplot_medeaf import matplotlib.font_manager as font_manager from matplotlib.font_manager import FontProperties TheMask=Mask('/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc') mask = TheMask.mask_at_level(0) jpk,jpj,jpi = TheMask.shape TheMask.cut_at_level(0) SUB_matrix=np.zeros((jpj,jpi),np.float32) SUBLIST = OGS.Pred.basin_list nSub = len(SUBLIST) xC = np.zeros(nSub,np.float32) yC = np.zeros(nSub,np.float32) colortext=['w','w','w','w','w','k','k','k','k','k','k','k','k','k','w','w'] cmap=pl.get_cmap('jet',nSub) for isub, sub in enumerate(SUBLIST): m = SubMask(sub,maskobject=TheMask) submask = m.mask[0,:,:] SUB_matrix[submask] =isub xC[isub]=TheMask.xlevels[submask].mean() yC[isub]=TheMask.ylevels[submask].mean()
from commons.time_interval import TimeInterval from commons.Timelist import TimeList from commons.mask import Mask from commons.layer import Layer from layer_integral.mapbuilder import MapBuilder from layer_integral.mapplot import mapplot,pl from commons.dataextractor import DataExtractor from commons.time_averagers import TimeAverager3D from layer_integral import coastline import commons.timerequestors as requestors from commons.utils import addsep clon,clat = coastline.get() TheMask=Mask(args.maskfile) INPUTDIR = addsep(args.inputdir) OUTPUTDIR = addsep(args.outdir) var = args.varname ldtype=[('top',np.float32),('bottom',np.float32),('mapdepthfilter',np.float32)] LF = np.loadtxt(args.layerfile,ldtype,ndmin=1) LAYERLIST=[ Layer(l['top'], l['bottom']) for l in LF ] UNITS_DICT={ 'ppn' : 'gC/m^2/y', 'N1p' : 'mmol /m^3',
from commons.mask import Mask from commons.submask import SubMask from basins import OGS from commons.layer import Layer import pickle import os def weighted_mean(Conc, Weight): Weight_sum = Weight.sum() Mass = (Conc * Weight).sum() Weighted_Mean = Mass/Weight_sum return Weighted_Mean maskfile = os.getenv('MASKFILE') TheMask = Mask(maskfile) Timestart=os.getenv("START_DATE") Time__end=os.getenv("END_DATE") MODEL_DIR= args.moddir REF_DIR = args.satdir outfile = args.outfile TI = TimeInterval(Timestart,Time__end,"%Y%m%d") sat_TL = TimeList.fromfilenames(TI, REF_DIR ,"*.nc", prefix="", dateformat="%Y%m%d") model_TL = TimeList.fromfilenames(TI, MODEL_DIR,"*.nc") IOname = IOnames.IOnames('IOnames_sat.xml')