# of CMEMS-Med-biogeochemistry-ScCP-1.0.pdf
from commons.Timelist import TimeList
from commons.time_interval import TimeInterval
import commons.IOnames as IOnames
import numpy as np
import SatManager as Sat
import matchup.matchup as matchup
from commons.dataextractor import DataExtractor
from layer_integral.mapbuilder import MapBuilder
from commons.mask import Mask
from commons.submask import SubMask
from basins import OGS
from commons.layer import Layer
TheMask=Mask('/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc')
MODEL_DIR="/pico/scratch/userexternal/gbolzon0/Carbonatic-17/wrkdir/POSTPROC/output/AVE_FREQ_1/TMP/"
REF_DIR = "/pico/scratch/userexternal/gbolzon0/Carbonatic-01/SAT16/"
Timestart="20140404"
Time__end="20150701"
TI = TimeInterval(Timestart,Time__end,"%Y%m%d")
sat_TL = TimeList.fromfilenames(TI, REF_DIR,"*.nc",prefix='',dateformat='%Y%m%d')
model_TL = TimeList.fromfilenames(TI, MODEL_DIR,"*.nc")
IonamesFile = '../postproc/IOnames_sat.xml'
IOname = IOnames.IOnames(IonamesFile)
nFrames = model_TL.nTimes
from commons.mask import Mask
import numpy as np
import netCDF4
TheMask=Mask('/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc')
CLIM_FILE="Climatology_KD490.nc"
tmask = TheMask.mask_at_level(0)
ncIN = netCDF4.Dataset(CLIM_FILE,'r')
CLIM16 = np.array( ncIN.variables['Mean'])
ncIN.close()
jpk,jpj,jpi = TheMask.shape
start_i=53
CLIM16[CLIM16==-999] = np.nan
tmask=tmask[:, start_i:]
VOIDS=np.zeros((365),np.int)
for julian in range(365):
clim16=CLIM16[julian,:,start_i:]
ii = (np.isnan(clim16) & tmask)
J,I = np.nonzero(ii)
print I
VOIDS[julian] = ii.sum()
import sys
sys.exit()
import pylab as pl
def readMeshMask(self, fname):
'''
Reads the meshmask.nc file and extracts the dimensions of the mesh
as well as the longitude, latitude and nav_lev vectors.
Requires NetCDF4 and bit.sea to work.
Inputs:
> fname: full path to the meshmask.nc file
'''
# Load the meshmask (requires bit.sea)
# This first mask is for loading the cell centered masks
self._mask = Mask(fname,
zlevelsvar="nav_lev",
ylevelsmatvar="gphit",
xlevelsmatvar="glamt")
# This mask is for generating the mesh points
mask = Mask(fname,
zlevelsvar="gdepw",
ylevelsmatvar="gphif",
xlevelsmatvar="glamf")
# Set variables
dims = list(mask.shape) # We need to add the missing points
dims[0] += 1
dims[1] += 1
dims[2] += 1
# Longitudinal coordinates, add one in the right
Lon = np.insert(mask.xlevels,
0,
mask.xlevels[0, :] -
(mask.xlevels[0, :] - mask.xlevels[1, :]) / 2.,
axis=0)
Lon = np.insert(Lon,
0,
Lon[:, 0] - (Lon[:, 0] - Lon[:, 1]) / 2.,
axis=1)
# Latitudinal coordinates, add one in the right
Lat = np.insert(mask.ylevels,
0,
mask.ylevels[0, :] +
(mask.ylevels[0, :] - mask.ylevels[1, :]) / 2.,
axis=0)
Lat = np.insert(Lat,
0,
Lat[:, 0] - (Lat[:, 0] - Lat[:, 1]) / 2.,
axis=1)
# Depth coordinates, add the one in the left
nav_lev = np.append(
mask.zlevels,
mask.zlevels[-1] + (mask.zlevels[-1] - mask.zlevels[-2]) / 2.)
# Return
return dims, Lon, Lat, nav_lev
year = int(idate0[0:4])
month = int(idate0[4:6])
day = int(idate0[6:8])
idate1 = timerequestors.Weekly_req(year, month, day)
print idate1
idate2 = timerequestors.Daily_req(year, month, day)
print idate2
# Variable name
VARLIST = ['P_l'] #'N3n','O2o']
read_adjusted = [True] #,False,False]
# MASK of the domain
TheMask = Mask(
"/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc"
)
nav_lev = TheMask.zlevels
layer = Layer(0, 200) #layer of the Float profile????
# new depth from 0 to 200 meters with 1 meter of resolution
NewPres = np.linspace(0, 200, 201)
dimnewpress = len(NewPres) # data interpolated on the vertical Z
f = open(idate0 + "." + VARLIST[0] + "_arg_mis.dat", "w") # OUTPUT x il 3DVAR
iniz = " \n"
f.writelines(iniz)
# LIST of profiles for the selected variable in VARLIST
# in the interval idate1.time_interval in the mediterranean area
import os
from commons.time_interval import TimeInterval
from commons.Timelist import TimeList
from commons.mask import Mask
from commons.layer import Layer
from layer_integral.mapplot import mapplot
import commons.timerequestors as requestors
import Sat.SatManager as Sat
import pylab as pl
from layer_integral import coastline
clon, clat = coastline.get()
maskfile = os.getenv("MASKFILE")
TheMask = Mask(maskfile)
_, jpj, jpi = TheMask.shape
Timestart = os.getenv("START_DATE")
Time__end = os.getenv("END_DATE")
INPUTDIR = args.inputdir
OUTPUTDIR = args.outdir
TI = TimeInterval(Timestart, Time__end, "%Y%m%d")
TL = TimeList.fromfilenames(TI, INPUTDIR, "*.nc", prefix="", dateformat="%Y%m%d")
# MY_YEAR = TimeInterval('20130101','20140101',"%Y%m%d")
import numpy as np
from commons.time_interval import TimeInterval
from commons.Timelist import TimeList
from commons.mask import Mask
from commons.layer import Layer
from layer_integral.mapbuilder import MapBuilder
from layer_integral.mapplot import mapplot, pl
from commons.dataextractor import DataExtractor
from commons.time_averagers import TimeAverager3D
from layer_integral import coastline
import commons.timerequestors as requestors
clon,clat = coastline.get()
TheMask=Mask('/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc')
INPUTDIR = args.inputdir
OUTPUTDIR = args.outdir
var = args.varname
LIMIT_PER_MASK=[5,5,5]
LAYERLIST=[Layer(0,50)]
#VARLIST=['ppn','N1p','N3n','PH_','pCO','P_l'] # saved as mg/m3/d --> * Heigh * 365/1000 #VARLIST=['DIC','AC_','PH_','pCO']
UNITS_DICT={
'ppn' : 'gC/m^2/y',
'N1p' : 'mmol /m^3',
'N3n' : 'mmol /m^3',
'PH' : '',
class OGSmesh(object):
'''
Python wrapper to the OGS C++ class that lets us interface with the
OGS mesh (.ogsmsh) file.
This class needs the libOGS.so that is generally deployed along with
this class by the deployment scripts of the OGSParaView Suite.
'''
def __init__(self,
maskpath,
maskname="meshmask.nc",
maptype='merc',
lib='libOGS'):
'''
Class constructor for OGSmesh.
Inputs:
> maskpath: Full path to meshmask.
> maptype: Kind of map projection (default: merc).
> lib: Full path to the OGSmesh.so library.
'''
# Class variables
self._maskpath = maskpath
self._maskname = maskname
self._mask = None # meshmask will update after being read
self._map = maptype
# Interface with the C functions
lib += '.dylib' if sys.platform == 'darwin' else '.so'
self._OGSlib = ct.cdll.LoadLibrary(lib)
@property
def map(self):
return self._map
@map.setter
def map(self, maptype):
self._map = maptype
def readMeshMask(self, fname):
'''
Reads the meshmask.nc file and extracts the dimensions of the mesh
as well as the longitude, latitude and nav_lev vectors.
Requires NetCDF4 and bit.sea to work.
Inputs:
> fname: full path to the meshmask.nc file
'''
# Load the meshmask (requires bit.sea)
# This first mask is for loading the cell centered masks
self._mask = Mask(fname,
zlevelsvar="nav_lev",
ylevelsmatvar="gphit",
xlevelsmatvar="glamt")
# This mask is for generating the mesh points
mask = Mask(fname,
zlevelsvar="gdepw",
ylevelsmatvar="gphif",
xlevelsmatvar="glamf")
# Set variables
dims = list(mask.shape) # We need to add the missing points
dims[0] += 1
dims[1] += 1
dims[2] += 1
# Longitudinal coordinates, add one in the right
Lon = np.insert(mask.xlevels,
0,
mask.xlevels[0, :] -
(mask.xlevels[0, :] - mask.xlevels[1, :]) / 2.,
axis=0)
Lon = np.insert(Lon,
0,
Lon[:, 0] - (Lon[:, 0] - Lon[:, 1]) / 2.,
axis=1)
# Latitudinal coordinates, add one in the right
Lat = np.insert(mask.ylevels,
0,
mask.ylevels[0, :] +
(mask.ylevels[0, :] - mask.ylevels[1, :]) / 2.,
axis=0)
Lat = np.insert(Lat,
0,
Lat[:, 0] - (Lat[:, 0] - Lat[:, 1]) / 2.,
axis=1)
# Depth coordinates, add the one in the left
nav_lev = np.append(
mask.zlevels,
mask.zlevels[-1] + (mask.zlevels[-1] - mask.zlevels[-2]) / 2.)
# Return
return dims, Lon, Lat, nav_lev
def getMeshResolution(self, dims):
'''
Returns the mesh resolution as a string (either "low", "mid" or "high")
according to the dimensions of the mesh read.
Inputs:
> dims: dimensions.
'''
if dims == (43, 160, 394): return "low"
if dims == (72, 253, 722): return "mid"
if dims == (125, 380, 1085): return "high"
return None
def generateBasinsMask(self):
'''
Generate the basins_mask field where all basins are numbered from 1
to the number of basins.
'''
return np.array([
SubMask(sub, maskobject=self._mask).mask.ravel()
for sub in OGS.P.basin_list[:-1]
],
dtype=c_uint8).T
def generateCoastsMask(self):
'''
Generate the continetnal shelf mask field where coastal areas
are separated from the open sea (at depth 200m).
FIX AMAL: the intersection with the med mask and mask_at_level
returns zero for the last value of the mask, even if the depth is less
than 200. To avoid that, we get the next depth value from 200.
'''
dims = self._mask.shape
# Extract mask at level 200
# This is all the places that have water at depth = 200 m
jk_m = self._mask.getDepthIndex(200.)
mask200_2D = self._mask.mask[jk_m + 1, :, :].copy() # FIX AMAL
mask200_3D = np.array([mask200_2D for i in xrange(dims[0])])
# Extract mask for mediterranean sea
# We want all the places that belong to the MED and that are water from 0 to 200 m
s = SubMask(OGS.P.basin_list[-1], maskobject=self._mask)
# Define coasts mask
coasts_mask = np.zeros(dims, dtype=c_uint8)
coasts_mask[~mask200_3D & s.mask] = 1 # Coast
coasts_mask[mask200_3D & s.mask] = 2 # Open sea
return coasts_mask
def applyProjection(self, dims, Lon, Lat, Lon0=0., Lat0=0.):
'''
Applies a map projection to the longitude and latitude
vectors. The kind of map projection is defined in the
class definition.
Inputs:
> dims: dimensions.
> Lon: longitude vector.
> Lat: latitude vector.
'''
# Define map projection
mproj = Basemap(projection = self._map, \
lat_0 = Lat0, \
lon_0 = Lon0, \
llcrnrlon = -5.3, \
llcrnrlat = 28.0, \
urcrnrlon = 37, \
urcrnrlat = 46.0, \
resolution = 'l'
)
# Initialize arrays
nLon = dims[2]
nLat = dims[1]
Lon2Meters = np.zeros((nLon, ), np.double)
Lat2Meters = np.zeros((nLat, ), np.double)
# Perform projection
for ii in xrange(0, nLon):
xpt, ypt = mproj(Lon[60, ii], Lat[60, 0]) # FIXED NEW
Lon2Meters[
ii] = xpt if not self._map == 'cyl' else 6371e3 * np.deg2rad(
xpt)
for jj in xrange(0, nLat):
xpt, ypt = mproj(Lon[0, nLon / 2], Lat[jj, nLon / 2])
Lat2Meters[
jj] = ypt if not self._map == 'cyl' else 6371e3 * np.deg2rad(
ypt)
# Return
return np.sort(Lon2Meters), np.sort(Lat2Meters)
def OGSwriteMesh(self, fname, wrkdir, Lon2Meters, Lat2Meters, nav_lev,
basins_mask, coast_mask):
'''
Wrapper for the C function writeOGSMesh inside the OGSmesh.so library.
'''
writeMesh = self._OGSlib.OGSWriteMesh
OGS.argtypes = [
c_char_p, c_char_p, c_int, c_int, c_int, c_double_p, c_double_p,
c_uint8_p, c_uint8_p
]
OGS.restype = c_int
# Compute sizes of vectors
nLon = Lon2Meters.shape[0]
nLat = Lat2Meters.shape[0]
nLev = nav_lev.shape[0]
# Return class instance
return writeMesh(fname,wrkdir,c_int(nLon),c_int(nLat),c_int(nLev),Lon2Meters.ctypes.data_as(c_double_p),\
Lat2Meters.ctypes.data_as(c_double_p), nav_lev.ctypes.data_as(c_double_p),\
basins_mask.ctypes.data_as(c_uint8_p),coast_mask.ctypes.data_as(c_uint8_p)
)
def createOGSMesh(self, fname="mesh.ogsmsh", path="."):
'''
creates a binary file containing all the mesh information for ParaView as well as
the basins and coasts masks.
Inputs:
> fname: name of the output file to write. It will be written at the same path
as the meshmask.nc file (Default: mesh.ogsmsh).
'''
# Read the mesh mask
dims, Lon, Lat, nav_lev = self.readMeshMask(
os.path.join(self._maskpath, self._maskname))
# Obtain the coasts_mask and the basins_mask
basins_mask = self.generateBasinsMask().ravel()
coasts_mask = self.generateCoastsMask().ravel()
# Project latitude and longitude according to map specifics
Lon2Meters, Lat2Meters = self.applyProjection(dims, Lon, Lat)
# Save into file
self.OGSwriteMesh(fname, path, Lon2Meters, Lat2Meters, nav_lev,
basins_mask, coasts_mask)
import numpy as np
import scipy.io.netcdf as NC
import glob,os
import pickle
import datetime
from commons.mask import Mask
from commons.submask import SubMask
from basins import OGS
maskfile = os.getenv('MASKFILE')
TheMask = Mask(maskfile)
tk_m = TheMask.getDepthIndex(200)
mask200 = TheMask.mask_at_level(200)
jpk,jpj,jpi =TheMask.shape
dtype = [(sub.name, np.bool) for sub in OGS.P]
SUB = np.zeros((jpj,jpi),dtype=dtype)
for sub in OGS.P:
sbmask = SubMask(sub,maskobject=TheMask).mask
SUB[sub.name] = sbmask[0,:,:]
SUB_LIST = ['alb', 'sww', 'swe', 'nwm', 'tyr', 'adn', 'ads', 'aeg', 'ion', 'lev', 'med']
numsub = len(SUB_LIST)
def stats_open_coast(filelist,var):
numd = len(filelist)
default=None,
required=True,
help=''' Path of maskfile''')
return parser.parse_args()
args = argument()
nproc = args.mpiprocs
max_proc_i = args.max_proc_i
max_proc_j = args.max_proc_j
from domdec import *
from commons.mask import Mask
TheMask = Mask(args.maskfile, dzvarname="e3t_0")
tmask = TheMask.mask_at_level(0)
jpjglo, jpiglo = tmask.shape
USED_PROCS, COMMUNICATION = candidate_decompositions(tmask, max_proc_i,
max_proc_j, nproc)
choosen_procs, nproci, nprocj = get_best_decomposition(USED_PROCS,
COMMUNICATION, nproc,
jpiglo, jpjglo)
dump_outfile(tmask, choosen_procs, nproci, nprocj)
fig, ax = plot_decomposition(tmask, nproci, nprocj)
fig.set_dpi(150)
outfile = 'domdec_' + str(choosen_procs) + ".png"
# generates submask.nc for createGridDA
# to be executed in MODEL/ or where meshmask.nc is
from basins import V0 as OGS
from commons.submask import SubMask
from commons.mask import Mask
TheMask = Mask('meshmask.nc', dzvarname="e3t_0")
for sub in OGS.Pred:
S = SubMask(sub, maskobject=TheMask)
S.save_as_netcdf('submask.nc', maskvarname=sub.name)
annaCoast = False
maskfile = os.getenv("MASKFILE");
if maskfile is None :
print "Error: Environment variable MASKFILE must be defined "
sys.exit(1)
if annaCoast:
kcoastfile = os.getenv( "KCOASTFILE");
if kcoastfile is None:
print "Error: Environment variable KCOASTFILE must be defined "
sys.exit(1)
TheMask=Mask(maskfile,ylevelsmatvar="gphit", xlevelsmatvar="glamt")
jpk,jpj,jpi = TheMask.shape
tmask = TheMask.mask
nav_lev = TheMask.zlevels
Lon = TheMask.xlevels
Lat = TheMask.ylevels
area = TheMask.area
e3t = TheMask.dz
MEDmask = tmask.copy()
MEDmask[:, Lon < -5.3] = False# Atlantic buffer
mask200_2D = TheMask.mask_at_level(200.0)
# Author Giorgio Bolzon
# useful to check differences between 2 directories
# when md5sum cannot be used because of differences in NetCDF files
import numpy as np
from commons.dataextractor import DataExtractor
from commons.mask import Mask
import os, glob
TheMask = Mask(
'/gpfs/work/IscrC_MEDCOAST_0/test_swp/TEST01/wrkdir/MODEL/meshmask.nc')
DIR1 = "/gpfs/work/IscrC_MEDCOAST_0/test_swp/test_FSA/ogstm/testcase/TEST01/wrkdir/MODEL/AVE_FREQ_2/"
DIR2 = "/gpfs/work/IscrC_MEDCOAST_0/test_swp/TEST01/wrkdir/MODEL/AVE_FREQ_2/"
searchstr = "ave.20000101-12:00:00*nc"
filelist = glob.glob(DIR1 + searchstr)
for file1 in filelist:
file2 = DIR2 + os.path.basename(file1)
prefix, datestr, varname, _ = os.path.basename(file2).rsplit(".")
VAR1 = DataExtractor(TheMask, file1, varname).values
VAR2 = DataExtractor(TheMask, file2, varname).values
d = VAR2 - VAR1
print varname, (d**2).sum()
if varname == 'O3h': break
from commons.time_interval import TimeInterval
from commons.Timelist import TimeList
from commons.mask import Mask
from commons.layer import Layer
from layer_integral.mapbuilder import MapBuilder
from layer_integral.mapplot import mapplot, pl
from commons.dataextractor import DataExtractor
from commons.time_averagers import TimeAverager3D
from layer_integral import coastline
import commons.timerequestors as requestors
clon,clat = coastline.get()
maskfile = os.getenv('MASKFILE')
TheMask = Mask(maskfile)
Timestart=os.getenv("START_DATE")
Time__end=os.getenv("END_DATE")
INPUTDIR = args.inputdir
OUTPUTDIR = args.outdir
var = args.varname
if args.optype=='mean':
LAYERLIST=[Layer( 0, 10), \
Layer( 10, 50), \
Layer( 50, 100), \
Layer(100, 150), \
Layer(150, 300), \
import numpy as np
from basins import V2 as OGS
from commons.mask import Mask
from commons.submask import SubMask
import pylab as pl
from layer_integral.mapplot import generic_mapplot_medeaf
import matplotlib.font_manager as font_manager
from matplotlib.font_manager import FontProperties
TheMask=Mask('/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc')
mask = TheMask.mask_at_level(0)
jpk,jpj,jpi = TheMask.shape
TheMask.cut_at_level(0)
SUB_matrix=np.zeros((jpj,jpi),np.float32)
SUBLIST = OGS.Pred.basin_list
nSub = len(SUBLIST)
xC = np.zeros(nSub,np.float32)
yC = np.zeros(nSub,np.float32)
colortext=['w','w','w','w','w','k','k','k','k','k','k','k','k','k','w','w']
cmap=pl.get_cmap('jet',nSub)
for isub, sub in enumerate(SUBLIST):
m = SubMask(sub,maskobject=TheMask)
submask = m.mask[0,:,:]
SUB_matrix[submask] =isub
xC[isub]=TheMask.xlevels[submask].mean()
yC[isub]=TheMask.ylevels[submask].mean()
from commons.time_interval import TimeInterval
from commons.Timelist import TimeList
from commons.mask import Mask
from commons.layer import Layer
from layer_integral.mapbuilder import MapBuilder
from layer_integral.mapplot import mapplot,pl
from commons.dataextractor import DataExtractor
from commons.time_averagers import TimeAverager3D
from layer_integral import coastline
import commons.timerequestors as requestors
from commons.utils import addsep
clon,clat = coastline.get()
TheMask=Mask(args.maskfile)
INPUTDIR = addsep(args.inputdir)
OUTPUTDIR = addsep(args.outdir)
var = args.varname
ldtype=[('top',np.float32),('bottom',np.float32),('mapdepthfilter',np.float32)]
LF = np.loadtxt(args.layerfile,ldtype,ndmin=1)
LAYERLIST=[ Layer(l['top'], l['bottom']) for l in LF ]
UNITS_DICT={
'ppn' : 'gC/m^2/y',
'N1p' : 'mmol /m^3',
from commons.mask import Mask
from commons.submask import SubMask
from basins import OGS
from commons.layer import Layer
import pickle
import os
def weighted_mean(Conc, Weight):
Weight_sum = Weight.sum()
Mass = (Conc * Weight).sum()
Weighted_Mean = Mass/Weight_sum
return Weighted_Mean
maskfile = os.getenv('MASKFILE')
TheMask = Mask(maskfile)
Timestart=os.getenv("START_DATE")
Time__end=os.getenv("END_DATE")
MODEL_DIR= args.moddir
REF_DIR = args.satdir
outfile = args.outfile
TI = TimeInterval(Timestart,Time__end,"%Y%m%d")
sat_TL = TimeList.fromfilenames(TI, REF_DIR ,"*.nc", prefix="", dateformat="%Y%m%d")
model_TL = TimeList.fromfilenames(TI, MODEL_DIR,"*.nc")
IOname = IOnames.IOnames('IOnames_sat.xml')
