TI = TimeInterval(Timestart,Time__end,"%Y%m%d")
TLCheck = TimeList.fromfilenames(TI, CHECKDIR,"*.nc",prefix='',dateformat='%Y%m%d')
#IonamesFile = '../../../postproc/IOnames_sat.cci.xml'
#IOname = IOnames.IOnames(IonamesFile)
suffix = os.path.basename(TLCheck.filelist[0])[8:]
WEEK_reqs=TLCheck.getWeeklyList(2)
jpi = Sat.masks.KD490mesh.jpi
jpj = Sat.masks.KD490mesh.jpj
for req in WEEK_reqs:
outfile = req.string + suffix
outpathfile = WEEKLYDIR + outfile
conditionToSkip = (os.path.exists(outpathfile)) and (not reset)
if conditionToSkip: continue
print outfile
ii, w = TLCheck.select(req)
nFiles = len(ii)
M = np.zeros((nFiles,jpj,jpi),np.float32)
for iFrame, j in enumerate(ii):
inputfile = TLCheck.filelist[j]
Kext = Sat.readfromfile(inputfile,'KD490')
M[iFrame,:,:] = Kext
Kext_OUT = Sat.averager(M)
Sat.dump_KD490_nativefile(outpathfile, Kext_OUT)
forecasts =TS.get_forecast_days(rundays=[2])
forecasts_sublist=TS.get_sublist(forecasts,[2,3,4]) #forecast tuesday and wed,thu
sat_archive="/gss/gss_work/DRES_OGS_BiGe/Observations/TIME_RAW_DATA/ONLINE/SAT/MODIS/DAILY/CHECKED/"
DAILY_SAT_LIST=TS.get_daily_sat(forecasts_sublist,sat_archive)
# float aggregator already done by others
day=0
surf_layer=Layer(0,10)
for time,archived_file,satfile in DAILY_SAT_LIST:
avefile=INPUTDIR + os.path.basename(archived_file)[:-3]
day=day+1
outfile=OUTDIR + "misfit+%dh.nc" % (day*24)
print avefile
continue
Sat16 = Sat.convertinV4format( Sat.readfromfile(satfile) )
De = DataExtractor(TheMask,filename=avefile, varname='P_i')
Model = MapBuilder.get_layer_average(De, surf_layer)
Misfit = Sat16-Model
cloudsLand = (np.isnan(Sat16)) #| (Sat16 > 1.e19) | (Sat16<0)
modelLand = np.isnan(Model) #lands are nan
nodata = cloudsLand | modelLand
selection = ~nodata # & TheMask.mask_at_level(200.0)
Misfit[nodata] = np.NaN
netcdf3.write_2d_file(Misfit, 'chl_misfit', outfile, TheMask)
TI = TimeInterval(Timestart, Time__end, "%Y%m%d")
TL = TimeList.fromfilenames(TI, INPUTDIR, "*.nc", prefix="", dateformat="%Y%m%d")
# MY_YEAR = TimeInterval('20130101','20140101',"%Y%m%d")
req_label = Timestart[0:4] #'Ave:2013'
req = requestors.Generic_req(TI)
indexes, weights = TL.select(req)
nFrames = len(indexes)
SAT_3D = np.zeros((nFrames, jpj, jpi), np.float32)
for iFrame, k in enumerate(indexes):
t = TL.Timelist[k]
inputfile = INPUTDIR + t.strftime("%Y%m%d") + "_d-OC_CNR-L4-CHL-MedOC4_SAM_7KM-MED-REP-v02.nc"
CHL = Sat.readfromfile(inputfile, "lchlm")
SAT_3D[iFrame, :, :] = CHL
Sat2d = Sat.averager(SAT_3D)
masknan = TheMask.mask_at_level(0)
Sat2d[~masknan] = np.NaN
var = "SATchl"
layer = Layer(0, 10)
fig, ax = mapplot(
{"varname": var, "clim": [0, 0.4], "layer": layer, "data": Sat2d, "date": "annual"},
fig=None,
ax=None,
mask=TheMask,
coastline_lon=clon,
Time__end="20160101"
TI = TimeInterval(Timestart,Time__end,"%Y%m%d")
TLCheck = TimeList.fromfilenames(TI, CHECKDIR,"*.nc",prefix='',dateformat='%Y%m%d')
IonamesFile = '../../postproc/IOnames_sat.xml'
IOname = IOnames.IOnames(IonamesFile)
MONTHLY_reqs=TLCheck.getMonthlist()
jpi = Sat.NativeMesh.jpi
jpj = Sat.NativeMesh.jpj
for req in MONTHLY_reqs:
outfile = req.string + IOname.Output.suffix + ".nc"
outpathfile = MONTHLYDIR + outfile
conditionToSkip = (os.path.exists(outpathfile)) and (not reset)
if conditionToSkip: continue
print outfile
ii, w = TLCheck.select(req)
nFiles = len(ii)
M = np.zeros((nFiles,jpj,jpi),np.float32)
for iFrame, j in enumerate(ii):
inputfile = TLCheck.filelist[j]
CHL = Sat.readfromfile(inputfile)
M[iFrame,:,:] = CHL
CHL_OUT = Sat.logAverager(M)
Sat.dumpV4file(outpathfile, CHL_OUT)
if __name__== "__main__":
from commons.mask import Mask
from Sat import SatManager as Sat
TheMask=Mask('/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc')
tmask = TheMask.mask_at_level(0)
jpk,jpj,jpi = TheMask.shape
x = TheMask.xlevels[0,:]
y = TheMask.ylevels[:,0]
x1km = Sat.masks.KD490mesh.lon
y1km = Sat.masks.KD490mesh.lat
I_START, I_END = array_of_indices_for_slicing(x, x1km)
J_START, J_END = array_of_indices_for_slicing(y, y1km)
inputfile="/gss/gss_work/DRES_OGS_BiGe/Observations/TIME_RAW_DATA/STATIC/SAT/KD490/WEEKLY/ORIGMESH/20010522_d-OC_CNR-L3-KD490-MedOC4AD4_SAM_1KM-MED-REP-v01.nc"
Kext = Sat.readfromfile(inputfile,'KD490')
KEXT_16=interp_2d_by_cells_slices(Kext, TheMask, I_START, I_END, J_START, J_END)
Sat.dump_simple_V4file('test.nc',KEXT_16,'KD490')
Timestart="19990101"
Time__end="20160101"
TI = TimeInterval(Timestart,Time__end,"%Y%m%d")
TL_orig = TimeList.fromfilenames(TI, ORIGDIR ,"*.nc",prefix='',dateformat='%Y%m%d')
somecheck = False
for filename in TL_orig.filelist:
outfile = CHECKDIR + os.path.basename(filename)
if not os.path.exists(outfile) :
somecheck = True
break
if somecheck:
MEAN,STD = Sat.readClimatology(CLIM_FILE)
else:
print "All checks done"
for iTime, filename in enumerate(TL_orig.filelist):
outfile = CHECKDIR + os.path.basename(filename)
exit_condition = os.path.exists(outfile) and (not reset)
if exit_condition:
continue
julian = int( TL_orig.Timelist[iTime].strftime("%j") )
if julian==366: julian=365
DAILY_REF_MEAN = MEAN[julian-1,:,:]
DAILY_REF_STD = STD[julian-1,:,:]
CHL_IN = Sat.readfromfile(filename,'KD490')
#CHL_IN[581:,164:] = Sat.fillValue # BLACK SEA
from commons.mask import Mask
from Sat import SatManager as Sat
import numpy as np
from Sat.KD490 import interp2d
import netCDF4
TheMask=Mask('/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc')
CLIM_FILE="/gss/gss_work/DRES_OGS_BiGe/Observations/CLIMATOLOGY/SAT/KD490/Climatology_KD490.nc"
MEAN,STD = Sat.readClimatology(CLIM_FILE)
jpk,jpj,jpi = TheMask.shape
x = TheMask.xlevels[0,:]
y = TheMask.ylevels[:,0]
x1km = Sat.masks.KD490mesh.lon
y1km = Sat.masks.KD490mesh.lat
I_START, I_END = interp2d.array_of_indices_for_slicing(x, x1km)
J_START, J_END = interp2d.array_of_indices_for_slicing(y, y1km)
CLIM = np.zeros((365, jpj,jpi), dtype=[('MEAN',np.float32)])
for julian in range(365):
print julian
Mfine=MEAN[julian,:,:]
M16 = interp2d.interp_2d_by_cells_slices(Mfine, TheMask, I_START, I_END, J_START, J_END)
CLIM[julian,:,:] = M16
ncOUT = netCDF4.Dataset('Climatology_KD490.nc','w')
ncOUT.createDimension('lon',jpi)
ncOUT.createDimension('lat',jpj)
TheMask=Mask('/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc')
jpk,jpj,jpi = TheMask.shape
x = TheMask.xlevels[0,:]
y = TheMask.ylevels[:,0]
x1km = Sat.masks.KD490mesh.lon
y1km = Sat.masks.KD490mesh.lat
I_START, I_END = interp2d.array_of_indices_for_slicing(x, x1km)
J_START, J_END = interp2d.array_of_indices_for_slicing(y, y1km)
INPUTDIR="/gss/gss_work/DRES_OGS_BiGe/Observations/TIME_RAW_DATA/STATIC/SAT/KD490/MONTHLY/ORIGMESH/"
OUTPUTDIR="/gss/gss_work/DRES_OGS_BiGe/Observations/TIME_RAW_DATA/STATIC/SAT/KD490/MONTHLY/V4/"
dateformat="%Y%m"
reset = False
Timestart="19990102"
Time__end="20160101"
TI = TimeInterval(Timestart,Time__end,"%Y%m%d")
TL = TimeList.fromfilenames(TI, INPUTDIR,"*.nc",prefix='',dateformat=dateformat)
for filename in TL.filelist:
outfile = OUTPUTDIR + os.path.basename(filename)
exit_condition = os.path.exists(outfile) and (not reset)
if exit_condition:
continue
Mfine = Sat.readfromfile(filename,'KD490')
M16 = interp2d.interp_2d_by_cells_slices(Mfine, TheMask, I_START, I_END, J_START, J_END)
Sat.dump_simple_V4file(outfile, M16, 'KD490')
Timestart="19990101"
Time__end="20160101"
TI = TimeInterval(Timestart,Time__end,"%Y%m%d")
TL_orig = TimeList.fromfilenames(TI, ORIGDIR ,"*.nc",prefix='',dateformat='%Y%m%d')
somecheck = False
for filename in TL_orig.filelist:
outfile = CHECKDIR + os.path.basename(filename)
if not os.path.exists(outfile) :
somecheck = True
break
if somecheck:
MEAN,STD = Sat.readClimatology(CLIM_FILE)
else:
print "All checks done"
for iTime, filename in enumerate(TL_orig.filelist):
outfile = CHECKDIR + os.path.basename(filename)
exit_condition = os.path.exists(outfile) and (not reset)
if exit_condition:
continue
julian = int( TL_orig.Timelist[iTime].strftime("%j") )
DAILY_REF_MEAN = MEAN[julian-1,:,:]
DAILY_REF_STD = STD[julian-1,:,:]
CHL_IN = Sat.readfromfile(filename)
CHL_IN[581:,164:] = Sat.fillValue # BLACK SEA
cloudsLandTIME = CHL_IN == Sat.fillValue
for itime, modeltime in enumerate(model_TL.Timelist):
print modeltime
CoupledList = sat_TL.couple_with([modeltime])
sattime = CoupledList[0][0]
satfile = REF_DIR + sattime.strftime(IOname.Input.dateformat) + IOname.Output.suffix + ".nc"
modfile = model_TL.filelist[itime]
De = DataExtractor(TheMask,filename=modfile, varname='P_i')
Model = MapBuilder.get_layer_average(De, surf_layer)
#ncIN = NC.netcdf_file(modfile,'r')
#Model = ncIN.variables['P_i'].data[0,0,:,:].copy()#.astype(np.float64)
#Model = ncIN.variables['lchlm'].data.copy()
#ncIN.close()
Sat16 = Sat.readfromfile(satfile,var='lchlm') #.astype(np.float64)
cloudsLand = (np.isnan(Sat16)) | (Sat16 > 1.e19) | (Sat16<0)
modelLand = np.isnan(Model) #lands are nan
nodata = cloudsLand | modelLand
selection = ~nodata & masksel_2D
M = matchup.matchup(Model[selection], Sat16[selection])
for isub, sub in enumerate(OGS.P):
selection = SUB[sub.name] & (~nodata) & masksel_2D
M = matchup.matchup(Model[selection], Sat16[selection])
BGC_CLASS4_CHL_RMS_SURF_BASIN[itime,isub] = M.RMSE()
BGC_CLASS4_CHL_BIAS_SURF_BASIN[itime,isub] = M.bias()
weight = TheMask.area[selection]
MODEL_MEAN[itime,isub] = weighted_mean( M.Model,weight)
COASTNESS_LIST=['coast','open_sea','everywhere']
nCOAST = len(COASTNESS_LIST)
dtype = [(coast, np.bool) for coast in COASTNESS_LIST]
COASTNESS = np.ones((jpj,jpi),dtype=dtype)
COASTNESS['coast'] = ~mask200_2D;
COASTNESS['open_sea'] = mask200_2D;
#COASTNESS['everywhere'] = True;
satfile = glob.glob(REF_DIR+date+"*")[0]
modfile = MODELDIR + "ave." + date + "-12:00:00.nc"
Model = netcdf3.read_3d_file(modfile, 'P_l')[0,:,:]
try:
Sat16 = Sat.readfromfile(satfile,'lchlm') # weekly
except:
Sat16 = Sat.convertinV4format(Sat.readfromfile(satfile, 'CHL')) # daily
cloudsLand = np.isnan(Sat16)
Sat16[cloudsLand] = -999.0
cloudsLand = Sat16==-999.0;
modelLand = Model > 1.0e+19
nodata = cloudsLand | modelLand
BGC_CLASS4_CHL_RMS_SURF_BASIN = np.zeros((nSUB,nCOAST),np.float32)
BGC_CLASS4_CHL_BIAS_SURF_BASIN = np.zeros((nSUB,nCOAST),np.float32)
VALID_POINTS = np.zeros((nSUB,nCOAST),np.float32)
MODEL_MEAN = np.zeros((nSUB,nCOAST),np.float32)
