from commons.time_interval import TimeInterval
from commons import IOnames
import numpy as np
import os
import Sat.SatManager as Sat
CHECKDIR ="/gss/gss_work/DRES_OGS_BiGe/Observations/TIME_RAW_DATA/STATIC/SAT/KD490/DAILY/CHECKED/"
WEEKLYDIR="/gss/gss_work/DRES_OGS_BiGe/Observations/TIME_RAW_DATA/STATIC/SAT/KD490/WEEKLY/ORIGMESH/"
reset = False
Timestart="19990102"
Time__end="20160101"
TI = TimeInterval(Timestart,Time__end,"%Y%m%d")
TLCheck = TimeList.fromfilenames(TI, CHECKDIR,"*.nc",prefix='',dateformat='%Y%m%d')
#IonamesFile = '../../../postproc/IOnames_sat.cci.xml'
#IOname = IOnames.IOnames(IonamesFile)
suffix = os.path.basename(TLCheck.filelist[0])[8:]
WEEK_reqs=TLCheck.getWeeklyList(2)
jpi = Sat.masks.KD490mesh.jpi
jpj = Sat.masks.KD490mesh.jpj
for req in WEEK_reqs:
outfile = req.string + suffix
outpathfile = WEEKLYDIR + outfile
conditionToSkip = (os.path.exists(outpathfile)) and (not reset)
from layer_integral.mapbuilder import MapBuilder
from commons.mask import Mask
from commons.submask import SubMask
from basins import OGS
from commons.layer import Layer
TheMask=Mask('/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc')
MODEL_DIR="/pico/scratch/userexternal/gbolzon0/Carbonatic-17/wrkdir/POSTPROC/output/AVE_FREQ_1/TMP/"
REF_DIR = "/pico/scratch/userexternal/gbolzon0/Carbonatic-01/SAT16/"
Timestart="20140404"
Time__end="20150701"
TI = TimeInterval(Timestart,Time__end,"%Y%m%d")
sat_TL = TimeList.fromfilenames(TI, REF_DIR,"*.nc",prefix='',dateformat='%Y%m%d')
model_TL = TimeList.fromfilenames(TI, MODEL_DIR,"*.nc")
IonamesFile = '../postproc/IOnames_sat.xml'
IOname = IOnames.IOnames(IonamesFile)
nFrames = model_TL.nTimes
nSUB = len(OGS.P.basin_list)
jpk,jpj,jpi =TheMask.shape
dtype = [(sub.name, np.bool) for sub in OGS.P]
SUB = np.zeros((jpj,jpi),dtype=dtype)
for sub in OGS.P:
sbmask = SubMask(sub,maskobject=TheMask).mask
SUB[sub.name] = sbmask[0,:,:]
layer = Layer(0, 200) #layer of the Float profile????
# new depth from 0 to 200 meters with 1 meter of resolution
NewPres = np.linspace(0, 200, 201)
dimnewpress = len(NewPres) # data interpolated on the vertical Z
f = open(idate0 + "." + VARLIST[0] + "_arg_mis.dat", "w") # OUTPUT x il 3DVAR
iniz = " \n"
f.writelines(iniz)
# LIST of profiles for the selected variable in VARLIST
# in the interval idate1.time_interval in the mediterranean area
Profilelist_1 = bio_float.FloatSelector(LOVFLOATVARS[VARLIST[0]],
idate1.time_interval, OGS.med)
TL = TimeList.fromfilenames(idate2.time_interval,
INPUTDIR,
"ave.*.nc",
filtervar="P_l")
TL.inputFrequency = 'weekly'
M = Matchup_Manager(Profilelist_1, TL, BASEDIR)
#import sys
#sys.exit()
# LIST of FLOATS (WMO) that are in Profilelist_1
WMOlist = bio_float.get_wmo_list(Profilelist_1)
nWMO = len(WMOlist) #NUMBER OF float IN THE SELECTED PERIOD
totallines = 0
for wmo in WMOlist:
#for ind in np.arange(0,1):
# wmo=WMOlist[ind]
'pCO2': [300,480],
'P_i' : [0, 0.4]
}
CONVERSION_DICT={
'ppn' : 365./100,
'N1p' : 1,
'N3n' : 1,
'PH' : 1,
'pCO2': 1,
'P_i' : 1
}
TI = TimeInterval('20000101','20121230',"%Y%m%d") # VALID FOR REANALYSIS RUN
TL = TimeList.fromfilenames(TI, INPUTDIR,"ave*.nc")
req = requestors.Clim_month(args.month)
req_label=req.longname()
indexes,weights = TL.select(req)
VARCONV=CONVERSION_DICT[var]
# setting up filelist for requested period -----------------
filelist=[]
for k in indexes:
t = TL.Timelist[k]
filename = INPUTDIR + "ave." + t.strftime("%Y%m%d-%H:%M:%S") + ".nc"
print filename
filelist.append(filename)
clon, clat = coastline.get()
maskfile = os.getenv("MASKFILE")
TheMask = Mask(maskfile)
_, jpj, jpi = TheMask.shape
Timestart = os.getenv("START_DATE")
Time__end = os.getenv("END_DATE")
INPUTDIR = args.inputdir
OUTPUTDIR = args.outdir
TI = TimeInterval(Timestart, Time__end, "%Y%m%d")
TL = TimeList.fromfilenames(TI, INPUTDIR, "*.nc", prefix="", dateformat="%Y%m%d")
# MY_YEAR = TimeInterval('20130101','20140101',"%Y%m%d")
req_label = Timestart[0:4] #'Ave:2013'
req = requestors.Generic_req(TI)
indexes, weights = TL.select(req)
nFrames = len(indexes)
SAT_3D = np.zeros((nFrames, jpj, jpi), np.float32)
for iFrame, k in enumerate(indexes):
t = TL.Timelist[k]
inputfile = INPUTDIR + t.strftime("%Y%m%d") + "_d-OC_CNR-L4-CHL-MedOC4_SAM_7KM-MED-REP-v02.nc"
CHL = Sat.readfromfile(inputfile, "lchlm")
SAT_3D[iFrame, :, :] = CHL
from commons.time_interval import TimeInterval
import numpy as np
import os
import SatManager as Sat
ORIGDIR = "/gss/gss_work/DRES_OGS_BiGe/Observations/TIME_RAW_DATA/ONLINE/SAT/MODIS/DAILY/ORIG/"
CHECKDIR = "/gss/gss_work/DRES_OGS_BiGe/Observations/TIME_RAW_DATA/ONLINE/SAT/MODIS/DAILY/CHECKED/"
CLIM_FILE = "/gss/gss_work/DRES_OGS_BiGe/Observations/CLIMATOLOGY/SAT/SeaWifs/SatClimatology.nc"
reset = False
Timestart = "19500101"
Time__end = "20500101"
TI = TimeInterval(Timestart, Time__end, "%Y%m%d")
TL_orig = TimeList.fromfilenames(TI, ORIGDIR, "*.nc", prefix="", dateformat="%Y%m%d")
somecheck = False
for filename in TL_orig.filelist:
outfile = CHECKDIR + os.path.basename(filename)
if not os.path.exists(outfile):
somecheck = True
break
if somecheck:
MEAN, STD = Sat.readClimatology(CLIM_FILE)
else:
print "All checks done"
for iTime, filename in enumerate(TL_orig.filelist):
outfile = CHECKDIR + os.path.basename(filename)
from commons.time_interval import TimeInterval
import numpy as np
import os
from Sat import SatManager as Sat
ORIGDIR ="/gss/gss_work/DRES_OGS_BiGe/Observations/TIME_RAW_DATA/STATIC/SAT/KD490/DAILY/ORIG/"
CHECKDIR ="/gss/gss_work/DRES_OGS_BiGe/Observations/TIME_RAW_DATA/STATIC/SAT/KD490/DAILY/CHECKED/"
CLIM_FILE="/gss/gss_work/DRES_OGS_BiGe/Observations/CLIMATOLOGY/SAT/KD490/Climatology_KD490.nc"
reset = False
Timestart="19990101"
Time__end="20160101"
TI = TimeInterval(Timestart,Time__end,"%Y%m%d")
TL_orig = TimeList.fromfilenames(TI, ORIGDIR ,"*.nc",prefix='',dateformat='%Y%m%d')
somecheck = False
for filename in TL_orig.filelist:
outfile = CHECKDIR + os.path.basename(filename)
if not os.path.exists(outfile) :
somecheck = True
break
if somecheck:
MEAN,STD = Sat.readClimatology(CLIM_FILE)
else:
print "All checks done"
for iTime, filename in enumerate(TL_orig.filelist):
from instruments.matchup_manager import Matchup_Manager
import basins.OGS as OGS
from instruments import lovbio_float as bio_float
from commons.mask import Mask
from commons.utils import addsep
from datetime import timedelta
from datetime import datetime
from commons import timerequestors
meantime = datetime.strptime(args.time, '%Y%m%d')
INPUTDIR = addsep(args.inputdir)
BASEDIR = addsep(args.basedir)
TI_week = timerequestors.Weekly_req(meantime.year, meantime.month,
meantime.day).time_interval
TI_day = timerequestors.Daily_req(meantime.year, meantime.month,
meantime.day).time_interval
Profilelist = bio_float.FloatSelector(None, TI_week, OGS.med)
TL = TimeList.fromfilenames(TI_day, INPUTDIR, "ave*.nc", filtervar="P_l")
TL.inputFrequency = 'weekly'
M = Matchup_Manager(Profilelist, TL, BASEDIR)
profilerscript = BASEDIR + 'jobProfiler.sh'
descriptor = "/pico/scratch/userexternal/lmariott/FLOAT_DA_01/wrkdir/float_preproc/VarDescriptor.xml"
M.writefiles_for_profiling(
descriptor, profilerscript,
aggregatedir=INPUTDIR) # preparation of data for aveScan
M.dumpModelProfiles(profilerscript) # sequential launch of aveScan
TheMask=Mask('/pico/home/usera07ogs/a07ogs00/OPA/V2C/etc/static-data/MED1672_cut/MASK/meshmask.nc')
jpk,jpj,jpi = TheMask.shape
x = TheMask.xlevels[0,:]
y = TheMask.ylevels[:,0]
x1km = Sat.masks.KD490mesh.lon
y1km = Sat.masks.KD490mesh.lat
I_START, I_END = interp2d.array_of_indices_for_slicing(x, x1km)
J_START, J_END = interp2d.array_of_indices_for_slicing(y, y1km)
INPUTDIR="/gss/gss_work/DRES_OGS_BiGe/Observations/TIME_RAW_DATA/STATIC/SAT/KD490/MONTHLY/ORIGMESH/"
OUTPUTDIR="/gss/gss_work/DRES_OGS_BiGe/Observations/TIME_RAW_DATA/STATIC/SAT/KD490/MONTHLY/V4/"
dateformat="%Y%m"
reset = False
Timestart="19990102"
Time__end="20160101"
TI = TimeInterval(Timestart,Time__end,"%Y%m%d")
TL = TimeList.fromfilenames(TI, INPUTDIR,"*.nc",prefix='',dateformat=dateformat)
for filename in TL.filelist:
outfile = OUTPUTDIR + os.path.basename(filename)
exit_condition = os.path.exists(outfile) and (not reset)
if exit_condition:
continue
Mfine = Sat.readfromfile(filename,'KD490')
M16 = interp2d.interp_2d_by_cells_slices(Mfine, TheMask, I_START, I_END, J_START, J_END)
Sat.dump_simple_V4file(outfile, M16, 'KD490')
}
CONVERSION_DICT={
'ppn' : 365./1000,
'N1p' : 1,
'N3n' : 1,
'PH' : 1,
'pCO2': 1,
'P_l' : 1,
'P_i' : 1
}
TI = TimeInterval('20010101','20141230',"%Y%m%d") # VALID FOR REANALYSIS RUN
req_label='Ave.2001-2014' #official
TL = TimeList.fromfilenames(TI, INPUTDIR,"ave*.nc",filtervar=var)
req = requestors.Generic_req(TI)
indexes,weights = TL.select(req)
VARCONV=CONVERSION_DICT[var]
# setting up filelist for requested period -----------------
filelist=[]
for k in indexes:
t = TL.Timelist[k]
filename = INPUTDIR + "ave." + t.strftime("%Y%m%d-%H:%M:%S") + "." + var + ".nc"
filelist.append(filename)
# ----------------------------------------------------------
M3d = TimeAverager3D(filelist, weights, var, TheMask)
except:
rank = 0
nranks = 1
INPUTDIR=addsep(args.inputdir)
OUTDIR =addsep(args.outdir)
varname = args.varname
starttime=args.starttime
end__time=args.endtime
TheMask = Mask(args.maskfile)
TI = TimeInterval(args.starttime,args.endtime,"%Y%m%d")
TL_in = TimeList.fromfilenames(TI, INPUTDIR,args.input_glob_pattern, prefix=args.input_prefix)
TLout = TimeList.fromfilenames(TI, OUTDIR, args.output_glob_pattern,prefix=args.output_prefix)
jpk,jpj,jpi=TheMask.shape
FILELIST_OUT=TLout.filelist[rank::nranks]
FILELIST__IN=TL_in.filelist[rank::nranks]
Timelist_OUT=TLout.Timelist[rank::nranks]
for iFrame, outfile in enumerate(FILELIST_OUT):
inputfile = FILELIST__IN[iFrame]
print "rank ", rank, inputfile, outfile
Weight_sum = Weight.sum()
Mass = (Conc * Weight).sum()
Weighted_Mean = Mass/Weight_sum
return Weighted_Mean
TheMask=Mask(args.maskfile)
MODEL_DIR= addsep(args.inputmodeldir)
REF_DIR = addsep(args.satdir)
outfile = args.outfile
Timestart="19990101"
Time__end="20150901"
TI = TimeInterval(Timestart,Time__end,"%Y%m%d")
sat_TL = TimeList.fromfilenames(TI, REF_DIR ,"*.nc", prefix="", dateformat="%Y%m")
model_TL = TimeList.fromfilenames(TI, MODEL_DIR,"*P_l.nc")
IOname = IOnames.IOnames('IOnames_sat_monthly.xml')
nFrames = model_TL.nTimes
nSUB = len(OGS.P.basin_list)
jpk,jpj,jpi =TheMask.shape
dtype = [(sub.name, np.bool) for sub in OGS.P]
SUB = np.zeros((jpj,jpi),dtype=dtype)
for sub in OGS.P:
sbmask = SubMask(sub,maskobject=TheMask).mask
SUB[sub.name] = sbmask[0,:,:]
mask200_2D = TheMask.mask_at_level(200.0)