def fromfilenames(timeinterval, inputdir,searchstring, filtervar=None, prefix='ave.', dateformat="%Y%m%d-%H:%M:%S"):
'''
Generates a TimeList object by reading a directory
HYPOTHESIS:
- the Input directory has files containing a date in their names
- every file refers to a single time frame
- The date of in the file name can be considered as centered in their period
The generated datetime list has all the files concerning the period indicated in the timeinterval.
Some of these files can have the centered date out of that period.
Example:
INPUTDIR="/pico/scratch/userexternal/gbolzon0/Carbonatic/wrkdir/MODEL/AVE_FREQ_1/"
Time_int = TimeInterval('20141201-00:00:00','20150701-00:00:00',"%Y%m%d-%H:%M:%S")
TL = TimeList.fromfilenames(Time_int, INPUTDIR,"ave*N1p.nc")
TL = TimeList.fromfilenames(Time_int, INPUTDIR,"ave*nc",filtervar="N1p")
For Sat data
TL = TimeList.fromfilenames(Time_int, INPUTDIR,"*nc", prefix='',dateformat='%Y%m%d')
For physical forcings
TL = TimeList.fromfilenames(Time_int, INPUTDIR,"T*.nc",prefix='T',dateformat='%Y%m%d')
'''
IOname = IOnames.filenamer(prefix,dateformat)
if not os.path.exists(inputdir):
raise NameError("Not existing directory " + inputdir)
inputdir = addsep(inputdir)
filelist_ALL = glob.glob(inputdir + searchstring)
if not filtervar is None:
filelist_ALL=[f for f in filelist_ALL if filtervar in f ]
assert len(filelist_ALL) > 0
filenamelist=[]
datetimelist=[]
External_filelist=[]
External_timelist=[]
for pathfile in filelist_ALL:
filename = os.path.basename(pathfile)
datestr = filename[IOname.date_startpos:IOname.date_endpos]
actualtime = datetime.datetime.strptime(datestr,IOname.dateformat)
if timeinterval.contains(actualtime) :
filenamelist.append(pathfile)
datetimelist.append(actualtime)
else:
External_filelist.append(pathfile)
External_timelist.append(actualtime)
TimeListObj = TimeList(datetimelist)
filenamelist.sort()
TimeListObj.timeinterval = timeinterval
TimeListObj.inputdir = inputdir
TimeListObj.searchstring = searchstring
TimeListObj.prefix = prefix
TimeListObj.filtervar = filtervar
TimeListObj.filelist = filenamelist
if TimeListObj.inputFrequency is not None:
for iFrame, t in enumerate(External_timelist):
TimInt = computeTimeWindow(TimeListObj.inputFrequency, t)
if TimeListObj.timeinterval.isInWindow(TimInt):
TimeListObj.filelist.append(External_filelist[iFrame])
TimeListObj.Timelist.append(External_timelist[iFrame])
TimeListObj.filelist.sort()
TimeListObj.Timelist.sort()
TimeListObj.nTimes =len(TimeListObj.filelist)
# we force daily datetimes to have hours = 12
if TimeListObj.inputFrequency == 'daily':
for iFrame, t in enumerate(TimeListObj.Timelist):
if t.hour==0:
newt = datetime.datetime(t.year,t.month,t.day,12,0,0)
TimeListObj.Timelist[iFrame] = newt
return TimeListObj
import pylab as pl
import math, operator
import glob
from dateutil.relativedelta import relativedelta
import datetime
from matplotlib.dates import date2num, num2date, YearLocator, MonthLocator, DateFormatter
from commons.utils import addsep
def movingaverage(interval, window_size):
window = np.ones(int(window_size))/float(window_size)
return np.convolve(interval, window, 'same')
DATADIR = addsep(args.inputdir)
FIGDIR = addsep(args.outdir)
os.system('mkdir -p ' + FIGDIR)
pl.close('all')
var='ppn'
SUBlist_subset=['alb','sww','swe','nwm','tyr','adn','ads','ion','lev','med']
subreg_mod=SUBlist_subset
#TEXTylabel="mgC/m2/day"
TEXTylabel="gC/m2/yr"
# define dates
dd=[]
#datelist=glob.glob(DATADIR + "ave." + in_yr + "*-12:00:00.col_integrals.nc")
datelist=glob.glob(DATADIR + "ave.*.col_integrals.nc")
datelist.sort()
return parser.parse_args()
args = argument()
from biofloats_ms_timeseries import timelistcontainer
from commons.time_interval import TimeInterval
import matplotlib
matplotlib.use('Agg')
import pylab as pl
import numpy as np
from commons.layer import Layer
from basins import V2 as OGS
from commons.utils import addsep
ARCHIVEDIR = addsep(args.archivedir)
ARCHIVE_PREV = addsep(args.previous_archive)
VALID_WRKDIR = addsep(args.validation_dir)
OUTFIG_DIR = addsep(args.outdir)
TI_V1 = TimeInterval("20150608","20160101","%Y%m%d")
TI_V2 = TimeInterval('20160412', args.date,'%Y%m%d')
V4_data = timelistcontainer(TI_V1,ARCHIVE_PREV,postfix_dir="")
V2C_data = timelistcontainer(TI_V2,ARCHIVEDIR ,postfix_dir="POSTPROC/AVE_FREQ_1/validation/biofloats_ms/")
V2C_data.append_dir(VALID_WRKDIR)
def single_plot(longvar, var, sub, layer ):
varV4 = var
if var == 'P_l': varV4 = 'P_i'
from commons import timerequestors from commons.Timelist import TimeList from commons.layer import Layer import pylab as pl from commons.utils import addsep import basins.OGS as OGS from instruments import lovbio_float as bio_float from instruments.var_conversions import LOVFLOATVARS from commons.mask import Mask from instruments.matchup_manager import Matchup_Manager import sys #import makeplots import qualitycheck idate0 = args.inputdate BASEDIR = addsep(args.basedir) INPUTDIR = addsep(args.inputdir) # DATE INTERVAL year = int(idate0[0:4]) month = int(idate0[4:6]) day = int(idate0[6:8]) idate1 = timerequestors.Weekly_req(year, month, day) print idate1 idate2 = timerequestors.Daily_req(year, month, day) print idate2 # Variable name VARLIST = ['P_l'] #'N3n','O2o'] read_adjusted = [True] #,False,False]
for date in S:
LINES.append("<date day=\"" + date + "\">\n")
for filename in analysis_forecast_basenames:
if filename[:8] == date:
LINES.append("<float wmo=\"" + filename[9:-3] + "\"></float>\n")
LINES.append("</date>\n")
LINES.append("</root>\n")
fid=open(filexml,'w')
fid.writelines(LINES)
fid.close()
PREVIOUS_DIR= addsep(args.previous_dir) #"/pico/home/usera07ogs/a07ogs00/OPA/V2C-dev/wrkdir/2/POSTPROC/AVE_FREQ_1/online_validation/PREVIOUS/matchup_outputs/"
ACTUAL_DIR = addsep(args.actual_dir) #"/pico/home/usera07ogs/a07ogs00/OPA/V2C-dev/wrkdir/2/POSTPROC/AVE_FREQ_1/online_validation/ACTUAL/matchup_outputs/"
OUTDIR = addsep(args.outdir)
xmlfile = args.xmlfile
ANALYSIS_FORECAST_LIST = glob.glob(PREVIOUS_DIR + "*nc")
ONLY_ANALYSIS_LIST = glob.glob(ACTUAL_DIR + "*nc")
analysis_forecast_basenames = [os.path.basename(filename) for filename in ANALYSIS_FORECAST_LIST]
analysis_forecast_basenames.sort()
only_analyis_basenames = [os.path.basename(filename) for filename in ONLY_ANALYSIS_LIST]
only_analyis_basenames.sort()
print xmlfile
dump_xml(xmlfile)
return parser.parse_args()
args = argument()
from sat_timeseries import timelistcontainer
from commons.time_interval import TimeInterval
from commons.utils import addsep
import matplotlib
matplotlib.use('Agg')
import pylab as pl
import matplotlib.dates as mdates
from basins import V2 as OGS
TI_V1 = TimeInterval('20150407','20160607','%Y%m%d')
TI_V2 = TimeInterval('20160412', args.date,'%Y%m%d')
ARCHIVE_DIR = addsep(args.archivedir) #"/gpfs/work/IscrC_MYMEDBIO/COPERNICUS/V2-dev"
ARCHIVE_PREV = addsep(args.previous_archive) #r"/gpfs/work/IscrC_MYMEDBIO/COPERNICUS/V4"
VALID_WRKDIR = addsep(args.validation_dir)
OUTFIG_DIR = addsep(args.outdir) # "/pico/home/userexternal/gcossari/COPERNICUS/CATENA/FIG_VALIDATION_ONLINE"
F0v1 = timelistcontainer(TI_V1,ARCHIVE_PREV,'f0', postfix_dir="")
F1v1 = timelistcontainer(TI_V1,ARCHIVE_PREV,'f1', postfix_dir="")
F2v1 = timelistcontainer(TI_V1,ARCHIVE_PREV,'f2', postfix_dir="")
F0v2 = timelistcontainer(TI_V2,ARCHIVE_DIR, 'f0', postfix_dir="POSTPROC/AVE_FREQ_1/validation/Sat/")
F1v2 = timelistcontainer(TI_V2,ARCHIVE_DIR, 'f1', postfix_dir="POSTPROC/AVE_FREQ_1/validation/Sat/")
F2v2 = timelistcontainer(TI_V2,ARCHIVE_DIR, 'f2', postfix_dir="POSTPROC/AVE_FREQ_1/validation/Sat/")
F0v2.append_dir(VALID_WRKDIR)
F1v2.append_dir(VALID_WRKDIR)
F1v2.append_dir(VALID_WRKDIR)
args = argument()
import matplotlib
matplotlib.use('Agg')
from commons.time_interval import TimeInterval
from commons.Timelist import TimeList
from instruments.matchup_manager import Matchup_Manager
import basins.OGS as OGS
from instruments import lovbio_float as bio_float
from commons.mask import Mask
from commons.utils import addsep
from datetime import timedelta
starttime=args.starttime
end__time=args.endtime
INPUTDIR=addsep(args.inputdir)
BASEDIR=addsep(args.basedir)
TI=TimeInterval(starttime,end__time,'%Y%m%d')
TI.end_time = TI.end_time + timedelta(1)
TheMask=Mask(args.maskfile)
Profilelist=bio_float.FloatSelector(None,TI,OGS.med)
TL = TimeList.fromfilenames(TI, INPUTDIR,"ave*.nc")
M = Matchup_Manager(Profilelist,TL,BASEDIR)
profilerscript = BASEDIR + 'jobProfiler.sh'
M.writefiles_for_profiling('VarDescriptor_valid_online.xml', profilerscript) # preparation of data for aveScan
M.dumpModelProfiles(profilerscript) # sequential launch of aveScan
args = argument()
import scipy.io.netcdf as NC
import glob
import os
import numpy as np
import read_descriptor
import IOnames as IOname
from maskload import *
import GB_lib
from commons.utils import addsep
INPUT_AVEDIR = addsep(args.inputdir)
AGGREGATE_AVEDIR = addsep(args.aggregatedir)
TMPDIR = addsep(args.tmpdir)
BASEDIR = addsep(args.outdir)
ionamesfile = args.ionames
IOnames = IOname.IOnames(ionamesfile)
filtervar = args.var
try:
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nranks =comm.size
isParallel = True
except:
rank = 0
try :
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nranks = comm.size
except:
rank = 0
nranks = 1
args = argument()
BIOAVEDIR = addsep(args.biodir)
PHYS_DIR = addsep(args.physdir)
PATH_NAME = BIOAVEDIR + "ave.*nc"
if rank==0 :
print "BIO_INPUT_DIR =", BIOAVEDIR
TMPOUTdir = addsep(args.tmpdir)
if rank==0 : print "TMPOUTDIR= ", TMPOUTdir
os.system("mkdir -p " + TMPOUTdir)
archived_filelist=glob.glob(PATH_NAME)
archived_filelist.sort()
for filename in archived_filelist[rank::nranks]:
dailyAve = os.path.basename(filename)
from commons.mask import Mask
from commons.layer import Layer
from layer_integral.mapbuilder import MapBuilder
from layer_integral.mapplot import mapplot,pl
from commons.dataextractor import DataExtractor
from commons.time_averagers import TimeAverager3D
from layer_integral import coastline
import commons.timerequestors as requestors
from commons.utils import addsep
clon,clat = coastline.get()
TheMask=Mask(args.maskfile)
INPUTDIR = addsep(args.inputdir)
OUTPUTDIR = addsep(args.outdir)
var = args.varname
ldtype=[('top',np.float32),('bottom',np.float32),('mapdepthfilter',np.float32)]
LF = np.loadtxt(args.layerfile,ldtype,ndmin=1)
LAYERLIST=[ Layer(l['top'], l['bottom']) for l in LF ]
UNITS_DICT={
'ppn' : 'gC/m^2/y',
'N1p' : 'mmol /m^3',
'N3n' : 'mmol /m^3',
'PH' : '',
'pCO2': 'ppm',
from commons.submask import SubMask
from basins import V2 as OGS
import glob
import scipy.io.netcdf as NC
def weighted_mean(Conc, Weight):
Weight_sum = Weight.sum()
Mass = (Conc * Weight).sum()
Weighted_Mean = Mass/Weight_sum
return Weighted_Mean
date=args.date
MODELDIR=addsep(args.forecastdir)
REF_DIR=addsep(args.satdir)
outfile = args.outfile
TheMask=Mask(args.maskfile)
nSUB = len(OGS.P.basin_list)
jpk,jpj,jpi =TheMask.shape
mask200_2D = TheMask.mask_at_level(200.0)
dtype = [(sub.name, np.bool) for sub in OGS.P]
SUB = np.zeros((jpj,jpi),dtype=dtype)
for sub in OGS.Pred:
SUB[sub.name] = SubMask(sub,maskobject=TheMask).mask_at_level(0)
parser.add_argument( '--type',
type = str,
choices = ['analysis','forecast'],
required = True)
return parser.parse_args()
args = argument()
from commons.timeseries import TimeSeries
from commons.time_interval import TimeInterval
from commons.utils import addsep
starttime=args.starttime
end__time=args.endtime
LOC = addsep(args.outdir)
archive_dir= args.arcdir
TI=TimeInterval(starttime,end__time,'%Y%m%d')
if args.type=='analysis':
T_bio = TimeSeries(TI, archive_dir,postfix_dir='POSTPROC/AVE_FREQ_1/ARCHIVE/',glob_pattern="ave*gz")
T_phys= TimeSeries(TI, archive_dir,postfix_dir='OPAOPER_A/' ,glob_pattern="*gz" )
T_bio.extract_analysis( LOC + 'output_bio/')
T_phys.extract_analysis(LOC + 'output_phys/');
if args.type =='forecast':
T_bio = TimeSeries(TI, archive_dir,postfix_dir='POSTPROC/AVE_FREQ_1/ARCHIVE/',glob_pattern="ave*gz")
T_phys_s= TimeSeries(TI, archive_dir,postfix_dir='OPAOPER_A/' ,glob_pattern="*gz" )
from commons.mask import Mask
from commons.submask import SubMask
from basins import V2 as OGS
from commons.layer import Layer
from commons.utils import addsep
import pickle
def weighted_mean(Conc, Weight):
Weight_sum = Weight.sum()
Mass = (Conc * Weight).sum()
Weighted_Mean = Mass/Weight_sum
return Weighted_Mean
TheMask=Mask(args.maskfile)
MODEL_DIR= addsep(args.inputmodeldir)
REF_DIR = addsep(args.satdir)
outfile = args.outfile
Timestart="19990101"
Time__end="20150901"
TI = TimeInterval(Timestart,Time__end,"%Y%m%d")
sat_TL = TimeList.fromfilenames(TI, REF_DIR ,"*.nc", prefix="", dateformat="%Y%m")
model_TL = TimeList.fromfilenames(TI, MODEL_DIR,"*P_l.nc")
IOname = IOnames.IOnames('IOnames_sat_monthly.xml')
nFrames = model_TL.nTimes
nSUB = len(OGS.P.basin_list)
