textUsage='SYNOPSIS:\n\tmake_ensemble_Mean_tzyx.py -v VARIABLE -path PATHIN -outdir PATHOUT [-tmpdir TMPPATH] [keepTmp] \n\t-minVar MINVAL -maxVar MAXVAL\tn-model MODELLIST -startYear STARTYEAR -endYear ENDYEAR [-monthList MONTHLIST]\n\t[-regridFirst REGRIDBOOL] [-deleteGrid DELETEBOOL] -rcp RCP\n'

textUsage=textUsage+'\tVARIABLE: a netcdf CMIP5 variable name, such as tos, zos, so, thetao;\n'

textUsage=textUsage+'\tPATHIN: input data directory (does not support sub-directories);\n'

textUsage=textUsage+'\tPATHOUT: output directory, created if does not exist;\n'

textUsage=textUsage+'\tTMPPATH: temporary path. Default: a random pathname is defined at runtime, as a leaf of PATHOUT;\n'

textUsage=textUsage+'\tkeepTmp: do not remove temporary directories;\n'

textUsage=textUsage+'\tMINVAL: any value below minVar is considered as nodata;\n'

textUsage=textUsage+'\tMAXVAL: any value above maxVar is considered as nodata;\n'

textUsage=textUsage+'\tMODELLIST: a text file with a model name per name, the model name is used to select the files to process;\n'

textUsage=textUsage+'\tSTARTYEAR: first year in the series of dates to process;\n'

textUsage=textUsage+'\tENDYEAR: last year in the series of date to process;\n'

textUsage=textUsage+'\tMONTHLIST: a comma separated list of month, such as "1,2,3" or "1,6,12". Values range is [1, 12].\n'

textUsage=textUsage+'In first place, the programme will average model output per model (if a model output has several rXiYpZ ensemble, they are averaged. Then, the averages are averaged to produce the ensemble mean;\n'

textUsage=textUsage+'\tREGRIDBOOL\n'

textUsage=textUsage+'\tDELETEBOOL\n'

textUsage=textUsage+'\tRCP a string corresponding to the RCP string to match in filenames.\n'

textUsage=textUsage+'Averages are computed for each month of the year.\n'

thisLogger.critical(msg)

print msg

print

print usage()

if code=='0':

return False

if code.lower()=='false':

return False

if code.lower()=='no':

return False

if code=='1':

return True

if code.lower()=='true':

return True

if code.lower()=='yes':

listMonth = [int(x) for x in parameter.strip().split(',')]

for ii in listMonth:

if ii<1 or ii>12:

print 'month defined in the month list must be in [1, 12]. Exit(100).'

sys.exit(100)

for x in foo:

if hasattr(x, '__iter__'):

for y in flatten(x):

yield y

else:

if refDict is None and newDict is None:

return None

# get list of all keys

if refDict is None:

return newDict

if len(refDict)==0:

return newDict

if newDict is None:

return refDict

if len(newDict)==0:

return refDict

keyList = sorted(set(refDict.keys() + newDict.keys()))

result={}

for ikey in keyList:

val = []

if ikey in refDict.keys(): val.append( refDict[ikey] )

if ikey in newDict.keys(): val.append( newDict[ikey] )

result[ikey] = [ x for x in flatten(val) ]

#del val

#gc.collect()

values = [3.3, 10, 20, 30, 50, 75, 100, 125, 150, 200, 250, 300, 400, 500]

levelAxis = cdms2.createAxis( values )

bounds = [0]

for ii in xrange(len(values)-1):

bounds.append( 0.5*(values[ii] + values[ii+1]) )

bounds.append( values[-1] + 0.5 * (values[-1] + values[-2]) )

levelAxis.setBounds(numpy.array(bounds))

levelAxis.id='levels'

levelAxis.designateLevel(True)

levelAxis.units='meters'

xstart=0

xend=360

xstep=thisStep

ystart=-85

yend=85

ystep=thisStep

lon_bnds=[]

lon=[]

for ii in numpy.arange(xstart, xend, xstep):

lon_bnds.append( [ii, ii + xstep] )

lon.append(ii+0.5*xstep)

lon_bnds=numpy.array(lon_bnds)

lon=numpy.array(lon)

lat_bnds=[]

lat=[]

for ii in numpy.arange(ystart, yend, ystep):

lat_bnds.append([ii, ii + ystep])

lat.append(ii+0.5*ystep)

lat_bnds=numpy.array(lat_bnds)

lat=numpy.array(lat)

latAxis = cdms2.createAxis(lat, lat_bnds)

latAxis.designateLatitude(True)

latAxis.units='degrees_north'

latAxis.id='latitude'

latAxis.long_name='Latitude'

lonAxis = cdms2.createAxis(lon, lon_bnds)

lonAxis.designateLongitude(True, xend)

lonAxis.designateCircular(xend)

lonAxis.units='degrees_east'

lonAxis.id='longitude'

lonAxis.long_name='Longitude'

oneSlice = numpy.squeeze(var[:,:,0])

refShape=oneSlice.shape

# where are the nodata vertical lines?

# 1./ transform the slice: 0=data, 1=nodata

test = numpy.zeros(oneSlice.shape)

wto1 = oneSlice >= nodata

if wto1.any():

test[wto1] = 1

else:

thisLogger.info('do_cleanNodataLines: no-data is missing from this dataset. Return.')

return var

# 2./ multiplications: if there are only nodata, results is 1

line = numpy.array(oneSlice[0, :]) # copy first line

for il in range(oneSlice.shape[1]):

line = line * oneSlice[il, :]

# 3./ do we have a 1 somewhere? It means that there was only nodata along the line

wone = line == 1

if wone.any():

thisLogger.info('do_cleanNodataLines: found {0} lines to correct.'.format(len(wone)))

else:

thisLogger.info('do_cleanNodataLines: found no line to correct.')

refshape = var.shape

if len(refshape)==3:

tmp = numpy.reshape(var, (refshape[0], refshape[1] * refshape[2]) )

elif len(refshape)==4:

tmp = numpy.reshape(var, (refshape[0], refshape[1] * refshape[2] * refshape[3]) )

wtnodata = (tmp.max(axis=0) - tmp.min(axis=0)) < 0.001

if wtnodata.any():

for ii in range(refshape[0]):

tmp[ii, wtnodata] = nodata

var[:] = numpy.reshape(tmp, refshape)

#del tmp, wtnodata

#gc.collect()

if data is None:

print 'no data passed!'

sys.exit()

return [accum, N, mini, maxi]

dim = numpy.squeeze(data[:]).shape

if accum is None:

accum = numpy.zeros(dim) + nodata

N = numpy.zeros(dim) + nodata

mini = data.copy()

maxi = data.copy()

wtadd = (data >= minVar ) * (data < maxVar) * (accum < nodata) # add where not nodata

wtreplace = (data >= minVar) * (data < maxVar) * (accum >= nodata) # replace if no data

wmax = (data >= maxi) * (data < nodata) * (data >= minVar) * (data < maxVar)

wmaxReplace = (mini >= nodata) * (data < nodata) * (data >= minVar)

wmin = (data <= mini) * (data >= minVar) * ( data < maxVar) * ( maxi < nodata )

wminReplace = (mini >= nodata) * (data < nodata) * (data >= minVar)

if wtadd.any():

accum[wtadd] = accum[wtadd] + data[wtadd]

N[wtadd] = N[wtadd] + 1 #numpy.ones(dim)

print '!!N is not null'

if wtreplace.any():

accum[wtreplace] = data[wtreplace]

N[wtreplace] = 1 #numpy.ones(dim)

print 'xx N is not null'

if wmax.any():

maxi[wmax] = data[wmax]

if wmin.any():

mini[wmin] = data[wmin]

if wmaxReplace.any():

maxi[wmaxReplace] = data[wmaxReplace]

if wminReplace.any():

mini[wminReplace] = data[wminReplace]

#del wtadd, wtreplace, wmax, wmaxReplace, wmin, wminReplace

#gc.collect()

if N is None:

print 'N is None'

createdFiles=[]

nodata=1.e20

if lstInFile is None:

thisLogger.info( 'do_regrid: No file to process. Return' )

sys.exit()

return None

if len(lstInFile)==0:

print 'lstInFile'

thisLogger.info('do_regrid: Found no file to process, consider revising search pattern. Return.')

sys.exit()

return None

(newGrid, latAxis, lonAxis, lat_bnds, lon_bnds) = makeGrid()

for fileName in lstInFile:

thisLogger.info('Regriding file: {0}'.format(fileName))

thisFile = cdms2.open(fileName)

# to reduce output file size and memory use, collect start/end times according to internal file encoding

startTime = [t for t in thisFile[variable].getTime().asComponentTime() if (t.year==startYear)]

endTime = [t for t in thisFile[variable].getTime().asComponentTime() if (t.year==endYear)]

if len(startTime)==0 and len(endTime)==0: # this file does not contain useful data, next iteration

continue

if len(startTime)==0: # the first date is not in this file, process from the start

startTime = thisFile[variable].getTime().asComponentTime()

if len(endTime)==0: # the last date is not in this file, process up to the end

endTime = thisFile[variable].getTime().asComponentTime()

thisLogger.info('start time = {0}-{1:02}'.format(startTime[0].year, startTime[0].month) )

thisLogger.info('end time = {0}-{1:02}'.format(endTime[-1].year, endTime[-1].month))

if thisFile[variable].getLevel() is None:

# some files do not have nodata set to 1.e20 (EC-EARTH), some have masked values set to something else (0 and 1.e20, for MRI):

# let's process our mask by identifying unchanged values

tmp = cdms2.createVariable(thisFile[variable].subRegion( time=(startTime[0], endTime[-1], 'cc'), level=(topLevel, bottomLevel,'cc') ))

data = autoMask(tmp, nodata)

#del tmp

gc.collect()

else:

verticalGrid = make_levels()

# print dir(verticalGrid)

# print verticalGrid.getBounds()

print verticalGrid.getBounds().min() , verticalGrid.getBounds().max()

topLevel = verticalGrid.getBounds().min()

bottomLevel = verticalGrid.getBounds().max()

if thisFile[variable].getMissing() is None:

tmp = cdms2.createVariable(thisFile[variable].subRegion( time=(startTime[0], endTime[-1], 'cc'), level=(topLevel, bottomLevel,'cc') ))

data = autoMask(tmp, nodata)

#del tmp

#gc.collect()

else:

data = cdms2.createVariable(thisFile[variable].subRegion( time=(startTime[0], endTime[-1], 'cc'), level=(topLevel, bottomLevel,'cc') ))

mask = numpy.array(data) < nodata

if thisFile[variable].getLevel() is None:

regrided = data.regrid(newGrid, missing=nodata, order=thisFile[variable].getOrder(), mask=mask)

else:

tmp = data.regrid(newGrid, missing=nodata, order=thisFile[variable].getOrder(), mask=mask)

regrided = tmp.pressureRegrid( verticalGrid, method='linear')

regrided.id=variable

outfilename = '{0}/{1}{2}'.format(outdir, stringBefore, os.path.basename(fileName))

createdFiles.append(outfilename )

if os.path.exists(outfilename): os.remove(outfilename)

outfile = cdms2.open(outfilename, 'w')

outfile.write(regrided)

outfile.close()

thisFile.close()

#del mask, regrided

#gc.collect()

#del newGrid, latAxis, lonAxis, lat_bnds, lon_bnds

#gc.collect()

if validYearList is None:

exitMessage('List of years to process is undefined, edit code. Exit 5.',5)

createdFiles={}

nodata=1.e20

print 'in do_stats, variable={0}'.format(variable)

if lstInFile is None:

thisLogger.info('do_stats: No file to process. Return.')

return

if len(lstInFile)==0:

thisLogger.info('do_stats: Found no file to process, consider revising search pattern.')

return

# open all files

listFID=[]

if type(lstInFile)==type([]):

if len(lstInFile[0]) == 1: #the first element is a char, so there is only 1 file

ifile = ''.join(lstInFile)

thisLogger.debug('Case 2, lstInFile={0}'.format(ifile))

if not os.path.isfile(ifile):

exitMessage('File {0} not found. Exit 202'.format(lstInFile), 202)

listFID.append(cdms2.open(ifile, 'r'))

else:

for ifile in lstInFile:

thisLogger.debug('Case 1, ifile={0}'.format(ifile))

if not os.path.isfile(ifile):

exitMessage('File {0} not found. Exit 201.'.format(ifile), 201)

listFID.append(cdms2.open(ifile, 'r'))

else:

exitMessage('Unknown type for object lstInFile. Exit(200)',200)

# go through the list of dates, compute ensemble average

for iyear in validYearList:

thisLogger.info('Processing year {0}'.format(iyear))

for imonth in monthList:

accumVar=None

accumN=None

mini=None

maxi=None

refGrid=None

dims=None

units=None

for ifile in listFID:

print 'in do_stats, ifile loop,'

print 'ready to get data for time: ',iyear, imonth

if ifile[variable].getTime() is None: # no time reference

print 'no time found'

if refGrid is None:

print 'no refgrid'

refGrid = ifile[variable].getGrid()

# axis=ifile[variable].getAxisList(omit='time')

dims=numpy.squeeze(ifile[variable]).shape

[accumVar, accumN, mini, maxi] = updateCounters( accumVar, accumN, mini, maxi,

numpy.array(ifile[variable]).ravel(),

minVar, maxVar, nodata)

else: # we can do some time slice

print "GETTTING TIME COMPONENET for ",iyear, imonth

thisTime = [ii for ii in ifile[variable].getTime().asComponentTime() if (ii.year==iyear and ii.month==imonth)]

for ii in ifile[variable].getTime().asComponentTime():

print ii, ii.year , ii.month

if len(thisTime)==1: # it must be one

if refGrid is None:

refGrid = ifile[variable].getGrid()

dims = numpy.squeeze(ifile[variable].subRegion(time=thisTime[0])).shape

units= ifile[variable].units

[accumVar, accumN, mini, maxi]= updateCounters(accumVar, accumN, mini, maxi,

numpy.array( ifile[variable].subRegion(time=thisTime[0])).ravel(),

minVar, maxVar, nodata )

else:

print ifile[variable].getTime().asComponentTime()

print '================'

print monthList

print 'iyear',iyear, ' imonth',imonth, len(thisTime), thisTime.shape

for ii in ifile[variable].getTime().asComponentTime():

print '::',ii, ii.year, ii.month

print '________'

exitMessage('Found more that 1 date!!! Stop processing')

units= ifile[variable].units

# compute average

# it can happen that there is no data to process: if the input files for the current model has an ending date before the current date

# in this case, accumN is None: do not save stats, and do not add a file name in createdFiles

# compute average

if accumN is not None:

print 'accumN is not None'

wtdivide = (accumN < nodata) * (accumN > 0)

if wtdivide.any():

accumVar[wtdivide] = accumVar[wtdivide] / accumN[wtdivide]

# compute std

if doSTD:

thisLogger.info('Computing std: to be implemented')

# create and save variables

meanVar = cdms2.createVariable( accumVar.reshape(dims), typecode='f', id='mean_{0}'.format(variable), fill_value=nodata, attributes=dict(long_name='mean', units=units) )

meanVar.setGrid(refGrid)

counter = cdms2.createVariable(accumN.reshape(dims), typecode='i', id='count', fill_value=nodata, attributes=dict(long_name='count', units='None') )

counter.setGrid(refGrid)

miniVar = cdms2.createVariable(mini.reshape(dims), typecode='f', id='minimum', fill_value=nodata, attributes=dict(long_name='minimum', units=units) )

miniVar.setGrid(refGrid)

maxiVar = cdms2.createVariable(maxi.reshape(dims), typecode='f', id='maximum', fill_value=nodata, attributes=dict(long_name='maximum', units=units) )

maxiVar.setGrid(refGrid)

outfilename = '{0}/{1}_{2}_{3}{4:02}.nc'.format(outdir, stringBefore, outnameBase, iyear, imonth )

if os.path.exists(outfilename): os.remove(outfilename)

thisLogger.debug('Saving stats to file {0}'.format(outfilename))

outfile = cdms2.open(outfilename, 'w')

outfile.write(meanVar)

outfile.write(counter)

outfile.write(miniVar)

outfile.write(maxiVar)

outfile.close()

createdFiles['{0}{1:02}'.format(iyear,imonth)] = outfilename

print 'in do_stats, variable={0}, adding in createdFiles {1}'.format(variable, outfilename)

#del wtdivide

#gc.collect()

else:

print 'accumN is None, nothing to do'

sys.exit()

#del accumVar, mini, maxi, accumN

#gc.collect()

# close input files

for ii in listFID: ii.close()

print 'in do_stats, variable={0}, returning with'.format(variable)

print createdFiles