def readOper(self):
"""
(postes geom only) read pickle version of the operational PRO files.
(pickles generated by extract_postes_add_mocage.py)
"""
pathOper = os.environ['CROCOPATH'] + '/s2m/' + self.options.vconf + '/oper_{0}/crocO/beaufix/oper.pkl'.format(self.begprodates[0].strftime('%Y'))
self.oper = load_pickle2(pathOper)
def loadEnsPro(self, kind, catPro = False, isOl = False):
if kind == 'Cl':
pathPkl = self.xpiddir + '../clim/crocO/clim.pkl'
elif (kind == 'Ol' and isOl is False):
if not os.path.exists(self.xpidoldir + '/crocO/' + self.options.saverep + '/'):
os.makedirs(self.xpidoldir + '/crocO/' + self.options.saverep + '/')
pathPkl = self.xpidoldir + '/crocO/' + self.options.saverep + '/' + 'ensPro' + kind + '.pkl'
else:
pathPkl = 'ensPro' + kind + '.pkl'
# with pickling on beaufix (february 2020 on), pickle files are in xpdir/crocO and with .foo extension added.
pathpklbeauf = pathPkl + '.foo'
if not os.path.islink(pathPkl):
if os.path.exists(pathpklbeauf):
try:
os.symlink(pathpklbeauf, pathPkl)
except FileExistsError:
pass
if not os.path.exists(pathPkl):
locPro = self.nc_to_pkl(pathPkl, kind, catPro = catPro, isOl = isOl)
else:
locPro = load_pickle2(pathPkl)
if not all(l in locPro.keys() for l in self.listvar):
print('Some of the ppvars you mentioned are not present in ', pathPkl,
'\n listvars:', self.listvar, '\npickle file keys:', [locPro.keys()], '\n reloading')
# mutualize variables because the pkl will be overwritten.
lkeys = list(locPro.keys())
lkeys.remove('time')
self.listvar = set(lkeys + self.listvar)
locPro = self.nc_to_pkl(pathPkl, kind, catPro = catPro, isOl = isOl)
return locPro
def loadEnsPrep(self, kind, isOl = False):
# if local pp (e.g. pp of a run on local machine)
if self.options.todo == 'pfpp' or self.options.todo == 'pf':
fromArch = False
else:
fromArch = True
# on beaufix, cannot straightforwardly parallelize the reading of preps
if 'beaufix' in os.uname()[1]:
if kind == 'bg':
locEns = {dd: PrepEnsBg(self.options, dd, fromArch=fromArch) for dd in self.options.dates}
elif kind == 'an':
print('didodidodi', self.options.dates, self.datefin.strftime('%Y%m%d%H'), type(self.options.dates[0]), type(self.datefin.strftime('%Y%m%d%H')))
locEns = {dd: PrepEnsAn(self.options, dd, fromArch=fromArch) for dd in self.options.dates if dd != self.datefin.strftime('%Y%m%d%H')}
else:
locEns = {dd: PrepEnsOl(self.options, dd, isOl=isOl, fromArch=fromArch) for dd in self.options.dates}
for dd in locEns.keys():
if (kind == 'ol' and isOl is False):
pathPkl = self.xpidoldir + '/crocO/' + self.options.saverep + '/' +\
kind + '_' + dd + '.pkl'
else:
pathPkl = kind + '_' + dd + '.pkl'
pathpklbeauf = pathPkl + '.foo'
if not os.path.islink(pathPkl):
if os.path.exists(pathpklbeauf):
try:
os.symlink(pathpklbeauf, pathPkl)
except FileExistsError:
pass
if not os.path.exists(pathPkl):
print(' unsuccesfull loading', pathPkl, )
print(('loading ' + kind + ' ens for date : ', dd))
locEns[dd].stackit()
with open(pathPkl, 'wb') as f:
print(('saaving ' + kind + ' to pickle !'))
pickle.dump(locEns[dd].stack, f, protocol=pickle.HIGHEST_PROTOCOL)
else:
locEns[dd].stack = load_pickle2(pathPkl)
if not all(l in locEns[dd].stack.keys() for l in self.listvar):
raise Exception('Some of the ppvars you mentioned are not present in ', pathPkl,
'\n listvars:', self.listvar, '\npickle file keys:', locEns[dd].stack.keys())
locEns[dd].isstacked = True
return locEns
else:
# BC june 2020: manager is used to share the dict between nodes
try:
manager = multiprocessing.Manager()
locEns = manager.dict()
p = multiprocessing.Pool(min(multiprocessing.cpu_count(), len(self.options.dates)))
p.map(loadEnsPrepDate_parallel, [[self, locEns, dd, kind, isOl, fromArch] for dd in self.options.dates])
except Exception as _:
p.close()
p.close()
p.join()
return dict(locEns)
def loadEnsPrepDate_parallel(largs):
"""
BC 24/06/20 externalized this fucntion for python2 compatibility (if class method, throws this exception):
https://stackoverflow.com/questions/1816958/cant-pickle-type-instancemethod-when-using-multiprocessing-pool-map/7309686#7309686
"""
pp = largs[0]
locEns = largs[1]
dd = largs[2]
kind = largs[3]
isOl = largs[4]
fromArch = largs[5]
if kind == 'bg':
local = PrepEnsBg(pp.options, dd, fromArch=fromArch)
elif kind == 'an':
local = PrepEnsAn(pp.options, dd, fromArch=fromArch)
else:
local = PrepEnsOl(pp.options, dd, isOl=isOl, fromArch=fromArch)
if (kind == 'ol' and isOl is False):
pathPkl = pp.xpidoldir + '/crampon/' + pp.options.saverep + '/' +\
kind + '_' + dd + '.pkl'
else:
pathPkl = kind + '_' + dd + '.pkl'
pathpklbeauf = pathPkl + '.foo'
if not os.path.islink(pathPkl):
if os.path.exists(pathpklbeauf):
try:
os.symlink(pathpklbeauf, pathPkl)
except FileExistsError:
pass
if not os.path.exists(pathPkl):
print(' unsuccesfull loading', pathPkl, )
print(('loading ' + kind + ' ens for date : ', dd))
local.stackit()
with open(pathPkl, 'wb') as f:
print(('saaving ' + kind + ' to pickle !'))
pickle.dump(local.stack, f, protocol=pickle.HIGHEST_PROTOCOL)
else:
local.stack = load_pickle2(pathPkl)
if not all(l in local.stack.keys() for l in pp.listvar):
raise Exception('Some of the ppvars you mentioned are not present in ', os.getcwd() + '/' + pathPkl,
'\n listvars:', pp.listvar, '\npickle file keys:', local.stack.keys())
local.isstacked = True
locEns[dd] = local
def readTruth(self):
"""
BC 24/10/19 : function to read the truth in the OL from the corresponding run.
- Sometimes, if the OL is a subset of a bigger OL, and the truth is in this bigger OL, need to read in it.
k is the variable.
- find the baseline experiment if necessary.
"""
def load_from_dict(ddict, mbsynth):
out = dict()
for var in ddict.keys():
if 'time' not in var:
out[var] = ddict[var][:, :, mbsynth]
else:
out['time'] = ddict['time']
return out
try:
self.truth = load_from_dict(self.ensProOl, self.mbsynth)
except IndexError:
print('\n\nWARNING : there is no corresponding mbsynth in this openloop not enough members\n looking in the bigger OL xp.\n\n')
print('loading ' + self.xpidoldir[0:-4] + '/crocO/' + self.options.saverep + '/ensProOl.pkl')
pathPklBigOl = self.xpidoldir[0:-4] + '/crocO/' + self.options.saverep + '/ensProOl.pkl'
gg = load_pickle2(pathPklBigOl)
self.truth = load_from_dict(gg, self.mbsynth)
def readObs(self):
# paths settings (ol or not, real obs or not).
# if performing pfpp, it is after a locla run and obs are properly archived.
if self.options.todo == 'pfpp':
if self.options.synth is not None:
self.pathArch = self.xpiddir + '/crocO/ARCH/' + self.options.sensor + '/'
self.pathSynth = self.xpiddir + '/crocO/SYNTH/' + self.options.sensor + '/'
else:
self.pathReal = self.options.crocOpath + '/s2m/' + self.options.vconf + '/obs/' + self.options.sensor + '/'
else:
if str(self.options.openloop) == 'on':
if self.options.synth is not None:
self.pathArch = self.xpidoldir + '/crocO/ARCH/' + self.options.sensor + '/'
self.pathSynth = self.xpidoldir + '/crocO/SYNTH/' + self.options.sensor + '/'
else:
self.pathReal = self.options.crocOpath + '/s2m/' + self.options.vconf + '/obs/' + self.options.sensor + '/'
else:
if hasattr(self.options, 'xpidoldir'):
if self.options.synth is not None:
self.pathArch = self.xpidoldir + '/crocO/ARCH/' + self.options.sensor + '/'
self.pathSynth = self.xpidoldir + '/crocO/SYNTH/' + self.options.sensor + '/'
else:
self.pathReal = self.options.crocOpath + '/s2m/' + self.options.vconf + '/obs/' + self.options.sensor + '/'
else:
# if no xpidoldir has been prescribed, obs must be real.
self.pathReal = self.options.crocOpath + '/s2m/' + self.options.vconf + '/obs/' + self.options.sensor + '/'
# proper loading (synth or real)
if self.options.todo == 'pfpp':
if self.options.synth is not None:
print('reading synthetic assimilated obs. in ' + self.pathArch)
self.obsArch = {dd: FromXp(self.pathArch, dd, self.options) for dd in self.options.dates}
self.obsSynth = {dd: FromXp(self.pathSynth, dd, self.options) for dd in self.options.dates}
for dd in self.options.dates:
self.obsArch[dd].load()
self.obsSynth[dd].load()
else:
print('reading real assimilated obs. in ' + self.pathReal)
self.obsReal = {dd: Real(self.pathReal, dd, self.options) for dd in self.options.dates}
for dd in self.options.dates:
self.obsReal[dd].load()
else:
if self.options.synth is not None:
print('sssynth', self.options.synth)
print('reading synthetic truth obs. in ' + self.pathArch)
print('reading synthetic assimilated obs. in ' + self.pathSynth)
self.obsArch = {dd: FromXp(self.pathArch, dd, self.options) for dd in self.options.dates}
self.obsSynth = {dd: FromXp(self.pathSynth, dd, self.options) for dd in self.options.dates}
for dd in self.options.dates:
self.obsArch[dd].load()
self.obsSynth[dd].load()
else:
print('reading real assimilated obs. in ' + self.pathReal)
self.obsReal = {dd: Real(self.pathReal, dd, self.options) for dd in self.options.dates}
for dd in self.options.dates:
self.obsReal[dd].load()
pathObsTs = self.options.crocOpath + '/s2m/' + self.options.vconf + '/obs/' + self.options.sensor +\
'/obs_{0}_{1}_{2}100106_{3}063006.pkl'.format(self.options.sensor,
self.options.vconf,
self.datedeb.strftime('%Y'),
int(self.datedeb.strftime('%Y')) + 1)
print('reading observation timeseries (pickle from csv file) in ')
print(pathObsTs)
if not os.path.exists(pathObsTs):
if '_X' in self.options.sensor:
sensorBase = self.options.sensor.split('X')[0][0:-1]
os.symlink(self.options.crocOpath + '/s2m/' + self.options.vconf +
'/obs/{0}/obs_{0}_{1}_{2}100106_{3}063006.pkl'.format(sensorBase,
self.options.vconf,
self.datedeb.strftime('%Y'),
int(self.datedeb.strftime('%Y')) + 1),
pathObsTs)
else:
# BC 30/06/20: some day replace with a link to a yearly file/
try:
os.symlink(self.options.crocOpath + '/s2m/' + self.options.vconf +
'/obs/{0}/obs_all_{0}_2013010106_2018123106.pkl'.format(self.options.vconf,
self.datedeb.strftime('%Y'),
int(self.datedeb.strftime('%Y')) + 1
),
pathObsTs)
except FileExistsError:
pass
self.obsTs = load_pickle2(pathObsTs)