def readOper(self): """ (postes geom only) read pickle version of the operational PRO files. (pickles generated by extract_postes_add_mocage.py) """ pathOper = os.environ['CROCOPATH'] + '/s2m/' + self.options.vconf + '/oper_{0}/crocO/beaufix/oper.pkl'.format(self.begprodates[0].strftime('%Y')) self.oper = load_pickle2(pathOper)
def loadEnsPro(self, kind, catPro = False, isOl = False): if kind == 'Cl': pathPkl = self.xpiddir + '../clim/crocO/clim.pkl' elif (kind == 'Ol' and isOl is False): if not os.path.exists(self.xpidoldir + '/crocO/' + self.options.saverep + '/'): os.makedirs(self.xpidoldir + '/crocO/' + self.options.saverep + '/') pathPkl = self.xpidoldir + '/crocO/' + self.options.saverep + '/' + 'ensPro' + kind + '.pkl' else: pathPkl = 'ensPro' + kind + '.pkl' # with pickling on beaufix (february 2020 on), pickle files are in xpdir/crocO and with .foo extension added. pathpklbeauf = pathPkl + '.foo' if not os.path.islink(pathPkl): if os.path.exists(pathpklbeauf): try: os.symlink(pathpklbeauf, pathPkl) except FileExistsError: pass if not os.path.exists(pathPkl): locPro = self.nc_to_pkl(pathPkl, kind, catPro = catPro, isOl = isOl) else: locPro = load_pickle2(pathPkl) if not all(l in locPro.keys() for l in self.listvar): print('Some of the ppvars you mentioned are not present in ', pathPkl, '\n listvars:', self.listvar, '\npickle file keys:', [locPro.keys()], '\n reloading') # mutualize variables because the pkl will be overwritten. lkeys = list(locPro.keys()) lkeys.remove('time') self.listvar = set(lkeys + self.listvar) locPro = self.nc_to_pkl(pathPkl, kind, catPro = catPro, isOl = isOl) return locPro
def loadEnsPrep(self, kind, isOl = False): # if local pp (e.g. pp of a run on local machine) if self.options.todo == 'pfpp' or self.options.todo == 'pf': fromArch = False else: fromArch = True # on beaufix, cannot straightforwardly parallelize the reading of preps if 'beaufix' in os.uname()[1]: if kind == 'bg': locEns = {dd: PrepEnsBg(self.options, dd, fromArch=fromArch) for dd in self.options.dates} elif kind == 'an': print('didodidodi', self.options.dates, self.datefin.strftime('%Y%m%d%H'), type(self.options.dates[0]), type(self.datefin.strftime('%Y%m%d%H'))) locEns = {dd: PrepEnsAn(self.options, dd, fromArch=fromArch) for dd in self.options.dates if dd != self.datefin.strftime('%Y%m%d%H')} else: locEns = {dd: PrepEnsOl(self.options, dd, isOl=isOl, fromArch=fromArch) for dd in self.options.dates} for dd in locEns.keys(): if (kind == 'ol' and isOl is False): pathPkl = self.xpidoldir + '/crocO/' + self.options.saverep + '/' +\ kind + '_' + dd + '.pkl' else: pathPkl = kind + '_' + dd + '.pkl' pathpklbeauf = pathPkl + '.foo' if not os.path.islink(pathPkl): if os.path.exists(pathpklbeauf): try: os.symlink(pathpklbeauf, pathPkl) except FileExistsError: pass if not os.path.exists(pathPkl): print(' unsuccesfull loading', pathPkl, ) print(('loading ' + kind + ' ens for date : ', dd)) locEns[dd].stackit() with open(pathPkl, 'wb') as f: print(('saaving ' + kind + ' to pickle !')) pickle.dump(locEns[dd].stack, f, protocol=pickle.HIGHEST_PROTOCOL) else: locEns[dd].stack = load_pickle2(pathPkl) if not all(l in locEns[dd].stack.keys() for l in self.listvar): raise Exception('Some of the ppvars you mentioned are not present in ', pathPkl, '\n listvars:', self.listvar, '\npickle file keys:', locEns[dd].stack.keys()) locEns[dd].isstacked = True return locEns else: # BC june 2020: manager is used to share the dict between nodes try: manager = multiprocessing.Manager() locEns = manager.dict() p = multiprocessing.Pool(min(multiprocessing.cpu_count(), len(self.options.dates))) p.map(loadEnsPrepDate_parallel, [[self, locEns, dd, kind, isOl, fromArch] for dd in self.options.dates]) except Exception as _: p.close() p.close() p.join() return dict(locEns)
def loadEnsPrepDate_parallel(largs): """ BC 24/06/20 externalized this fucntion for python2 compatibility (if class method, throws this exception): https://stackoverflow.com/questions/1816958/cant-pickle-type-instancemethod-when-using-multiprocessing-pool-map/7309686#7309686 """ pp = largs[0] locEns = largs[1] dd = largs[2] kind = largs[3] isOl = largs[4] fromArch = largs[5] if kind == 'bg': local = PrepEnsBg(pp.options, dd, fromArch=fromArch) elif kind == 'an': local = PrepEnsAn(pp.options, dd, fromArch=fromArch) else: local = PrepEnsOl(pp.options, dd, isOl=isOl, fromArch=fromArch) if (kind == 'ol' and isOl is False): pathPkl = pp.xpidoldir + '/crampon/' + pp.options.saverep + '/' +\ kind + '_' + dd + '.pkl' else: pathPkl = kind + '_' + dd + '.pkl' pathpklbeauf = pathPkl + '.foo' if not os.path.islink(pathPkl): if os.path.exists(pathpklbeauf): try: os.symlink(pathpklbeauf, pathPkl) except FileExistsError: pass if not os.path.exists(pathPkl): print(' unsuccesfull loading', pathPkl, ) print(('loading ' + kind + ' ens for date : ', dd)) local.stackit() with open(pathPkl, 'wb') as f: print(('saaving ' + kind + ' to pickle !')) pickle.dump(local.stack, f, protocol=pickle.HIGHEST_PROTOCOL) else: local.stack = load_pickle2(pathPkl) if not all(l in local.stack.keys() for l in pp.listvar): raise Exception('Some of the ppvars you mentioned are not present in ', os.getcwd() + '/' + pathPkl, '\n listvars:', pp.listvar, '\npickle file keys:', local.stack.keys()) local.isstacked = True locEns[dd] = local
def readTruth(self): """ BC 24/10/19 : function to read the truth in the OL from the corresponding run. - Sometimes, if the OL is a subset of a bigger OL, and the truth is in this bigger OL, need to read in it. k is the variable. - find the baseline experiment if necessary. """ def load_from_dict(ddict, mbsynth): out = dict() for var in ddict.keys(): if 'time' not in var: out[var] = ddict[var][:, :, mbsynth] else: out['time'] = ddict['time'] return out try: self.truth = load_from_dict(self.ensProOl, self.mbsynth) except IndexError: print('\n\nWARNING : there is no corresponding mbsynth in this openloop not enough members\n looking in the bigger OL xp.\n\n') print('loading ' + self.xpidoldir[0:-4] + '/crocO/' + self.options.saverep + '/ensProOl.pkl') pathPklBigOl = self.xpidoldir[0:-4] + '/crocO/' + self.options.saverep + '/ensProOl.pkl' gg = load_pickle2(pathPklBigOl) self.truth = load_from_dict(gg, self.mbsynth)
def readObs(self): # paths settings (ol or not, real obs or not). # if performing pfpp, it is after a locla run and obs are properly archived. if self.options.todo == 'pfpp': if self.options.synth is not None: self.pathArch = self.xpiddir + '/crocO/ARCH/' + self.options.sensor + '/' self.pathSynth = self.xpiddir + '/crocO/SYNTH/' + self.options.sensor + '/' else: self.pathReal = self.options.crocOpath + '/s2m/' + self.options.vconf + '/obs/' + self.options.sensor + '/' else: if str(self.options.openloop) == 'on': if self.options.synth is not None: self.pathArch = self.xpidoldir + '/crocO/ARCH/' + self.options.sensor + '/' self.pathSynth = self.xpidoldir + '/crocO/SYNTH/' + self.options.sensor + '/' else: self.pathReal = self.options.crocOpath + '/s2m/' + self.options.vconf + '/obs/' + self.options.sensor + '/' else: if hasattr(self.options, 'xpidoldir'): if self.options.synth is not None: self.pathArch = self.xpidoldir + '/crocO/ARCH/' + self.options.sensor + '/' self.pathSynth = self.xpidoldir + '/crocO/SYNTH/' + self.options.sensor + '/' else: self.pathReal = self.options.crocOpath + '/s2m/' + self.options.vconf + '/obs/' + self.options.sensor + '/' else: # if no xpidoldir has been prescribed, obs must be real. self.pathReal = self.options.crocOpath + '/s2m/' + self.options.vconf + '/obs/' + self.options.sensor + '/' # proper loading (synth or real) if self.options.todo == 'pfpp': if self.options.synth is not None: print('reading synthetic assimilated obs. in ' + self.pathArch) self.obsArch = {dd: FromXp(self.pathArch, dd, self.options) for dd in self.options.dates} self.obsSynth = {dd: FromXp(self.pathSynth, dd, self.options) for dd in self.options.dates} for dd in self.options.dates: self.obsArch[dd].load() self.obsSynth[dd].load() else: print('reading real assimilated obs. in ' + self.pathReal) self.obsReal = {dd: Real(self.pathReal, dd, self.options) for dd in self.options.dates} for dd in self.options.dates: self.obsReal[dd].load() else: if self.options.synth is not None: print('sssynth', self.options.synth) print('reading synthetic truth obs. in ' + self.pathArch) print('reading synthetic assimilated obs. in ' + self.pathSynth) self.obsArch = {dd: FromXp(self.pathArch, dd, self.options) for dd in self.options.dates} self.obsSynth = {dd: FromXp(self.pathSynth, dd, self.options) for dd in self.options.dates} for dd in self.options.dates: self.obsArch[dd].load() self.obsSynth[dd].load() else: print('reading real assimilated obs. in ' + self.pathReal) self.obsReal = {dd: Real(self.pathReal, dd, self.options) for dd in self.options.dates} for dd in self.options.dates: self.obsReal[dd].load() pathObsTs = self.options.crocOpath + '/s2m/' + self.options.vconf + '/obs/' + self.options.sensor +\ '/obs_{0}_{1}_{2}100106_{3}063006.pkl'.format(self.options.sensor, self.options.vconf, self.datedeb.strftime('%Y'), int(self.datedeb.strftime('%Y')) + 1) print('reading observation timeseries (pickle from csv file) in ') print(pathObsTs) if not os.path.exists(pathObsTs): if '_X' in self.options.sensor: sensorBase = self.options.sensor.split('X')[0][0:-1] os.symlink(self.options.crocOpath + '/s2m/' + self.options.vconf + '/obs/{0}/obs_{0}_{1}_{2}100106_{3}063006.pkl'.format(sensorBase, self.options.vconf, self.datedeb.strftime('%Y'), int(self.datedeb.strftime('%Y')) + 1), pathObsTs) else: # BC 30/06/20: some day replace with a link to a yearly file/ try: os.symlink(self.options.crocOpath + '/s2m/' + self.options.vconf + '/obs/{0}/obs_all_{0}_2013010106_2018123106.pkl'.format(self.options.vconf, self.datedeb.strftime('%Y'), int(self.datedeb.strftime('%Y')) + 1 ), pathObsTs) except FileExistsError: pass self.obsTs = load_pickle2(pathObsTs)