def test(directory, sourcefiles, cwd): """ Makes a file in /testdata/ name testclasses_file.txt whcih has the ensemble info after using match ensembles and match_realizations """ # first change to a scratch dir and link in test data. The paths are CCCma specific. ctt.loadtestfiles(directory, sourcefiles) f = cwd + '/testdata/testclasses_file.txt' # ========================================================================= # mkensemble # ========================================================================= fp = '*historical*' ens = cd.mkensemble(fp, prefix='') ens.fulldetails_tofile(f) # Create a second cmipdata ensemble fp2 = '*ts*' ens2 = cd.mkensemble(fp2, prefix='') ens.fulldetails_tofile(f) # ========================================================================= # match_ensembles # ========================================================================= print 'Matching ensembles...' enso1, enso2 = cd.match_ensembles(ens, ens2) enso1.sinfo() enso2.sinfo() ens.fulldetails_tofile(f) # ========================================================================= # match_realizations # ========================================================================= print 'Matching realizations...' ensem1, ensem2 = cd.match_realizations(ens, ens2) ensem1.sinfo() ensem2.sinfo() ens.fulldetails_tofile(f)
def test(directory, sourcefiles, cwd): """ Makes a file in /testdata/ name testclasses_file.txt whcih has the ensemble info after using match ensembles and match_realizations """ # first change to a scratch dir and link in test data. The paths are CCCma specific. ctt.loadtestfiles(directory,sourcefiles) f = cwd + '/testdata/testclasses_file.txt' # ========================================================================= # mkensemble # ========================================================================= fp = '*historical*' ens = cd.mkensemble(fp, prefix='') ens.fulldetails_tofile(f) # Create a second cmipdata ensemble fp2 = '*ts*' ens2 = cd.mkensemble(fp2, prefix='') ens.fulldetails_tofile(f) # ========================================================================= # match_ensembles # ========================================================================= print 'Matching ensembles...' enso1, enso2 = cd.match_ensembles(ens, ens2) enso1.sinfo() enso2.sinfo() ens.fulldetails_tofile(f) # ========================================================================= # match_realizations # ========================================================================= print 'Matching realizations...' ensem1, ensem2 = cd.match_realizations(ens, ens2) ensem1.sinfo() ensem2.sinfo() ens.fulldetails_tofile(f)
def test_ens_stats(directory, sourcefiles, var): ctt.loadtestfiles(directory, [sourcefiles[0]]) filepattern = "*.nc" ens = cd.mkensemble(filepattern) cd.ens_stats(ens, var) filepattern = "ENS*" ens = cd.mkensemble(filepattern) data = ctt.sha(ens) return data
def test_ens_stats(directory, sourcefiles, var): ctt.loadtestfiles(directory, [sourcefiles[0]]) filepattern = '*.nc' ens = cd.mkensemble(filepattern) cd.ens_stats(ens, var) filepattern = 'ENS*' ens = cd.mkensemble(filepattern) data = ctt.sha(ens) return data
def test_my_operator(directory, sourcefiles, var, e1name, e2name): ctt.loadtestfiles(directory, sourcefiles) filepattern = "*.nc" ens = cd.mkensemble(filepattern) ens = cd.cat_experiments(ens, var, exp1_name=e1name, exp2_name=e2name) filepattern = "*.nc" ens = cd.mkensemble(filepattern) my_cdo_str = "cdo remapdis,r1x180 -zonmean -seldate,1950-01-01,2000-12-31\ -sub {infile} -timmean -seldate,1950-01-01,2000-12-31 {infile}\ {outfile}" ens = cd.my_operator(ens, my_cdo_str, output_prefix="test_", delete=True) data = ctt.sha(ens) return data
def test_my_operator(directory, sourcefiles, var, e1name, e2name): ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) ens = cd.cat_experiments(ens, var, exp1_name=e1name, exp2_name=e2name) filepattern = '*.nc' ens = cd.mkensemble(filepattern) my_cdo_str = 'cdo remapdis,r1x180 -zonmean -seldate,1950-01-01,2000-12-31\ -sub {infile} -timmean -seldate,1950-01-01,2000-12-31 {infile}\ {outfile}' ens = cd.my_operator(ens, my_cdo_str, output_prefix='test_', delete=True) data = ctt.sha(ens) return data
def test_remap(directory, sourcefiles): ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) ens = cd.remap(ens, remap='r1x180') data = ctt.sha(ens) return data
def test_zonal_mean(directory, sourcefiles): ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) ens = cd.zonmean(ens, delete=True) data = ctt.sha(ens) return data
def test_cat_experiments(directory, sourcefiles, var, e1name, e2name): ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) ens = cd.cat_experiments(ens, var, exp1_name=e1name, exp2_name=e2name) data = ctt.sha(ens) return data
def test_cat_exp_slices(directory, sourcefiles): ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) ens = cd.cat_exp_slices(ens) data = ctt.sha(ens) return data
def test_climatology(directory, sourcefiles): ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) ens = cd.climatology(ens) data = ctt.sha(ens) return data
def test_areamean(directory, sourcefiles): ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) ens = cd.areamean(ens) data = ctt.sha(ens) return data
def model_files(var, model, expname, frequency, cmipdir): prefix = cmipdir + '/' + var + '/' ensstring = prefix + var + '_*' + frequency + '_*' + model + '_' + expname + '_*.nc' ens = cd.mkensemble(ensstring, prefix=prefix) ens = cd.cat_exp_slices(ens, delete=False, output_prefix='cmipfiles/') mfiles = ens.lister('ncfile') return mfiles, ens
def test_ensembleenvelope(directory, sourcefiles, var, sd, ed, cwd): """ Tests ensemble_envelope_timeseries Parameters ---------- directory : string the directory to link the files to sourcefiles : list a list of the locations of the files to link var : string the variable plotted sd : int start date for time slice ed : int end date for time slice cwd : string directory of the test modules """ ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) ens = cd.time_slice(ens, sd, ed) ens = cd.remap(ens, 'r360x180', delete=True) ens = cd.my_operator(ens, my_cdo_str='cdo -fldmean -yearmean {infile} {outfile}', output_prefix='processed_', delete=True) cd.ens_stats(ens, var) cd.ensemble_envelope_timeseries(ens, 'ENS-MEAN_processed_remap_ts_Amon_CanESM2_historical_R-MEAN_195001-200012.nc', 'ENS-STD_processed_remap_ts_Amon_CanESM2_historical_STD_195001-200012.nc', var, pdf='ensenv') plotcompare(directory, cwd, 'ensenv') os.system('rm ensenv.pdf')
def test_1dtimeseries(directory, sourcefiles, var, cwd): """ Tests plot_realizations_1d for a timeseries plot Parameters ---------- directory : string the directory to link the files to sourcefiles : list a list of the locations of the files to link var : string the variable plotted cwd : string directory of the test modules """ ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) data = cd.loadfiles(ens, var, cdostr='-fldmean -yearmean', toDatetime=True) fig, axt = plt.subplots(1, 1, figsize=(8, 8)) cd.plot_realizations_1d(data, var, 'time', ax=axt, pdf='tsplot', kwargs={ 'color': 'r', 'alpha': 0.3 }) plotcompare(directory, cwd, 'tsplot') os.system('rm tsplot.pdf')
def test_time_slice(directory, sourcefiles, sd, ed): ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) ens = cd.time_slice(ens, start_date=sd, end_date=ed) data = ctt.sha(ens) return data
def test_time_slice(directory, sourcefiles, sd, ed): ctt.loadtestfiles(directory, sourcefiles) filepattern = "*.nc" ens = cd.mkensemble(filepattern) ens = cd.time_slice(ens, start_date=sd, end_date=ed) data = ctt.sha(ens) return data
def test_remap(directory, sourcefiles): ctt.loadtestfiles(directory, sourcefiles) filepattern = "*.nc" ens = cd.mkensemble(filepattern) ens = cd.remap(ens, remap="r1x180") data = ctt.sha(ens) return data
def test_cat_experiments(directory, sourcefiles, var, e1name, e2name): ctt.loadtestfiles(directory, sourcefiles) filepattern = "*.nc" ens = cd.mkensemble(filepattern) ens = cd.cat_experiments(ens, var, exp1_name=e1name, exp2_name=e2name) data = ctt.sha(ens) return data
def test_zonal_mean(directory, sourcefiles): ctt.loadtestfiles(directory, sourcefiles) filepattern = "*.nc" ens = cd.mkensemble(filepattern) ens = cd.zonmean(ens, delete=True) data = ctt.sha(ens) return data
def test_climatology(directory, sourcefiles): ctt.loadtestfiles(directory, sourcefiles) filepattern = "*.nc" ens = cd.mkensemble(filepattern) ens = cd.climatology(ens) data = ctt.sha(ens) return data
def test_cat_exp_slices(directory, sourcefiles): ctt.loadtestfiles(directory, sourcefiles) filepattern = "*.nc" ens = cd.mkensemble(filepattern) ens = cd.cat_exp_slices(ens) data = ctt.sha(ens) return data
def test_areamean(directory, sourcefiles): ctt.loadtestfiles(directory, sourcefiles) filepattern = "*.nc" ens = cd.mkensemble(filepattern) ens = cd.areamean(ens) data = ctt.sha(ens) return data
def test_loadingtools(directory, sourcefiles, var): """ Loads the data from a set of files """ ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) data = hashlib.sha1(cd.loadfiles(ens,var)['data']).hexdigest() return {var: data}
def test_time_anomaly(directory, sourcefiles, sd, ed): ctt.loadtestfiles(directory, sourcefiles) filepattern = "*.nc" ens = cd.mkensemble(filepattern) ens = cd.time_anomaly(ens, start_date=sd, end_date=ed, delete=True) ens.fulldetails() data = ctt.sha(ens) return data
def test_time_anomaly(directory, sourcefiles, sd, ed): ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) ens = cd.time_anomaly(ens, start_date=sd, end_date=ed, delete=True) ens.fulldetails() data = ctt.sha(ens) return data
def process_local_cmip5_data(var): """time-merge """ # 2. Join the time-slices # First build a cmipdata ensemble object filepattern = var + '*.nc' ens = cd.mkensemble(filepattern) # Join the time-slices and limit to years between 1880 and 2012 ens = cd.cat_experiments(ens, var, 'historical', 'rcp45') ens = cd.time_slice(ens, '1881-01-01', '2012-12-31')
def mk_cmip5_trends(datapath='./'): """Compute trends for uas, uflx and slp over various periods. """ # remove old trends if they exist ot = datapath + 'cmip5_trends.h5' if os.path.isfile(ot): os.remove(ot) # list of models being used model_list = ['ACCESS1-0', 'ACCESS1-3', 'BNU-ESM', 'CMCC-CMS', 'CMCC-CM', 'CNRM-CM5', 'CSIRO-Mk3-6-0', 'CanESM2', 'GISS-E2-H-CC', 'GISS-E2-H', 'GISS-E2-R-CC', 'GISS-E2-R', 'HadCM3', 'HadGEM2-AO', 'HadGEM2-CC', 'HadGEM2-ES', 'IPSL-CM5A-LR', 'IPSL-CM5A-MR', 'IPSL-CM5B-LR', 'MIROC-ESM-CHEM', 'MIROC-ESM', 'MIROC5', 'MPI-ESM-LR','MPI-ESM-MR','MRI-CGCM3', 'NorESM1-ME', 'NorESM1-M', 'bcc-csm1-1-m', 'bcc-csm1-1','inmcm4' ] # Where to find the data prefix = os.path.join(datapath, 'remap_') # Do psl # make an ensemble object and the calculate the trends for req. dates filepattern = prefix + 'psl*' + '_historical-rcp45_r1i1p1_188101-201212.nc' ens = cd.mkensemble(filepattern, prefix=prefix) save_cmip5_trends(ens, 'psl', '1951-01-01', '2004-12-31', datapath=datapath) ens = cd.mkensemble(filepattern, prefix=prefix) save_cmip5_trends(ens, 'psl', '1979-01-01', '2004-12-31', datapath=datapath) # Do uas # make an ensemble object and the calculate the trends for req. dates filepattern = prefix + 'uas*_historical-rcp45_r1i1p1_188101-201212.nc' ens = cd.mkensemble(filepattern, prefix=prefix) save_cmip5_trends(ens, 'uas', '1951-01-01', '2011-12-31', datapath=datapath) ens = cd.mkensemble(filepattern, prefix=prefix) save_cmip5_trends(ens, 'uas', '1988-01-01', '2011-12-31', datapath=datapath) # Do tauu # make an ensemble object and the calculate the trends for req. dates filepattern = prefix + 'tauu*_historical-rcp45_r1i1p1_188101-201212.nc' ens = cd.mkensemble(filepattern, prefix=prefix) save_cmip5_trends(ens, 'tauu', '1988-01-01', '2011-12-31', datapath=datapath)
def cmip_average(files, var): """ Creates and stores a netCDF file with the average data across all the models provided. Returns the name of the created file. """ ens = cd.mkensemble('', filenames=files) out = 'netcdf/' + var + '_' + 'cmip5.nc' # skip if the new file was already made if not os.path.isfile(out): ens = cd.remap(ens, output_prefix='netcdf/forens-') ensemble_means, _ = cd.ens_stats(ens, var, output_prefix='netcdf/') os.rename(ensemble_means[0], out) return out
def preprocess_models(datapath='./'): # where we are starting from cwd = os.getcwd() # move to where the data is os.chdir(datapath) # time-merge, remap and zonal mean the model data variables = ['uas', 'psl', 'tauu'] for var in variables: # time-merge the data across the historical and rcp45 experiments filepattern = var + '_Amon*.nc' ens = cd.mkensemble(filepattern) # remap to a 1x1 grid ens_remap = cd.remap(ens,remap='r360x180', delete=True) # Compute zonal means ens_zonmean = cd.zonmean(ens_remap, delete=False) # move back os.chdir(cwd)
def test_1dtimeseries(directory, sourcefiles, var, cwd): """ Tests plot_realizations_1d for a timeseries plot Parameters ---------- directory : string the directory to link the files to sourcefiles : list a list of the locations of the files to link var : string the variable plotted cwd : string directory of the test modules """ ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) data = cd.loadfiles(ens, var, cdostr='-fldmean -yearmean', toDatetime=True) fig, axt = plt.subplots(1, 1, figsize=(8, 8)) cd.plot_realizations_1d(data, var, 'time', ax=axt, pdf='tsplot', kwargs={'color': 'r', 'alpha': 0.3}) plotcompare(directory, cwd, 'tsplot') os.system('rm tsplot.pdf')
def test_ensembleenvelope(directory, sourcefiles, var, sd, ed, cwd): """ Tests ensemble_envelope_timeseries Parameters ---------- directory : string the directory to link the files to sourcefiles : list a list of the locations of the files to link var : string the variable plotted sd : int start date for time slice ed : int end date for time slice cwd : string directory of the test modules """ ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) ens = cd.time_slice(ens, sd, ed) ens = cd.remap(ens, 'r360x180', delete=True) ens = cd.my_operator( ens, my_cdo_str='cdo -fldmean -yearmean {infile} {outfile}', output_prefix='processed_', delete=True) cd.ens_stats(ens, var) cd.ensemble_envelope_timeseries( ens, 'ENS-MEAN_processed_remap_ts_Amon_CanESM2_historical_R-MEAN_195001-200012.nc', 'ENS-STD_processed_remap_ts_Amon_CanESM2_historical_STD_195001-200012.nc', var, pdf='ensenv') plotcompare(directory, cwd, 'ensenv') os.system('rm ensenv.pdf')
def make_ensemble(files): ens = cd.mkensemble()
def getcmipfiles(plots, expname, cmipdir): """ Loop through the plots and create the comparison files if cdo operations are needed and map the keys in the compare dictionary to the correct file names. """ # get the date ranges need for each variable startdates = min_start_dates(plots) enddates = max_end_dates(plots) cmip5_variables = {} for p in plots: p['model_files'] = {} p['model_file'] = {} p['cmip5_files'] = [] p['cmip5_file'] = None if p['comp_models'] or p['comp_cmips']: # map the file names of the comparison files to the model names for model in p['comp_models'][:]: p['modelfiles_for_log'] = {} try: if model not in p['model_files']: p['model_files'][model], ens = model_files( p['variable'], model, expname, p['frequency'], cmipdir) else: ens = cd.mkensemble('', filenames=p['model_files'][model]) p['model_file'][model] = model_average( ens, p['variable'], model) p['modelfiles_for_log'][model] = p['model_files'][model] except: with open('logs/log.txt', 'a') as outfile: outfile.write('No cmip5 files were found for ' + p['variable'] + ': ' + model + '\n\n') print 'No cmip5 files were found for ' + p[ 'variable'] + ': ' + model p['comp_models'].remove(model) try: p['comp_cmips'].remove(model) except: pass for model in p['comp_cmips'][:]: if model not in p['comp_models']: try: p['model_files'][model], ens = model_files( p['variable'], model, expname, p['frequency'], cmipdir) except: with open('logs/log.txt', 'a') as outfile: outfile.write('No cmip5 files were found for ' + p['variable'] + ': ' + model + '\n\n') print 'No cmip5 files were found for ' + p[ 'variable'] + ': ' + model # remove the model from the list if no comparison files were found p['comp_cmips'].remove(model) for p in plots: if p['comp_cmips']: # map the file name of the comparison file to cmip5 in compare dictionary try: files = {} for f in p['comp_cmips']: files[f] = p['model_files'][f] p['cmip5_files'] = cmip_files(files) # p['cmip5_file'] = cmip_average(p['variable'], p['frequency'], p['cmip5_files'], str(startdates[p['variable']]) + '-01', str(enddates[p['variable']]) + '-01', expname) except: p['comp_cmips'] = [] get_cmip_average(plots, MEANDIR)
def test_starting_files(directory, sourcefiles): ctt.loadtestfiles(directory, sourcefiles) filepattern = '*.nc' ens = cd.mkensemble(filepattern) data = ctt.sha(ens) return data
def test_starting_files(directory, sourcefiles): ctt.loadtestfiles(directory, sourcefiles) filepattern = "*.nc" ens = cd.mkensemble(filepattern) data = ctt.sha(ens) return data