Example #1
0
File: cla.py Project: wydh/cmipdata
def test(directory, sourcefiles, cwd):
    """ Makes a file in /testdata/ name testclasses_file.txt whcih has the ensemble info
        after using match ensembles and match_realizations
    """
    # first change to a scratch dir and link in test data. The paths are CCCma specific.
    ctt.loadtestfiles(directory, sourcefiles)
    f = cwd + '/testdata/testclasses_file.txt'

    # =========================================================================
    #     mkensemble
    # =========================================================================
    fp = '*historical*'
    ens = cd.mkensemble(fp, prefix='')
    ens.fulldetails_tofile(f)

    # Create a second cmipdata ensemble
    fp2 = '*ts*'
    ens2 = cd.mkensemble(fp2, prefix='')
    ens.fulldetails_tofile(f)
    # =========================================================================
    #     match_ensembles
    # =========================================================================
    print 'Matching ensembles...'
    enso1, enso2 = cd.match_ensembles(ens, ens2)
    enso1.sinfo()
    enso2.sinfo()
    ens.fulldetails_tofile(f)
    # =========================================================================
    #     match_realizations
    # =========================================================================
    print 'Matching realizations...'
    ensem1, ensem2 = cd.match_realizations(ens, ens2)
    ensem1.sinfo()
    ensem2.sinfo()
    ens.fulldetails_tofile(f)
Example #2
0
def test(directory, sourcefiles, cwd):
    """ Makes a file in /testdata/ name testclasses_file.txt whcih has the ensemble info
        after using match ensembles and match_realizations
    """
    # first change to a scratch dir and link in test data. The paths are CCCma specific.
    ctt.loadtestfiles(directory,sourcefiles)
    f = cwd + '/testdata/testclasses_file.txt'

    # =========================================================================
    #     mkensemble
    # =========================================================================
    fp = '*historical*'
    ens = cd.mkensemble(fp, prefix='')
    ens.fulldetails_tofile(f)

    # Create a second cmipdata ensemble
    fp2 = '*ts*'
    ens2 = cd.mkensemble(fp2, prefix='')
    ens.fulldetails_tofile(f)
    # =========================================================================
    #     match_ensembles
    # =========================================================================
    print 'Matching ensembles...'
    enso1, enso2 = cd.match_ensembles(ens, ens2)
    enso1.sinfo()
    enso2.sinfo()
    ens.fulldetails_tofile(f)
    # =========================================================================
    #     match_realizations
    # =========================================================================
    print 'Matching realizations...'
    ensem1, ensem2 = cd.match_realizations(ens, ens2)
    ensem1.sinfo()
    ensem2.sinfo()
    ens.fulldetails_tofile(f)
Example #3
0
def test_ens_stats(directory, sourcefiles, var):
    ctt.loadtestfiles(directory, [sourcefiles[0]])
    filepattern = "*.nc"
    ens = cd.mkensemble(filepattern)
    cd.ens_stats(ens, var)
    filepattern = "ENS*"
    ens = cd.mkensemble(filepattern)
    data = ctt.sha(ens)
    return data
Example #4
0
def test_ens_stats(directory, sourcefiles, var):
    ctt.loadtestfiles(directory, [sourcefiles[0]])
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    cd.ens_stats(ens, var)
    filepattern = 'ENS*'
    ens = cd.mkensemble(filepattern)
    data = ctt.sha(ens)
    return data
Example #5
0
def test_my_operator(directory, sourcefiles, var, e1name, e2name):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = "*.nc"
    ens = cd.mkensemble(filepattern)
    ens = cd.cat_experiments(ens, var, exp1_name=e1name, exp2_name=e2name)
    filepattern = "*.nc"
    ens = cd.mkensemble(filepattern)
    my_cdo_str = "cdo remapdis,r1x180 -zonmean -seldate,1950-01-01,2000-12-31\
             -sub {infile} -timmean -seldate,1950-01-01,2000-12-31 {infile}\
             {outfile}"
    ens = cd.my_operator(ens, my_cdo_str, output_prefix="test_", delete=True)
    data = ctt.sha(ens)
    return data
Example #6
0
def test_my_operator(directory, sourcefiles, var, e1name, e2name):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    ens = cd.cat_experiments(ens, var, exp1_name=e1name, exp2_name=e2name)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    my_cdo_str = 'cdo remapdis,r1x180 -zonmean -seldate,1950-01-01,2000-12-31\
             -sub {infile} -timmean -seldate,1950-01-01,2000-12-31 {infile}\
             {outfile}'

    ens = cd.my_operator(ens, my_cdo_str, output_prefix='test_', delete=True)
    data = ctt.sha(ens)
    return data
Example #7
0
def test_remap(directory, sourcefiles):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    ens = cd.remap(ens, remap='r1x180')
    data = ctt.sha(ens)
    return data
Example #8
0
def test_zonal_mean(directory, sourcefiles):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    ens = cd.zonmean(ens, delete=True)
    data = ctt.sha(ens)
    return data
Example #9
0
def test_cat_experiments(directory, sourcefiles, var, e1name, e2name):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    ens = cd.cat_experiments(ens, var, exp1_name=e1name, exp2_name=e2name)
    data = ctt.sha(ens)
    return data
Example #10
0
def test_cat_exp_slices(directory, sourcefiles):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    ens = cd.cat_exp_slices(ens)
    data = ctt.sha(ens)
    return data
Example #11
0
def test_climatology(directory, sourcefiles):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    ens = cd.climatology(ens)
    data = ctt.sha(ens)
    return data
Example #12
0
def test_areamean(directory, sourcefiles):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    ens = cd.areamean(ens)
    data = ctt.sha(ens)
    return data
Example #13
0
def model_files(var, model, expname, frequency, cmipdir):
    prefix = cmipdir + '/' + var + '/'
    ensstring = prefix + var + '_*' + frequency + '_*' + model + '_' + expname + '_*.nc'
    ens = cd.mkensemble(ensstring, prefix=prefix)
    ens = cd.cat_exp_slices(ens, delete=False, output_prefix='cmipfiles/')
    mfiles = ens.lister('ncfile')
    return mfiles, ens
Example #14
0
def test_ensembleenvelope(directory, sourcefiles, var, sd, ed, cwd):
    """ Tests ensemble_envelope_timeseries
    
    Parameters
    ----------
    directory : string
                the directory to link the files to
    sourcefiles : list
                  a list of the locations of the files to link
    var :  string
           the variable plotted
    sd : int
         start date for time slice
    ed : int
         end date for time slice
    cwd : string 
          directory of the test modules
    """
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    ens = cd.time_slice(ens, sd, ed)
    ens = cd.remap(ens, 'r360x180', delete=True)
    ens = cd.my_operator(ens, my_cdo_str='cdo -fldmean -yearmean {infile} {outfile}', output_prefix='processed_', delete=True)
    cd.ens_stats(ens, var)
    cd.ensemble_envelope_timeseries(ens, 'ENS-MEAN_processed_remap_ts_Amon_CanESM2_historical_R-MEAN_195001-200012.nc', 'ENS-STD_processed_remap_ts_Amon_CanESM2_historical_STD_195001-200012.nc', var, pdf='ensenv')    
    plotcompare(directory, cwd, 'ensenv')
    os.system('rm  ensenv.pdf')               
Example #15
0
def test_1dtimeseries(directory, sourcefiles, var, cwd):
    """ Tests plot_realizations_1d for a timeseries plot
    
    Parameters
    ----------
    directory : string
                the directory to link the files to
    sourcefiles : list
                  a list of the locations of the files to link
    var :  string
           the variable plotted
    cwd : string 
          directory of the test modules
    """
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    data = cd.loadfiles(ens, var, cdostr='-fldmean -yearmean', toDatetime=True)
    fig, axt = plt.subplots(1, 1, figsize=(8, 8))
    cd.plot_realizations_1d(data,
                            var,
                            'time',
                            ax=axt,
                            pdf='tsplot',
                            kwargs={
                                'color': 'r',
                                'alpha': 0.3
                            })
    plotcompare(directory, cwd, 'tsplot')
    os.system('rm  tsplot.pdf')
Example #16
0
def test_time_slice(directory, sourcefiles, sd, ed):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    ens = cd.time_slice(ens, start_date=sd, end_date=ed)
    data = ctt.sha(ens)
    return data
Example #17
0
def test_time_slice(directory, sourcefiles, sd, ed):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = "*.nc"
    ens = cd.mkensemble(filepattern)
    ens = cd.time_slice(ens, start_date=sd, end_date=ed)
    data = ctt.sha(ens)
    return data
Example #18
0
def test_remap(directory, sourcefiles):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = "*.nc"
    ens = cd.mkensemble(filepattern)
    ens = cd.remap(ens, remap="r1x180")
    data = ctt.sha(ens)
    return data
Example #19
0
def test_cat_experiments(directory, sourcefiles, var, e1name, e2name):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = "*.nc"
    ens = cd.mkensemble(filepattern)
    ens = cd.cat_experiments(ens, var, exp1_name=e1name, exp2_name=e2name)
    data = ctt.sha(ens)
    return data
Example #20
0
def test_zonal_mean(directory, sourcefiles):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = "*.nc"
    ens = cd.mkensemble(filepattern)
    ens = cd.zonmean(ens, delete=True)
    data = ctt.sha(ens)
    return data
Example #21
0
def test_climatology(directory, sourcefiles):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = "*.nc"
    ens = cd.mkensemble(filepattern)
    ens = cd.climatology(ens)
    data = ctt.sha(ens)
    return data
Example #22
0
def test_cat_exp_slices(directory, sourcefiles):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = "*.nc"
    ens = cd.mkensemble(filepattern)
    ens = cd.cat_exp_slices(ens)
    data = ctt.sha(ens)
    return data
Example #23
0
def model_files(var, model, expname, frequency, cmipdir):
    prefix = cmipdir + '/' + var + '/'
    ensstring = prefix + var + '_*' + frequency + '_*' + model + '_' + expname + '_*.nc'
    ens = cd.mkensemble(ensstring, prefix=prefix)
    ens = cd.cat_exp_slices(ens, delete=False, output_prefix='cmipfiles/')
    mfiles = ens.lister('ncfile')
    return mfiles, ens
Example #24
0
def test_areamean(directory, sourcefiles):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = "*.nc"
    ens = cd.mkensemble(filepattern)
    ens = cd.areamean(ens)
    data = ctt.sha(ens)
    return data
Example #25
0
File: loa.py Project: wydh/cmipdata
def test_loadingtools(directory, sourcefiles, var):
    """ Loads the data from a set of files 
    """
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    data = hashlib.sha1(cd.loadfiles(ens,var)['data']).hexdigest()
    return {var: data}
Example #26
0
def test_time_anomaly(directory, sourcefiles, sd, ed):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = "*.nc"
    ens = cd.mkensemble(filepattern)
    ens = cd.time_anomaly(ens, start_date=sd, end_date=ed, delete=True)
    ens.fulldetails()
    data = ctt.sha(ens)
    return data
Example #27
0
def test_time_anomaly(directory, sourcefiles, sd, ed):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    ens = cd.time_anomaly(ens, start_date=sd, end_date=ed, delete=True)
    ens.fulldetails()
    data = ctt.sha(ens)
    return data
def process_local_cmip5_data(var):
    """time-merge
    """
    # 2. Join the time-slices
    # First build a cmipdata ensemble object
    filepattern = var + '*.nc' 
    ens = cd.mkensemble(filepattern)
    # Join the time-slices and limit to years between 1880 and 2012
    ens = cd.cat_experiments(ens, var, 'historical', 'rcp45')    
    ens = cd.time_slice(ens, '1881-01-01', '2012-12-31')    
Example #29
0
def mk_cmip5_trends(datapath='./'):
    """Compute trends for uas, uflx and slp over various periods.
    """
        # remove old trends if they exist
    ot = datapath + 'cmip5_trends.h5'
    if os.path.isfile(ot):
        os.remove(ot)
        
    # list of models being used
    model_list = ['ACCESS1-0', 'ACCESS1-3', 'BNU-ESM', 'CMCC-CMS', 'CMCC-CM',
                  'CNRM-CM5', 'CSIRO-Mk3-6-0', 'CanESM2', 
                  'GISS-E2-H-CC', 'GISS-E2-H', 'GISS-E2-R-CC', 'GISS-E2-R',
                  'HadCM3', 'HadGEM2-AO', 'HadGEM2-CC', 'HadGEM2-ES', 
                  'IPSL-CM5A-LR', 'IPSL-CM5A-MR', 'IPSL-CM5B-LR', 'MIROC-ESM-CHEM',
                  'MIROC-ESM', 'MIROC5', 'MPI-ESM-LR','MPI-ESM-MR','MRI-CGCM3',
                  'NorESM1-ME', 'NorESM1-M', 'bcc-csm1-1-m', 'bcc-csm1-1','inmcm4'
                  ]
    
    # Where to find the data
    prefix = os.path.join(datapath, 'remap_')
    
    # Do psl
    # make an ensemble object and the calculate the trends for req. dates
    filepattern = prefix + 'psl*' +  '_historical-rcp45_r1i1p1_188101-201212.nc'
    ens = cd.mkensemble(filepattern, prefix=prefix)
    save_cmip5_trends(ens, 'psl', '1951-01-01', '2004-12-31', datapath=datapath)
    ens = cd.mkensemble(filepattern, prefix=prefix)
    save_cmip5_trends(ens, 'psl', '1979-01-01', '2004-12-31', datapath=datapath)

    # Do uas
    # make an ensemble object and the calculate the trends for req. dates
    filepattern = prefix + 'uas*_historical-rcp45_r1i1p1_188101-201212.nc'
    ens = cd.mkensemble(filepattern, prefix=prefix)
    save_cmip5_trends(ens, 'uas', '1951-01-01', '2011-12-31', datapath=datapath)
    ens = cd.mkensemble(filepattern, prefix=prefix)
    save_cmip5_trends(ens, 'uas', '1988-01-01', '2011-12-31', datapath=datapath)

    # Do tauu
    # make an ensemble object and the calculate the trends for req. dates
    filepattern = prefix + 'tauu*_historical-rcp45_r1i1p1_188101-201212.nc'
    ens = cd.mkensemble(filepattern, prefix=prefix)
    save_cmip5_trends(ens, 'tauu', '1988-01-01', '2011-12-31', datapath=datapath)
Example #30
0
def cmip_average(files, var):
    """ Creates and stores a netCDF file with the average data
        across all the models provided.
        Returns the name of the created file.
    """
    ens = cd.mkensemble('', filenames=files)
    out = 'netcdf/' + var + '_' + 'cmip5.nc'
    # skip if the new file was already made
    if not os.path.isfile(out):
        ens = cd.remap(ens, output_prefix='netcdf/forens-')
        ensemble_means, _ = cd.ens_stats(ens, var, output_prefix='netcdf/')
        os.rename(ensemble_means[0], out)
    return out
def preprocess_models(datapath='./'):
    # where we are starting from
    cwd = os.getcwd()
    # move to where the data is
    os.chdir(datapath)

    # time-merge, remap and zonal mean the model data
    variables = ['uas', 'psl', 'tauu']
    for var in variables:
        # time-merge the data across the historical and rcp45 experiments
        filepattern = var + '_Amon*.nc'
        ens = cd.mkensemble(filepattern)
       
        # remap to a 1x1 grid
        ens_remap = cd.remap(ens,remap='r360x180', delete=True)

        # Compute zonal means
        ens_zonmean = cd.zonmean(ens_remap, delete=False)

    # move back
    os.chdir(cwd)
Example #32
0
def test_1dtimeseries(directory, sourcefiles, var, cwd):
    """ Tests plot_realizations_1d for a timeseries plot
    
    Parameters
    ----------
    directory : string
                the directory to link the files to
    sourcefiles : list
                  a list of the locations of the files to link
    var :  string
           the variable plotted
    cwd : string 
          directory of the test modules
    """  
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    data = cd.loadfiles(ens, var, cdostr='-fldmean -yearmean', toDatetime=True)
    fig, axt = plt.subplots(1, 1, figsize=(8, 8))
    cd.plot_realizations_1d(data, var, 'time', ax=axt, pdf='tsplot',
                        kwargs={'color': 'r', 'alpha': 0.3})
    plotcompare(directory, cwd, 'tsplot')
    os.system('rm  tsplot.pdf') 
Example #33
0
def test_ensembleenvelope(directory, sourcefiles, var, sd, ed, cwd):
    """ Tests ensemble_envelope_timeseries
    
    Parameters
    ----------
    directory : string
                the directory to link the files to
    sourcefiles : list
                  a list of the locations of the files to link
    var :  string
           the variable plotted
    sd : int
         start date for time slice
    ed : int
         end date for time slice
    cwd : string 
          directory of the test modules
    """
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    ens = cd.time_slice(ens, sd, ed)
    ens = cd.remap(ens, 'r360x180', delete=True)
    ens = cd.my_operator(
        ens,
        my_cdo_str='cdo -fldmean -yearmean {infile} {outfile}',
        output_prefix='processed_',
        delete=True)
    cd.ens_stats(ens, var)
    cd.ensemble_envelope_timeseries(
        ens,
        'ENS-MEAN_processed_remap_ts_Amon_CanESM2_historical_R-MEAN_195001-200012.nc',
        'ENS-STD_processed_remap_ts_Amon_CanESM2_historical_STD_195001-200012.nc',
        var,
        pdf='ensenv')
    plotcompare(directory, cwd, 'ensenv')
    os.system('rm  ensenv.pdf')
Example #34
0
def make_ensemble(files):
    ens = cd.mkensemble()
Example #35
0
def getcmipfiles(plots, expname, cmipdir):
    """ Loop through the plots and create the comparison files if cdo operations are needed 
        and map the keys in the compare dictionary to the correct file names.
    """
    # get the date ranges need for each variable
    startdates = min_start_dates(plots)
    enddates = max_end_dates(plots)
    cmip5_variables = {}
    for p in plots:
        p['model_files'] = {}
        p['model_file'] = {}
        p['cmip5_files'] = []
        p['cmip5_file'] = None
        if p['comp_models'] or p['comp_cmips']:
            # map the file names of the comparison files to the model names
            for model in p['comp_models'][:]:
                p['modelfiles_for_log'] = {}
                try:
                    if model not in p['model_files']:
                        p['model_files'][model], ens = model_files(
                            p['variable'], model, expname, p['frequency'],
                            cmipdir)
                    else:
                        ens = cd.mkensemble('',
                                            filenames=p['model_files'][model])
                    p['model_file'][model] = model_average(
                        ens, p['variable'], model)
                    p['modelfiles_for_log'][model] = p['model_files'][model]
                except:
                    with open('logs/log.txt', 'a') as outfile:
                        outfile.write('No cmip5 files were found for ' +
                                      p['variable'] + ': ' + model + '\n\n')
                    print 'No cmip5 files were found for ' + p[
                        'variable'] + ': ' + model
                    p['comp_models'].remove(model)
                    try:
                        p['comp_cmips'].remove(model)
                    except:
                        pass

            for model in p['comp_cmips'][:]:
                if model not in p['comp_models']:
                    try:
                        p['model_files'][model], ens = model_files(
                            p['variable'], model, expname, p['frequency'],
                            cmipdir)
                    except:
                        with open('logs/log.txt', 'a') as outfile:
                            outfile.write('No cmip5 files were found for ' +
                                          p['variable'] + ': ' + model +
                                          '\n\n')
                        print 'No cmip5 files were found for ' + p[
                            'variable'] + ': ' + model
                        # remove the model from the list if no comparison files were found
                        p['comp_cmips'].remove(model)

    for p in plots:
        if p['comp_cmips']:
            # map the file name of the comparison file to cmip5 in compare dictionary
            try:
                files = {}
                for f in p['comp_cmips']:
                    files[f] = p['model_files'][f]
                p['cmip5_files'] = cmip_files(files)


#                p['cmip5_file'] = cmip_average(p['variable'], p['frequency'], p['cmip5_files'], str(startdates[p['variable']]) + '-01', str(enddates[p['variable']]) + '-01', expname)
            except:
                p['comp_cmips'] = []

    get_cmip_average(plots, MEANDIR)
Example #36
0
def test_starting_files(directory, sourcefiles):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = '*.nc'
    ens = cd.mkensemble(filepattern)
    data = ctt.sha(ens)
    return data
Example #37
0
def test_starting_files(directory, sourcefiles):
    ctt.loadtestfiles(directory, sourcefiles)
    filepattern = "*.nc"
    ens = cd.mkensemble(filepattern)
    data = ctt.sha(ens)
    return data