Пример #1
0
    return kwargs

# Calling parse_input() function to build kwargs from external arguments paased by user 
kwargs=assign_constraints()
# open output file
outfile=kwargs.pop("output",["search_result.txt"])
fout=open(outfile[0],'w')
# if checksum_type has been passed add checksum to output
checksum=False
cks = kwargs.pop("checksum",["None"])
if cks[0] in  ["md5","sha256"]:
    checksum=True
    cks_type=cks[0]

# open connection to local database and intiate SQLalchemy session 
cmip5 = CMIP5.connect()

# get constraints combination
combs=combine_constraints(**kwargs)
# for each constraints combination
for constraints in combs:
    db_results=[]
    print(constraints)
# search on local DB, return instance_ids
    outputs=cmip5.outputs(**constraints)
# loop through returned Instance objects
    db_results=[v for o in outputs for v in o.versions if v.is_latest]
    if db_results==[]:
        db_results=[v for o in outputs for v in o.versions if v==o.latest()[0]]
# write result to output file
    if db_results==[]:
Пример #2
0
def choose_models():

    cmip5 = CMIP5.connect()
    # Get a list of all models in the database
    all_models = cmip5.models()
    good_models = []

    # Variables we care about
    var_names = [
        'ps', 'tas', 'huss', 'clt', 'uas', 'vas', 'pr', 'prsn', 'evspsbl',
        'rsds', 'rlds', 'thetao', 'so', 'vo'
    ]

    # Loop over models and find out which ones have all the data
    for model_name in all_models:
        keep = True
        # Loop over variables
        for var in var_names:
            # Monthly averaged variables
            if var in [
                    'ps', 'tas', 'huss', 'clt', 'uas', 'vas', 'pr', 'prsn',
                    'evspsbl', 'rsds', 'rlds'
            ]:
                realm = 'Amon'
            elif var in ['thetao', 'so', 'vo']:
                realm = 'Omon'
            else:
                print 'Unknown variable'
            # Find all output fields
            outputs = cmip5.outputs(experiment='historical',
                                    variable=var,
                                    mip=realm,
                                    model=model_name)
            # Get file path
            try:
                dir = outputs[0].drstree_path()
            except (IndexError):
                dir = ''
            # Special case
            if var == 'clt' and model_name == 'bcc-csm1-1-m':
                dir = '/g/data1/ua6/unofficial-ESG-replica/tmp/tree/esgf2.dkrz.de/thredds/fileServer/cmip5/output1/BCC/bcc-csm1-1-m/historical/mon/atmos/Amon/r1i1p1/v20120709/clt/'
            if len(dir) == 0:
                # Missing data
                keep = False
                print 'Skipping model ' + model_name + ' because missing ' + var
        if keep:
            # All the data was there
            good_models.append(str(model_name))

    # Alphabetise and print to screen
    good_models = sorted(good_models, key=str.lower)
    print 'Models that have all variables: '
    for model_name in good_models:
        print model_name

    # Check if any of the remaining models have incomplete r1i1p1 ensemble
    ensemble_problems = []
    for model_name in good_models:
        keep = True
        for var in var_names:
            # Call the function in cmip5_paths which assumes r1i1p1
            dir = get_directory(model_name, 'historical', var)
            if len(dir) == 0:
                keep = False
        if not keep:
            ensemble_problems.append(model_name)

    # Print to screen
    if len(ensemble_problems) > 0:
        print 'These models do not have a complete r1i1p1; check if there is another complete ensemble member:'
        for model_name in ensemble_problems:
            print model_name
Пример #3
0
    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

from __future__ import print_function

from ARCCSSive import CMIP5
from ARCCSSive.CMIP5.Model import Instance 
# connect to the database
db=CMIP5.connect()
#search database instances
outputs=db.outputs(variable='tas',model='MIROC5',experiment='historical',mip='Amon',ensemble='r1i1p1')

# loop through result instance objects returned by search
for o in outputs:
    model = o.model
    print(str(model)) 
    files = o.filenames()
    print(files) 
    fpath = o.drstree_path()
    print(str(fpath))
# loops through result version objects related to instance
    for v in o.versions:
        if v.is_latest: print("latest available version on ESGF as of ",str(v.checked_on))
Пример #4
0
def session(request, tmpdir_factory):
    session = CMIP5.connect('sqlite:///:memory:')

    dira = tmpdir_factory.mktemp('a')
    dirb = tmpdir_factory.mktemp('b')

    # Create some example entries
    db = session.session
    added_on=date.today()
    inst1_id = add_instance_item(db,
        variable   = 'a',
        mip        = '6hrLev',
        model      = 'c',
        experiment = 'd',
        ensemble   = 'e',
        realm      = 'realm')
    v11_id = add_version_item(db,
        instance_id = inst1_id,
        path        = dira.strpath,
        is_latest   = False,
        checked_on  = added_on,
        to_update  = False,
        dataset_id  = 'someid',
        version     = 'v20111201')
    v12_id = add_version_item(db,
        instance_id = inst1_id,
        path        = dira.strpath,
        is_latest   = False,
        checked_on  = added_on,
        to_update  = False,
        dataset_id  = 'someid',
        version     = 'v20120101')
    v13_id = add_version_item(db,
        instance_id = inst1_id,
        path        = dira.strpath,
        is_latest   = False,
        checked_on  = added_on,
        to_update  = False,
        dataset_id  = 'someid',
        version     = 'NA')

    inst2_id = add_instance_item(db,
        variable   = 'f',
        mip        = 'cfMon',
        model      = 'c',
        experiment = 'd',
        ensemble   = 'e',
        realm      = 'realm')
    v21_id = add_version_item(db,
        instance_id = inst2_id,
        path        = dirb.strpath,
        is_latest   = True,
        checked_on  = added_on,
        to_update  = False,
        dataset_id  = 'someid',
        version     = 'v20111201')
    v22_id = add_version_item(db,
        instance_id = inst2_id,
        path        = dirb.strpath,
        is_latest   = False,
        checked_on  = added_on,
        to_update  = False,
        dataset_id  = 'someid',
        version     = 'v20120101')
    add_warning_item(db,
        version_id    = v11_id,
        warning    = 'Test warning for inst1 v20111201',
        added_by    = '*****@*****.**',
        added_on    = added_on)
    add_warning_item(db,
        version_id    = v12_id,
        warning    = 'Test warning for inst1 v20120101',
        added_by   = '*****@*****.**',
        added_on   = added_on)
    add_file_item(db,
        version_id    = v22_id,
        filename   = 'Somefilename',
        md5        = 'Somemd5',
        sha256     = 'Somesha256')
    add_file_item(db,
        version_id    = v22_id,
        filename   = 'Anotherfilename',
        md5        = 'Anothermd5',
        sha256     = 'Anothersha256')
    add_warning_item(db,
        version_id    = v21_id,
        warning    = 'Test warning for inst2 v20111201',
        added_by    = '*****@*****.**',
        added_on    = added_on)

    inst = add_instance_item(db,
        variable   = 'tas',
        mip        = 'Amon',
        model      = 'ACCESS1-3',
        experiment = 'rcp45',
        ensemble   = 'r1i1p1',
        realm      = 'realm')
    vers = add_version_item(db,
        instance_id = inst,
        path        = dirb.strpath,
        is_latest   = False,
        checked_on  = added_on,
        to_update   = False,
        dataset_id  = 'someid',
        version     = 'v20130507')
    add_file_item(db,
        version_id    = vers,
        filename   = 'example.nc',
        md5        = None,
        sha256     = None)
    db.commit()

    # Close the session
    def fin():
        db.close()
    request.addfinalizer(fin)

    return session
Пример #5
0
def choose_models_future ():

    # Get list of models that passed choose_models.py
    all_models = build_model_list()

    cmip5 = CMIP5.connect()

    var_names = ['ps', 'tas', 'huss', 'clt', 'uas', 'vas', 'pr', 'prsn', 'evspsbl', 'rsds', 'rlds', 'thetao', 'so', 'vo']

    # Loop over models and find out which ones have all the data
    good_models = []
    for model_name in all_models:
        keep = True
        # Loop over RCPs
        for expt in ['rcp45', 'rcp85']:
            # Loop over variables
            for var in var_names:               
                if var in ['ps', 'tas', 'uas', 'vas','huss', 'clt', 'pr', 'prsn', 'evspsbl', 'rsds', 'rlds']:
                    realm = 'Amon'
                elif var in ['thetao', 'so', 'vo']:
                    realm = 'Omon'
                else:
                    print 'Unknown variable'
                # Find all output fields
                outputs=cmip5.outputs(experiment=expt, variable=var, mip=realm, model=model_name)
                # Get file path
                try:
                    dir = outputs[0].drstree_path()
                except (IndexError):
                    dir = ''
                if len(dir) == 0:
                    # Missing data
                    keep = False
                    print 'Skipping model ' + model_name + ' because missing ' + var + ' for experiment ' + expt
        if keep:
            # All the data was there
            good_models.append(model_name)

    # Print to screen
    print 'Models that have everything we need for RCPs 4.5 and 8.5:'
    for model_name in good_models:
        print model_name

    # Check if any of the remaining models have incomplete r1i1p1 ensemble
    # Repeat the above procedure but specify r1i1p1
    ensemble_problems = []
    for model_name in good_models:
        keep = True
        for expt in ['rcp45', 'rcp85']:
            for var in var_names:
                if var in ['ps', 'tas', 'uas', 'vas', 'huss', 'clt', 'pr', 'prsn', 'evspsbl', 'rsds', 'rlds']:
                    realm = 'Amon'
                elif var in ['thetao', 'so', 'vo']:
                    realm = 'Omon'
                else:
                    print 'Unknown variable'
                outputs = cmip5.outputs(experiment=expt, variable=var, mip=realm, model=model_name,ensemble='r1i1p1')
                try:
                    dir = outputs[0].drstree_path()
                except(IndexError):
                    dir = ''
                if len(dir) == 0:
                    keep = False
        if not keep:
            ensemble_problems.append(model_name)

    # Print to screen
    if len(ensemble_problems) > 0:
        print 'These models do not have a complete r1i1p1; check if there is another complete ensemble member: '
        for model_name in ensemble_problems:
            print model_name