return kwargs
# Calling parse_input() function to build kwargs from external arguments paased by user
kwargs=assign_constraints()
# open output file
outfile=kwargs.pop("output",["search_result.txt"])
fout=open(outfile[0],'w')
# if checksum_type has been passed add checksum to output
checksum=False
cks = kwargs.pop("checksum",["None"])
if cks[0] in ["md5","sha256"]:
checksum=True
cks_type=cks[0]
# open connection to local database and intiate SQLalchemy session
cmip5 = CMIP5.connect()
# get constraints combination
combs=combine_constraints(**kwargs)
# for each constraints combination
for constraints in combs:
db_results=[]
print(constraints)
# search on local DB, return instance_ids
outputs=cmip5.outputs(**constraints)
# loop through returned Instance objects
db_results=[v for o in outputs for v in o.versions if v.is_latest]
if db_results==[]:
db_results=[v for o in outputs for v in o.versions if v==o.latest()[0]]
# write result to output file
if db_results==[]:
def choose_models():
cmip5 = CMIP5.connect()
# Get a list of all models in the database
all_models = cmip5.models()
good_models = []
# Variables we care about
var_names = [
'ps', 'tas', 'huss', 'clt', 'uas', 'vas', 'pr', 'prsn', 'evspsbl',
'rsds', 'rlds', 'thetao', 'so', 'vo'
]
# Loop over models and find out which ones have all the data
for model_name in all_models:
keep = True
# Loop over variables
for var in var_names:
# Monthly averaged variables
if var in [
'ps', 'tas', 'huss', 'clt', 'uas', 'vas', 'pr', 'prsn',
'evspsbl', 'rsds', 'rlds'
]:
realm = 'Amon'
elif var in ['thetao', 'so', 'vo']:
realm = 'Omon'
else:
print 'Unknown variable'
# Find all output fields
outputs = cmip5.outputs(experiment='historical',
variable=var,
mip=realm,
model=model_name)
# Get file path
try:
dir = outputs[0].drstree_path()
except (IndexError):
dir = ''
# Special case
if var == 'clt' and model_name == 'bcc-csm1-1-m':
dir = '/g/data1/ua6/unofficial-ESG-replica/tmp/tree/esgf2.dkrz.de/thredds/fileServer/cmip5/output1/BCC/bcc-csm1-1-m/historical/mon/atmos/Amon/r1i1p1/v20120709/clt/'
if len(dir) == 0:
# Missing data
keep = False
print 'Skipping model ' + model_name + ' because missing ' + var
if keep:
# All the data was there
good_models.append(str(model_name))
# Alphabetise and print to screen
good_models = sorted(good_models, key=str.lower)
print 'Models that have all variables: '
for model_name in good_models:
print model_name
# Check if any of the remaining models have incomplete r1i1p1 ensemble
ensemble_problems = []
for model_name in good_models:
keep = True
for var in var_names:
# Call the function in cmip5_paths which assumes r1i1p1
dir = get_directory(model_name, 'historical', var)
if len(dir) == 0:
keep = False
if not keep:
ensemble_problems.append(model_name)
# Print to screen
if len(ensemble_problems) > 0:
print 'These models do not have a complete r1i1p1; check if there is another complete ensemble member:'
for model_name in ensemble_problems:
print model_name
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from __future__ import print_function
from ARCCSSive import CMIP5
from ARCCSSive.CMIP5.Model import Instance
# connect to the database
db=CMIP5.connect()
#search database instances
outputs=db.outputs(variable='tas',model='MIROC5',experiment='historical',mip='Amon',ensemble='r1i1p1')
# loop through result instance objects returned by search
for o in outputs:
model = o.model
print(str(model))
files = o.filenames()
print(files)
fpath = o.drstree_path()
print(str(fpath))
# loops through result version objects related to instance
for v in o.versions:
if v.is_latest: print("latest available version on ESGF as of ",str(v.checked_on))
def session(request, tmpdir_factory):
session = CMIP5.connect('sqlite:///:memory:')
dira = tmpdir_factory.mktemp('a')
dirb = tmpdir_factory.mktemp('b')
# Create some example entries
db = session.session
added_on=date.today()
inst1_id = add_instance_item(db,
variable = 'a',
mip = '6hrLev',
model = 'c',
experiment = 'd',
ensemble = 'e',
realm = 'realm')
v11_id = add_version_item(db,
instance_id = inst1_id,
path = dira.strpath,
is_latest = False,
checked_on = added_on,
to_update = False,
dataset_id = 'someid',
version = 'v20111201')
v12_id = add_version_item(db,
instance_id = inst1_id,
path = dira.strpath,
is_latest = False,
checked_on = added_on,
to_update = False,
dataset_id = 'someid',
version = 'v20120101')
v13_id = add_version_item(db,
instance_id = inst1_id,
path = dira.strpath,
is_latest = False,
checked_on = added_on,
to_update = False,
dataset_id = 'someid',
version = 'NA')
inst2_id = add_instance_item(db,
variable = 'f',
mip = 'cfMon',
model = 'c',
experiment = 'd',
ensemble = 'e',
realm = 'realm')
v21_id = add_version_item(db,
instance_id = inst2_id,
path = dirb.strpath,
is_latest = True,
checked_on = added_on,
to_update = False,
dataset_id = 'someid',
version = 'v20111201')
v22_id = add_version_item(db,
instance_id = inst2_id,
path = dirb.strpath,
is_latest = False,
checked_on = added_on,
to_update = False,
dataset_id = 'someid',
version = 'v20120101')
add_warning_item(db,
version_id = v11_id,
warning = 'Test warning for inst1 v20111201',
added_by = '*****@*****.**',
added_on = added_on)
add_warning_item(db,
version_id = v12_id,
warning = 'Test warning for inst1 v20120101',
added_by = '*****@*****.**',
added_on = added_on)
add_file_item(db,
version_id = v22_id,
filename = 'Somefilename',
md5 = 'Somemd5',
sha256 = 'Somesha256')
add_file_item(db,
version_id = v22_id,
filename = 'Anotherfilename',
md5 = 'Anothermd5',
sha256 = 'Anothersha256')
add_warning_item(db,
version_id = v21_id,
warning = 'Test warning for inst2 v20111201',
added_by = '*****@*****.**',
added_on = added_on)
inst = add_instance_item(db,
variable = 'tas',
mip = 'Amon',
model = 'ACCESS1-3',
experiment = 'rcp45',
ensemble = 'r1i1p1',
realm = 'realm')
vers = add_version_item(db,
instance_id = inst,
path = dirb.strpath,
is_latest = False,
checked_on = added_on,
to_update = False,
dataset_id = 'someid',
version = 'v20130507')
add_file_item(db,
version_id = vers,
filename = 'example.nc',
md5 = None,
sha256 = None)
db.commit()
# Close the session
def fin():
db.close()
request.addfinalizer(fin)
return session
def choose_models_future ():
# Get list of models that passed choose_models.py
all_models = build_model_list()
cmip5 = CMIP5.connect()
var_names = ['ps', 'tas', 'huss', 'clt', 'uas', 'vas', 'pr', 'prsn', 'evspsbl', 'rsds', 'rlds', 'thetao', 'so', 'vo']
# Loop over models and find out which ones have all the data
good_models = []
for model_name in all_models:
keep = True
# Loop over RCPs
for expt in ['rcp45', 'rcp85']:
# Loop over variables
for var in var_names:
if var in ['ps', 'tas', 'uas', 'vas','huss', 'clt', 'pr', 'prsn', 'evspsbl', 'rsds', 'rlds']:
realm = 'Amon'
elif var in ['thetao', 'so', 'vo']:
realm = 'Omon'
else:
print 'Unknown variable'
# Find all output fields
outputs=cmip5.outputs(experiment=expt, variable=var, mip=realm, model=model_name)
# Get file path
try:
dir = outputs[0].drstree_path()
except (IndexError):
dir = ''
if len(dir) == 0:
# Missing data
keep = False
print 'Skipping model ' + model_name + ' because missing ' + var + ' for experiment ' + expt
if keep:
# All the data was there
good_models.append(model_name)
# Print to screen
print 'Models that have everything we need for RCPs 4.5 and 8.5:'
for model_name in good_models:
print model_name
# Check if any of the remaining models have incomplete r1i1p1 ensemble
# Repeat the above procedure but specify r1i1p1
ensemble_problems = []
for model_name in good_models:
keep = True
for expt in ['rcp45', 'rcp85']:
for var in var_names:
if var in ['ps', 'tas', 'uas', 'vas', 'huss', 'clt', 'pr', 'prsn', 'evspsbl', 'rsds', 'rlds']:
realm = 'Amon'
elif var in ['thetao', 'so', 'vo']:
realm = 'Omon'
else:
print 'Unknown variable'
outputs = cmip5.outputs(experiment=expt, variable=var, mip=realm, model=model_name,ensemble='r1i1p1')
try:
dir = outputs[0].drstree_path()
except(IndexError):
dir = ''
if len(dir) == 0:
keep = False
if not keep:
ensemble_problems.append(model_name)
# Print to screen
if len(ensemble_problems) > 0:
print 'These models do not have a complete r1i1p1; check if there is another complete ensemble member: '
for model_name in ensemble_problems:
print model_name