def get_inventory(context, verbose=False):
'''
Generates an inventory of CMIP6 data
:context: dict of search terms
:returns: dict repr of json data structure
Example
-------
context = {'experiment_id': 'historical', 'variable': ['siconc', 'sivol'], 'frequency': 'mon'}
table = get_inventory(context)
'''
conn = SearchConnection('https://esgf-data.dkrz.de/esg-search',
distrib=True)
ctx = conn.new_context(project='CMIP6',
experiment_id='historical',
variable=['siconc', 'sivol'],
frequency='mon')
if verbose:
print(f'# Hits: {ctx.hit_count}')
data = {} # initialize dict
if ctx.hit_count > 0:
data['ensembles'] = [r.json for r in ctx.search()]
return data
def node_info(request):
if request.method == 'POST':
''' For demo purposes this is loading a local file '''
try:
from xml.etree.ElementTree import parse
tree = parse('scripts/registration.xml')
root = tree.getroot()
name = json.loads(request.body)['node']
response = {}
for node in root:
if node.attrib['shortName'] == name:
response['org'] = node.attrib['organization']
response['namespace'] = node.attrib['namespace']
response['email'] = node.attrib['supportEmail']
response['ip'] = node.attrib['ip']
response['longName'] = node.attrib['longName']
response['version'] = node.attrib['version']
response['shortName'] = name
response['adminPeer'] = node.attrib['adminPeer']
response['hostname'] = node.attrib['hostname']
for child in list(node):
if child.tag[-len('AuthorizationService'):] == "AuthorizationService":
response['authService'] = child.attrib["endpoint"]
if child.tag[-len('GeoLocation'):] == "GeoLocation":
response['location'] = child.attrib["city"]
if child.tag[-len('Metrics'):] == "Metrics":
for gchild in list(child):
if gchild.tag[-len('DownloadedData'):] == "DownloadedData":
response['dataDownCount'] = gchild.attrib['count']
response['dataDownSize'] = gchild.attrib['size']
response['dataDownUsers'] = gchild.attrib['users']
if gchild.tag[-len('RegisteredUsers'):] == "RegisteredUsers":
response['registeredUsers'] = gchild.attrib['count']
from pyesgf.search import SearchConnection
print 'attempting to connect to ' + 'http://' + response['hostname'] + 'esg-search/'
conn = SearchConnection('http://' + response['hostname'] + '/esg-search/', distrib=True)
try:
conn.get_shard_list()
response['status'] = 'up'
except Exception as e:
print repr(e)
response['status'] = 'down'
return HttpResponse(json.dumps(response))
except Exception as e:
import traceback
print '1', e.__doc__
print '2', sys.exc_info()
print '3', sys.exc_info()[0]
print '4', sys.exc_info()[1]
print '5', traceback.tb_lineno(sys.exc_info()[2])
ex_type, ex, tb = sys.exc_info()
print '6', traceback.print_tb(tb)
return HttpResponse(status=500)
elif request.method == 'POST':
print "Unexpected POST request"
return HttpResponse(status=500)
def get_esgf_cordex_info(self):
res_dict = {}
models = {}
driving_models = {}
conn = SearchConnection('http://esgf-data.dkrz.de/esg-search',distrib=True)
ctx = conn.new_context(project='CORDEX',replica=False)
#print ctx.hit_count
domains = ctx.facet_counts['domain'].keys()
#print domains
for key in domains:
ctx2 = conn.new_context(project='CORDEX',domain=key,replica=False)
#print ctx2.hit_count
models[key] = ctx2.facet_counts['rcm_name'].keys()
#print models[key]
driving_models[key]={}
for thismodel in models[key]:
ctx3 = conn.new_context(project='CORDEX',domain=key,rcm_name=thismodel,replica=False)
#print key,thismodel
#print ctx3.hit_count
driving_models[key][thismodel] = ctx3.facet_counts['driving_model'].keys()
return driving_models
def get_esgf_cordex_info(self):
res_dict = {}
models = {}
driving_models = {}
conn = SearchConnection('http://esgf-data.dkrz.de/esg-search',
distrib=True)
ctx = conn.new_context(project='CORDEX', replica=False)
#print ctx.hit_count
domains = ctx.facet_counts['domain'].keys()
#print domains
for key in domains:
ctx2 = conn.new_context(project='CORDEX',
domain=key,
replica=False)
#print ctx2.hit_count
models[key] = ctx2.facet_counts['rcm_name'].keys()
#print models[key]
driving_models[key] = {}
for thismodel in models[key]:
ctx3 = conn.new_context(project='CORDEX',
domain=key,
rcm_name=thismodel,
replica=False)
#print key,thismodel
#print ctx3.hit_count
driving_models[key][thismodel] = ctx3.facet_counts[
'driving_model'].keys()
return driving_models
def __init__(self,
url='http://localhost:8081/esg-search',
distrib=False,
replica=False,
latest=True,
monitor=None):
# replica is boolean defining whether to return master records
# or replicas, or None to return both.
if replica is True:
self.replica = None # master + replica
else:
self.replica = False # only master
# latest: A boolean defining whether to return only latest versions
# or only non-latest versions, or None to return both.
if latest is True:
self.latest = True # only latest versions
else:
self.latest = None # all versions
self.monitor = monitor
from pyesgf.search import SearchConnection
self.conn = SearchConnection(url, distrib=distrib)
self.fields = 'id,instance_id,number_of_files,number_of_aggregations,size,url'
# local context has *all* local datasets
local_conn = SearchConnection(url, distrib=False)
self.local_ctx = local_conn.new_context(fields=self.fields,
replica=True,
latest=None)
def fetch_cmip5(experiment, variable, time_frequency, models, ensembles):
""" Download CMIP5 data for a specified set of facets
Uses pyesgf module to query the ESGF nodes RESTful API, and generate a
wget script which is then executed.
Input parameters defining facets can be strings or lists of strings.
"""
conn = SearchConnection('http://pcmdi9.llnl.gov/esg-search', distrib=True)
ctx = conn.new_context(project='CMIP5', experiment=experiment,
time_frequency=time_frequency, variable=variable,
model=models, ensemble=ensembles, latest=True,
download_emptypath='unknown')
a = ctx.get_download_script()
# write the download script out
with open('getc5.sh','w') as f:
f.write(a)
# run the download script
subprocess.Popen(['chmod', 'u+x', 'getc5.sh']).wait()
subprocess.Popen(['./getc5.sh']).wait()
# delete any empty files.
os.system('find ./*.nc -type f -size 0 -delete')
def get_esgf_data_and_nodes():
PCMDI_SERVICE = 'http://pcmdi9.llnl.gov/esg-search/search'
conn = SearchConnection(PCMDI_SERVICE,distrib=True)
ctx = conn.new_context(project='CMIP5',replica=False)
#ctx = conn.new_context()
#ctx = ctx.constrain(model='MPI-ESM-P',experiment='piControl',time_frequency='mon', data_node='aims3.llnl.gov')
my_result=[]
ctx = ctx.constrain(project='CMIP5',model='MPI-ESM-P',experiment='piControl', data_node='aims3.llnl.gov')
data_nodes = ctx.facet_counts['data_node'].keys()
models = ctx.facet_counts['model'].keys()
print "data nodes:", data_nodes
print "models: ", models
#constrain search to get to datasets
#ctx = ctx.constrain(model='MPI-ESM-P',experiment='piControl',time_frequency='mon', data_node='aims3.llnl.gov')
results = ctx.search()
print 'Hits:', ctx.hit_count
print 'Realms:', ctx.facet_counts['realm']
print 'Ensembles:', ctx.facet_counts['ensemble']
for i in range(0,ctx.hit_count):
file_ctx = results[i].file_context()
files = file_ctx.search()
size = files.batch_size
for j in range(0,len(files)):
my_result.append(files[j].file_id)
print "files:", size
return my_result
def __init__(
self,
url='http://localhost:8081/esg-search',
distrib=False,
replica=False,
latest=True,
monitor=None):
# replica is boolean defining whether to return master records
# or replicas, or None to return both.
if replica is True:
self.replica = None # master + replica
else:
self.replica = False # only master
# latest: A boolean defining whether to return only latest versions
# or only non-latest versions, or None to return both.
if latest is True:
self.latest = True # only latest versions
else:
self.latest = None # all versions
self.monitor = monitor
from pyesgf.search import SearchConnection
self.conn = SearchConnection(url, distrib=distrib)
self.fields = 'id,instance_id,number_of_files,number_of_aggregations,size,url'
# local context has *all* local datasets
local_conn = SearchConnection(url, distrib=False)
self.local_ctx = local_conn.new_context(fields=self.fields, replica=True, latest=None)
def fetch_cmip5(experiment, variable, time_frequency, models, ensembles):
""" Download CMIP5 data for a specified set of facets
Uses pyesgf module to query the ESGF nodes RESTful API, and generate a
wget script which is then executed.
Input parameters defining facets can be strings or lists of strings.
"""
conn = SearchConnection('http://pcmdi9.llnl.gov/esg-search', distrib=True)
ctx = conn.new_context(project='CMIP5',
experiment=experiment,
time_frequency=time_frequency,
variable=variable,
model=models,
ensemble=ensembles,
latest=True,
download_emptypath='unknown')
a = ctx.get_download_script()
# write the download script out
with open('getc5.sh', 'w') as f:
f.write(a)
# run the download script
subprocess.Popen(['chmod', 'u+x', 'getc5.sh']).wait()
subprocess.Popen(['./getc5.sh']).wait()
# delete any empty files.
os.system('find ./*.nc -type f -size 0 -delete')
def __init__(self, request, url=None):
self.request = request
self._parse_params()
if not url:
settings = self.request.registry.settings
url = settings.get('esgfsearch.url')
self.conn = SearchConnection(url, distrib=self.distrib)
def check_index(index_node, dataset_name, publish):
"""
Check whether a dataset is published to the Solr Index or not.
:param str index_node: The index node to check
:param str dataset_name: The dataset name
:param bool publish: True if check for existence, False otherwise
:returns: True iff the dataset was successfully published or unpublished from Solr, otherwise returns False
:rtype: *boolean*
"""
if publish:
hit_count_num = 1
else:
hit_count_num = 0
conn = SearchConnection('http://%s/esg-search' % index_node, distrib=False)
limit = 50
i = 0
while i < limit:
i += 1
ctx = conn.new_context(master_id=dataset_name, data_node=socket.gethostname())
if ctx.hit_count == hit_count_num:
return True
elif i < limit:
print '.',
time.sleep(10)
return False
def test_context_facets1():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5')
context2 = context.constrain(model="IPSL-CM5A-LR")
assert context2.facet_constraints['project'] == 'CMIP5'
assert context2.facet_constraints['model'] == 'IPSL-CM5A-LR'
def test_context_facets1():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5')
context2 = context.constrain(model="IPSL-CM5A-LR")
assert context2.facet_constraints['project'] == 'CMIP5'
assert context2.facet_constraints['model'] == 'IPSL-CM5A-LR'
def test_context_facets2(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5')
context2 = context.constrain(model="IPSL-CM5A-LR")
self.assertTrue(context2.facet_constraints['project'] == 'CMIP5')
self.assertTrue(context2.facet_constraints['model'] == 'IPSL-CM5A-LR')
def main():
#--- input user credential --
username='******'
password='******'
#----------------------------
#--- input search criteria --
project='ISIMIP2b'
model='GFDL-ESM2M'
impact_model='WaterGAP2'
experiment ='rcp45'
variable='Discharge'
# Loging in
lm = LogonManager()
lm.logoff()
lm.is_logged_on()
OPENID = 'https://esgf-data.dkrz.de/esgf-idp/openid/'+username
myproxy_host = 'esgf-data.dkrz.de'
lm.logon_with_openid(openid=OPENID, password=password, bootstrap=True)
lm.logon(hostname=myproxy_host,interactive=False, username=username,password=password,bootstrap=True)
lm.is_logged_on()
# Open connection with potsam node (ISIMIP)
conn = SearchConnection('http://esg.pik-potsdam.de/esg-search', distrib=False)
# Search datasets
# can do general searches e.g., search for all datasets in ISIMIP2b with experiment='rcp45'. This will return all instances in ctx
ctx = conn.new_context(
project=project,
model=model,
impact_model=impact_model,
experiment=experiment,
variable_long_name=variable)
# list number of counts
print('Founds '+str(ctx.hit_count)+' matching datasets')
# grab search results and display them
a=ctx.search()
cnt=1
for i in a :
print('['+str(cnt)+']'+' -----> '+i.dataset_id)
print('- - - - - - - - - - - - - - - - - - - - - - - -')
cnt = cnt + 1
# Ask user to choose a dataset or to download all
num = input("Which one should I download master? [Type -1 for all, 3 for third listed dataset.]")
# Case where user select a specific dataset
if num != -1:
print("Downloading dataset "+str(num)+".")
wget_makeNrun(a[num-1].file_context())
# case where user selects all
elif num == -1:
print("Downloading all "+str(ctx.hit_count)+" datasets.")
for i in a:
print("Downloading all datasets returned in search.")
wget_makeNrun(i.file_context())
return 0
def _search(cls, **search_kwargs):
conn = SearchConnection(cls.ESGF_SEARCH_URI)
ctx = conn.new_context(project=cls.ESGF_SEARCH_CCI_PROJ_NAME,
**search_kwargs)
results = ctx.search()
return results
def esgf_search(esgf_search_uri, esgf_search_proj_name, var_standard_name,
total_exp_hits, var_exp_hits):
conn = SearchConnection(esgf_search_uri)
ctx = conn.new_context(project=esgf_search_proj_name, )
# project search
results = ctx.search()
hits = results.context.hit_count
if hits >= total_exp_hits:
ret_val = 1
else:
return 0
# variable
if var_standard_name is not None:
ctx = conn.new_context(project=esgf_search_proj_name,
cf_standard_name=var_standard_name)
results = ctx.search()
hits = results.context.hit_count
if hits >= var_exp_hits:
ret_val = 1
else:
return 0
return ret_val
def test_context_facets2(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5')
context2 = context.constrain(model="IPSL-CM5A-LR")
assert context2.facet_constraints['project'] == 'CMIP5'
assert context2.facet_constraints['model'] == 'IPSL-CM5A-LR'
def test_constrain_freetext():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5', query='humidity')
assert context.freetext_constraint == 'humidity'
context = context.constrain(experiment='historical')
assert context.freetext_constraint == 'humidity'
def test_constrain_regression1():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5', model='IPSL-CM5A-LR')
assert 'experiment' not in context.facet_constraints
context2 = context.constrain(experiment='historical')
assert 'experiment' not in context.facet_constraints
def test_constrain_regression1(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5', model='IPSL-CM5A-LR')
self.assertTrue('experiment' not in context.facet_constraints)
context2 = context.constrain(experiment='historical')
self.assertTrue('experiment' in context2.facet_constraints)
def test_constrain_freetext(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5', query='humidity')
self.assertTrue(context.freetext_constraint == 'humidity')
context = context.constrain(experiment='historical')
self.assertTrue(context.freetext_constraint == 'humidity')
def test_context_facet_multivalue2():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5', model='IPSL-CM5A-MR')
assert context.facet_constraints.getall('model') == ['IPSL-CM5A-MR']
context2 = context.constrain(model=['IPSL-CM5A-MR', 'IPSL-CM5A-LR'])
assert sorted(context2.facet_constraints.getall('model')) == ['IPSL-CM5A-LR', 'IPSL-CM5A-MR']
def test_constrain_regression1(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5', model='IPSL-CM5A-LR')
assert 'experiment' not in context.facet_constraints
context2 = context.constrain(experiment='historical')
assert 'experiment' in context2.facet_constraints
def test_context_facet_options():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5', model='IPSL-CM5A-LR',
ensemble='r1i1p1', experiment='rcp60',
realm='seaIce')
assert context.get_facet_options().keys() == ['data_node', 'cf_standard_name', 'variable_long_name',
'cmor_table', 'time_frequency', 'variable']
def test_constrain_freetext(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5', query='humidity')
assert context.freetext_constraint == 'humidity'
context = context.constrain(experiment='historical')
assert context.freetext_constraint == 'humidity'
def test_constrain_regression1():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5', model='IPSL-CM5A-LR')
assert 'experiment' not in context.facet_constraints
context2 = context.constrain(experiment='historical')
assert 'experiment' not in context.facet_constraints
def test_download_script():
conn = SearchConnection(TEST_SERVICE, distrib=False)
ctx = conn.new_context(project='CMIP5', ensemble='r1i1p1', model='IPSL-CM5A-LR', realm='seaIce',
experiment='historicalGHG')
script = ctx.get_download_script()
assert '# ESG Federation download script' in script
assert '# Search URL: %s' % TEST_SERVICE in script
def test_constrain_freetext():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5', query='humidity')
assert context.freetext_constraint == 'humidity'
context = context.constrain(experiment='historical')
assert context.freetext_constraint == 'humidity'
def test_constrain():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5')
count1 = context.hit_count
context = context.constrain(model="IPSL-CM5A-LR")
count2 = context.hit_count
assert count1 > count2
def test_constrain(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5')
count1 = context.hit_count
context = context.constrain(model="IPSL-CM5A-LR")
count2 = context.hit_count
assert count1 > count2
def test_constrain(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5')
count1 = context.hit_count
context = context.constrain(model="IPSL-CM5A-LR")
count2 = context.hit_count
self.assertTrue(count1 > count2)
def test_facet_count():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5')
context2 = context.constrain(model="IPSL-CM5A-LR")
counts = context2.facet_counts
assert counts['model'].keys() == ['IPSL-CM5A-LR']
assert counts['project'].keys() == ['CMIP5']
def test_context_project_cmip6(self):
test_service = 'https://esgf-node.ipsl.upmc.fr/esg-search'
conn = SearchConnection(test_service)
context = conn.new_context(project='CMIP6')
self.assertEqual(context.hit_count, 1973)
context2 = context.constrain(realm="atmosChem")
self.assertEqual(context2.hit_count, 10)
def test_context_facet_multivalue2(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5', model='IPSL-CM5A-MR')
self.assertTrue(
context.facet_constraints.getall('model') == ['IPSL-CM5A-MR'])
context2 = context.constrain(model=['IPSL-CM5A-MR', 'IPSL-CM5A-LR'])
self.assertTrue(sorted(context2.facet_constraints.getall('model'))
== ['IPSL-CM5A-LR', 'IPSL-CM5A-MR'])
def test_context_facets_multivalue():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5')
context2 = context.constrain(model=['IPSL-CM5A-LR', 'IPSL-CM5A-MR'])
assert context2.hit_count > 0
assert context2.facet_constraints['project'] == 'CMIP5'
assert sorted(context2.facet_constraints.getall('model')) == ['IPSL-CM5A-LR', 'IPSL-CM5A-MR']
def test_facet_count(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5')
context2 = context.constrain(model="IPSL-CM5A-LR")
counts = context2.facet_counts
self.assertTrue(list(counts['model'].keys()) == ['IPSL-CM5A-LR'])
self.assertTrue(list(counts['project'].keys()) == ['CMIP5'])
def test_context_facet_multivalue2(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5', model='IPSL-CM5A-MR')
assert context.facet_constraints.getall('model') == ['IPSL-CM5A-MR']
context2 = context.constrain(model=['IPSL-CM5A-MR', 'IPSL-CM5A-LR'])
assert sorted(context2
.facet_constraints
.getall('model')) == ['IPSL-CM5A-LR', 'IPSL-CM5A-MR']
def test_facet_count():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5')
context2 = context.constrain(model="IPSL-CM5A-LR")
counts = context2.facet_counts
assert counts['model'].keys() == ['IPSL-CM5A-LR']
assert counts['project'].keys() == ['CMIP5']
def test_context_facet_multivalue2():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5', model='IPSL-CM5A-MR')
assert context.facet_constraints.getall('model') == ['IPSL-CM5A-MR']
context2 = context.constrain(model=['IPSL-CM5A-MR', 'IPSL-CM5A-LR'])
assert sorted(context2.facet_constraints.getall('model')) == [
'IPSL-CM5A-LR', 'IPSL-CM5A-MR'
]
def test_facet_count(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5')
context2 = context.constrain(model="IPSL-CM5A-LR")
counts = context2.facet_counts
assert list(counts['model'].keys()) == ['IPSL-CM5A-LR']
assert list(counts['project'].keys()) == ['CMIP5']
def test_constrain():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5')
count1 = context.hit_count
context = context.constrain(model="IPSL-CM5A-LR")
count2 = context.hit_count
assert count1 > count2
def test_context_facet_options(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5', model='IPSL-CM5A-LR',
ensemble='r1i1p1', experiment='rcp60',
realm='seaIce')
expected = sorted([u'access', u'index_node', u'data_node', u'format',
u'cf_standard_name', u'variable_long_name',
u'cmor_table', u'time_frequency', u'variable'])
self.assertTrue(sorted(context.get_facet_options().keys()) == expected)
def test_context_facet_multivalue3():
conn = SearchConnection(TEST_SERVICE)
ctx = conn.new_context(project='CMIP5', query='humidity', experiment='rcp45')
hits1 = ctx.hit_count
assert hits1 > 0
ctx2 = conn.new_context(project='CMIP5', query='humidity',
experiment=['rcp45','rcp85'])
hits2 = ctx2.hit_count
assert hits2 > hits1
def test_context_facet_options(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5', model='IPSL-CM5A-LR',
ensemble='r1i1p1', experiment='rcp60',
realm='seaIce')
expected = sorted([u'access', u'index_node', u'data_node', u'format',
u'cf_standard_name', u'variable_long_name',
u'cmor_table', u'time_frequency', u'variable'])
assert sorted(context.get_facet_options().keys()) == expected
def test_context_facets3():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5')
context2 = context.constrain(model="IPSL-CM5A-LR")
results = context2.search()
result = results[0]
assert result.json['project'] == ['CMIP5']
assert result.json['model'] == ['IPSL-CM5A-LR']
def test_context_facets3():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5')
context2 = context.constrain(model="IPSL-CM5A-LR")
results = context2.search()
result = results[0]
assert result.json['project'] == ['CMIP5']
assert result.json['model'] == ['IPSL-CM5A-LR']
def test_context_facets_multivalue():
conn = SearchConnection(TEST_SERVICE)
context = conn.new_context(project='CMIP5')
context2 = context.constrain(model=['IPSL-CM5A-LR', 'IPSL-CM5A-MR'])
assert context2.hit_count > 0
assert context2.facet_constraints['project'] == 'CMIP5'
assert sorted(context2.facet_constraints.getall('model')) == [
'IPSL-CM5A-LR', 'IPSL-CM5A-MR'
]
def test_context_facet_multivalue3(self):
conn = SearchConnection(self.test_service, cache=self.cache)
ctx = conn.new_context(project='CMIP5', query='humidity',
experiment='rcp45')
hits1 = ctx.hit_count
self.assertTrue(hits1 > 0)
ctx2 = conn.new_context(project='CMIP5', query='humidity',
experiment=['rcp45', 'rcp85'])
hits2 = ctx2.hit_count
self.assertTrue(hits2 > hits1)
def test_context_facets3(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5')
context2 = context.constrain(model="IPSL-CM5A-LR")
results = context2.search()
result = results[0]
self.assertTrue(result.json['project'] == ['CMIP5'])
self.assertTrue(result.json['model'] == ['IPSL-CM5A-LR'])
def esgfSearch(mip_era,
experiment,
variable,
frequency=None,
table=None,
model=None,
member=None,
latest=True):
"""
ctx = esgfSearch(mip_era, experiment, variable)
Perform an esgfSearch and return DatasetSearchContext.
Inputs:
mip_era (str): CMIP3, CMIP5, or CMIP6
experiment (str): experiment id to search for (e.g., 'historical')
variable (str): variable id to search for (e.g., 'tas,ta')
Optional arguments:
frequency (str): frequency to search for (e.g., mon)
table (str): table id to search for
Returns:
ctx: DatasetSearchContext
"""
# Import esgf-pyclient library
from pyesgf.search import SearchConnection
# Moved to function import
# Create search connection
conn = SearchConnection('https://esgf-node.llnl.gov/esg-search/',
distrib=True)
# search based on mip controlled vocabulary
if mip_era.upper() == 'CMIP6':
ctx = conn.new_context(variable=variable,
frequency=frequency,
experiment_id=experiment,
table_id=table,
source_id=model,
variant_label=member,
latest=latest,
facets='source_id')
else:
ctx = conn.new_context(project=mip_era.upper(),
variable=variable,
time_frequency=frequency,
experiment=experiment,
cmor_table=table,
model=model,
ensemble=member,
latest=latest,
facets='model')
return ctx
def test_context_facets3(self):
conn = SearchConnection(self.test_service, cache=self.cache)
context = conn.new_context(project='CMIP5')
context2 = context.constrain(model="IPSL-CM5A-LR")
results = context2.search()
result = results[0]
assert result.json['project'] == ['CMIP5']
assert result.json['model'] == ['IPSL-CM5A-LR']
def test_context_facet_multivalue3(self):
conn = SearchConnection(self.test_service, cache=self.cache)
ctx = conn.new_context(project='CMIP5', query='humidity',
experiment='rcp45')
hits1 = ctx.hit_count
assert hits1 > 0
ctx2 = conn.new_context(project='CMIP5', query='humidity',
experiment=['rcp45', 'rcp85'])
hits2 = ctx2.hit_count
assert hits2 > hits1
def test_distrib():
conn = SearchConnection(TEST_SERVICE, distrib=False)
context = conn.new_context(project='CMIP5')
count1 = context.hit_count
conn2 = SearchConnection(TEST_SERVICE, distrib=True)
context = conn2.new_context(project='CMIP5')
count2 = context.hit_count
assert count1 < count2
def test_temporal_search_CMIP5():
conn = SearchConnection('http://esgf-index1.ceda.ac.uk/esg-search',
distrib=False)
ctx1 = conn.new_context(project = "CMIP5", model = "HadGEM2-ES",
time_frequency = "mon", realm = "atmos", ensemble = "r1i1p1", latest = True)
ctx2 = conn.new_context(project = "CMIP5", model = "HadGEM2-ES",
time_frequency = "mon", realm = "atmos", ensemble = "r1i1p1", latest = True,
from_timestamp = "2100-12-30T23:23:59Z", to_timestamp = "2200-01-01T00:00:00Z")
assert ctx2.hit_count < ctx1.hit_count
def test_temporal_search_CORDEX(self):
conn = SearchConnection(self.test_service2, distrib=True)
ctx1 = conn.new_context(project='CORDEX',
from_timestamp="1990-01-01T12:00:00Z",
to_timestamp="2100-12-31T12:00:00Z")
ctx2 = conn.new_context(project='CORDEX',
from_timestamp="2011-01-01T12:00:00Z",
to_timestamp="2100-12-31T12:00:00Z")
assert ctx2.hit_count < ctx1.hit_count
def test_temporal_search_CORDEX():
conn = SearchConnection("http://esgf-data.dkrz.de/esg-search", distrib=True)
ctx1 = conn.new_context(project='CORDEX',
from_timestamp="1990-01-01T12:00:00Z",
to_timestamp="2100-12-31T12:00:00Z")
ctx2 = conn.new_context(project='CORDEX',
from_timestamp="2011-01-01T12:00:00Z",
to_timestamp="2100-12-31T12:00:00Z")
assert ctx2.hit_count < ctx1.hit_count