def main():
#--- input user credential --
username='******'
password='******'
#----------------------------
#--- input search criteria --
project='ISIMIP2b'
model='GFDL-ESM2M'
impact_model='WaterGAP2'
experiment ='rcp45'
variable='Discharge'
# Loging in
lm = LogonManager()
lm.logoff()
lm.is_logged_on()
OPENID = 'https://esgf-data.dkrz.de/esgf-idp/openid/'+username
myproxy_host = 'esgf-data.dkrz.de'
lm.logon_with_openid(openid=OPENID, password=password, bootstrap=True)
lm.logon(hostname=myproxy_host,interactive=False, username=username,password=password,bootstrap=True)
lm.is_logged_on()
# Open connection with potsam node (ISIMIP)
conn = SearchConnection('http://esg.pik-potsdam.de/esg-search', distrib=False)
# Search datasets
# can do general searches e.g., search for all datasets in ISIMIP2b with experiment='rcp45'. This will return all instances in ctx
ctx = conn.new_context(
project=project,
model=model,
impact_model=impact_model,
experiment=experiment,
variable_long_name=variable)
# list number of counts
print('Founds '+str(ctx.hit_count)+' matching datasets')
# grab search results and display them
a=ctx.search()
cnt=1
for i in a :
print('['+str(cnt)+']'+' -----> '+i.dataset_id)
print('- - - - - - - - - - - - - - - - - - - - - - - -')
cnt = cnt + 1
# Ask user to choose a dataset or to download all
num = input("Which one should I download master? [Type -1 for all, 3 for third listed dataset.]")
# Case where user select a specific dataset
if num != -1:
print("Downloading dataset "+str(num)+".")
wget_makeNrun(a[num-1].file_context())
# case where user selects all
elif num == -1:
print("Downloading all "+str(ctx.hit_count)+" datasets.")
for i in a:
print("Downloading all datasets returned in search.")
wget_makeNrun(i.file_context())
return 0
def logon(host=DKRZ_HOST):
"""logon to ESGF Host."""
from pyesgf.logon import LogonManager
print("logon to: ".format(host))
lm = LogonManager()
lm.logoff()
lm.logon(hostname=host, interactive=True, bootstrap=True)
print("logged on: {}".format(lm.is_logged_on()))
return lm.is_logged_on()
def logon(username=None, password=None, hostname=None, interactive=False, outdir=None):
"""
Logon to MyProxy and fetch proxy certificate.
"""
outdir = outdir or os.curdir
lm = LogonManager(esgf_dir=outdir, dap_config=os.path.join(outdir, 'dodsrc'))
lm.logoff()
# logon
lm.logon(username=username, password=password, hostname=hostname,
bootstrap=True, update_trustroots=False, interactive=interactive)
return os.path.join(outdir, ESGF_CREDENTIALS)
def test_logoff():
lm = LogonManager(esgf_dir)
# Only re-logon if credentials are not valid
if not lm.is_logged_on():
lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY, bootstrap=True)
assert lm.is_logged_on()
lm.logoff()
assert not op.exists(op.join(esgf_dir, 'credentials.pem'))
assert not lm.is_logged_on()
assert lm.state == lm.STATE_NO_CREDENTIALS
def test_logoff():
lm = LogonManager(esgf_dir)
# Only re-logon if credentials are not valid
if not lm.is_logged_on():
lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY, bootstrap=True)
assert lm.is_logged_on()
lm.logoff()
assert not op.exists(op.join(esgf_dir, 'credentials.pem'))
assert not lm.is_logged_on()
assert lm.state == lm.STATE_NO_CREDENTIALS
def myproxy_logon_with_openid(openid, password=None, interactive=False, outdir=None):
"""
Tries to get MyProxy parameters from OpenID and calls :meth:`logon`.
:param openid: OpenID used to login at ESGF node.
"""
outdir = outdir or os.curdir
username, hostname, port = parse_openid(openid)
lm = LogonManager(esgf_dir=outdir, dap_config=os.path.join(outdir, 'dodsrc'))
lm.logoff()
lm.logon(username=username, password=password, hostname=hostname,
bootstrap=True, update_trustroots=False, interactive=interactive)
return os.path.join(outdir, ESGF_CREDENTIALS)
def myproxy_logon_with_openid(openid,
password=None,
interactive=False,
outdir=None):
"""
Tries to get MyProxy parameters from OpenID and calls :meth:`logon`.
:param openid: OpenID used to login at ESGF node.
"""
outdir = outdir or os.curdir
username, hostname, port = parse_openid(openid)
lm = LogonManager(esgf_dir=outdir,
dap_config=os.path.join(outdir, 'dodsrc'))
lm.logoff()
lm.logon(username=username,
password=password,
hostname=hostname,
bootstrap=True,
update_trustroots=False,
interactive=interactive)
return os.path.join(outdir, ESGF_CREDENTIALS)
def logon(username=None,
password=None,
hostname=None,
interactive=False,
outdir=None):
"""
Logon to MyProxy and fetch proxy certificate.
"""
outdir = outdir or os.curdir
# use myproxy patch
# TODO: update to myproxyclient 2.x
from phoenix.patch import patch_myproxy_client
patch_myproxy_client()
# end patch
lm = LogonManager(esgf_dir=outdir,
dap_config=os.path.join(outdir, 'dodsrc'))
lm.logoff()
lm.logon(username=username,
password=password,
hostname=hostname,
bootstrap=True,
update_trustroots=False,
interactive=interactive)
return os.path.join(outdir, ESGF_CREDENTIALS)
def download_ESGF_data(Open_ID, password, server, project, experiment,
time_frequency, variable, domain, path_output):
"""Esta función nos permite descargar masivamente mediante WGET los diferentes ficheros netcdf que contienen los servidrores de ESGF sobre cambio climático.
El servidor por defecto que se va a utilizar es: https://esgf-data.dkrz.de/projects/esgf-dkrz/.
Parámetros:
---------------------
Open_ID : string. ID de tu usuario para acceder a la base de datos correspondiente del servidor
password : string. Contraseña correspondiente a la ID
server : string. Servidor del que se desea descargar la información. Ejemplo: https://esgf-data.dkrz.de/esg-search
project : string. Proyecto dentro del servidor del que se quiere descargar los datos. Ejemplo: CORDEX, CMIP5, CMIP6
experiment : string. Escenarios de cambio climático. Ejemplo: historical, rcp26, rcp45, rcp85
time_frequency : string. Frecuencia de la base de datos que se quiere. Ejemplo: 1hr, 6hr, day, mon
variable : string. Variable que se desea descargar: tasmax, tasmin, pr
domain : string. En el caso de que se desee descargar CORDEX, se debe de incluir el nombre de la malla. Ejemplo: EUR-11
path_output : string. Directorio donde se desean guardar los ficheros
Salidas:
----------------------
Ficheros netcdf para cada uno de los escenarios y modelos solicitados
"""
dir_file = __file__
os.chdir(dir_file[:-16])
print(dir_file)
conn = SearchConnection('https://' + server + '/esg-search', distrib=True)
lm = LogonManager()
lm.logoff()
lm.is_logged_on()
lm.logon_with_openid(Open_ID, password, bootstrap=True)
lm.is_logged_on()
if project == 'CORDEX':
ctx = conn.new_context(
project=project,
experiment=experiment,
time_frequency=time_frequency,
variable=variable,
domain=domain,
)
else:
ctx = conn.new_context(
project=project,
experiment=experiment,
time_frequency=time_frequency,
variable=variable,
)
with open('wget-plantilla-ESGF.sh', "r+") as out_file:
lines = out_file.readlines()
for ct in tqdm.tqdm(range(ctx.hit_count)):
files_list = list()
result = ctx.search()[ct]
lines[22] = "openId='" + Open_ID + "'\n"
lines[
23] = "search_url='https://" + server + "/esg-search/wget/?distrib=false&dataset_id=" + result.dataset_id + "'\n"
lines_first = lines[:27]
lines_end = lines[28:]
files = result.file_context().search()
ntcf_ds = list()
ntcf_name = list()
for file in files:
try:
if variable in file.opendap_url:
files_list.append("'" + file.filename + "'" + ' ' + "'" +
file.download_url + "'" + ' ' + "'" +
file.checksum_type + "'" + ' ' + "'" +
file.checksum + "'" + '\n')
except:
continue
if len(files_list) == 0:
continue
else:
with open(path_output + "Download.sh", "w") as fh:
for line in (lines_first + files_list + lines_end):
fh.write(line)
conn = SearchConnection('https://' + server + '/esg-search',
distrib=True)
lm = LogonManager()
lm.logoff()
lm.is_logged_on()
lm.logon_with_openid(Open_ID, password)
lm.is_logged_on()
os.chdir(path_output)
os.system('bash ' + path_output + 'Download.sh' + ' H ' + Open_ID +
' ' + password)
"""Example for plotting all orographies with OoPlot by accessing the ESGF database.
"""
import cordex.plot as crxplt
from pyesgf.search import SearchConnection
from pyesgf.logon import LogonManager
# logon to ESGF node
print('logon to ESGF')
lm = LogonManager()
lm.logoff()
lm.logon(hostname='esgf-data.dkrz.de', interactive=True, bootstrap=True)
print('logged on: {}'.format(lm.is_logged_on()))
def plot_orog(filename, output):
"""plots the orog variable to output file.
"""
var = 'orog'
crxplt.contour2(filename, var, output)
# search CORDEX project for REMO2015 fx orog variables
conn = SearchConnection('http://esgf-data.dkrz.de/esg-search', distrib=False)
ctx = conn.new_context(project='CORDEX',
experiment='evaluation',
time_frequency='fx',
rcm_name='REMO2015',
variable='orog')
result = ctx.search()
