def main(): #--- input user credential -- username='******' password='******' #---------------------------- #--- input search criteria -- project='ISIMIP2b' model='GFDL-ESM2M' impact_model='WaterGAP2' experiment ='rcp45' variable='Discharge' # Loging in lm = LogonManager() lm.logoff() lm.is_logged_on() OPENID = 'https://esgf-data.dkrz.de/esgf-idp/openid/'+username myproxy_host = 'esgf-data.dkrz.de' lm.logon_with_openid(openid=OPENID, password=password, bootstrap=True) lm.logon(hostname=myproxy_host,interactive=False, username=username,password=password,bootstrap=True) lm.is_logged_on() # Open connection with potsam node (ISIMIP) conn = SearchConnection('http://esg.pik-potsdam.de/esg-search', distrib=False) # Search datasets # can do general searches e.g., search for all datasets in ISIMIP2b with experiment='rcp45'. This will return all instances in ctx ctx = conn.new_context( project=project, model=model, impact_model=impact_model, experiment=experiment, variable_long_name=variable) # list number of counts print('Founds '+str(ctx.hit_count)+' matching datasets') # grab search results and display them a=ctx.search() cnt=1 for i in a : print('['+str(cnt)+']'+' -----> '+i.dataset_id) print('- - - - - - - - - - - - - - - - - - - - - - - -') cnt = cnt + 1 # Ask user to choose a dataset or to download all num = input("Which one should I download master? [Type -1 for all, 3 for third listed dataset.]") # Case where user select a specific dataset if num != -1: print("Downloading dataset "+str(num)+".") wget_makeNrun(a[num-1].file_context()) # case where user selects all elif num == -1: print("Downloading all "+str(ctx.hit_count)+" datasets.") for i in a: print("Downloading all datasets returned in search.") wget_makeNrun(i.file_context()) return 0
def logon(host=DKRZ_HOST): """logon to ESGF Host.""" from pyesgf.logon import LogonManager print("logon to: ".format(host)) lm = LogonManager() lm.logoff() lm.logon(hostname=host, interactive=True, bootstrap=True) print("logged on: {}".format(lm.is_logged_on())) return lm.is_logged_on()
def logon(username=None, password=None, hostname=None, interactive=False, outdir=None): """ Logon to MyProxy and fetch proxy certificate. """ outdir = outdir or os.curdir lm = LogonManager(esgf_dir=outdir, dap_config=os.path.join(outdir, 'dodsrc')) lm.logoff() # logon lm.logon(username=username, password=password, hostname=hostname, bootstrap=True, update_trustroots=False, interactive=interactive) return os.path.join(outdir, ESGF_CREDENTIALS)
def test_logoff(): lm = LogonManager(esgf_dir) # Only re-logon if credentials are not valid if not lm.is_logged_on(): lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY, bootstrap=True) assert lm.is_logged_on() lm.logoff() assert not op.exists(op.join(esgf_dir, 'credentials.pem')) assert not lm.is_logged_on() assert lm.state == lm.STATE_NO_CREDENTIALS
def myproxy_logon_with_openid(openid, password=None, interactive=False, outdir=None): """ Tries to get MyProxy parameters from OpenID and calls :meth:`logon`. :param openid: OpenID used to login at ESGF node. """ outdir = outdir or os.curdir username, hostname, port = parse_openid(openid) lm = LogonManager(esgf_dir=outdir, dap_config=os.path.join(outdir, 'dodsrc')) lm.logoff() lm.logon(username=username, password=password, hostname=hostname, bootstrap=True, update_trustroots=False, interactive=interactive) return os.path.join(outdir, ESGF_CREDENTIALS)
def logon(username=None, password=None, hostname=None, interactive=False, outdir=None): """ Logon to MyProxy and fetch proxy certificate. """ outdir = outdir or os.curdir # use myproxy patch # TODO: update to myproxyclient 2.x from phoenix.patch import patch_myproxy_client patch_myproxy_client() # end patch lm = LogonManager(esgf_dir=outdir, dap_config=os.path.join(outdir, 'dodsrc')) lm.logoff() lm.logon(username=username, password=password, hostname=hostname, bootstrap=True, update_trustroots=False, interactive=interactive) return os.path.join(outdir, ESGF_CREDENTIALS)
def download_ESGF_data(Open_ID, password, server, project, experiment, time_frequency, variable, domain, path_output): """Esta función nos permite descargar masivamente mediante WGET los diferentes ficheros netcdf que contienen los servidrores de ESGF sobre cambio climático. El servidor por defecto que se va a utilizar es: https://esgf-data.dkrz.de/projects/esgf-dkrz/. Parámetros: --------------------- Open_ID : string. ID de tu usuario para acceder a la base de datos correspondiente del servidor password : string. Contraseña correspondiente a la ID server : string. Servidor del que se desea descargar la información. Ejemplo: https://esgf-data.dkrz.de/esg-search project : string. Proyecto dentro del servidor del que se quiere descargar los datos. Ejemplo: CORDEX, CMIP5, CMIP6 experiment : string. Escenarios de cambio climático. Ejemplo: historical, rcp26, rcp45, rcp85 time_frequency : string. Frecuencia de la base de datos que se quiere. Ejemplo: 1hr, 6hr, day, mon variable : string. Variable que se desea descargar: tasmax, tasmin, pr domain : string. En el caso de que se desee descargar CORDEX, se debe de incluir el nombre de la malla. Ejemplo: EUR-11 path_output : string. Directorio donde se desean guardar los ficheros Salidas: ---------------------- Ficheros netcdf para cada uno de los escenarios y modelos solicitados """ dir_file = __file__ os.chdir(dir_file[:-16]) print(dir_file) conn = SearchConnection('https://' + server + '/esg-search', distrib=True) lm = LogonManager() lm.logoff() lm.is_logged_on() lm.logon_with_openid(Open_ID, password, bootstrap=True) lm.is_logged_on() if project == 'CORDEX': ctx = conn.new_context( project=project, experiment=experiment, time_frequency=time_frequency, variable=variable, domain=domain, ) else: ctx = conn.new_context( project=project, experiment=experiment, time_frequency=time_frequency, variable=variable, ) with open('wget-plantilla-ESGF.sh', "r+") as out_file: lines = out_file.readlines() for ct in tqdm.tqdm(range(ctx.hit_count)): files_list = list() result = ctx.search()[ct] lines[22] = "openId='" + Open_ID + "'\n" lines[ 23] = "search_url='https://" + server + "/esg-search/wget/?distrib=false&dataset_id=" + result.dataset_id + "'\n" lines_first = lines[:27] lines_end = lines[28:] files = result.file_context().search() ntcf_ds = list() ntcf_name = list() for file in files: try: if variable in file.opendap_url: files_list.append("'" + file.filename + "'" + ' ' + "'" + file.download_url + "'" + ' ' + "'" + file.checksum_type + "'" + ' ' + "'" + file.checksum + "'" + '\n') except: continue if len(files_list) == 0: continue else: with open(path_output + "Download.sh", "w") as fh: for line in (lines_first + files_list + lines_end): fh.write(line) conn = SearchConnection('https://' + server + '/esg-search', distrib=True) lm = LogonManager() lm.logoff() lm.is_logged_on() lm.logon_with_openid(Open_ID, password) lm.is_logged_on() os.chdir(path_output) os.system('bash ' + path_output + 'Download.sh' + ' H ' + Open_ID + ' ' + password)
"""Example for plotting all orographies with OoPlot by accessing the ESGF database. """ import cordex.plot as crxplt from pyesgf.search import SearchConnection from pyesgf.logon import LogonManager # logon to ESGF node print('logon to ESGF') lm = LogonManager() lm.logoff() lm.logon(hostname='esgf-data.dkrz.de', interactive=True, bootstrap=True) print('logged on: {}'.format(lm.is_logged_on())) def plot_orog(filename, output): """plots the orog variable to output file. """ var = 'orog' crxplt.contour2(filename, var, output) # search CORDEX project for REMO2015 fx orog variables conn = SearchConnection('http://esgf-data.dkrz.de/esg-search', distrib=False) ctx = conn.new_context(project='CORDEX', experiment='evaluation', time_frequency='fx', rcm_name='REMO2015', variable='orog') result = ctx.search()