def main(): #--- input user credential -- username='******' password='******' #---------------------------- #--- input search criteria -- project='ISIMIP2b' model='GFDL-ESM2M' impact_model='WaterGAP2' experiment ='rcp45' variable='Discharge' # Loging in lm = LogonManager() lm.logoff() lm.is_logged_on() OPENID = 'https://esgf-data.dkrz.de/esgf-idp/openid/'+username myproxy_host = 'esgf-data.dkrz.de' lm.logon_with_openid(openid=OPENID, password=password, bootstrap=True) lm.logon(hostname=myproxy_host,interactive=False, username=username,password=password,bootstrap=True) lm.is_logged_on() # Open connection with potsam node (ISIMIP) conn = SearchConnection('http://esg.pik-potsdam.de/esg-search', distrib=False) # Search datasets # can do general searches e.g., search for all datasets in ISIMIP2b with experiment='rcp45'. This will return all instances in ctx ctx = conn.new_context( project=project, model=model, impact_model=impact_model, experiment=experiment, variable_long_name=variable) # list number of counts print('Founds '+str(ctx.hit_count)+' matching datasets') # grab search results and display them a=ctx.search() cnt=1 for i in a : print('['+str(cnt)+']'+' -----> '+i.dataset_id) print('- - - - - - - - - - - - - - - - - - - - - - - -') cnt = cnt + 1 # Ask user to choose a dataset or to download all num = input("Which one should I download master? [Type -1 for all, 3 for third listed dataset.]") # Case where user select a specific dataset if num != -1: print("Downloading dataset "+str(num)+".") wget_makeNrun(a[num-1].file_context()) # case where user selects all elif num == -1: print("Downloading all "+str(ctx.hit_count)+" datasets.") for i in a: print("Downloading all datasets returned in search.") wget_makeNrun(i.file_context()) return 0
def logon(host=DKRZ_HOST): """logon to ESGF Host.""" from pyesgf.logon import LogonManager print("logon to: ".format(host)) lm = LogonManager() lm.logoff() lm.logon(hostname=host, interactive=True, bootstrap=True) print("logged on: {}".format(lm.is_logged_on())) return lm.is_logged_on()
def auth(self): ''' Authenticate with the auth server specified on object creation. ''' # Check that we're logged on lm = LogonManager() log.debug('Logon manager started') if not lm.is_logged_on(): log.debug(self.username, self.password, self.auth_server) lm.logon(self.username, self.password, self.auth_server) if not lm.is_logged_on(): raise Exception('NOAUTH')
def test_logoff(): lm = LogonManager(esgf_dir) # Only re-logon if credentials are not valid if not lm.is_logged_on(): lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY, bootstrap=True) assert lm.is_logged_on() lm.logoff() assert not op.exists(op.join(esgf_dir, 'credentials.pem')) assert not lm.is_logged_on() assert lm.state == lm.STATE_NO_CREDENTIALS
def test_logon(): _clear_creds() _load_creds(certificates_tarball='pcmdi9-certs.tar.gz') lm = LogonManager(esgf_dir) lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY) assert lm.is_logged_on()
def check_credentials(request): if request.method == 'POST': try: data = json.loads(request.body) response = {} creds = Credential.objects.filter(site_user_name=request.user) if len(creds) != 0: for c in creds: try: if c.service == 'esgf': import pyesgf from pyesgf.logon import LogonManager lm = LogonManager() lm.logon_with_openid( c.service_user_name, c.password) if lm.is_logged_on(): response[s] = 'success' print 'esgf log in successful' else: print 'esgf log in failed' response[s] = 'fail' if c.service == 'velo': user, password = c.service_user_name, c.password velo_creds = { "velo_user": user, "velo_pass": password, "command": "init" } result = velo_request(velo_creds) # TODO: Extract values out to CAPITAL_NAMED_CONSTANTS if result == "Success": print "velo login successful" else: print "velo login failed" response[s] = "fail" if c.service == 'github': import github3 from github3 import login gh = login(c.site_user_name, password=c.password) if gh.user() == c.site_user_name: print 'Github login successful' response[s] = 'success' else: print 'Github login failure' response[s] = 'fail' if c.service == 'jira': print 'Working on jira....' except: print_debug(c) return HttpResponse(status=500) return HttpResponse(json.dumps(response)) except Exception as e: print_debug(e) return HttpResponse(status=500) else: return HttpResponse(status=404)
def test_logon_openid(): _clear_creds() _load_creds(certificates_tarball='pcmdi9-certs.tar.gz') lm = LogonManager(esgf_dir) lm.logon_with_openid(TEST_OPENID, TEST_PASSWORD, interactive=False) assert lm.is_logged_on()
def logon(request): credential = { 'username': request.GET.get('username'), 'password': request.GET.get('password') } if not credential['username']: print "[-] No username in logon request" return HttpResponse(status=403) elif not credential['password']: print "[-] No password in logon request" return HttpResponse(status=403) lm = LogonManager() bootstrap = False if not os.path.exists(ESGF_CREDENTIALS): bootstrap = True try: lm.logon_with_openid(credential['username'], credential['password'], bootstrap=bootstrap) except Exception as e: print_message('Unable to log in user {}'.format(credential.get('username'))) return HttpResponse(status=403) if lm.is_logged_on(): return HttpResponse(status=200) else: return HttpResponse(status=403)
def test_logon(self, extra_args=None): _clear_creds(self.esgf_dir) _load_creds(self.esgf_dir, certificates_tarball='pcmdi9-certs.tar.gz') if not extra_args: extra_args = {} lm = LogonManager(self.esgf_dir, **extra_args) lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY) assert lm.is_logged_on()
def test_logon_openid(self): _clear_creds(self.esgf_dir) _load_creds(self.esgf_dir, certificates_tarball='pcmdi9-certs.tar.gz') lm = LogonManager(self.esgf_dir) # NOTE: for many users the OpenID lookup might not provide the username # in which case this test will fail because it needs interactive # prompting for a username. lm.logon_with_openid(TEST_OPENID, TEST_PASSWORD, interactive=False) assert lm.is_logged_on()
def check_credentials(request): if request.method == 'POST': try: data = json.loads(request.body) response = {} for s in data: if s == 'esgf': import pyesgf from pyesgf.logon import LogonManager lm = LogonManager() lm.logon_with_openid(data[s]['username'], data[s]['password']) if lm.is_logged_on() != True: response[s] = 'failed' else: response[s] = 'success' if s == 'velo': lib_path = os.path.abspath(os.path.join('apps', 'velo')) sys.path.append(lib_path) import VeloAPI velo_api = VeloAPI.Velo() velo_api.start_jvm() ''' Using the test credentials for the time being, simply uncomment to use the users credentials velo_api.init_velo(data[s]['username'], data[s]['password']) ''' # res = velo_api.init_velo("acmetest", "acmetest") response[s] = 'success' ''' if res.logged_on() != True: resonse[s] = 'failed' else: response[s] = 'success ''' return HttpResponse(json.dumps(response)) except Exception as e: import traceback print '1', e.__doc__ print '2', sys.exc_info() print '3', sys.exc_info()[0] print '4', sys.exc_info()[1] print '5', traceback.tb_lineno(sys.exc_info()[2]) ex_type, ex, tb = sys.exc_info() print '6', traceback.print_tb(tb) return HttpResponse(status=500) else: return HttpResponse(status=404)
def logon(openid, password): ''' Function to retrieve a short-term X.509 certificate that can be used to authenticate with ESGF. The certificate is written in the location ~/.esg/credentials.pem. The trusted CA certificates are written in the directory ~/.esg/certificates. ''' # Must configure the DN of the JPL MyProxy server if using a JPL openid if "esg-datanode.jpl.nasa.gov" in openid: os.environ['MYPROXY_SERVER_DN'] = JPL_MYPROXY_SERVER_DN lm = LogonManager() lm.logon_with_openid(openid,password) return lm.is_logged_on()
def logon(openid, password): """ Function to retrieve a short-term X.509 certificate that can be used to authenticate with ESGF. The certificate is written in the location ~/.esg/credentials.pem. The trusted CA certificates are written in the directory ~/.esg/certificates. """ # Must configure the DN of the JPL MyProxy server if using a JPL openid if JPL_HOSTNAME in openid: os.environ['MYPROXY_SERVER_DN'] = JPL_MYPROXY_SERVER_DN logon_manager = LogonManager() logon_manager.logon_with_openid(openid, password, bootstrap=True) return logon_manager.is_logged_on()
def download(request): credential = { 'username': '******', 'password': '******' } lm = LogonManager() bootstrap = False if not os.path.exists(ESGF_CREDENTIALS): bootstrap = True try: print "logging in" lm.logon_with_openid(credential['username'], credential['password'], bootstrap=bootstrap) except Exception as e: # print_debug(e) return HttpResponse(status=403) if lm.is_logged_on(): print "...login success" try: if not os.path.exists('/tmp'): os.mkdir('/tmp') url = request.GET.get('url') # esgf_download(request.GET.get('url')) print "[+] Downloading " + url # response = urllib2.urlopen(url) print "opening connection to server" response = requests.get(url, verify='/Users/sterling/.esg/credentials.pem',stream=True) response.raw.decode_content = True with open("someClimateFile.nc", 'wb') as f: shutil.copyfileobj(response.raw, f) print "...download success" return HttpResponse(status=200) except Exception as e: # print_debug(e) return HttpResponse(status=400) else: return HttpResponse(status=403) return HttpResponse(status=200)
def check_credentials(request): if request.method == 'POST': try: data = json.loads(request.body) response = {} creds = Credential.objects.filter(site_user_name=request.user) velo_started = False if len(creds) != 0: for c in creds: try: if c.service == 'esgf': import pyesgf from pyesgf.logon import LogonManager lm = LogonManager() lm.logon_with_openid( c.service_user_name, c.password) if lm.is_logged_on(): response[s] = 'success' print 'esgf log in successful' else: print 'esgf log in failed' response[s] = 'fail' if c.service == 'velo': lib_path = os.path.abspath( os.path.join('apps', 'velo')) sys.path.append(lib_path) ''' For production, replace below with: rm = velo_api.init(c.service_user_name, c.password) ''' rm = velo_api.init('acmetest', 'acmetest') if rm.getRepositoryUrlBase() == 'u\'http://acmetest.ornl.gov:80/alfresco\'': response[s] = 'success' velo_api.shutdown_jvm() print 'velo log in successful' else: velo_api.shutdown_jvm() response[s] = 'fail' print 'Error in velo initialization', rm.getRepositoryUrlBase() if c.service == 'github': import github3 from github3 import login gh = login(c.site_user_name, password=c.password) if gh.user() == c.site_user_name: print 'Github login successful' response[s] = 'success' else: print 'Github login failure' response[s] = 'fail' if c.service == 'jira': print 'Working on jira....' except: if velo_started: velo_api.shutdown_jvm() import traceback print '1', e.__doc__ print '2', sys.exc_info() print '3', sys.exc_info()[0] print '4', sys.exc_info()[1] print '5', traceback.tb_lineno(sys.exc_info()[2]) ex_type, ex, tb = sys.exc_info() print '6', traceback.print_tb(tb) return HttpResponse(status=500) return HttpResponse(json.dumps(response)) except Exception as e: import traceback print '1', e.__doc__ print '2', sys.exc_info() print '3', sys.exc_info()[0] print '4', sys.exc_info()[1] print '5', traceback.tb_lineno(sys.exc_info()[2]) ex_type, ex, tb = sys.exc_info() print '6', traceback.print_tb(tb) return HttpResponse(status=500) else: return HttpResponse(status=404)
'tasmin', 'tasmax', 'uas', 'vas', 'tas', 'pr', 'ps', 'rsds', 'rlds'] # ,'sfcWind' expers = ['rcp26', 'historical', 'rcp85'] # logon manager lm = LogonManager() if not lm.is_logged_on(): lm.logoff() lm.is_logged_on() lm.logon_with_openid(openid=openid, password=None, bootstrap=True) lm.is_logged_on() lm.logon(hostname='esgf-data.dkrz.de', interactive=True, bootstrap=True) lm.is_logged_on() conn = SearchConnection('https://esgf-data.dkrz.de/esg-search', distrib=True) for exper in expers: logging.info("Experimet: " + exper)
def test_bootstrap(): _clear_creds() lm = LogonManager(esgf_dir) lm.logon(TEST_USER, TEST_PASSWORD, TEST_MYPROXY, bootstrap=True) assert lm.is_logged_on()
def test_no_logon(self): _clear_creds(self.esgf_dir) lm = LogonManager(self.esgf_dir) assert not lm.is_logged_on()
def test_no_logon(): _clear_creds() lm = LogonManager(esgf_dir) assert lm.is_logged_on() == False
def grid(request): ''' For demo purposes this is loading a local file ''' from xml.etree.ElementTree import parse import requests from StringIO import StringIO try: r = requests.get('http://pcmdi9.llnl.gov/esgf-node-manager/registration.xml') f = StringIO(r.content) out = open('scripts/registration.xml', 'w') out.write(f.read()) out.close() except Exception as e: print repr(e) return HttpResponse(status=404) tree = parse('scripts/registration.xml') node_name_list = [] node_peer_list = [] node_url_list = [] node_location_list = [] for node in tree.getroot(): attrs = node.attrib node_name_list.append(attrs["shortName"]) node_peer_list.append(attrs["adminPeer"]) node_url_list.append(attrs["hostname"]) for child in node: if child.tag[-11:] == "GeoLocation": node_location_list.append(child.attrib["city"]) node_list = zip(node_peer_list, node_url_list, node_name_list, node_location_list) creds = Credential.objects.filter(site_user_name=request.user) if len(creds) != 0: for c in creds: try: if c.service == 'esgf': import pyesgf from pyesgf import LogonManager lm = LogonManager() lm.logon_with_openid(c.service_user_name, c.password) if lm.is_logged_on(): request.session['esgf_login'] = lm print 'esgf log in successful' if c.service == 'velo': lib_path = os.path.abspath(os.path.join('apps', 'velo')) sys.path.append(lib_path) import VeloAPI velo_api = VeloAPI.Velo() velo_api.start_jvm() res = velo_api.init('acmetest', 'acmetest') request.session['velo_login'] = res print 'velo log in successful' ''' For production, uncomment res = velo_api.init(c.service_user_name, c.password) ''' except: import traceback print '1', e.__doc__ print '2', sys.exc_info() print '3', sys.exc_info()[0] print '4', sys.exc_info()[1] print '5', traceback.tb_lineno(sys.exc_info()[2]) ex_type, ex, tb = sys.exc_info() print '6', traceback.print_tb(tb) return HttpResponse(status=500) return HttpResponse(render_template(request, "web_fe/grid.html", {'nodes': node_list}))
"""Example for plotting all orographies with OoPlot by accessing the ESGF database. """ import cordex.plot as crxplt from pyesgf.search import SearchConnection from pyesgf.logon import LogonManager # logon to ESGF node print('logon to ESGF') lm = LogonManager() lm.logoff() lm.logon(hostname='esgf-data.dkrz.de', interactive=True, bootstrap=True) print('logged on: {}'.format(lm.is_logged_on())) def plot_orog(filename, output): """plots the orog variable to output file. """ var = 'orog' crxplt.contour2(filename, var, output) # search CORDEX project for REMO2015 fx orog variables conn = SearchConnection('http://esgf-data.dkrz.de/esg-search', distrib=False) ctx = conn.new_context(project='CORDEX', experiment='evaluation', time_frequency='fx', rcm_name='REMO2015', variable='orog') result = ctx.search()
def dataset_download(message, data, user): print 'got a dataset_download request' username = data.get('params').get('openid_username') password = data.get('params').get('openid_password') search_string = data.get('params').get('search_string') nodes = data.get('params').get('nodes') data_type = data.get('params').get('data_type') data_name = data.get('params').get('data_name').replace(' ', '_') if not username: print_message('No username given') return -1 if not password: print_message('No password given') return -1 if not search_string: print_message('No search_string given') return -1 if not nodes: print_message('No nodes given') return -1 if not data_type: print_message('No data_type given') return -1 if not data_name: print_message('No data_name given') return -1 lm = LogonManager() lm.logon_with_openid(username, password, bootstrap=True) if not lm.is_logged_on(): print_message('User {user} is not logged in during download request'.format(user=user)) return -1 for node in nodes: try: path = os.path.abspath(os.path.dirname(__file__)) + '/../../' print '[+] searching {node} for {string}'.format(node=node, string=search_string) conn_string = 'http://{node}{suffix}'.format(node=node, suffix=ESGF_SEARCH_SUFFIX) conn = SearchConnection(conn_string, distrib=True) context = conn.new_context(**search_string) rs = context.search() print_message('got reply from {node}'.format(node=node)) if len(rs) == 0: continue script_text = context.get_download_script() script_path = '' if data_type == 'observation': script_path = path + 'userdata/' + user + '/observations/' + data_name elif data_type == 'model': script_path = path + 'userdata/' + user + '/model_output/' + data_name script_name = '{path}/{name}_download_script.sh'.format(path=script_path, name=data_name) if not os.path.exists(script_path): print_message('creating directory {}'.format(script_path)) os.makedirs(script_path) try: with open(script_name, 'w') as script: script.write(script_text) script.close() except Exception, e: print_debug(e) word_count_cmd = 'cat {script_name} | grep \\.nc.*http:// | wc -l'.format(path=script_path, script_name=script_name) print_message('running command {}'.format(word_count_cmd)) p = subprocess.check_output(word_count_cmd, shell=True) number_of_downloads = int(p) percent_complete = 0 number_complete = 0.0 print_message('Number of .nc files to download: {}'.format(number_of_downloads)) try: subprocess.call(['chmod', '+x', script_name]) p = subprocess.Popen('exec {name} > output_status.txt 2>&1'.format(name=script_name), shell=True, cwd=script_path, preexec_fn=os.setsid) should_break = False for line in tail('-f', '{path}/output_status.txt'.format(path=script_path), _iter=True): update_message = None if 'ERROR' in line: os.killpg(os.getpgid(p.pid), signal.SIGTERM) print_message('error downloading file') update_message = { 'text': json.dumps({ 'user': user, 'data_name': data_name, 'percent_complete': percent_complete, 'message': line, 'destination': 'esgf_download_status' }) } if 'saved' in line or 'Saving to' in line: number_complete += 1 percent_complete = number_complete / number_of_downloads * 100 print_message('percent complete: {}%'.format(percent_complete), 'ok') update_message = { 'text': json.dumps({ 'user': user, 'data_name': data_name, 'percent_complete': percent_complete, 'message': 'downloading', 'destination': 'esgf_download_status' }) } if percent_complete > 99: print_message('Download complete', 'ok') update_message = { 'text': json.dumps({ 'user': user, 'data_name': data_name, 'percent_complete': 100.0, 'message': 'complete', 'destination': 'esgf_download_status' }) } should_break = True if update_message: print_message("sending to active group {}".format(update_message)) Group('active').send(update_message) if should_break: break out, err = p.communicate() print out, err except Exception, e: print_debug(e) return -1 break
def download_ESGF_data(Open_ID, password, server, project, experiment, time_frequency, variable, domain, path_output): """Esta función nos permite descargar masivamente mediante WGET los diferentes ficheros netcdf que contienen los servidrores de ESGF sobre cambio climático. El servidor por defecto que se va a utilizar es: https://esgf-data.dkrz.de/projects/esgf-dkrz/. Parámetros: --------------------- Open_ID : string. ID de tu usuario para acceder a la base de datos correspondiente del servidor password : string. Contraseña correspondiente a la ID server : string. Servidor del que se desea descargar la información. Ejemplo: https://esgf-data.dkrz.de/esg-search project : string. Proyecto dentro del servidor del que se quiere descargar los datos. Ejemplo: CORDEX, CMIP5, CMIP6 experiment : string. Escenarios de cambio climático. Ejemplo: historical, rcp26, rcp45, rcp85 time_frequency : string. Frecuencia de la base de datos que se quiere. Ejemplo: 1hr, 6hr, day, mon variable : string. Variable que se desea descargar: tasmax, tasmin, pr domain : string. En el caso de que se desee descargar CORDEX, se debe de incluir el nombre de la malla. Ejemplo: EUR-11 path_output : string. Directorio donde se desean guardar los ficheros Salidas: ---------------------- Ficheros netcdf para cada uno de los escenarios y modelos solicitados """ dir_file = __file__ os.chdir(dir_file[:-16]) print(dir_file) conn = SearchConnection('https://' + server + '/esg-search', distrib=True) lm = LogonManager() lm.logoff() lm.is_logged_on() lm.logon_with_openid(Open_ID, password, bootstrap=True) lm.is_logged_on() if project == 'CORDEX': ctx = conn.new_context( project=project, experiment=experiment, time_frequency=time_frequency, variable=variable, domain=domain, ) else: ctx = conn.new_context( project=project, experiment=experiment, time_frequency=time_frequency, variable=variable, ) with open('wget-plantilla-ESGF.sh', "r+") as out_file: lines = out_file.readlines() for ct in tqdm.tqdm(range(ctx.hit_count)): files_list = list() result = ctx.search()[ct] lines[22] = "openId='" + Open_ID + "'\n" lines[ 23] = "search_url='https://" + server + "/esg-search/wget/?distrib=false&dataset_id=" + result.dataset_id + "'\n" lines_first = lines[:27] lines_end = lines[28:] files = result.file_context().search() ntcf_ds = list() ntcf_name = list() for file in files: try: if variable in file.opendap_url: files_list.append("'" + file.filename + "'" + ' ' + "'" + file.download_url + "'" + ' ' + "'" + file.checksum_type + "'" + ' ' + "'" + file.checksum + "'" + '\n') except: continue if len(files_list) == 0: continue else: with open(path_output + "Download.sh", "w") as fh: for line in (lines_first + files_list + lines_end): fh.write(line) conn = SearchConnection('https://' + server + '/esg-search', distrib=True) lm = LogonManager() lm.logoff() lm.is_logged_on() lm.logon_with_openid(Open_ID, password) lm.is_logged_on() os.chdir(path_output) os.system('bash ' + path_output + 'Download.sh' + ' H ' + Open_ID + ' ' + password)