def test_logon_openid():
_clear_creds()
_load_creds(certificates_tarball='pcmdi9-certs.tar.gz')
lm = LogonManager(esgf_dir)
lm.logon_with_openid(TEST_OPENID, TEST_PASSWORD, interactive=False)
assert lm.is_logged_on()
def check_credentials(request):
if request.method == 'POST':
try:
data = json.loads(request.body)
response = {}
creds = Credential.objects.filter(site_user_name=request.user)
if len(creds) != 0:
for c in creds:
try:
if c.service == 'esgf':
import pyesgf
from pyesgf.logon import LogonManager
lm = LogonManager()
lm.logon_with_openid(
c.service_user_name, c.password)
if lm.is_logged_on():
response[s] = 'success'
print 'esgf log in successful'
else:
print 'esgf log in failed'
response[s] = 'fail'
if c.service == 'velo':
user, password = c.service_user_name, c.password
velo_creds = {
"velo_user": user,
"velo_pass": password,
"command": "init"
}
result = velo_request(velo_creds)
# TODO: Extract values out to CAPITAL_NAMED_CONSTANTS
if result == "Success":
print "velo login successful"
else:
print "velo login failed"
response[s] = "fail"
if c.service == 'github':
import github3
from github3 import login
gh = login(c.site_user_name, password=c.password)
if gh.user() == c.site_user_name:
print 'Github login successful'
response[s] = 'success'
else:
print 'Github login failure'
response[s] = 'fail'
if c.service == 'jira':
print 'Working on jira....'
except:
print_debug(c)
return HttpResponse(status=500)
return HttpResponse(json.dumps(response))
except Exception as e:
print_debug(e)
return HttpResponse(status=500)
else:
return HttpResponse(status=404)
def main():
#--- input user credential --
username='******'
password='******'
#----------------------------
#--- input search criteria --
project='ISIMIP2b'
model='GFDL-ESM2M'
impact_model='WaterGAP2'
experiment ='rcp45'
variable='Discharge'
# Loging in
lm = LogonManager()
lm.logoff()
lm.is_logged_on()
OPENID = 'https://esgf-data.dkrz.de/esgf-idp/openid/'+username
myproxy_host = 'esgf-data.dkrz.de'
lm.logon_with_openid(openid=OPENID, password=password, bootstrap=True)
lm.logon(hostname=myproxy_host,interactive=False, username=username,password=password,bootstrap=True)
lm.is_logged_on()
# Open connection with potsam node (ISIMIP)
conn = SearchConnection('http://esg.pik-potsdam.de/esg-search', distrib=False)
# Search datasets
# can do general searches e.g., search for all datasets in ISIMIP2b with experiment='rcp45'. This will return all instances in ctx
ctx = conn.new_context(
project=project,
model=model,
impact_model=impact_model,
experiment=experiment,
variable_long_name=variable)
# list number of counts
print('Founds '+str(ctx.hit_count)+' matching datasets')
# grab search results and display them
a=ctx.search()
cnt=1
for i in a :
print('['+str(cnt)+']'+' -----> '+i.dataset_id)
print('- - - - - - - - - - - - - - - - - - - - - - - -')
cnt = cnt + 1
# Ask user to choose a dataset or to download all
num = input("Which one should I download master? [Type -1 for all, 3 for third listed dataset.]")
# Case where user select a specific dataset
if num != -1:
print("Downloading dataset "+str(num)+".")
wget_makeNrun(a[num-1].file_context())
# case where user selects all
elif num == -1:
print("Downloading all "+str(ctx.hit_count)+" datasets.")
for i in a:
print("Downloading all datasets returned in search.")
wget_makeNrun(i.file_context())
return 0
def test_logon_openid():
_clear_creds()
_load_creds(certificates_tarball='pcmdi9-certs.tar.gz')
lm = LogonManager(esgf_dir)
lm.logon_with_openid(TEST_OPENID, TEST_PASSWORD, interactive=False)
assert lm.is_logged_on()
def logon(request):
credential = {
'username': request.GET.get('username'),
'password': request.GET.get('password')
}
if not credential['username']:
print "[-] No username in logon request"
return HttpResponse(status=403)
elif not credential['password']:
print "[-] No password in logon request"
return HttpResponse(status=403)
lm = LogonManager()
bootstrap = False
if not os.path.exists(ESGF_CREDENTIALS):
bootstrap = True
try:
lm.logon_with_openid(credential['username'], credential['password'], bootstrap=bootstrap)
except Exception as e:
print_message('Unable to log in user {}'.format(credential.get('username')))
return HttpResponse(status=403)
if lm.is_logged_on():
return HttpResponse(status=200)
else:
return HttpResponse(status=403)
def logon(openid, password):
''' Login on ESGF with credentials, returns connection object '''
lm=LogonManager()
try:
lm.logon_with_openid(openid, password, bootstrap=True, update_trustroots=True)
except:
e = sys.exc_info()[1]
print("Logon Error: ", e)
return lm
def test_logon_openid(self):
_clear_creds(self.esgf_dir)
_load_creds(self.esgf_dir, certificates_tarball='pcmdi9-certs.tar.gz')
lm = LogonManager(self.esgf_dir)
# NOTE: for many users the OpenID lookup might not provide the username
# in which case this test will fail because it needs interactive
# prompting for a username.
lm.logon_with_openid(TEST_OPENID, TEST_PASSWORD, interactive=False)
assert lm.is_logged_on()
def test_logon_openid(self):
_clear_creds(self.esgf_dir)
_load_creds(self.esgf_dir, certificates_tarball='pcmdi9-certs.tar.gz')
lm = LogonManager(self.esgf_dir)
# NOTE: for many users the OpenID lookup might not provide the username
# in which case this test will fail because it needs interactive
# prompting for a username.
lm.logon_with_openid(TEST_OPENID, TEST_PASSWORD, interactive=False)
assert lm.is_logged_on()
def logon(openid, password):
'''
Function to retrieve a short-term X.509 certificate that can be used to authenticate with ESGF.
The certificate is written in the location ~/.esg/credentials.pem.
The trusted CA certificates are written in the directory ~/.esg/certificates.
'''
# Must configure the DN of the JPL MyProxy server if using a JPL openid
if "esg-datanode.jpl.nasa.gov" in openid:
os.environ['MYPROXY_SERVER_DN'] = JPL_MYPROXY_SERVER_DN
lm = LogonManager()
lm.logon_with_openid(openid,password)
return lm.is_logged_on()
def check_credentials(request):
if request.method == 'POST':
try:
data = json.loads(request.body)
response = {}
for s in data:
if s == 'esgf':
import pyesgf
from pyesgf.logon import LogonManager
lm = LogonManager()
lm.logon_with_openid(data[s]['username'], data[s]['password'])
if lm.is_logged_on() != True:
response[s] = 'failed'
else:
response[s] = 'success'
if s == 'velo':
lib_path = os.path.abspath(os.path.join('apps', 'velo'))
sys.path.append(lib_path)
import VeloAPI
velo_api = VeloAPI.Velo()
velo_api.start_jvm()
'''
Using the test credentials for the time being, simply uncomment to
use the users credentials
velo_api.init_velo(data[s]['username'], data[s]['password'])
'''
# res = velo_api.init_velo("acmetest", "acmetest")
response[s] = 'success'
'''
if res.logged_on() != True:
resonse[s] = 'failed'
else:
response[s] = 'success
'''
return HttpResponse(json.dumps(response))
except Exception as e:
import traceback
print '1', e.__doc__
print '2', sys.exc_info()
print '3', sys.exc_info()[0]
print '4', sys.exc_info()[1]
print '5', traceback.tb_lineno(sys.exc_info()[2])
ex_type, ex, tb = sys.exc_info()
print '6', traceback.print_tb(tb)
return HttpResponse(status=500)
else:
return HttpResponse(status=404)
def logon(openid, password):
"""
Function to retrieve a short-term X.509 certificate that can be used to authenticate with ESGF.
The certificate is written in the location ~/.esg/credentials.pem.
The trusted CA certificates are written in the directory ~/.esg/certificates.
"""
# Must configure the DN of the JPL MyProxy server if using a JPL openid
if JPL_HOSTNAME in openid:
os.environ['MYPROXY_SERVER_DN'] = JPL_MYPROXY_SERVER_DN
logon_manager = LogonManager()
logon_manager.logon_with_openid(openid, password, bootstrap=True)
return logon_manager.is_logged_on()
def download(request):
credential = {
'username': '******',
'password': '******'
}
lm = LogonManager()
bootstrap = False
if not os.path.exists(ESGF_CREDENTIALS):
bootstrap = True
try:
print "logging in"
lm.logon_with_openid(credential['username'], credential['password'], bootstrap=bootstrap)
except Exception as e:
# print_debug(e)
return HttpResponse(status=403)
if lm.is_logged_on():
print "...login success"
try:
if not os.path.exists('/tmp'):
os.mkdir('/tmp')
url = request.GET.get('url')
# esgf_download(request.GET.get('url'))
print "[+] Downloading " + url
# response = urllib2.urlopen(url)
print "opening connection to server"
response = requests.get(url, verify='/Users/sterling/.esg/credentials.pem',stream=True)
response.raw.decode_content = True
with open("someClimateFile.nc", 'wb') as f:
shutil.copyfileobj(response.raw, f)
print "...download success"
return HttpResponse(status=200)
except Exception as e:
# print_debug(e)
return HttpResponse(status=400)
else:
return HttpResponse(status=403)
return HttpResponse(status=200)
def download_ESGF_data(Open_ID, password, server, project, experiment,
time_frequency, variable, domain, path_output):
"""Esta función nos permite descargar masivamente mediante WGET los diferentes ficheros netcdf que contienen los servidrores de ESGF sobre cambio climático.
El servidor por defecto que se va a utilizar es: https://esgf-data.dkrz.de/projects/esgf-dkrz/.
Parámetros:
---------------------
Open_ID : string. ID de tu usuario para acceder a la base de datos correspondiente del servidor
password : string. Contraseña correspondiente a la ID
server : string. Servidor del que se desea descargar la información. Ejemplo: https://esgf-data.dkrz.de/esg-search
project : string. Proyecto dentro del servidor del que se quiere descargar los datos. Ejemplo: CORDEX, CMIP5, CMIP6
experiment : string. Escenarios de cambio climático. Ejemplo: historical, rcp26, rcp45, rcp85
time_frequency : string. Frecuencia de la base de datos que se quiere. Ejemplo: 1hr, 6hr, day, mon
variable : string. Variable que se desea descargar: tasmax, tasmin, pr
domain : string. En el caso de que se desee descargar CORDEX, se debe de incluir el nombre de la malla. Ejemplo: EUR-11
path_output : string. Directorio donde se desean guardar los ficheros
Salidas:
----------------------
Ficheros netcdf para cada uno de los escenarios y modelos solicitados
"""
dir_file = __file__
os.chdir(dir_file[:-16])
print(dir_file)
conn = SearchConnection('https://' + server + '/esg-search', distrib=True)
lm = LogonManager()
lm.logoff()
lm.is_logged_on()
lm.logon_with_openid(Open_ID, password, bootstrap=True)
lm.is_logged_on()
if project == 'CORDEX':
ctx = conn.new_context(
project=project,
experiment=experiment,
time_frequency=time_frequency,
variable=variable,
domain=domain,
)
else:
ctx = conn.new_context(
project=project,
experiment=experiment,
time_frequency=time_frequency,
variable=variable,
)
with open('wget-plantilla-ESGF.sh', "r+") as out_file:
lines = out_file.readlines()
for ct in tqdm.tqdm(range(ctx.hit_count)):
files_list = list()
result = ctx.search()[ct]
lines[22] = "openId='" + Open_ID + "'\n"
lines[
23] = "search_url='https://" + server + "/esg-search/wget/?distrib=false&dataset_id=" + result.dataset_id + "'\n"
lines_first = lines[:27]
lines_end = lines[28:]
files = result.file_context().search()
ntcf_ds = list()
ntcf_name = list()
for file in files:
try:
if variable in file.opendap_url:
files_list.append("'" + file.filename + "'" + ' ' + "'" +
file.download_url + "'" + ' ' + "'" +
file.checksum_type + "'" + ' ' + "'" +
file.checksum + "'" + '\n')
except:
continue
if len(files_list) == 0:
continue
else:
with open(path_output + "Download.sh", "w") as fh:
for line in (lines_first + files_list + lines_end):
fh.write(line)
conn = SearchConnection('https://' + server + '/esg-search',
distrib=True)
lm = LogonManager()
lm.logoff()
lm.is_logged_on()
lm.logon_with_openid(Open_ID, password)
lm.is_logged_on()
os.chdir(path_output)
os.system('bash ' + path_output + 'Download.sh' + ' H ' + Open_ID +
' ' + password)
def grid(request):
''' For demo purposes this is loading a local file '''
from xml.etree.ElementTree import parse
import requests
from StringIO import StringIO
try:
r = requests.get('http://pcmdi9.llnl.gov/esgf-node-manager/registration.xml')
f = StringIO(r.content)
out = open('scripts/registration.xml', 'w')
out.write(f.read())
out.close()
except Exception as e:
print repr(e)
return HttpResponse(status=404)
tree = parse('scripts/registration.xml')
node_name_list = []
node_peer_list = []
node_url_list = []
node_location_list = []
for node in tree.getroot():
attrs = node.attrib
node_name_list.append(attrs["shortName"])
node_peer_list.append(attrs["adminPeer"])
node_url_list.append(attrs["hostname"])
for child in node:
if child.tag[-11:] == "GeoLocation":
node_location_list.append(child.attrib["city"])
node_list = zip(node_peer_list, node_url_list, node_name_list, node_location_list)
creds = Credential.objects.filter(site_user_name=request.user)
if len(creds) != 0:
for c in creds:
try:
if c.service == 'esgf':
import pyesgf
from pyesgf import LogonManager
lm = LogonManager()
lm.logon_with_openid(c.service_user_name, c.password)
if lm.is_logged_on():
request.session['esgf_login'] = lm
print 'esgf log in successful'
if c.service == 'velo':
lib_path = os.path.abspath(os.path.join('apps', 'velo'))
sys.path.append(lib_path)
import VeloAPI
velo_api = VeloAPI.Velo()
velo_api.start_jvm()
res = velo_api.init('acmetest', 'acmetest')
request.session['velo_login'] = res
print 'velo log in successful'
'''
For production, uncomment
res = velo_api.init(c.service_user_name, c.password)
'''
except:
import traceback
print '1', e.__doc__
print '2', sys.exc_info()
print '3', sys.exc_info()[0]
print '4', sys.exc_info()[1]
print '5', traceback.tb_lineno(sys.exc_info()[2])
ex_type, ex, tb = sys.exc_info()
print '6', traceback.print_tb(tb)
return HttpResponse(status=500)
return HttpResponse(render_template(request, "web_fe/grid.html", {'nodes': node_list}))
'pr',
'ps',
'rsds',
'rlds'] # ,'sfcWind'
expers = ['rcp26', 'historical', 'rcp85']
# logon manager
lm = LogonManager()
if not lm.is_logged_on():
lm.logoff()
lm.is_logged_on()
lm.logon_with_openid(openid=openid, password=None, bootstrap=True)
lm.is_logged_on()
lm.logon(hostname='esgf-data.dkrz.de', interactive=True, bootstrap=True)
lm.is_logged_on()
conn = SearchConnection('https://esgf-data.dkrz.de/esg-search', distrib=True)
for exper in expers:
logging.info("Experimet: " + exper)
print("Experimet: " + exper)
for var in vars:
myvar = var
logging.info("Variable: " + var)
print("Variable: " + var)
def dataset_download(message, data, user):
print 'got a dataset_download request'
username = data.get('params').get('openid_username')
password = data.get('params').get('openid_password')
search_string = data.get('params').get('search_string')
nodes = data.get('params').get('nodes')
data_type = data.get('params').get('data_type')
data_name = data.get('params').get('data_name').replace(' ', '_')
if not username:
print_message('No username given')
return -1
if not password:
print_message('No password given')
return -1
if not search_string:
print_message('No search_string given')
return -1
if not nodes:
print_message('No nodes given')
return -1
if not data_type:
print_message('No data_type given')
return -1
if not data_name:
print_message('No data_name given')
return -1
lm = LogonManager()
lm.logon_with_openid(username, password, bootstrap=True)
if not lm.is_logged_on():
print_message('User {user} is not logged in during download request'.format(user=user))
return -1
for node in nodes:
try:
path = os.path.abspath(os.path.dirname(__file__)) + '/../../'
print '[+] searching {node} for {string}'.format(node=node, string=search_string)
conn_string = 'http://{node}{suffix}'.format(node=node, suffix=ESGF_SEARCH_SUFFIX)
conn = SearchConnection(conn_string, distrib=True)
context = conn.new_context(**search_string)
rs = context.search()
print_message('got reply from {node}'.format(node=node))
if len(rs) == 0:
continue
script_text = context.get_download_script()
script_path = ''
if data_type == 'observation':
script_path = path + 'userdata/' + user + '/observations/' + data_name
elif data_type == 'model':
script_path = path + 'userdata/' + user + '/model_output/' + data_name
script_name = '{path}/{name}_download_script.sh'.format(path=script_path, name=data_name)
if not os.path.exists(script_path):
print_message('creating directory {}'.format(script_path))
os.makedirs(script_path)
try:
with open(script_name, 'w') as script:
script.write(script_text)
script.close()
except Exception, e:
print_debug(e)
word_count_cmd = 'cat {script_name} | grep \\.nc.*http:// | wc -l'.format(path=script_path, script_name=script_name)
print_message('running command {}'.format(word_count_cmd))
p = subprocess.check_output(word_count_cmd, shell=True)
number_of_downloads = int(p)
percent_complete = 0
number_complete = 0.0
print_message('Number of .nc files to download: {}'.format(number_of_downloads))
try:
subprocess.call(['chmod', '+x', script_name])
p = subprocess.Popen('exec {name} > output_status.txt 2>&1'.format(name=script_name),
shell=True,
cwd=script_path,
preexec_fn=os.setsid)
should_break = False
for line in tail('-f', '{path}/output_status.txt'.format(path=script_path), _iter=True):
update_message = None
if 'ERROR' in line:
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
print_message('error downloading file')
update_message = {
'text': json.dumps({
'user': user,
'data_name': data_name,
'percent_complete': percent_complete,
'message': line,
'destination': 'esgf_download_status'
})
}
if 'saved' in line or 'Saving to' in line:
number_complete += 1
percent_complete = number_complete / number_of_downloads * 100
print_message('percent complete: {}%'.format(percent_complete), 'ok')
update_message = {
'text': json.dumps({
'user': user,
'data_name': data_name,
'percent_complete': percent_complete,
'message': 'downloading',
'destination': 'esgf_download_status'
})
}
if percent_complete > 99:
print_message('Download complete', 'ok')
update_message = {
'text': json.dumps({
'user': user,
'data_name': data_name,
'percent_complete': 100.0,
'message': 'complete',
'destination': 'esgf_download_status'
})
}
should_break = True
if update_message:
print_message("sending to active group {}".format(update_message))
Group('active').send(update_message)
if should_break:
break
out, err = p.communicate()
print out, err
except Exception, e:
print_debug(e)
return -1
break
def check_credentials(request):
if request.method == 'POST':
try:
data = json.loads(request.body)
response = {}
creds = Credential.objects.filter(site_user_name=request.user)
velo_started = False
if len(creds) != 0:
for c in creds:
try:
if c.service == 'esgf':
import pyesgf
from pyesgf.logon import LogonManager
lm = LogonManager()
lm.logon_with_openid(
c.service_user_name, c.password)
if lm.is_logged_on():
response[s] = 'success'
print 'esgf log in successful'
else:
print 'esgf log in failed'
response[s] = 'fail'
if c.service == 'velo':
lib_path = os.path.abspath(
os.path.join('apps', 'velo'))
sys.path.append(lib_path)
'''
For production, replace below with:
rm = velo_api.init(c.service_user_name, c.password)
'''
rm = velo_api.init('acmetest', 'acmetest')
if rm.getRepositoryUrlBase() == 'u\'http://acmetest.ornl.gov:80/alfresco\'':
response[s] = 'success'
velo_api.shutdown_jvm()
print 'velo log in successful'
else:
velo_api.shutdown_jvm()
response[s] = 'fail'
print 'Error in velo initialization', rm.getRepositoryUrlBase()
if c.service == 'github':
import github3
from github3 import login
gh = login(c.site_user_name, password=c.password)
if gh.user() == c.site_user_name:
print 'Github login successful'
response[s] = 'success'
else:
print 'Github login failure'
response[s] = 'fail'
if c.service == 'jira':
print 'Working on jira....'
except:
if velo_started:
velo_api.shutdown_jvm()
import traceback
print '1', e.__doc__
print '2', sys.exc_info()
print '3', sys.exc_info()[0]
print '4', sys.exc_info()[1]
print '5', traceback.tb_lineno(sys.exc_info()[2])
ex_type, ex, tb = sys.exc_info()
print '6', traceback.print_tb(tb)
return HttpResponse(status=500)
return HttpResponse(json.dumps(response))
except Exception as e:
import traceback
print '1', e.__doc__
print '2', sys.exc_info()
print '3', sys.exc_info()[0]
print '4', sys.exc_info()[1]
print '5', traceback.tb_lineno(sys.exc_info()[2])
ex_type, ex, tb = sys.exc_info()
print '6', traceback.print_tb(tb)
return HttpResponse(status=500)
else:
return HttpResponse(status=404)
