def get_api(conf):
username = conf.get_go_username()
if username is None:
print("Globus Username: "******"Globus Password: "******"Test":
globusonline.transfer.api_client.goauth.HOST = \
"graph.api.test.globuscs.info"
nexus_cert = os.path.join(
os.path.dirname(
globus.connect.security.__file__),
"graph.api.test.globuscs.info.pem")
api_ca = os.path.join(
os.path.dirname(
globus.connect.security.__file__),
"go-ca-cert.pem")
globusonline.transfer.api_client.verified_https.match_hostname = \
lambda cert, hostname: True
base_url = "https://transfer.test.api.globusonline.org/" + globusonline.transfer.api_client.API_VERSION
socket.setdefaulttimeout(300)
for tries in range(0,10):
try:
auth_result = get_access_token(
username=username,
password=password,
ca_certs=nexus_cert)
if auth_result is not None:
break
except ssl.SSLError as e:
if "timed out" not in str(e):
raise e
time.sleep(30)
except GOCredentialsError as e:
print("Globus Username: "******"Globus Password: ")
api = TransferAPIClient(
username=auth_result.username,
goauth=auth_result.token,
base_url=base_url,
server_ca_file=api_ca,
max_attempts=10)
api.password = password
return api
def get_api(conf):
username = conf.get_go_username()
if username is None:
print("Globus Id: ", end=' ')
username = sys.stdin.readline().strip()
atglobusidorg = username.rfind("@globusid.org")
if atglobusidorg != -1:
username = username[:atglobusidorg]
password = conf.get_go_password()
if password is None:
password = getpass.getpass("Password: "******"Production":
globusonline.transfer.api_client.goauth.HOST = \
"nexus.api.%s.globuscs.info" % (go_instance)
base_url = \
"https://transfer.%s.api.globusonline.org/%s" \
% (go_instance, globusonline.transfer.api_client.API_VERSION)
nexus_cert = os.path.join(
os.path.dirname(globus.connect.security.__file__),
"lets_encrypt_fullchain.pem")
api_cert = nexus_cert
globusonline.transfer.api_client.verified_https.match_hostname = \
lambda cert, hostname: True
socket.setdefaulttimeout(300)
for tries in range(0, 10):
try:
auth_result = get_access_token(username=username,
password=password,
ca_certs=nexus_cert)
if auth_result is not None:
break
except ssl.SSLError as e:
if "timed out" not in str(e):
raise e
time.sleep(30)
except GOCredentialsError as e:
print("Globus Id: ", end=' ')
username = sys.stdin.readline().strip()
password = getpass.getpass("Globus Password: ")
api = TransferAPIClient(username=auth_result.username,
goauth=auth_result.token,
base_url=base_url,
server_ca_file=api_ca,
timeout=300.0,
max_attempts=10)
api.password = password
return api
def main():
""" Main function to test the library """
pass
username = '******'
cert_file = '/home/fiameni/Documents/Personal/reserved/certificates/infn/expires_2015/usercert.pem'
key_file = '/home/fiameni/Documents/Personal/reserved/certificates/infn/expires_2015/userkey.pem'
api = TransferAPIClient(username, cert_file, key_file)
print api.endpoint_list()
def remove(request):
"""
removes files on a remote Globus Online endpoint - API is not complete, so transfer
0 byte files instead of actually deleting anything
"""
# global so that we can use it in signal handlers
global api
global task_id
# connect to the service
api = TransferAPIClient(request["globus_username"],
cert_file=request["x509_proxy"])
# make sure we can auto-activate the endpoints
ep = activate_ep(api, request["endpoint"])
label = None
if "PEGASUS_WF_UUID" in os.environ and "PEGASUS_DAG_JOB_ID" in os.environ:
label = os.environ["PEGASUS_WF_UUID"] + " - " + os.environ[
"PEGASUS_DAG_JOB_ID"]
# set up a new transfer
code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(hours=24)
t = Transfer(submission_id,
request["endpoint"],
request["endpoint"],
deadline=deadline,
label=label,
notify_on_succeeded=False,
notify_on_failed=False,
notify_on_inactive=False)
for f in request["files"]:
t.add_item("/dev/null", f)
# finalize and submit the transfer
code, reason, data = api.transfer(t)
task_id = data["task_id"]
# how many faults will we accept before giving up?
acceptable_faults = min(100, len(request["files"]) * 3)
# wait for the task to complete, and see the tasks and
# endpoint ls change
try:
status = wait_for_task(api, task_id, acceptable_faults)
except Exception, err:
logger.error(err)
cancel_task(api, task_id)
sys.exit(1)
def main():
# Derive command line options from the method args, making
# the endpoint name positional and the rest options.
argspec = inspect.getargspec(TransferAPIClient.endpoint_create)
fargs = [
arg.replace("_", "-") for arg in argspec.args
if arg not in ("self", "endpoint_name")
]
parser = OptionParser(usage="Usage: %prog [options] endpoint_name")
for i, arg in enumerate(fargs):
default = argspec.defaults[i]
action = "store"
type_ = "string"
if arg in "public is-globus-connect".split():
action = "store_true"
type_ = None
elif arg == "port":
type_ = "int"
if default not in (None, ""):
doc = "[DEFAULT: %s]" % default
else:
doc = None
parser.add_option("--%s" % arg,
action=action,
type=type_,
default=default,
help=doc)
options, args = parser.parse_args(sys.argv)
if len(args) != 2:
parser.error("requires one positional argument with endpoint name")
# Convert options object into a keyword dict we can pass to the method
kw = {}
for arg in fargs:
name = arg.replace("-", "_")
value = getattr(options, name)
if value is not None:
kw[name] = value
# will prompt for username and password
auth_result = get_access_token()
api = TransferAPIClient(username=auth_result.username,
goauth=auth_result.token)
_, _, data = api.endpoint_create(args[1], **kw)
setup_key = data.get("globus_connect_setup_key")
if setup_key:
print "GC Setup Key: %s" % setup_key
print "Endpoint Name: %s" % data["canonical_name"]
print data["message"]
def activateEndpoints():
src = config['globus']["source_endpoint_id"]
dest = config['globus']["destination_endpoint_id"]
generateAuthToken()
api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token'])
# TODO: Can't use autoactivate; must populate credentials
"""try:
actList = api.endpoint_activation_requirements(src)[2]
actList.set_requirement_value('myproxy', 'username', 'data_mover')
actList.set_requirement_value('myproxy', 'passphrase', 'terraref2016')
actList.set_requirement_value('delegate_proxy', 'proxy_chain', 'some PEM cert w public key')
except:"""
api.endpoint_autoactivate(src)
api.endpoint_autoactivate(dest)
def __init__(self, auth=['', ''], resource_file_path='', debug=False):
"""
Initialize the Globus session: login with Globus username and password
Credentials can also be passed through a resource json file like this:
{
"globus_username":"",
"globus_password":"",
"myproxy_username": "",
"myproxy_password": "",
"PEM_passphrase": ""
}
:param auth: a list containing username[0], password[1], certfile[2]
and keyfile[3]
:param resource_file_path: path to a json file containing the credentials
:param debug: set True to enable debug messages
"""
if debug:
LOGGER.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
LOGGER.addHandler(ch)
# Read resources from file:
self.globus_init = None
if resource_file_path and os.path.isfile(resource_file_path):
self.globus_init = json.load(open(resource_file_path))
if not auth[0]:
auth[0] = self.globus_init['globus_username']
if not auth[1]:
auth[1] = self.globus_init['globus_password']
if not auth[0] or not auth[1]:
print "Can not init session without a username and password.."
return
self.api = None
self.proxy_name = 'credential-' + auth[0] + '.pem'
try:
result = get_access_token(username=auth[0], password=auth[1])
goauth = result.token
self.api = TransferAPIClient(username=auth[0], goauth=goauth)
self.api.set_debug_print(False, False)
LOGGER.info("Successfully logged in with Globus Online!")
except Exception as e:
print "Globus Online authentication failed: {0}".format(e)
def __init__(self, auth, http_session=None):
"""
Initialize the Globus session: login with username and certificates
:param auth: a list containing username[0], certfile[1] and keyfile[2]
:param http_session:
"""
self.auth = auth
if len(self.auth) < 4: #or !self.auth[0] or !self.auth[2] or !self.auth[3]:
print "Can not init session: some parameters are missing.."
self.auth = auth
self.http_session = http_session
self.myproxy_username = ''
self.api = None
if len(self.auth) > 0:
self.proxy_name = 'credential-' + self.auth[0] + '.pem'
try:
self.api = TransferAPIClient(username=self.auth[0],
cert_file=self.auth[2],
key_file=self.auth[3])
self.api.set_debug_print(False, False)
LOGGER.info("Successfully logged in with Globus online!")
except Exception as e:
raise Exception("GSI authentication failed: {0}".format(e))
def run(self):
self.debug("thread for <%i>: %s" % (self.id, self.task))
from globusonline.transfer.api_client \
import TransferAPIClient
token = self.get_token()
api = TransferAPIClient(self.user, goauth=token)
while True:
code, reason, data = api.task(self.task, fields="status")
status = data["status"]
print(status)
if status in ("SUCCEEDED", "FAILED"):
break
self.debug("Globus: done " + status)
self.q.put(status)
def main():
# Derive command line options from the method args, making
# the endpoint name positional and the rest options.
argspec = inspect.getargspec(TransferAPIClient.endpoint_create)
fargs = [arg.replace("_", "-") for arg in argspec.args
if arg not in ("self", "endpoint_name")]
parser = OptionParser(usage="Usage: %prog [options] endpoint_name")
for i, arg in enumerate(fargs):
default = argspec.defaults[i]
action = "store"
type_ = "string"
if arg in "public is-globus-connect".split():
action = "store_true"
type_ = None
elif arg == "port":
type_ = "int"
if default not in (None, ""):
doc = "[DEFAULT: %s]" % default
else:
doc = None
parser.add_option("--%s" % arg, action=action, type=type_,
default=default, help=doc)
options, args = parser.parse_args(sys.argv)
if len(args) != 2:
parser.error("requires one positional argument with endpoint name")
# Convert options object into a keyword dict we can pass to the method
kw = {}
for arg in fargs:
name = arg.replace("-", "_")
value = getattr(options, name)
if value is not None:
kw[name] = value
# will prompt for username and password
auth_result = get_access_token()
api = TransferAPIClient(username=auth_result.username,
goauth=auth_result.token)
_, _, data = api.endpoint_create(args[1], **kw)
setup_key = data.get("globus_connect_setup_key")
if setup_key:
print "GC Setup Key: %s" % setup_key
print "Endpoint Name: %s" % data["canonical_name"]
print data["message"]
def transfer(options):
# Map legacy endpoint names to UUIDs
srcendpoint = options.srcendpoint
dstendpoint = options.dstendpoint
if '#' in srcendpoint:
srcendpoint = config['endpoint'][srcendpoint]
if '#' in dstendpoint:
dstendpoint = config['endpoint'][dstendpoint]
# Get access token (This method of getting an acces token is deprecated and should be replaced by OAuth2 calls).
auth_result = get_access_token(config['globus']['username'], config['globus']['password'])
# Create a transfer submission
api_client = TransferAPIClient(config['globus']['username'], goauth=auth_result.token)
activate_endpoint(api_client, srcendpoint)
activate_endpoint(api_client, dstendpoint)
code, message, data = api_client.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(days=10)
transfer_task = Transfer(submission_id, srcendpoint, dstendpoint, deadline)
# Add srcpath to the transfer task
if options.srcpath:
transfer_task.add_item(options.srcpath,
get_destination_path(options.srcpath, options.dstpath, options.recursive),
recursive=options.recursive)
# Add srclist to the transfer task
if options.srclist:
with open(options.srclist) as f:
srcpath = f.readline().rstrip('\n')
transfer_task.add_item(srcpath,
get_destination_path(srcpath, options.dstpath, options.recursive),
recursive=options.recursive)
# Start the transfer
task_id = None
try:
code, reason, data = api_client.transfer(transfer_task)
task_id = data["task_id"]
print 'task_id %s' % task_id
except Exception as e:
msg = "Could not submit the transfer. Error: %s" % str(e)
sys.exit(1)
# Check a status of the transfer every minute (60 secs)
while True:
code, reason, data = api_client.task(task_id)
if data['status'] == 'SUCCEEDED':
print 'progress %d/%d' % (data['files_transferred'], data['files'])
return ('success', '')
elif data['status'] == 'FAILED':
return ('error', data['nice_status_details'])
elif data['status'] == 'ACTIVE':
print 'progress %d/%d' % (data['files_transferred'], data['files'])
time.sleep(10)
def generateGlobusSubmissionID():
try:
api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token'])
status_code, status_message, submission_id = api.submission_id()
except:
try:
# Try activating endpoints and retrying
activateEndpoints()
api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token'])
status_code, status_message, submission_id = api.submission_id()
except:
logger.error("- exception generating submission ID for Globus transfer")
return None
if status_code == 200:
logger.debug("- generated new Globus submission ID %s" % submission_id['value'])
return submission_id['value']
else:
logger.error("- could not generate new Globus submission ID (%s: %s)" % (status_code, status_message))
return None
def submit(request):
'''View to submit a Globus transfer request.
The access token and files to download are retrieved from the session. '''
openid = request.user.profile.openid()
openid_parsed = urlparse(openid)
hostname = openid_parsed.hostname
# get a username and password if autoactivation failed and a user was asked for a password
if hostname == "ceda.ac.uk":
myproxy_server = discover_myproxy(openid)
esgf_username = request.POST.get(ESGF_USERNAME)
else:
myproxy_server = hostname
esgf_username = os.path.basename(openid_parsed.path)
esgf_password = request.POST.get(ESGF_PASSWORD)
# retrieve all data transfer request parameters from session
username = request.session.get(GLOBUS_USERNAME)
access_token = request.session.get(GLOBUS_ACCESS_TOKEN)
download_map = request.session.get(GLOBUS_DOWNLOAD_MAP)
target_endpoint = request.session.get(TARGET_ENDPOINT)
target_folder = request.session.get(TARGET_FOLDER)
#print 'Downloading files=%s' % download_map.items()
print 'User selected destionation endpoint:%s, folder: %s' % (target_endpoint, target_folder)
api_client = TransferAPIClient(username, goauth=access_token)
# loop over source endpoints and autoactivate them
# if the autoactivation fails, redirect to a form asking for a password
activateEndpoint(api_client, target_endpoint)
for source_endpoint, source_files in download_map.items():
status, message = activateEndpoint(
api_client, source_endpoint,
myproxy_server=myproxy_server, username=esgf_username, password=esgf_password)
if not status:
print hostname
return render(request,
'cog/globus/password.html',
{ 'openid': openid, 'username': hostname=='ceda.ac.uk', 'message': message })
# loop over source endpoints, submit one transfer for each source endpoint
task_ids = [] # list of submitted task ids
for source_endpoint, source_files in download_map.items():
# submit transfer request
task_id = submitTransfer(
api_client, source_endpoint, source_files,
target_endpoint, target_folder)
task_ids.append(task_id)
# display confirmation page
return render(request,
'cog/globus/confirmation.html',
{ 'task_ids':task_ids, 'title':'Globus Download Confirmation' })
def getGlobusTaskData(task):
authToken = config['globus']['valid_users'][task['user']]['auth_token']
api = TransferAPIClient(username=task['user'], goauth=authToken)
try:
logger.debug("%s requesting task data from Globus" % task['globus_id'])
status_code, status_message, task_data = api.task(task['globus_id'])
except (APIError, ClientError) as e:
try:
# Refreshing auth tokens and retry
generateAuthTokens()
authToken = config['globus']['valid_users'][task['user']]['auth_token']
api = TransferAPIClient(username=task['user'], goauth=authToken)
status_code, status_message, task_data = api.task(task['globus_id'])
except (APIError, ClientError) as e:
logger.error("%s error checking with Globus for transfer status" % task['globus_id'])
status_code = 503
if status_code == 200:
return task_data
else:
return None
def transfer(src_endpoint, src_path, dst_endpoint, dst_path, access_token,
openid):
(server, username) = decompose_openid(openid)
client = TransferAPIClient(username, goauth=access_token)
client.connect()
globus_submission_id = client.submission_id()
transfer = Transfer(globus_submission_id, src_endpoint, dst_endpoint)
transfer.add_item(src_path, dst_path, recursive=True)
client.transfer(transfer)
return {'status': 'Success', 'message': ''}
def generateGlobusSubmissionID():
try:
api = TransferAPIClient(username=config['globus']['username'],
goauth=config['globus']['auth_token'])
status_code, status_message, submission_id = api.submission_id()
except:
try:
# Try activating endpoints and retrying
activateEndpoints()
api = TransferAPIClient(username=config['globus']['username'],
goauth=config['globus']['auth_token'])
status_code, status_message, submission_id = api.submission_id()
except:
logger.error(
"- exception generating submission ID for Globus transfer")
return None
if status_code == 200:
logger.debug("- generated new Globus submission ID %s" %
submission_id['value'])
return submission_id['value']
else:
logger.error("- could not generate new Globus submission ID (%s: %s)" %
(status_code, status_message))
return None
def getGlobusTaskData(task):
authToken = config['globus']['auth_token']
if len(task['user']) == 0:
guser = config['globus']['username']
else:
guser = task['user']
api = TransferAPIClient(username=guser, goauth=authToken)
try:
logger.debug("%s requesting task data from Globus as %s" %
(task['globus_id'], guser))
status_code, status_message, task_data = api.task(task['globus_id'])
except:
try:
# Refreshing auth tokens and retry
generateAuthToken()
authToken = config['globus']['auth_token']
api = TransferAPIClient(username=guser, goauth=authToken)
status_code, status_message, task_data = api.task(
task['globus_id'])
except:
logger.error("%s error checking with Globus for transfer status" %
task['globus_id'])
status_code = 503
if status_code == 200:
return task_data
else:
return None
def __init__(self, auth=['', ''], resource_file_path='', debug=False):
"""
Initialize the Globus session: login with Globus username and password
Credentials can also be passed through a resource json file like this:
{
"globus_username":"",
"globus_password":"",
"myproxy_username": "",
"myproxy_password": "",
"PEM_passphrase": ""
}
:param auth: a list containing username[0], password[1], certfile[2]
and keyfile[3]
:param resource_file_path: path to a json file containing the credentials
:param debug: set True to enable debug messages
"""
if debug:
LOGGER.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
LOGGER.addHandler(ch)
# Read resources from file:
self.globus_init = None
if resource_file_path and os.path.isfile(resource_file_path):
self.globus_init = json.load(open(resource_file_path))
if not auth[0]:
auth[0] = self.globus_init['globus_username']
if not auth[1]:
auth[1] = self.globus_init['globus_password']
if not auth[0] or not auth[1]:
print "Can not init session without a username and password.."
return
self.api = None
self.proxy_name = 'credential-' + auth[0] + '.pem'
try:
result = get_access_token(username=auth[0], password=auth[1])
goauth = result.token
self.api = TransferAPIClient(username=auth[0],
goauth=goauth)
self.api.set_debug_print(False, False)
LOGGER.info("Successfully logged in with Globus Online!")
except Exception as e:
print "Globus Online authentication failed: {0}".format(e)
def submitTransfer(openid, username, access_token, source_endpoint, source_files, target_endpoint, target_directory):
'''
Method to submit a data transfer request to Globus.
'''
# instantiate Globus Transfer API client
api_client = TransferAPIClient(username, goauth=access_token)
# must activate the endpoints using cached credentials
activateEndpoint(api_client, source_endpoint, openid)
activateEndpoint(api_client, target_endpoint)
# obtain a submission id from Globus
code, message, data = api_client.transfer_submission_id()
submission_id = data["value"]
print "Obtained transfer submission id: %s" % submission_id
# maximum time for completing the transfer
deadline = datetime.utcnow() + timedelta(days=10)
# create a transfer request
transfer_task = Transfer(submission_id, source_endpoint, target_endpoint, deadline)
for source_file in source_files:
source_directory, filename = os.path.split(source_file)
target_file = os.path.join(target_directory, filename)
transfer_task.add_item(source_file, target_file)
# submit the transfer request
try:
code, reason, data = api_client.transfer(transfer_task)
task_id = data["task_id"]
print "Submitted transfer task with id: %s" % task_id
except Exception as e:
print "Could not submit the transfer. Error: %s" % str(e)
task_id = "Could not submit the transfer. Please contact the ESGF node admin to investigate the issue."
return task_id
def submit(request):
'''View to submit a Globus transfer request.
The access token and files to download are retrieved from the session. '''
openid = request.user.profile.openid()
# get a password if authoactivation failed and a user was asked for a password
password = request.POST.get(ESGF_PASSWORD)
# retrieve all data transfer request parameters from session
username = request.session[GLOBUS_USERNAME]
access_token = request.session[GLOBUS_ACCESS_TOKEN]
download_map = request.session[GLOBUS_DOWNLOAD_MAP]
target_endpoint = request.session[TARGET_ENDPOINT]
target_folder = request.session[TARGET_FOLDER]
print 'Downloading files=%s' % download_map.items()
print 'User selected destionation endpoint:%s, folder: %s' % (
target_endpoint, target_folder)
api_client = TransferAPIClient(username, goauth=access_token)
# loop over source endpoints and autoactivate them
# if the autoactivation fails, redirect to a form asking for a password
activateEndpoint(api_client, target_endpoint)
for source_endpoint, source_files in download_map.items():
status, message = activateEndpoint(api_client, source_endpoint, openid,
password)
if not status:
return render(request, 'cog/globus/password.html', {
'openid': openid,
'message': message
})
# loop over source endpoints, submit one transfer for each source endpoint
task_ids = [] # list of submitted task ids
for source_endpoint, source_files in download_map.items():
# submit transfer request
task_id = submitTransfer(api_client, source_endpoint, source_files,
target_endpoint, target_folder)
task_ids.append(task_id)
# display confirmation page
return render(request, 'cog/globus/confirmation.html', {
'task_ids': task_ids,
'title': 'Globus Download Confirmation'
})
def activateEndpoints():
src = config['globus']["source_endpoint_id"]
dest = config['globus']["destination_endpoint_id"]
generateAuthToken()
api = TransferAPIClient(username=config['globus']['username'],
goauth=config['globus']['auth_token'])
# TODO: Can't use autoactivate; must populate credentials
"""try:
actList = api.endpoint_activation_requirements(src)[2]
actList.set_requirement_value('myproxy', 'username', 'data_mover')
actList.set_requirement_value('myproxy', 'passphrase', 'terraref2016')
actList.set_requirement_value('delegate_proxy', 'proxy_chain', 'some PEM cert w public key')
except:"""
api.endpoint_autoactivate(src)
api.endpoint_autoactivate(dest)
def getGlobusTaskData(task):
authToken = config['globus']['auth_token']
if len(task['user']) == 0:
guser = config['globus']['username']
else:
guser = task['user']
api = TransferAPIClient(username=guser, goauth=authToken)
try:
logger.debug("%s requesting task data from Globus as %s" %
(task['globus_id'], guser))
status_code, status_message, task_data = api.task(task['globus_id'])
except gaierror as e:
logger.error(
"%s gaierror checking with Globus for transfer status: %s" %
(task['globus_id'], e))
status_code = 404
except Exception as e:
if hasattr(e, 'status_code') and e.status_code == 404:
return {"status": "NOT FOUND"}
try:
# Refreshing auth tokens and retry
generateAuthToken()
authToken = config['globus']['destinations'][end_id]['auth_token']
api = TransferAPIClient(username=guser, goauth=authToken)
status_code, status_message, task_data = api.task(
task['globus_id'])
except gaierror as e:
logger.error(
"%s gaierror checking with Globus for transfer status: %s" %
(task['globus_id'], e))
status_code = 404
except Exception as e:
if hasattr(e, 'status_code') and e.status_code == 404:
return {"status": "NOT FOUND"}
logger.error("%s error checking with Globus for transfer status" %
task['globus_id'])
status_code = 503
if status_code == 200:
return task_data
else:
return None
def getGlobusTaskData(task):
authToken = config['globus']['valid_users'][task['user']]['auth_token']
api = TransferAPIClient(username=task['user'], goauth=authToken)
try:
logger.debug("%s requesting task data from Globus" % task['globus_id'])
status_code, status_message, task_data = api.task(task['globus_id'])
except (APIError, ClientError) as e:
try:
# Refreshing auth tokens and retry
generateAuthTokens()
authToken = config['globus']['valid_users'][
task['user']]['auth_token']
api = TransferAPIClient(username=task['user'], goauth=authToken)
status_code, status_message, task_data = api.task(
task['globus_id'])
except (APIError, ClientError) as e:
logger.error("%s error checking with Globus for transfer status" %
task['globus_id'])
status_code = 503
if status_code == 200:
return task_data
else:
return None
def initializeGlobusTransfer(globus_batch_obj):
globus_batch = globus_batch_obj['contents']
globus_batch_id = globus_batch_obj['id']
api = TransferAPIClient(username=config['globus']['username'],
goauth=config['globus']['auth_token'])
submissionID = generateGlobusSubmissionID()
if submissionID:
# Prepare transfer object
transferObj = Transfer(submissionID,
config['globus']['source_endpoint_id'],
config['globus']['destination_endpoint_id'],
verify_checksum=True)
queue_length = 0
for ds in globus_batch:
if 'files' in globus_batch[ds]:
for fkey in globus_batch[ds]['files']:
fobj = globus_batch[ds]['files'][fkey]
if 'src_path' not in fobj:
srcpath = fobj['orig_path']
if srcpath.startswith('/LemnaTec'):
srcpath = "/gantry_data" + srcpath
else:
srcpath = fobj['src_path']
transferObj.add_item(srcpath, fobj['path'])
queue_length += 1
# Send transfer to Globus
try:
logger.debug("- attempting to send new transfer")
status_code, status_message, transfer_data = api.transfer(
transferObj)
except:
try:
# Try refreshing endpoints and retrying
activateEndpoints()
api = TransferAPIClient(username=config['globus']['username'],
goauth=config['globus']['auth_token'])
status_code, status_message, transfer_data = api.transfer(
transferObj)
except (APIError, ClientError) as e:
logger.error("- problem initializing Globus transfer")
status_code = 503
status_message = e
except:
logger.error(
"- unexpected problem initializing Globus transfer")
status_code = 503
status_message = ""
if status_code == 200 or status_code == 202:
# Notify NCSA monitor of new task, and add to activeTasks for logging
globusID = transfer_data['task_id']
logger.info("%s new Globus transfer task started (%s files)" %
(globusID, queue_length),
extra={
"globus_id": globusID,
"action": "TRANSFER STARTED",
"contents": globus_batch
})
created_task = {
"globus_id": globusID,
"contents": globus_batch,
"started": str(datetime.datetime.now()),
"status": "CREATED"
}
writeTaskToDatabase(created_task)
removePendingTask(globus_batch_id)
return True
else:
# If failed, leave pending list as-is and try again on next iteration (e.g. in 180 seconds)
logger.error(
"- Globus transfer initialization failed for %s (%s: %s)" %
(ds, status_code, status_message))
return False
def transfer(request):
"""
takes a transfer specification parsed from json:
{
"globus_username": "******",
"x509_proxy": "/tmp/...",
"src_endpoint": "rynge#obelix",
"dst_endpoint": "rynge#workflow",
"files":[
{"src":"/etc/hosts","dst":"/tmp/foobar.txt"},
{"src":"/etc/hosts","dst":"/tmp/foobar-2.txt"}
],
}
"""
# global so that we can use it in signal handlers
global api
global task_id
# connect to the service
api = TransferAPIClient(request["globus_username"],
cert_file=request["x509_proxy"])
# make sure we can auto-activate the endpoints
src_ep = activate_ep(api, request["src_endpoint"])
dst_ep = activate_ep(api, request["dst_endpoint"])
label = None
if "PEGASUS_WF_UUID" in os.environ and "PEGASUS_DAG_JOB_ID" in os.environ:
label = os.environ["PEGASUS_WF_UUID"] + " - " + os.environ[
"PEGASUS_DAG_JOB_ID"]
# set up a new transfer
code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(hours=24)
t = Transfer(submission_id,
request["src_endpoint"],
request["dst_endpoint"],
deadline=deadline,
label=label,
notify_on_succeeded=False,
notify_on_failed=False,
notify_on_inactive=False)
for pair in request["files"]:
t.add_item(pair["src"], pair["dst"])
# finalize and submit the transfer
code, reason, data = api.transfer(t)
task_id = data["task_id"]
# how many faults will we accept before giving up?
acceptable_faults = min(100, len(request["files"]) * 3)
# wait for the task to complete, and see the tasks and
# endpoint ls change
try:
status = wait_for_task(api, task_id, acceptable_faults)
except Exception, err:
logger.error(err)
cancel_task(api, task_id)
sys.exit(1)
def initializeGlobusTransfer():
global pendingTransfers
global activeTasks
global status_numActive
global status_numPending
maxQueue = config['globus']['max_transfer_file_count']
api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token'])
submissionID = generateGlobusSubmissionID()
if submissionID:
# Prepare transfer object
transferObj = Transfer(submissionID,
config['globus']['source_endpoint_id'],
config['globus']['destination_endpoint_id'],
verify_checksum=True)
queueLength = 0
# Loop over a copy of the list instead of actual thing - other thread will be appending to actual thing
loopingTransfers = safeCopy(pendingTransfers)
currentTransferBatch = {}
logger.debug("- building transfer %s from pending queue" % submissionID)
for ds in loopingTransfers:
if "files" in loopingTransfers[ds]:
if ds not in currentTransferBatch:
currentTransferBatch[ds] = {}
currentTransferBatch[ds]['files'] = {}
# Add files from each dataset
for f in loopingTransfers[ds]['files']:
if queueLength < maxQueue:
fobj = loopingTransfers[ds]['files'][f]
if fobj["path"].find(config['globus']['source_path']) > -1:
src_path = os.path.join(fobj["path"], fobj["name"])
dest_path = os.path.join(config['globus']['destination_path'],
fobj["path"].replace(config['globus']['source_path'], ""),
fobj["name"])
else:
src_path = os.path.join(config['globus']['source_path'], fobj["path"], fobj["name"])
dest_path = os.path.join(config['globus']['destination_path'], fobj["path"], fobj["name"])
# Clean up dest path to new folder structure
# ua-mac/raw_data/LemnaTec/EnvironmentLogger/2016-01-01/2016-08-03_04-05-34_environmentlogger.json
# ua-mac/raw_data/LemnaTec/MovingSensor/co2Sensor/2016-01-01/2016-08-02__09-42-51-195/file.json
# ua-mac/raw_data/LemnaTec/3DScannerRawDataTopTmp/scanner3DTop/2016-01-01/2016-08-02__09-42-51-195/file.json
# ua-mac/raw_data/MAC/lightning/2016-01-01/weather_2016_06_29.dat
dest_path = dest_path.replace("LemnaTec/", "")
dest_path = dest_path.replace("MovingSensor/", "")
dest_path = dest_path.replace("MAC/", "")
dest_path = dest_path.replace("3DScannerRawDataTopTmp/", "")
dest_path = dest_path.replace("3DScannerRawDataLowerOnEastSideTmp/", "")
dest_path = dest_path.replace("3DScannerRawDataLowerOnWestSideTmp/", "")
# /ua-mac/raw_data/MovingSensor.reproc2016-8-18/scanner3DTop/2016-08-22/2016-08-22__15-13-01-672/6af8d63b-b5bb-49b2-8e0e-c26e719f5d72__Top-heading-east_0.png
# /ua-mac/raw_data/MovingSensor.reproc2016-8-18/scanner3DTop/2016-08-22/2016-08-22__15-13-01-672/6af8d63b-b5bb-49b2-8e0e-c26e719f5d72__Top-heading-east_0.ply
if dest_path.endswith(".ply") and (dest_path.find("scanner3DTop") or
dest_path.find("scanner3DLowerOnEastSide") or
dest_path.find("scanner3DLowerOnWestSide")) > -1:
dest_path = dest_path.replace("raw_data", "Level_1")
if dest_path.find("MovingSensor.reproc") > -1:
new_dest_path = ""
dirs = dest_path.split("/")
for dir_part in dirs:
if dir_part.find("MovingSensor.reproc") == -1:
new_dest_path = os.path.join(new_dest_path, dir_part)
dest_path = new_dest_path
# danforth/raw_data/sorghum_pilot_dataset/snapshot123456/file.png
transferObj.add_item(src_path, dest_path)
# remainingTransfers will have leftover data once max Globus transfer size is met
queueLength += 1
fobj["orig_path"] = fobj["path"]
fobj["path"] = dest_path
currentTransferBatch[ds]['files'][f] = fobj
else:
break
# Clean up placeholder entries once queue length is exceeded
if currentTransferBatch[ds]['files'] == {}:
del currentTransferBatch[ds]['files']
if currentTransferBatch[ds] == {}:
del currentTransferBatch[ds]
if queueLength >= maxQueue:
break
if queueLength > 0:
# Send transfer to Globus
try:
logger.debug("- attempting to send new transfer")
status_code, status_message, transfer_data = api.transfer(transferObj)
except (APIError, ClientError) as e:
try:
# Try refreshing endpoints and retrying
activateEndpoints()
api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token'])
status_code, status_message, transfer_data = api.transfer(transferObj)
except (APIError, ClientError) as e:
logger.error("- problem initializing Globus transfer")
status_code = 503
status_message = e
except:
logger.error("- unexpected problem initializing Globus transfer")
status_code = 503
status_message = e
if status_code == 200 or status_code == 202:
# Notify NCSA monitor of new task, and add to activeTasks for logging
globusID = transfer_data['task_id']
logger.info("%s new Globus transfer task started (%s files)" % (globusID, queueLength), extra={
"globus_id": globusID,
"action": "TRANSFER STARTED",
"contents": currentTransferBatch
})
activeTasks[globusID] = {
"globus_id": globusID,
"contents": currentTransferBatch,
"started": str(datetime.datetime.now()),
"status": "IN PROGRESS"
}
writeTasksToDisk(os.path.join(config['log_path'], "active_tasks.json"), activeTasks)
# Now that we've safely sent the pending transfers, remove them
for ds in currentTransferBatch:
if ds in pendingTransfers:
for f in currentTransferBatch[ds]['files']:
if f in pendingTransfers[ds]['files']:
del pendingTransfers[ds]['files'][f]
notifyMonitorOfNewTransfer(globusID, currentTransferBatch)
writeTasksToDisk(os.path.join(config['log_path'], "pending_transfers.json"), pendingTransfers)
else:
# If failed, leave pending list as-is and try again on next iteration (e.g. in 180 seconds)
logger.error("- Globus transfer initialization failed for %s (%s: %s)" % (ds, status_code, status_message))
return
status_numActive = len(activeTasks)
cleanPendingTransfers()
def mover(username, src_site, dst_site, dst_dir):
"""
Do a bunch of API calls and display the results. Does a small transfer
between tutorial endpoints, but otherwise does not modify user data.
Uses module global API client instance.
"""
global api
#activer=[username,"-c",os.getcwd()+"/credential.pem"]
#api, _ = create_client_from_args(activer)
user_credential_path=os.getcwd()+"/credential-"+username+".pem"
#print "user_credential_path=",user_credential_path
api = TransferAPIClient(username, cert_file=user_credential_path)
api.set_debug_print(False, False)
#print " Here (mover): ",api.task_list()
# See what is in the account before we make any submissions.
time.sleep(0.01)
print ""
print ""
print ""
print "=== Before transfer ==="
#display_tasksummary(); print
#display_task_list(); print
#display_endpoint_list(); print
print "=== Activate endpoints ==="
dest_directory= dst_dir
site_ep1 = src_site
site_ep2 = dst_site
print "Please enter your myproxy username (\'none\' if you do not have one)."
myproxy_username = sys.stdin.readline().rstrip()
preferred_activation(username, site_ep1, myproxy_username)
preferred_activation(username, site_ep2, myproxy_username)
print "=== Prepare transfer ==="
#raw_input("Press Enter to continue...")
# submit a transfer
oldstdout=sys.stdout
sys.stdout = open(os.devnull,'w')
code, message, data = api.transfer_submission_id()
sys.stdout = oldstdout # enable output
#code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(minutes=10)
t = Transfer(submission_id, site_ep1, site_ep2)#, deadline)
f=open('json_file','r')
json_results=f.read()
f.close
#print json_results,type(json_results)
results=json.loads(json_results)
#print results,type(results)
#print results[0],type(results[0])
for result in results:
#print "Result: ",result
if result[-1]=="/":
#print "Result: it is a directory"
t.add_item(result, dest_directory, recursive=True)
else:
#print "Result: it is a file"
file_name=re.split("/",result)[-1]
#print "Result: "+file_name
t.add_item(result, dest_directory+"/"+file_name)
print "=== Submit transfer ==="
oldstdout=sys.stdout
sys.stdout = open(os.devnull,'w')
code, reason, data = api.transfer(t)
sys.stdout = oldstdout # enable output
#code, reason, data = api.transfer(t)
task_id = data["task_id"]
#print " Task ID is %s " % (task_id)
# see the new transfer show up
#print "=== After submit ==="
#display_tasksummary(); print
#display_task(task_id); print
#raw_input("Press Enter to continue...")
# wait for the task to complete, and see the summary and lists
# update
print "=== Checking completion ==="
# To save the task_id for further check could be useful.
#wait_for_task(task_id)
max_wait = 10*1
waiting = True
if waiting:
print "Task %s is still under process..." % (task_id)
oldstdout=sys.stdout
sys.stdout = open(os.devnull,'w')
display_tasksummary(); print
sys.stdout = oldstdout # enable output
#display_task(task_id); print
#display_ls("cin0641a#GSI-PLX"); print
waiting=wait_for_task(task_id,max_wait)
print "=== Exiting ==="
#display_tasksummary(); print
print "The task ID for this operation is: "+task_id; print
oldstdout=sys.stdout
sys.stdout = open(os.devnull,'w')
status, reason, result = api.task(task_id)
sys.stdout = oldstdout # enable output
print "Its status is "+result["status"]; print
def initializeGlobusTransfer():
global pendingTransfers
global activeTasks
global status_numActive
global status_numPending
maxQueue = config['globus']['max_transfer_file_count']
api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token'])
submissionID = generateGlobusSubmissionID()
if submissionID:
# Prepare transfer object
transferObj = Transfer(submissionID,
config['globus']['source_endpoint_id'],
config['globus']['destination_endpoint_id'],
verify_checksum=True,
preserve_timestamp=True)
queueLength = 0
# Loop over a copy of the list instead of actual thing - other thread will be appending to actual thing
loopingTransfers = safeCopy(pendingTransfers)
currentTransferBatch = {}
logger.debug("- building transfer %s from pending queue" % submissionID)
for ds in loopingTransfers:
if "files" in loopingTransfers[ds]:
if ds not in currentTransferBatch:
currentTransferBatch[ds] = {}
currentTransferBatch[ds]['files'] = {}
# Add files from each dataset
for f in loopingTransfers[ds]['files']:
if queueLength < maxQueue:
fobj = loopingTransfers[ds]['files'][f]
src_path = os.path.join(config['globus']['source_path'], fobj["path"], fobj["name"])
dest_path = os.path.join(config['globus']['destination_path'], fobj["path"], fobj["name"])
transferObj.add_item(src_path, dest_path)
# remainingTransfers will have leftover data once max Globus transfer size is met
queueLength += 1
currentTransferBatch[ds]['files'][f] = fobj
else:
break
# Clean up placeholder entries once queue length is exceeded
if currentTransferBatch[ds]['files'] == {}:
del currentTransferBatch[ds]['files']
if currentTransferBatch[ds] == {}:
del currentTransferBatch[ds]
if queueLength >= maxQueue:
break
if queueLength > 0:
# Send transfer to Globus
try:
logger.debug("- attempting to send new transfer")
status_code, status_message, transfer_data = api.transfer(transferObj)
except (APIError, ClientError) as e:
try:
# Try refreshing endpoints and retrying
activateEndpoints()
api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token'])
status_code, status_message, transfer_data = api.transfer(transferObj)
except (APIError, ClientError) as e:
logger.error("- problem initializing Globus transfer")
status_code = 503
status_message = e
if status_code == 200 or status_code == 202:
# Notify NCSA monitor of new task, and add to activeTasks for logging
globusID = transfer_data['task_id']
logger.info("%s new Globus transfer task started (%s files)" % (globusID, queueLength), extra={
"globus_id": globusID,
"action": "TRANSFER STARTED",
"contents": currentTransferBatch
})
activeTasks[globusID] = {
"globus_id": globusID,
"contents": currentTransferBatch,
"started": str(datetime.datetime.now()),
"status": "IN PROGRESS"
}
writeTasksToDisk(config['active_tasks_path'], activeTasks)
# Now that we've safely sent the pending transfers, remove them
for ds in currentTransferBatch:
if ds in pendingTransfers:
for f in currentTransferBatch[ds]['files']:
if f in pendingTransfers[ds]['files']:
del pendingTransfers[ds]['files'][f]
notifyMonitorOfNewTransfer(globusID, currentTransferBatch)
writeTasksToDisk(config['pending_transfers_path'], pendingTransfers)
else:
# If failed, leave pending list as-is and try again on next iteration (e.g. in 180 seconds)
logger.error("- Globus transfer initialization failed for %s (%s: %s)" % (ds, status_code, status_message))
return
status_numActive = len(activeTasks)
cleanPendingTransfers()
class ClientGlobus():
def __init__(self, auth=['', ''], resource_file_path='', debug=False):
"""
Initialize the Globus session: login with Globus username and password
Credentials can also be passed through a resource json file like this:
{
"globus_username":"",
"globus_password":"",
"myproxy_username": "",
"myproxy_password": "",
"PEM_passphrase": ""
}
:param auth: a list containing username[0], password[1], certfile[2]
and keyfile[3]
:param resource_file_path: path to a json file containing the credentials
:param debug: set True to enable debug messages
"""
if debug:
LOGGER.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
LOGGER.addHandler(ch)
# Read resources from file:
self.globus_init = None
if resource_file_path and os.path.isfile(resource_file_path):
self.globus_init = json.load(open(resource_file_path))
if not auth[0]:
auth[0] = self.globus_init['globus_username']
if not auth[1]:
auth[1] = self.globus_init['globus_password']
if not auth[0] or not auth[1]:
print "Can not init session without a username and password.."
return
self.api = None
self.proxy_name = 'credential-' + auth[0] + '.pem'
try:
result = get_access_token(username=auth[0], password=auth[1])
goauth = result.token
self.api = TransferAPIClient(username=auth[0], goauth=goauth)
self.api.set_debug_print(False, False)
LOGGER.info("Successfully logged in with Globus Online!")
except Exception as e:
print "Globus Online authentication failed: {0}".format(e)
def display_endpoint_list(self):
code, reason, endpoint_list = self.api.endpoint_list(limit=100)
LOGGER.info("Found %d endpoints for user %s:" \
% (endpoint_list["length"], self.api.username))
for ep in endpoint_list["DATA"]:
self._print_endpoint(ep)
def _print_endpoint(self, ep):
name = ep["canonical_name"]
print name
if ep["activated"]:
print " activated (expires: %s)" % ep["expire_time"]
else:
print " not activated"
if ep["public"]:
print " public"
else:
print " not public"
if ep["myproxy_server"]:
print " default myproxy server: %s" % ep["myproxy_server"]
else:
print " no default myproxy server"
servers = ep.get("DATA", ())
print " servers:"
for s in servers:
uri = s["uri"]
if not uri:
uri = "GC endpoint, no uri available"
print " " + uri,
if s["subject"]:
print " (%s)" % s["subject"]
else:
print
def endpoint_activation(self,
endpoint_name,
myproxy_username='',
myproxy_password=''):
"""
Activate a GridFTP endpoint.
Different attempts will be made: autocativation, using myproxy and using
a delegate proxy (in the latter case \'grid-proxy-init\' is needed to
generate a X.509 proxy).
Myproxy username and password will be used to try the authentication
with myproxy: they will be prompted if not passed as parameter or not
found in the resource json file.
The passphrase of the user certificate (needed for generating an X.509
proxy certificate) will be prompted if not found in the resource json
file.
:param endpoint_name: name of the endpoint to activate
:param myproxy_username: myproxy user name (optional, if empty end needed
it will be prompted)
:param myproxy_password: myproxy user password (optional, if empty end
needed it will be prompted)
"""
if self.api is None:
print "You need to be authenticated with Globus Online to perform " \
"this operation. "
return False
LOGGER.debug("Checking if endpoint {0} is already activated..".format(
endpoint_name))
_, _, data = self.api.endpoint(endpoint_name)
if data["activated"]:
LOGGER.info(
"Endpoint {0} is already active!".format(endpoint_name))
return True
# Trying with autoactivation
LOGGER.debug("Trying autoactivation..")
code, message, data = self.api.endpoint_autoactivate(endpoint_name)
if not data["code"].startswith("AutoActivationFailed"):
LOGGER.info("Endpoint {0} activated!".format(endpoint_name))
LOGGER.debug("result: {0} ({1})".format(data["code"],
data["message"]))
return True
LOGGER.debug("Trying activating with myproxy..")
if not myproxy_username and self.globus_init:
myproxy_username = self.globus_init['myproxy_username']
if not myproxy_username and self.globus_init:
myproxy_username = raw_input(
"Please insert your \'myproxy\' username:"******"Timeout must be multiple of poll_interval"
timeout_left = timeout
while no_timeout or timeout_left >= 0:
code, reason, data = self.api.task(task_id, fields="status")
status = data["status"]
LOGGER.debug("Checking task {0} status: {1}".format(
task_id, status))
if status in ("SUCCEEDED", "FAILED"):
return status
if timeout_left > 0 or no_timeout:
time.sleep(poll_interval)
timeout_left -= poll_interval
return None
def display_task_summary(self):
code, reason, data = self.api.tasksummary()
LOGGER.info("Task Summary for %s:" % self.api.username)
for k, v in data.iteritems():
if k == "DATA_TYPE":
continue
LOGGER.info("%3d %s" % (int(v), k.upper().ljust(9)))
def _print_task(self, data, indent_level=0):
indent = " " * indent_level
indent += " " * 2
for k, v in data.iteritems():
if k in ("DATA_TYPE", "LINKS"):
continue
print indent + "%s: %s" % (k, v)
def display_task(self, task_id, show_successful_transfers=True):
"""
Prints the status of a transfer.
:param task_id: id of the transfer
:param show_successful_transfers:
"""
code, reason, data = self.api.task(task_id)
print 'Task {0}'.format(task_id)
self._print_task(data, 0)
if show_successful_transfers:
try:
code, reason, data = self.api.task_successful_transfers(
task_id)
transfer_list = data["DATA"]
print "Successful Transfers (src -> dst)"
for t in transfer_list:
print " %s -> %s" % (t[u'source_path'],
t[u'destination_path'])
except Exception as e:
"Error verifying successful transfer: {0}".format(e)
def display_successful_transfer(self, task_id):
"""
Print successful transfers given the task id
:param task_id:
"""
try:
code, reason, data = self.api.task_successful_transfers(task_id)
transfer_list = data["DATA"]
print "Successful Transfers (src -> dst)"
for t in transfer_list:
print " %s -> %s" % (t[u'source_path'], t[u'destination_path'])
except Exception as e:
"Error verifying successful transfer: {0}".format(e)
def check_proxy(self):
""" Check for a local x509 prox. If not found or expired it creates
a new one."""
grid_proxy_init_options = ' -out ' + self.proxy_name
passphrase = ''
if self.globus_init:
passphrase = self.globus_init['PEM_passphrase']
LOGGER.debug("Checking for a valid proxy")
if os.path.exists(self.proxy_name):
LOGGER.debug(self.proxy_name + ' exists..')
try:
ret_val = os.system('grid-proxy-info -exists -f ' +
self.proxy_name)
if ret_val == 0:
LOGGER.info("Proxy found!")
else:
# Remove interactive message: add arguments to manage the
# request for GRID pass phrase
LOGGER.debug(
"Proxy expired. I will try to create a new one..")
os.system('grid-proxy-init' + grid_proxy_init_options)
except:
print "Proxy invalid. New one, please!"
if passphrase:
os.system('echo ' + passphrase + ' | ' +
'grid-proxy-init' + ' -pwstdin' +
grid_proxy_init_options)
else:
os.system('grid-proxy-init' + grid_proxy_init_options)
else:
LOGGER.debug(self.proxy_name +
" does not exist. I'll try to create it..")
if passphrase:
os.system('echo \'' + passphrase + '\' | ' +
'grid-proxy-init' + ' -pwstdin' +
grid_proxy_init_options)
else:
os.system('grid-proxy-init' + grid_proxy_init_options)
# def get(self):
# pass
def get_endpoint_from_url(self, url):
"""
Read the endpoints.cfg configuration file to discover which endpoint
is associated to the url.
At the moment it is not possible to discover which is the Globus
endpoint that can be used to access a resource. Resolving the PID
it is possible to get the physical URL of the resource (iRODS URL),
but the information about the endpoint is missing. For now a
configuration file can be used to set the association between
iRODS URL and Globus endpoint.
:param url: physical URL of the resource
:return: Globus endpoint name
"""
# get minimal url
o = urlparse(url)
simple_url = o.netloc.rsplit(':')[0]
config = ConfigParser.SafeConfigParser()
config.read(os.path.join(os.getcwd(), "endpoints.cfg"))
res = ''
try:
res = config.get('irods_endpoints', simple_url)
LOGGER.info("Found endpoint {0} associated to URL {1}".format(
res, simple_url))
except ConfigParser.NoOptionError:
print "Endpoint not found for URL {0}. You can add more endpoints " \
"editing the config file 'endpoints.cfg'.".format(
simple_url)
return res
local_ep = options['LOCALENDPOINT']
local_dir = options['LOCALPATH']
remote_ep = options['REMOTEENDPOINT']
remote_dir = options['REMOTEPATH']
oauthtokenfile = options['OAUTHTOKEN']
if os.path.exists(oauthtokenfile):
authfile = open(oauthtokenfile, 'r')
auth = authfile.readlines()
goauth = auth[0].rstrip('\n')
logging.debug('Running gosync.py with the following options:')
logging.debug(options)
api = TransferAPIClient(username,
goauth=goauth,
httplib_debuglevel=1,
max_attempts=5)
logging.debug('Client created:')
logging.debug(api)
logging.debug('Starting initial transfer.')
do_transfer(local_ep, local_dir, remote_ep, remote_dir)
logging.debug('Watching for file system changes.')
event_handler = MyEventHandler(local_ep, local_dir, remote_ep, remote_dir)
observer = Observer()
logging.debug('Created observer to watch {}.'.format(local_dir))
observer.schedule(event_handler, local_dir, recursive=True)
observer.start()
logging.debug('Observer started.')
class GlobusOnlineAPIHelper(GlobusOnlineHelper):
def __init__(self, inst, go_cert_file, go_key_file, go_server_ca):
self.inst = inst
self.go_cert_file = go_cert_file
self.go_key_file = go_key_file
self.go_server_ca = go_server_ca
def connect(self, username):
self.api = TransferAPIClient(username, self.go_server_ca, self.go_cert_file, self.go_key_file)
def endpoint_exists(self, ep):
try:
(code, msg, data) = self.api.endpoint(ep.name)
if code == 200:
return True
elif code == 404:
return False
else:
raise GlobusOnlineException, "Unexpected error %i: %s" % (code, msg)
except ClientError as ce:
if ce.status_code == 404:
return False
else:
raise GlobusOnlineException, "Unexpected GO API exception: %s" % ce
def disconnect(self):
self.api.close()
def endpoint_remove(self, ep):
try:
(code, msg, data) = self.api.endpoint_delete(ep.name)
if code >= 400:
raise GlobusOnlineException, "Unexpected error %i: %s" % (code, msg)
except ClientError as ce:
raise GlobusOnlineException, "Unexpected GO API exception: %s" % ce
def endpoint_gc_create(self, ep, replace):
if self.endpoint_exists(ep):
if not replace:
raise EndpointExistsException, "An endpoint called '%s' already exists. Please choose a different name." % ep.name
else:
self.endpoint_remove(ep)
try:
(code, msg, data) = self.api.endpoint_create(ep.name, None, description="Globus Provision endpoint",
is_globus_connect=True)
if code >= 400:
raise GlobusOnlineException, "Unexpected error %i: %s" % (code, msg)
except ClientError as ce:
raise GlobusOnlineException, "Unexpected GO API exception: %s" % ce
gc_setupkey = data["globus_connect_setup_key"]
return gc_setupkey
def endpoint_gc_create_finalize(self, ep):
gridftp_hostname, gridftp_subject, myproxy_hostname, myproxy_subject = self._get_hostnames_subjects(ep)
# This is a kludge until http://jira.globus.org/browse/KOA-1624 gets fixed
username = self.inst.config.get("ec2-username")
keyfile = os.path.expanduser(self.inst.config.get("ec2-keyfile"))
gc_setupkey = self.inst.topology.get_node_by_id(ep.gridftp[5:]).gc_setupkey
ssh = SSH(username, gridftp_hostname, keyfile, default_outf = None, default_errf = None)
ssh.open()
outf = StringIO.StringIO()
errf = StringIO.StringIO()
rc = ssh.run("grid-cert-info -subject -file /etc/grid-security/gc-cert-%s.pem" % (gc_setupkey), outf=outf, errf=errf, exception_on_error=False)
if rc != 0:
raise GlobusOnlineException, "Could not create endpoint %s" % ep.name
ssh.close()
gridftp_subject = outf.getvalue().strip()
self.endpoint_remove(ep)
self._endpoint_create(ep.name, gridftp_hostname, gridftp_subject, myproxy_hostname, ep.public)
def _endpoint_create(self, ep_name, gridftp_hostname, gridftp_subject, myproxy_hostname, public):
try:
(code, msg, data) = self.api.endpoint_create(ep_name, gridftp_hostname, description="Globus Provision endpoint",
scheme="gsiftp", port=2811, subject=gridftp_subject,
myproxy_server=myproxy_hostname)
if code >= 400:
raise GlobusOnlineException, "Unexpected error %i: %s" % (code, msg)
except ClientError as ce:
raise GlobusOnlineException, "Unexpected GO API exception: %s" % ce
if public:
try:
(code, msg, data) = self.api.endpoint(ep_name)
if code >= 400:
raise GlobusOnlineException, "Unexpected error %i: %s" % (code, msg)
except ClientError as ce:
raise GlobusOnlineException, "Unexpected GO API exception: %s" % ce
data["public"] = True
try:
(code, msg, data) = self.api.endpoint_update(ep_name, data)
if code >= 400:
raise GlobusOnlineException, "Unexpected error %i: %s" % (code, msg)
except ClientError as ce:
raise GlobusOnlineException, "Unexpected GO API exception: %s" % ce
def preferred_activation(username, endpoint_name, myproxy_username):
user_credential_path=os.getcwd()+"/credential-"+username+".pem"
print "==Activating %s ==" % endpoint_name
api = TransferAPIClient(username, cert_file=user_credential_path)
api.set_debug_print(False, False)
try:
code, message, data = api.endpoint(endpoint_name)
if not data["activated"]:
try:
print "==Try autoactivation=="
code, message, data = api.endpoint_autoactivate(endpoint_name)
except:
print "Cannot autoactivate"
except:
pass
try:
code, message, data = api.endpoint(endpoint_name)
except:
data={'activated': "Unavailable"}
if not data["activated"]: # and data["activated"] == "Unavailable":
try:
if myproxy_username != "none":
print "==Try myproxy for %s ==" % myproxy_username
status, message, data = api.endpoint_autoactivate(endpoint_name)
data.set_requirement_value("myproxy", "username", myproxy_username)
from getpass import getpass
passphrase = getpass()
data.set_requirement_value("myproxy", "passphrase", passphrase)
api.endpoint_activate(endpoint_name, data)
#activer=[username,"-c",os.getcwd()+"/credential.pem"]
#api, _ = create_client_from_args(activer)
#conditional_activation(endpoint_name,myproxy_username)
code, message, data = api.endpoint(endpoint_name)
else:
raise
except:
print "==Local proxy activation=="
_, _, reqs = api.endpoint_activation_requirements(endpoint_name, type="delegate_proxy")
#print "endpoint_name",endpoint_name
#print reqs
public_key = reqs.get_requirement_value("delegate_proxy", "public_key")
#print public_key
proxy = x509_proxy.create_proxy_from_file(user_credential_path, public_key)
#print "proxy"
#print proxy
reqs.set_requirement_value("delegate_proxy", "proxy_chain", proxy)
#print reqs
result = api.endpoint_activate(endpoint_name, reqs)
servers = ep["DATA"]
print >> output, " servers:"
for s in servers:
print >> output, " " + s["uri"],
if s["subject"]:
print >> output, " (%s)" % s["subject"]
else:
print >> output, ""
def display_endpoints():
code, reason, endpoints = api.endpoints(limit=100)
print >> output, ("Found %d endpoints for user %s:" %
(endpoints["length"], api.username))
for ep in endpoints["DATA"]:
_print_endpoint(ep)
if __name__ == '__main__':
options, args = process_args()
api = TransferAPIClient(args[0],
server_ca_file=options.server_ca_file,
cert_file=options.cert_file,
key_file=options.key_file,
saml_cookie=options.saml_cookie,
base_url=options.base_url)
output = options.output
display_endpoints()
transfer(options.source_ep, options.source_path, options.destination_ep,
options.destination_path, options.deadline)
def connect(self, username):
self.api = TransferAPIClient(username, self.go_server_ca, self.go_cert_file, self.go_key_file)
class ClientGlobus():
def __init__(self, auth, http_session=None):
"""
Initialize the Globus session: login with username and certificates
:param auth: a list containing username[0], certfile[1] and keyfile[2]
:param http_session:
"""
self.auth = auth
if len(self.auth) < 4: #or !self.auth[0] or !self.auth[2] or !self.auth[3]:
print "Can not init session: some parameters are missing.."
self.auth = auth
self.http_session = http_session
self.myproxy_username = ''
self.api = None
if len(self.auth) > 0:
self.proxy_name = 'credential-' + self.auth[0] + '.pem'
try:
self.api = TransferAPIClient(username=self.auth[0],
cert_file=self.auth[2],
key_file=self.auth[3])
self.api.set_debug_print(False, False)
LOGGER.info("Successfully logged in with Globus online!")
except Exception as e:
raise Exception("GSI authentication failed: {0}".format(e))
def display_endpoint_list(self):
code, reason, endpoint_list = self.api.endpoint_list(limit=100)
LOGGER.info("Found %d endpoints for user %s:" \
% (endpoint_list["length"], self.api.username))
for ep in endpoint_list["DATA"]:
self._print_endpoint(ep)
def _print_endpoint(self, ep):
name = ep["canonical_name"]
print name
if ep["activated"]:
print " activated (expires: %s)" % ep["expire_time"]
else:
print " not activated"
if ep["public"]:
print " public"
else:
print " not public"
if ep["myproxy_server"]:
print " default myproxy server: %s" % ep["myproxy_server"]
else:
print " no default myproxy server"
servers = ep.get("DATA", ())
print " servers:"
for s in servers:
uri = s["uri"]
if not uri:
uri = "GC endpoint, no uri available"
print " " + uri,
if s["subject"]:
print " (%s)" % s["subject"]
else:
print
def endpoint_activation(self, endpoint_name, myproxy_username=''):
"""
Activate a GridFTP endpoint
:param endpoint_name: name of the endpoint to activate
:param myproxy_username: myproxy user name
"""
#code, reason, data = self.api.endpoint(endpoint_name)
#self._print_endpoint(data)
#self.display_endpoint_list();
LOGGER.debug("Checking if endpoint {0} is already activated".format(
endpoint_name))
_, _, data = self.api.endpoint(endpoint_name)
if data["activated"]:
LOGGER.info("Endpoint {0} is already active!".format(endpoint_name))
return True
# Trying with autoactivation
LOGGER.debug("Trying autoactivation")
code, message, data = self.api.endpoint_autoactivate(
endpoint_name)
if not data["code"].startswith("AutoActivationFailed"):
LOGGER.info("Endpoint {0} activated!".format(endpoint_name))
LOGGER.debug("result: {0} ({1})".format(data["code"],
data["message"]))
return True
LOGGER.debug("Trying activating with myproxy")
data.set_requirement_value("myproxy", "username", myproxy_username)
# Remove interactive messages: add arguments to manage this part
from getpass import getpass
passphrase = getpass()
data.set_requirement_value("myproxy", "passphrase", passphrase)
try:
status, message, data = self.api.endpoint_activate(endpoint_name,
data,
if_expires_in=600)
if not data["code"].startswith("AutoActivationFailed"):
LOGGER.info("Endpoint {0} activated!\n".format(endpoint_name))
LOGGER.debug("result: {0} ({1})\n".format(data["code"],
data["message"]))
return True
except Exception as e:
print "Error: {0}".format(e)
# Trying activating a delegate proxy
self.check_proxy()
user_credential_path = os.path.join(os.getcwd(), self.proxy_name)
LOGGER.debug("Trying delegate proxy activation")
_, _, reqs = self.api.endpoint_activation_requirements(
endpoint_name, type="delegate_proxy")
public_key = reqs.get_requirement_value("delegate_proxy", "public_key")
#print public_key
proxy = x509_proxy.create_proxy_from_file(user_credential_path,
public_key, lifetime_hours=3)
#print proxy
reqs.set_requirement_value("delegate_proxy", "proxy_chain", proxy)
try:
code, message, data = self.api.endpoint_activate(endpoint_name,
reqs)
if not data["code"].startswith("AutoActivationFailed"):
LOGGER.info("Endpoint {0} activated!\n".format(endpoint_name))
LOGGER.debug("result: {0} ({1})\n".format(data["code"],
data["message"]))
return True
except Exception as e:
print "Error: {0}".format(e)
print "Can not active the endpoint {0}\n".format(endpoint_name)
if "proxy is not valid until" in str(e):
print "This error may be related to clock time skew. " \
"Please, check if your client clock is server " \
"synchronized and not ahead (you could check with " \
"\"www.time.is\")"
return False
def transfer(self, src_endpoint, dst_endpoint, item, dst_dir):
"""
Transfer a file from one endpoint to another. Return the Globus
task_id.
:param src_endpoint: source endpoint name (i.e. user#endpoint)
:param dst_endpoint: destination endpoint name (i.e. user#endpoint)
:param item: object to be transferred
:param dst_dir: destination directory
:return: Globus task_id
"""
if src_endpoint:
self.endpoint_activation(src_endpoint)
if dst_endpoint:
self.endpoint_activation(dst_endpoint)
# submit a transfer
#oldstdout=sys.stdout
#sys.stdout = open(os.devnull,'w')
code, message, data = self.api.transfer_submission_id()
#sys.stdout = oldstdout # enable output
submission_id = data["value"]
#deadline = datetime.utcnow() + timedelta(minutes=10)
t = Transfer(submission_id, src_endpoint, dst_endpoint)#, deadline)
LOGGER.info("Transferring {0} from endpoint {1} to endpoint {2}".format(
item, src_endpoint, dst_endpoint))
t.add_item(item, os.path.join(dst_dir, os.path.basename(item)))
code, reason, data = self.api.transfer(t)
task_id = data["task_id"]
self.display_task(task_id)
return task_id
def display_tasksummary(self):
code, reason, data = self.api.tasksummary()
LOGGER.info("Task Summary for %s:" % self.api.username)
for k, v in data.iteritems():
if k == "DATA_TYPE":
continue
LOGGER.info("%3d %s" % (int(v), k.upper().ljust(9)))
def _print_task(self, data, indent_level=0):
indent = " " * indent_level
indent += " " * 2
for k, v in data.iteritems():
if k in ("DATA_TYPE", "LINKS"):
continue
LOGGER.info(indent + "%s: %s" % (k, v))
def display_task(self, task_id, show_successful_transfers=True):
code, reason, data = self.api.task(task_id)
LOGGER.info("Task %s:" % task_id)
self._print_task(data, 0)
if show_successful_transfers:
try:
code, reason, data = self.api.task_successful_transfers(
task_id)
transfer_list = data["DATA"]
LOGGER.info("Successful Transfers (src -> dst)")
for t in transfer_list:
LOGGER.info(" %s -> %s" % (t[u'source_path'],
t[u'destination_path']))
except Exception as e:
"Error verifying successful transfer: {0}".format(e)
def check_proxy(self):
""" Check for a local x509 prox. If not found or expired it creates
a new one."""
grid_proxy_init_options = ' -out ' + self.proxy_name
LOGGER.debug("Checking for a valid proxy")
if os.path.exists(self.proxy_name):
LOGGER.debug(self.proxy_name + ' exists..')
try:
ret_val = os.system('grid-proxy-info -exists -f ' + self.proxy_name)
if ret_val == 0:
LOGGER.info("Proxy found!")
else:
# Remove interactive message: add arguments to manage the
# request for GRID pass phrase
LOGGER.debug("Proxy expired. I will try to create a new one..")
os.system('grid-proxy-init' + grid_proxy_init_options)
except:
print "Proxy invalid. New one, please!"
os.system('grid-proxy-init' + grid_proxy_init_options)
else:
LOGGER.debug(self.proxy_name + " does not exist. I'll try to create it..")
os.system('grid-proxy-init' + grid_proxy_init_options)
#def get(self):
# pass
def get_endpoint_from_url(self, url):
"""
Read the endpoints.cfg configuration file to discover which endpoint
is associated to the url.
At the moment it is not possible to discover which is the Globus
endpoint that can be used to access a resource. Resolving the PID
it is possible to get the physical URL of the resource (iRODS URL),
but the information about the endpoint is missing. For now a
configuration file can be used to set the association between
iRODS URL and Globus endpoint.
:param url: physical URL of the resource
:return: Globus endpoint name
"""
# get minimal url
o = urlparse(url)
simple_url = o.netloc.rsplit(':')[0]
config = ConfigParser.SafeConfigParser()
config.read(os.path.join(os.getcwd(), "endpoints.cfg"))
res = ''
try:
res = config.get('irods_endpoints', simple_url)
LOGGER.info("Found endpoint {0} associated to URL {1}".format(res,
simple_url))
except ConfigParser.NoOptionError:
print "Endpoint not found for URL {0}. You can add more endpoints " \
"editing the config file 'endpoints.cfg'.".format(
simple_url)
return res
password=password,
ca_certs=nexus_cert)
if auth_result is not None:
break
except ssl.SSLError, e:
if "timed out" not in str(e):
raise e
time.sleep(30)
except GOCredentialsError, e:
print "Globus Online Username: "******"Globus Online Password: "******"timed out" not in str(e):
class ClientGlobus():
def __init__(self, auth=['', ''], resource_file_path='', debug=False):
"""
Initialize the Globus session: login with Globus username and password
Credentials can also be passed through a resource json file like this:
{
"globus_username":"",
"globus_password":"",
"myproxy_username": "",
"myproxy_password": "",
"PEM_passphrase": ""
}
:param auth: a list containing username[0], password[1], certfile[2]
and keyfile[3]
:param resource_file_path: path to a json file containing the credentials
:param debug: set True to enable debug messages
"""
if debug:
LOGGER.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
LOGGER.addHandler(ch)
# Read resources from file:
self.globus_init = None
if resource_file_path and os.path.isfile(resource_file_path):
self.globus_init = json.load(open(resource_file_path))
if not auth[0]:
auth[0] = self.globus_init['globus_username']
if not auth[1]:
auth[1] = self.globus_init['globus_password']
if not auth[0] or not auth[1]:
print "Can not init session without a username and password.."
return
self.api = None
self.proxy_name = 'credential-' + auth[0] + '.pem'
try:
result = get_access_token(username=auth[0], password=auth[1])
goauth = result.token
self.api = TransferAPIClient(username=auth[0],
goauth=goauth)
self.api.set_debug_print(False, False)
LOGGER.info("Successfully logged in with Globus Online!")
except Exception as e:
print "Globus Online authentication failed: {0}".format(e)
def display_endpoint_list(self):
code, reason, endpoint_list = self.api.endpoint_list(limit=100)
LOGGER.info("Found %d endpoints for user %s:" \
% (endpoint_list["length"], self.api.username))
for ep in endpoint_list["DATA"]:
self._print_endpoint(ep)
def _print_endpoint(self, ep):
name = ep["canonical_name"]
print name
if ep["activated"]:
print " activated (expires: %s)" % ep["expire_time"]
else:
print " not activated"
if ep["public"]:
print " public"
else:
print " not public"
if ep["myproxy_server"]:
print " default myproxy server: %s" % ep["myproxy_server"]
else:
print " no default myproxy server"
servers = ep.get("DATA", ())
print " servers:"
for s in servers:
uri = s["uri"]
if not uri:
uri = "GC endpoint, no uri available"
print " " + uri,
if s["subject"]:
print " (%s)" % s["subject"]
else:
print
def endpoint_activation(self, endpoint_name, myproxy_username='',
myproxy_password=''):
"""
Activate a GridFTP endpoint.
Different attempts will be made: autocativation, using myproxy and using
a delegate proxy (in the latter case \'grid-proxy-init\' is needed to
generate a X.509 proxy).
Myproxy username and password will be used to try the authentication
with myproxy: they will be prompted if not passed as parameter or not
found in the resource json file.
The passphrase of the user certificate (needed for generating an X.509
proxy certificate) will be prompted if not found in the resource json
file.
:param endpoint_name: name of the endpoint to activate
:param myproxy_username: myproxy user name (optional, if empty end needed
it will be prompted)
:param myproxy_password: myproxy user password (optional, if empty end
needed it will be prompted)
"""
if self.api is None:
print "You need to be authenticated with Globus Online to perform " \
"this operation. "
return False
LOGGER.debug("Checking if endpoint {0} is already activated..".format(
endpoint_name))
_, _, data = self.api.endpoint(endpoint_name)
if data["activated"]:
LOGGER.info(
"Endpoint {0} is already active!".format(endpoint_name))
return True
# Trying with autoactivation
LOGGER.debug("Trying autoactivation..")
code, message, data = self.api.endpoint_autoactivate(
endpoint_name)
if not data["code"].startswith("AutoActivationFailed"):
LOGGER.info("Endpoint {0} activated!".format(endpoint_name))
LOGGER.debug("result: {0} ({1})".format(data["code"],
data["message"]))
return True
LOGGER.debug("Trying activating with myproxy..")
if not myproxy_username and self.globus_init:
myproxy_username = self.globus_init['myproxy_username']
if not myproxy_username and self.globus_init:
myproxy_username = raw_input(
"Please insert your \'myproxy\' username:"******"Timeout must be multiple of poll_interval"
timeout_left = timeout
while no_timeout or timeout_left >= 0:
code, reason, data = self.api.task(task_id, fields="status")
status = data["status"]
LOGGER.debug("Checking task {0} status: {1}".format(task_id,
status))
if status in ("SUCCEEDED", "FAILED"):
return status
if timeout_left > 0 or no_timeout:
time.sleep(poll_interval)
timeout_left -= poll_interval
return None
def display_task_summary(self):
code, reason, data = self.api.tasksummary()
LOGGER.info("Task Summary for %s:" % self.api.username)
for k, v in data.iteritems():
if k == "DATA_TYPE":
continue
LOGGER.info("%3d %s" % (int(v), k.upper().ljust(9)))
def _print_task(self, data, indent_level=0):
indent = " " * indent_level
indent += " " * 2
for k, v in data.iteritems():
if k in ("DATA_TYPE", "LINKS"):
continue
print indent + "%s: %s" % (k, v)
def display_task(self, task_id, show_successful_transfers=True):
"""
Prints the status of a transfer.
:param task_id: id of the transfer
:param show_successful_transfers:
"""
code, reason, data = self.api.task(task_id)
print 'Task {0}'.format(task_id)
self._print_task(data, 0)
if show_successful_transfers:
try:
code, reason, data = self.api.task_successful_transfers(
task_id)
transfer_list = data["DATA"]
print "Successful Transfers (src -> dst)"
for t in transfer_list:
print " %s -> %s" % (t[u'source_path'],
t[u'destination_path'])
except Exception as e:
"Error verifying successful transfer: {0}".format(e)
def display_successful_transfer(self, task_id):
"""
Print successful transfers given the task id
:param task_id:
"""
try:
code, reason, data = self.api.task_successful_transfers(
task_id)
transfer_list = data["DATA"]
print "Successful Transfers (src -> dst)"
for t in transfer_list:
print " %s -> %s" % (t[u'source_path'], t[u'destination_path'])
except Exception as e:
"Error verifying successful transfer: {0}".format(e)
def check_proxy(self):
""" Check for a local x509 prox. If not found or expired it creates
a new one."""
grid_proxy_init_options = ' -out ' + self.proxy_name
passphrase = ''
if self.globus_init:
passphrase = self.globus_init['PEM_passphrase']
LOGGER.debug("Checking for a valid proxy")
if os.path.exists(self.proxy_name):
LOGGER.debug(self.proxy_name + ' exists..')
try:
ret_val = os.system(
'grid-proxy-info -exists -f ' + self.proxy_name)
if ret_val == 0:
LOGGER.info("Proxy found!")
else:
# Remove interactive message: add arguments to manage the
# request for GRID pass phrase
LOGGER.debug(
"Proxy expired. I will try to create a new one..")
os.system('grid-proxy-init' + grid_proxy_init_options)
except:
print "Proxy invalid. New one, please!"
if passphrase:
os.system('echo ' + passphrase + ' | ' + 'grid-proxy-init'
+ ' -pwstdin' + grid_proxy_init_options)
else:
os.system('grid-proxy-init' + grid_proxy_init_options)
else:
LOGGER.debug(
self.proxy_name + " does not exist. I'll try to create it..")
if passphrase:
os.system('echo \'' + passphrase + '\' | ' + 'grid-proxy-init'
+ ' -pwstdin' +grid_proxy_init_options)
else:
os.system('grid-proxy-init' + grid_proxy_init_options)
# def get(self):
# pass
def get_endpoint_from_url(self, url):
"""
Read the endpoints.cfg configuration file to discover which endpoint
is associated to the url.
At the moment it is not possible to discover which is the Globus
endpoint that can be used to access a resource. Resolving the PID
it is possible to get the physical URL of the resource (iRODS URL),
but the information about the endpoint is missing. For now a
configuration file can be used to set the association between
iRODS URL and Globus endpoint.
:param url: physical URL of the resource
:return: Globus endpoint name
"""
# get minimal url
o = urlparse(url)
simple_url = o.netloc.rsplit(':')[0]
config = ConfigParser.SafeConfigParser()
config.read(os.path.join(os.getcwd(), "endpoints.cfg"))
res = ''
try:
res = config.get('irods_endpoints', simple_url)
LOGGER.info("Found endpoint {0} associated to URL {1}".format(res,
simple_url))
except ConfigParser.NoOptionError:
print "Endpoint not found for URL {0}. You can add more endpoints " \
"editing the config file 'endpoints.cfg'.".format(
simple_url)
return res
auth_result = get_access_token(username=username,
password=password,
ca_certs=nexus_cert)
if auth_result is not None:
break
except ssl.SSLError, e:
if "timed out" not in str(e):
raise e
time.sleep(30)
except GOCredentialsError, e:
print "Globus Online Username: "******"Globus Online Password: "******"timed out" not in str(e):
raise e
