def generateGlobusSubmissionID():
try:
api = TransferAPIClient(username=config['globus']['username'],
goauth=config['globus']['auth_token'])
status_code, status_message, submission_id = api.submission_id()
except:
try:
# Try activating endpoints and retrying
activateEndpoints()
api = TransferAPIClient(username=config['globus']['username'],
goauth=config['globus']['auth_token'])
status_code, status_message, submission_id = api.submission_id()
except:
logger.error(
"- exception generating submission ID for Globus transfer")
return None
if status_code == 200:
logger.debug("- generated new Globus submission ID %s" %
submission_id['value'])
return submission_id['value']
else:
logger.error("- could not generate new Globus submission ID (%s: %s)" %
(status_code, status_message))
return None
def getGlobusTaskData(task):
authToken = config['globus']['auth_token']
if len(task['user']) == 0:
guser = config['globus']['username']
else:
guser = task['user']
api = TransferAPIClient(username=guser, goauth=authToken)
try:
logger.debug("%s requesting task data from Globus as %s" %
(task['globus_id'], guser))
status_code, status_message, task_data = api.task(task['globus_id'])
except:
try:
# Refreshing auth tokens and retry
generateAuthToken()
authToken = config['globus']['auth_token']
api = TransferAPIClient(username=guser, goauth=authToken)
status_code, status_message, task_data = api.task(
task['globus_id'])
except:
logger.error("%s error checking with Globus for transfer status" %
task['globus_id'])
status_code = 503
if status_code == 200:
return task_data
else:
return None
def get_api(conf):
username = conf.get_go_username()
if username is None:
print("Globus Id: ", end=' ')
username = sys.stdin.readline().strip()
atglobusidorg = username.rfind("@globusid.org")
if atglobusidorg != -1:
username = username[:atglobusidorg]
password = conf.get_go_password()
if password is None:
password = getpass.getpass("Password: "******"Production":
globusonline.transfer.api_client.goauth.HOST = \
"nexus.api.%s.globuscs.info" % (go_instance)
base_url = \
"https://transfer.%s.api.globusonline.org/%s" \
% (go_instance, globusonline.transfer.api_client.API_VERSION)
nexus_cert = os.path.join(
os.path.dirname(globus.connect.security.__file__),
"lets_encrypt_fullchain.pem")
api_cert = nexus_cert
globusonline.transfer.api_client.verified_https.match_hostname = \
lambda cert, hostname: True
socket.setdefaulttimeout(300)
for tries in range(0, 10):
try:
auth_result = get_access_token(username=username,
password=password,
ca_certs=nexus_cert)
if auth_result is not None:
break
except ssl.SSLError as e:
if "timed out" not in str(e):
raise e
time.sleep(30)
except GOCredentialsError as e:
print("Globus Id: ", end=' ')
username = sys.stdin.readline().strip()
password = getpass.getpass("Globus Password: ")
api = TransferAPIClient(username=auth_result.username,
goauth=auth_result.token,
base_url=base_url,
server_ca_file=api_ca,
timeout=300.0,
max_attempts=10)
api.password = password
return api
def transfer(options):
# Map legacy endpoint names to UUIDs
srcendpoint = options.srcendpoint
dstendpoint = options.dstendpoint
if '#' in srcendpoint:
srcendpoint = config['endpoint'][srcendpoint]
if '#' in dstendpoint:
dstendpoint = config['endpoint'][dstendpoint]
# Get access token (This method of getting an acces token is deprecated and should be replaced by OAuth2 calls).
auth_result = get_access_token(config['globus']['username'], config['globus']['password'])
# Create a transfer submission
api_client = TransferAPIClient(config['globus']['username'], goauth=auth_result.token)
activate_endpoint(api_client, srcendpoint)
activate_endpoint(api_client, dstendpoint)
code, message, data = api_client.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(days=10)
transfer_task = Transfer(submission_id, srcendpoint, dstendpoint, deadline)
# Add srcpath to the transfer task
if options.srcpath:
transfer_task.add_item(options.srcpath,
get_destination_path(options.srcpath, options.dstpath, options.recursive),
recursive=options.recursive)
# Add srclist to the transfer task
if options.srclist:
with open(options.srclist) as f:
srcpath = f.readline().rstrip('\n')
transfer_task.add_item(srcpath,
get_destination_path(srcpath, options.dstpath, options.recursive),
recursive=options.recursive)
# Start the transfer
task_id = None
try:
code, reason, data = api_client.transfer(transfer_task)
task_id = data["task_id"]
print 'task_id %s' % task_id
except Exception as e:
msg = "Could not submit the transfer. Error: %s" % str(e)
sys.exit(1)
# Check a status of the transfer every minute (60 secs)
while True:
code, reason, data = api_client.task(task_id)
if data['status'] == 'SUCCEEDED':
print 'progress %d/%d' % (data['files_transferred'], data['files'])
return ('success', '')
elif data['status'] == 'FAILED':
return ('error', data['nice_status_details'])
elif data['status'] == 'ACTIVE':
print 'progress %d/%d' % (data['files_transferred'], data['files'])
time.sleep(10)
def getGlobusTaskData(task):
authToken = config['globus']['auth_token']
if len(task['user']) == 0:
guser = config['globus']['username']
else:
guser = task['user']
api = TransferAPIClient(username=guser, goauth=authToken)
try:
logger.debug("%s requesting task data from Globus as %s" %
(task['globus_id'], guser))
status_code, status_message, task_data = api.task(task['globus_id'])
except gaierror as e:
logger.error(
"%s gaierror checking with Globus for transfer status: %s" %
(task['globus_id'], e))
status_code = 404
except Exception as e:
if hasattr(e, 'status_code') and e.status_code == 404:
return {"status": "NOT FOUND"}
try:
# Refreshing auth tokens and retry
generateAuthToken()
authToken = config['globus']['destinations'][end_id]['auth_token']
api = TransferAPIClient(username=guser, goauth=authToken)
status_code, status_message, task_data = api.task(
task['globus_id'])
except gaierror as e:
logger.error(
"%s gaierror checking with Globus for transfer status: %s" %
(task['globus_id'], e))
status_code = 404
except Exception as e:
if hasattr(e, 'status_code') and e.status_code == 404:
return {"status": "NOT FOUND"}
logger.error("%s error checking with Globus for transfer status" %
task['globus_id'])
status_code = 503
if status_code == 200:
return task_data
else:
return None
def transfer(src_endpoint, src_path, dst_endpoint, dst_path, access_token,
openid):
(server, username) = decompose_openid(openid)
client = TransferAPIClient(username, goauth=access_token)
client.connect()
globus_submission_id = client.submission_id()
transfer = Transfer(globus_submission_id, src_endpoint, dst_endpoint)
transfer.add_item(src_path, dst_path, recursive=True)
client.transfer(transfer)
return {'status': 'Success', 'message': ''}
def remove(request):
"""
removes files on a remote Globus Online endpoint - API is not complete, so transfer
0 byte files instead of actually deleting anything
"""
# global so that we can use it in signal handlers
global api
global task_id
# connect to the service
api = TransferAPIClient(request["globus_username"],
cert_file=request["x509_proxy"])
# make sure we can auto-activate the endpoints
ep = activate_ep(api, request["endpoint"])
label = None
if "PEGASUS_WF_UUID" in os.environ and "PEGASUS_DAG_JOB_ID" in os.environ:
label = os.environ["PEGASUS_WF_UUID"] + " - " + os.environ[
"PEGASUS_DAG_JOB_ID"]
# set up a new transfer
code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(hours=24)
t = Transfer(submission_id,
request["endpoint"],
request["endpoint"],
deadline=deadline,
label=label,
notify_on_succeeded=False,
notify_on_failed=False,
notify_on_inactive=False)
for f in request["files"]:
t.add_item("/dev/null", f)
# finalize and submit the transfer
code, reason, data = api.transfer(t)
task_id = data["task_id"]
# how many faults will we accept before giving up?
acceptable_faults = min(100, len(request["files"]) * 3)
# wait for the task to complete, and see the tasks and
# endpoint ls change
try:
status = wait_for_task(api, task_id, acceptable_faults)
except Exception, err:
logger.error(err)
cancel_task(api, task_id)
sys.exit(1)
def submit(request):
'''View to submit a Globus transfer request.
The access token and files to download are retrieved from the session. '''
openid = request.user.profile.openid()
openid_parsed = urlparse(openid)
hostname = openid_parsed.hostname
# get a username and password if autoactivation failed and a user was asked for a password
if hostname == "ceda.ac.uk":
myproxy_server = discover_myproxy(openid)
esgf_username = request.POST.get(ESGF_USERNAME)
else:
myproxy_server = hostname
esgf_username = os.path.basename(openid_parsed.path)
esgf_password = request.POST.get(ESGF_PASSWORD)
# retrieve all data transfer request parameters from session
username = request.session.get(GLOBUS_USERNAME)
access_token = request.session.get(GLOBUS_ACCESS_TOKEN)
download_map = request.session.get(GLOBUS_DOWNLOAD_MAP)
target_endpoint = request.session.get(TARGET_ENDPOINT)
target_folder = request.session.get(TARGET_FOLDER)
#print 'Downloading files=%s' % download_map.items()
print 'User selected destionation endpoint:%s, folder: %s' % (target_endpoint, target_folder)
api_client = TransferAPIClient(username, goauth=access_token)
# loop over source endpoints and autoactivate them
# if the autoactivation fails, redirect to a form asking for a password
activateEndpoint(api_client, target_endpoint)
for source_endpoint, source_files in download_map.items():
status, message = activateEndpoint(
api_client, source_endpoint,
myproxy_server=myproxy_server, username=esgf_username, password=esgf_password)
if not status:
print hostname
return render(request,
'cog/globus/password.html',
{ 'openid': openid, 'username': hostname=='ceda.ac.uk', 'message': message })
# loop over source endpoints, submit one transfer for each source endpoint
task_ids = [] # list of submitted task ids
for source_endpoint, source_files in download_map.items():
# submit transfer request
task_id = submitTransfer(
api_client, source_endpoint, source_files,
target_endpoint, target_folder)
task_ids.append(task_id)
# display confirmation page
return render(request,
'cog/globus/confirmation.html',
{ 'task_ids':task_ids, 'title':'Globus Download Confirmation' })
def main():
# Derive command line options from the method args, making
# the endpoint name positional and the rest options.
argspec = inspect.getargspec(TransferAPIClient.endpoint_create)
fargs = [
arg.replace("_", "-") for arg in argspec.args
if arg not in ("self", "endpoint_name")
]
parser = OptionParser(usage="Usage: %prog [options] endpoint_name")
for i, arg in enumerate(fargs):
default = argspec.defaults[i]
action = "store"
type_ = "string"
if arg in "public is-globus-connect".split():
action = "store_true"
type_ = None
elif arg == "port":
type_ = "int"
if default not in (None, ""):
doc = "[DEFAULT: %s]" % default
else:
doc = None
parser.add_option("--%s" % arg,
action=action,
type=type_,
default=default,
help=doc)
options, args = parser.parse_args(sys.argv)
if len(args) != 2:
parser.error("requires one positional argument with endpoint name")
# Convert options object into a keyword dict we can pass to the method
kw = {}
for arg in fargs:
name = arg.replace("-", "_")
value = getattr(options, name)
if value is not None:
kw[name] = value
# will prompt for username and password
auth_result = get_access_token()
api = TransferAPIClient(username=auth_result.username,
goauth=auth_result.token)
_, _, data = api.endpoint_create(args[1], **kw)
setup_key = data.get("globus_connect_setup_key")
if setup_key:
print "GC Setup Key: %s" % setup_key
print "Endpoint Name: %s" % data["canonical_name"]
print data["message"]
def preferred_activation(username, endpoint_name, myproxy_username):
user_credential_path=os.getcwd()+"/credential-"+username+".pem"
print "==Activating %s ==" % endpoint_name
api = TransferAPIClient(username, cert_file=user_credential_path)
api.set_debug_print(False, False)
try:
code, message, data = api.endpoint(endpoint_name)
if not data["activated"]:
try:
print "==Try autoactivation=="
code, message, data = api.endpoint_autoactivate(endpoint_name)
except:
print "Cannot autoactivate"
except:
pass
try:
code, message, data = api.endpoint(endpoint_name)
except:
data={'activated': "Unavailable"}
if not data["activated"]: # and data["activated"] == "Unavailable":
try:
if myproxy_username != "none":
print "==Try myproxy for %s ==" % myproxy_username
status, message, data = api.endpoint_autoactivate(endpoint_name)
data.set_requirement_value("myproxy", "username", myproxy_username)
from getpass import getpass
passphrase = getpass()
data.set_requirement_value("myproxy", "passphrase", passphrase)
api.endpoint_activate(endpoint_name, data)
#activer=[username,"-c",os.getcwd()+"/credential.pem"]
#api, _ = create_client_from_args(activer)
#conditional_activation(endpoint_name,myproxy_username)
code, message, data = api.endpoint(endpoint_name)
else:
raise
except:
print "==Local proxy activation=="
_, _, reqs = api.endpoint_activation_requirements(endpoint_name, type="delegate_proxy")
#print "endpoint_name",endpoint_name
#print reqs
public_key = reqs.get_requirement_value("delegate_proxy", "public_key")
#print public_key
proxy = x509_proxy.create_proxy_from_file(user_credential_path, public_key)
#print "proxy"
#print proxy
reqs.set_requirement_value("delegate_proxy", "proxy_chain", proxy)
#print reqs
result = api.endpoint_activate(endpoint_name, reqs)
def getGlobusTaskData(task):
authToken = config['globus']['valid_users'][task['user']]['auth_token']
api = TransferAPIClient(username=task['user'], goauth=authToken)
try:
logger.debug("%s requesting task data from Globus" % task['globus_id'])
status_code, status_message, task_data = api.task(task['globus_id'])
except (APIError, ClientError) as e:
try:
# Refreshing auth tokens and retry
generateAuthTokens()
authToken = config['globus']['valid_users'][
task['user']]['auth_token']
api = TransferAPIClient(username=task['user'], goauth=authToken)
status_code, status_message, task_data = api.task(
task['globus_id'])
except (APIError, ClientError) as e:
logger.error("%s error checking with Globus for transfer status" %
task['globus_id'])
status_code = 503
if status_code == 200:
return task_data
else:
return None
def __init__(self, auth=['', ''], resource_file_path='', debug=False):
"""
Initialize the Globus session: login with Globus username and password
Credentials can also be passed through a resource json file like this:
{
"globus_username":"",
"globus_password":"",
"myproxy_username": "",
"myproxy_password": "",
"PEM_passphrase": ""
}
:param auth: a list containing username[0], password[1], certfile[2]
and keyfile[3]
:param resource_file_path: path to a json file containing the credentials
:param debug: set True to enable debug messages
"""
if debug:
LOGGER.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
LOGGER.addHandler(ch)
# Read resources from file:
self.globus_init = None
if resource_file_path and os.path.isfile(resource_file_path):
self.globus_init = json.load(open(resource_file_path))
if not auth[0]:
auth[0] = self.globus_init['globus_username']
if not auth[1]:
auth[1] = self.globus_init['globus_password']
if not auth[0] or not auth[1]:
print "Can not init session without a username and password.."
return
self.api = None
self.proxy_name = 'credential-' + auth[0] + '.pem'
try:
result = get_access_token(username=auth[0], password=auth[1])
goauth = result.token
self.api = TransferAPIClient(username=auth[0], goauth=goauth)
self.api.set_debug_print(False, False)
LOGGER.info("Successfully logged in with Globus Online!")
except Exception as e:
print "Globus Online authentication failed: {0}".format(e)
def activateEndpoints():
src = config['globus']["source_endpoint_id"]
dest = config['globus']["destination_endpoint_id"]
generateAuthToken()
api = TransferAPIClient(username=config['globus']['username'],
goauth=config['globus']['auth_token'])
# TODO: Can't use autoactivate; must populate credentials
"""try:
actList = api.endpoint_activation_requirements(src)[2]
actList.set_requirement_value('myproxy', 'username', 'data_mover')
actList.set_requirement_value('myproxy', 'passphrase', 'terraref2016')
actList.set_requirement_value('delegate_proxy', 'proxy_chain', 'some PEM cert w public key')
except:"""
api.endpoint_autoactivate(src)
api.endpoint_autoactivate(dest)
def run(self):
self.debug("thread for <%i>: %s" % (self.id, self.task))
from globusonline.transfer.api_client \
import TransferAPIClient
token = self.get_token()
api = TransferAPIClient(self.user, goauth=token)
while True:
code, reason, data = api.task(self.task, fields="status")
status = data["status"]
print(status)
if status in ("SUCCEEDED", "FAILED"):
break
self.debug("Globus: done " + status)
self.q.put(status)
def submit(request):
'''View to submit a Globus transfer request.
The access token and files to download are retrieved from the session. '''
openid = request.user.profile.openid()
# get a password if authoactivation failed and a user was asked for a password
password = request.POST.get(ESGF_PASSWORD)
# retrieve all data transfer request parameters from session
username = request.session[GLOBUS_USERNAME]
access_token = request.session[GLOBUS_ACCESS_TOKEN]
download_map = request.session[GLOBUS_DOWNLOAD_MAP]
target_endpoint = request.session[TARGET_ENDPOINT]
target_folder = request.session[TARGET_FOLDER]
print 'Downloading files=%s' % download_map.items()
print 'User selected destionation endpoint:%s, folder: %s' % (
target_endpoint, target_folder)
api_client = TransferAPIClient(username, goauth=access_token)
# loop over source endpoints and autoactivate them
# if the autoactivation fails, redirect to a form asking for a password
activateEndpoint(api_client, target_endpoint)
for source_endpoint, source_files in download_map.items():
status, message = activateEndpoint(api_client, source_endpoint, openid,
password)
if not status:
return render(request, 'cog/globus/password.html', {
'openid': openid,
'message': message
})
# loop over source endpoints, submit one transfer for each source endpoint
task_ids = [] # list of submitted task ids
for source_endpoint, source_files in download_map.items():
# submit transfer request
task_id = submitTransfer(api_client, source_endpoint, source_files,
target_endpoint, target_folder)
task_ids.append(task_id)
# display confirmation page
return render(request, 'cog/globus/confirmation.html', {
'task_ids': task_ids,
'title': 'Globus Download Confirmation'
})
servers = ep["DATA"]
print >> output, " servers:"
for s in servers:
print >> output, " " + s["uri"],
if s["subject"]:
print >> output, " (%s)" % s["subject"]
else:
print >> output, ""
def display_endpoints():
code, reason, endpoints = api.endpoints(limit=100)
print >> output, ("Found %d endpoints for user %s:" %
(endpoints["length"], api.username))
for ep in endpoints["DATA"]:
_print_endpoint(ep)
if __name__ == '__main__':
options, args = process_args()
api = TransferAPIClient(args[0],
server_ca_file=options.server_ca_file,
cert_file=options.cert_file,
key_file=options.key_file,
saml_cookie=options.saml_cookie,
base_url=options.base_url)
output = options.output
display_endpoints()
transfer(options.source_ep, options.source_path, options.destination_ep,
options.destination_path, options.deadline)
def mover(username, src_site, dst_site, dst_dir):
"""
Do a bunch of API calls and display the results. Does a small transfer
between tutorial endpoints, but otherwise does not modify user data.
Uses module global API client instance.
"""
global api
#activer=[username,"-c",os.getcwd()+"/credential.pem"]
#api, _ = create_client_from_args(activer)
user_credential_path=os.getcwd()+"/credential-"+username+".pem"
#print "user_credential_path=",user_credential_path
api = TransferAPIClient(username, cert_file=user_credential_path)
api.set_debug_print(False, False)
#print " Here (mover): ",api.task_list()
# See what is in the account before we make any submissions.
time.sleep(0.01)
print ""
print ""
print ""
print "=== Before transfer ==="
#display_tasksummary(); print
#display_task_list(); print
#display_endpoint_list(); print
print "=== Activate endpoints ==="
dest_directory= dst_dir
site_ep1 = src_site
site_ep2 = dst_site
print "Please enter your myproxy username (\'none\' if you do not have one)."
myproxy_username = sys.stdin.readline().rstrip()
preferred_activation(username, site_ep1, myproxy_username)
preferred_activation(username, site_ep2, myproxy_username)
print "=== Prepare transfer ==="
#raw_input("Press Enter to continue...")
# submit a transfer
oldstdout=sys.stdout
sys.stdout = open(os.devnull,'w')
code, message, data = api.transfer_submission_id()
sys.stdout = oldstdout # enable output
#code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(minutes=10)
t = Transfer(submission_id, site_ep1, site_ep2)#, deadline)
f=open('json_file','r')
json_results=f.read()
f.close
#print json_results,type(json_results)
results=json.loads(json_results)
#print results,type(results)
#print results[0],type(results[0])
for result in results:
#print "Result: ",result
if result[-1]=="/":
#print "Result: it is a directory"
t.add_item(result, dest_directory, recursive=True)
else:
#print "Result: it is a file"
file_name=re.split("/",result)[-1]
#print "Result: "+file_name
t.add_item(result, dest_directory+"/"+file_name)
print "=== Submit transfer ==="
oldstdout=sys.stdout
sys.stdout = open(os.devnull,'w')
code, reason, data = api.transfer(t)
sys.stdout = oldstdout # enable output
#code, reason, data = api.transfer(t)
task_id = data["task_id"]
#print " Task ID is %s " % (task_id)
# see the new transfer show up
#print "=== After submit ==="
#display_tasksummary(); print
#display_task(task_id); print
#raw_input("Press Enter to continue...")
# wait for the task to complete, and see the summary and lists
# update
print "=== Checking completion ==="
# To save the task_id for further check could be useful.
#wait_for_task(task_id)
max_wait = 10*1
waiting = True
if waiting:
print "Task %s is still under process..." % (task_id)
oldstdout=sys.stdout
sys.stdout = open(os.devnull,'w')
display_tasksummary(); print
sys.stdout = oldstdout # enable output
#display_task(task_id); print
#display_ls("cin0641a#GSI-PLX"); print
waiting=wait_for_task(task_id,max_wait)
print "=== Exiting ==="
#display_tasksummary(); print
print "The task ID for this operation is: "+task_id; print
oldstdout=sys.stdout
sys.stdout = open(os.devnull,'w')
status, reason, result = api.task(task_id)
sys.stdout = oldstdout # enable output
print "Its status is "+result["status"]; print
def transfer(request):
"""
takes a transfer specification parsed from json:
{
"globus_username": "******",
"x509_proxy": "/tmp/...",
"src_endpoint": "rynge#obelix",
"dst_endpoint": "rynge#workflow",
"files":[
{"src":"/etc/hosts","dst":"/tmp/foobar.txt"},
{"src":"/etc/hosts","dst":"/tmp/foobar-2.txt"}
],
}
"""
# global so that we can use it in signal handlers
global api
global task_id
# connect to the service
api = TransferAPIClient(request["globus_username"],
cert_file=request["x509_proxy"])
# make sure we can auto-activate the endpoints
src_ep = activate_ep(api, request["src_endpoint"])
dst_ep = activate_ep(api, request["dst_endpoint"])
label = None
if "PEGASUS_WF_UUID" in os.environ and "PEGASUS_DAG_JOB_ID" in os.environ:
label = os.environ["PEGASUS_WF_UUID"] + " - " + os.environ[
"PEGASUS_DAG_JOB_ID"]
# set up a new transfer
code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(hours=24)
t = Transfer(submission_id,
request["src_endpoint"],
request["dst_endpoint"],
deadline=deadline,
label=label,
notify_on_succeeded=False,
notify_on_failed=False,
notify_on_inactive=False)
for pair in request["files"]:
t.add_item(pair["src"], pair["dst"])
# finalize and submit the transfer
code, reason, data = api.transfer(t)
task_id = data["task_id"]
# how many faults will we accept before giving up?
acceptable_faults = min(100, len(request["files"]) * 3)
# wait for the task to complete, and see the tasks and
# endpoint ls change
try:
status = wait_for_task(api, task_id, acceptable_faults)
except Exception, err:
logger.error(err)
cancel_task(api, task_id)
sys.exit(1)
def initializeGlobusTransfer(globus_batch_obj):
globus_batch = globus_batch_obj['contents']
globus_batch_id = globus_batch_obj['id']
api = TransferAPIClient(username=config['globus']['username'],
goauth=config['globus']['auth_token'])
submissionID = generateGlobusSubmissionID()
if submissionID:
# Prepare transfer object
transferObj = Transfer(submissionID,
config['globus']['source_endpoint_id'],
config['globus']['destination_endpoint_id'],
verify_checksum=True)
queue_length = 0
for ds in globus_batch:
if 'files' in globus_batch[ds]:
for fkey in globus_batch[ds]['files']:
fobj = globus_batch[ds]['files'][fkey]
if 'src_path' not in fobj:
srcpath = fobj['orig_path']
if srcpath.startswith('/LemnaTec'):
srcpath = "/gantry_data" + srcpath
else:
srcpath = fobj['src_path']
transferObj.add_item(srcpath, fobj['path'])
queue_length += 1
# Send transfer to Globus
try:
logger.debug("- attempting to send new transfer")
status_code, status_message, transfer_data = api.transfer(
transferObj)
except:
try:
# Try refreshing endpoints and retrying
activateEndpoints()
api = TransferAPIClient(username=config['globus']['username'],
goauth=config['globus']['auth_token'])
status_code, status_message, transfer_data = api.transfer(
transferObj)
except (APIError, ClientError) as e:
logger.error("- problem initializing Globus transfer")
status_code = 503
status_message = e
except:
logger.error(
"- unexpected problem initializing Globus transfer")
status_code = 503
status_message = ""
if status_code == 200 or status_code == 202:
# Notify NCSA monitor of new task, and add to activeTasks for logging
globusID = transfer_data['task_id']
logger.info("%s new Globus transfer task started (%s files)" %
(globusID, queue_length),
extra={
"globus_id": globusID,
"action": "TRANSFER STARTED",
"contents": globus_batch
})
created_task = {
"globus_id": globusID,
"contents": globus_batch,
"started": str(datetime.datetime.now()),
"status": "CREATED"
}
writeTaskToDatabase(created_task)
removePendingTask(globus_batch_id)
return True
else:
# If failed, leave pending list as-is and try again on next iteration (e.g. in 180 seconds)
logger.error(
"- Globus transfer initialization failed for %s (%s: %s)" %
(ds, status_code, status_message))
return False
auth_result = get_access_token(username=username,
password=password,
ca_certs=nexus_cert)
if auth_result is not None:
break
except ssl.SSLError, e:
if "timed out" not in str(e):
raise e
time.sleep(30)
except GOCredentialsError, e:
print "Globus Online Username: "******"Globus Online Password: "******"timed out" not in str(e):
raise e
local_ep = options['LOCALENDPOINT']
local_dir = options['LOCALPATH']
remote_ep = options['REMOTEENDPOINT']
remote_dir = options['REMOTEPATH']
oauthtokenfile = options['OAUTHTOKEN']
if os.path.exists(oauthtokenfile):
authfile = open(oauthtokenfile, 'r')
auth = authfile.readlines()
goauth = auth[0].rstrip('\n')
logging.debug('Running gosync.py with the following options:')
logging.debug(options)
api = TransferAPIClient(username,
goauth=goauth,
httplib_debuglevel=1,
max_attempts=5)
logging.debug('Client created:')
logging.debug(api)
logging.debug('Starting initial transfer.')
do_transfer(local_ep, local_dir, remote_ep, remote_dir)
logging.debug('Watching for file system changes.')
event_handler = MyEventHandler(local_ep, local_dir, remote_ep, remote_dir)
observer = Observer()
logging.debug('Created observer to watch {}.'.format(local_dir))
observer.schedule(event_handler, local_dir, recursive=True)
observer.start()
logging.debug('Observer started.')
