def generateGlobusSubmissionID(): try: api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) status_code, status_message, submission_id = api.submission_id() except: try: # Try activating endpoints and retrying activateEndpoints() api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) status_code, status_message, submission_id = api.submission_id() except: logger.error( "- exception generating submission ID for Globus transfer") return None if status_code == 200: logger.debug("- generated new Globus submission ID %s" % submission_id['value']) return submission_id['value'] else: logger.error("- could not generate new Globus submission ID (%s: %s)" % (status_code, status_message)) return None
def getGlobusTaskData(task): authToken = config['globus']['auth_token'] if len(task['user']) == 0: guser = config['globus']['username'] else: guser = task['user'] api = TransferAPIClient(username=guser, goauth=authToken) try: logger.debug("%s requesting task data from Globus as %s" % (task['globus_id'], guser)) status_code, status_message, task_data = api.task(task['globus_id']) except: try: # Refreshing auth tokens and retry generateAuthToken() authToken = config['globus']['auth_token'] api = TransferAPIClient(username=guser, goauth=authToken) status_code, status_message, task_data = api.task( task['globus_id']) except: logger.error("%s error checking with Globus for transfer status" % task['globus_id']) status_code = 503 if status_code == 200: return task_data else: return None
def get_api(conf): username = conf.get_go_username() if username is None: print("Globus Id: ", end=' ') username = sys.stdin.readline().strip() atglobusidorg = username.rfind("@globusid.org") if atglobusidorg != -1: username = username[:atglobusidorg] password = conf.get_go_password() if password is None: password = getpass.getpass("Password: "******"Production": globusonline.transfer.api_client.goauth.HOST = \ "nexus.api.%s.globuscs.info" % (go_instance) base_url = \ "https://transfer.%s.api.globusonline.org/%s" \ % (go_instance, globusonline.transfer.api_client.API_VERSION) nexus_cert = os.path.join( os.path.dirname(globus.connect.security.__file__), "lets_encrypt_fullchain.pem") api_cert = nexus_cert globusonline.transfer.api_client.verified_https.match_hostname = \ lambda cert, hostname: True socket.setdefaulttimeout(300) for tries in range(0, 10): try: auth_result = get_access_token(username=username, password=password, ca_certs=nexus_cert) if auth_result is not None: break except ssl.SSLError as e: if "timed out" not in str(e): raise e time.sleep(30) except GOCredentialsError as e: print("Globus Id: ", end=' ') username = sys.stdin.readline().strip() password = getpass.getpass("Globus Password: ") api = TransferAPIClient(username=auth_result.username, goauth=auth_result.token, base_url=base_url, server_ca_file=api_ca, timeout=300.0, max_attempts=10) api.password = password return api
def transfer(options): # Map legacy endpoint names to UUIDs srcendpoint = options.srcendpoint dstendpoint = options.dstendpoint if '#' in srcendpoint: srcendpoint = config['endpoint'][srcendpoint] if '#' in dstendpoint: dstendpoint = config['endpoint'][dstendpoint] # Get access token (This method of getting an acces token is deprecated and should be replaced by OAuth2 calls). auth_result = get_access_token(config['globus']['username'], config['globus']['password']) # Create a transfer submission api_client = TransferAPIClient(config['globus']['username'], goauth=auth_result.token) activate_endpoint(api_client, srcendpoint) activate_endpoint(api_client, dstendpoint) code, message, data = api_client.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(days=10) transfer_task = Transfer(submission_id, srcendpoint, dstendpoint, deadline) # Add srcpath to the transfer task if options.srcpath: transfer_task.add_item(options.srcpath, get_destination_path(options.srcpath, options.dstpath, options.recursive), recursive=options.recursive) # Add srclist to the transfer task if options.srclist: with open(options.srclist) as f: srcpath = f.readline().rstrip('\n') transfer_task.add_item(srcpath, get_destination_path(srcpath, options.dstpath, options.recursive), recursive=options.recursive) # Start the transfer task_id = None try: code, reason, data = api_client.transfer(transfer_task) task_id = data["task_id"] print 'task_id %s' % task_id except Exception as e: msg = "Could not submit the transfer. Error: %s" % str(e) sys.exit(1) # Check a status of the transfer every minute (60 secs) while True: code, reason, data = api_client.task(task_id) if data['status'] == 'SUCCEEDED': print 'progress %d/%d' % (data['files_transferred'], data['files']) return ('success', '') elif data['status'] == 'FAILED': return ('error', data['nice_status_details']) elif data['status'] == 'ACTIVE': print 'progress %d/%d' % (data['files_transferred'], data['files']) time.sleep(10)
def getGlobusTaskData(task): authToken = config['globus']['auth_token'] if len(task['user']) == 0: guser = config['globus']['username'] else: guser = task['user'] api = TransferAPIClient(username=guser, goauth=authToken) try: logger.debug("%s requesting task data from Globus as %s" % (task['globus_id'], guser)) status_code, status_message, task_data = api.task(task['globus_id']) except gaierror as e: logger.error( "%s gaierror checking with Globus for transfer status: %s" % (task['globus_id'], e)) status_code = 404 except Exception as e: if hasattr(e, 'status_code') and e.status_code == 404: return {"status": "NOT FOUND"} try: # Refreshing auth tokens and retry generateAuthToken() authToken = config['globus']['destinations'][end_id]['auth_token'] api = TransferAPIClient(username=guser, goauth=authToken) status_code, status_message, task_data = api.task( task['globus_id']) except gaierror as e: logger.error( "%s gaierror checking with Globus for transfer status: %s" % (task['globus_id'], e)) status_code = 404 except Exception as e: if hasattr(e, 'status_code') and e.status_code == 404: return {"status": "NOT FOUND"} logger.error("%s error checking with Globus for transfer status" % task['globus_id']) status_code = 503 if status_code == 200: return task_data else: return None
def transfer(src_endpoint, src_path, dst_endpoint, dst_path, access_token, openid): (server, username) = decompose_openid(openid) client = TransferAPIClient(username, goauth=access_token) client.connect() globus_submission_id = client.submission_id() transfer = Transfer(globus_submission_id, src_endpoint, dst_endpoint) transfer.add_item(src_path, dst_path, recursive=True) client.transfer(transfer) return {'status': 'Success', 'message': ''}
def remove(request): """ removes files on a remote Globus Online endpoint - API is not complete, so transfer 0 byte files instead of actually deleting anything """ # global so that we can use it in signal handlers global api global task_id # connect to the service api = TransferAPIClient(request["globus_username"], cert_file=request["x509_proxy"]) # make sure we can auto-activate the endpoints ep = activate_ep(api, request["endpoint"]) label = None if "PEGASUS_WF_UUID" in os.environ and "PEGASUS_DAG_JOB_ID" in os.environ: label = os.environ["PEGASUS_WF_UUID"] + " - " + os.environ[ "PEGASUS_DAG_JOB_ID"] # set up a new transfer code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(hours=24) t = Transfer(submission_id, request["endpoint"], request["endpoint"], deadline=deadline, label=label, notify_on_succeeded=False, notify_on_failed=False, notify_on_inactive=False) for f in request["files"]: t.add_item("/dev/null", f) # finalize and submit the transfer code, reason, data = api.transfer(t) task_id = data["task_id"] # how many faults will we accept before giving up? acceptable_faults = min(100, len(request["files"]) * 3) # wait for the task to complete, and see the tasks and # endpoint ls change try: status = wait_for_task(api, task_id, acceptable_faults) except Exception, err: logger.error(err) cancel_task(api, task_id) sys.exit(1)
def submit(request): '''View to submit a Globus transfer request. The access token and files to download are retrieved from the session. ''' openid = request.user.profile.openid() openid_parsed = urlparse(openid) hostname = openid_parsed.hostname # get a username and password if autoactivation failed and a user was asked for a password if hostname == "ceda.ac.uk": myproxy_server = discover_myproxy(openid) esgf_username = request.POST.get(ESGF_USERNAME) else: myproxy_server = hostname esgf_username = os.path.basename(openid_parsed.path) esgf_password = request.POST.get(ESGF_PASSWORD) # retrieve all data transfer request parameters from session username = request.session.get(GLOBUS_USERNAME) access_token = request.session.get(GLOBUS_ACCESS_TOKEN) download_map = request.session.get(GLOBUS_DOWNLOAD_MAP) target_endpoint = request.session.get(TARGET_ENDPOINT) target_folder = request.session.get(TARGET_FOLDER) #print 'Downloading files=%s' % download_map.items() print 'User selected destionation endpoint:%s, folder: %s' % (target_endpoint, target_folder) api_client = TransferAPIClient(username, goauth=access_token) # loop over source endpoints and autoactivate them # if the autoactivation fails, redirect to a form asking for a password activateEndpoint(api_client, target_endpoint) for source_endpoint, source_files in download_map.items(): status, message = activateEndpoint( api_client, source_endpoint, myproxy_server=myproxy_server, username=esgf_username, password=esgf_password) if not status: print hostname return render(request, 'cog/globus/password.html', { 'openid': openid, 'username': hostname=='ceda.ac.uk', 'message': message }) # loop over source endpoints, submit one transfer for each source endpoint task_ids = [] # list of submitted task ids for source_endpoint, source_files in download_map.items(): # submit transfer request task_id = submitTransfer( api_client, source_endpoint, source_files, target_endpoint, target_folder) task_ids.append(task_id) # display confirmation page return render(request, 'cog/globus/confirmation.html', { 'task_ids':task_ids, 'title':'Globus Download Confirmation' })
def main(): # Derive command line options from the method args, making # the endpoint name positional and the rest options. argspec = inspect.getargspec(TransferAPIClient.endpoint_create) fargs = [ arg.replace("_", "-") for arg in argspec.args if arg not in ("self", "endpoint_name") ] parser = OptionParser(usage="Usage: %prog [options] endpoint_name") for i, arg in enumerate(fargs): default = argspec.defaults[i] action = "store" type_ = "string" if arg in "public is-globus-connect".split(): action = "store_true" type_ = None elif arg == "port": type_ = "int" if default not in (None, ""): doc = "[DEFAULT: %s]" % default else: doc = None parser.add_option("--%s" % arg, action=action, type=type_, default=default, help=doc) options, args = parser.parse_args(sys.argv) if len(args) != 2: parser.error("requires one positional argument with endpoint name") # Convert options object into a keyword dict we can pass to the method kw = {} for arg in fargs: name = arg.replace("-", "_") value = getattr(options, name) if value is not None: kw[name] = value # will prompt for username and password auth_result = get_access_token() api = TransferAPIClient(username=auth_result.username, goauth=auth_result.token) _, _, data = api.endpoint_create(args[1], **kw) setup_key = data.get("globus_connect_setup_key") if setup_key: print "GC Setup Key: %s" % setup_key print "Endpoint Name: %s" % data["canonical_name"] print data["message"]
def preferred_activation(username, endpoint_name, myproxy_username): user_credential_path=os.getcwd()+"/credential-"+username+".pem" print "==Activating %s ==" % endpoint_name api = TransferAPIClient(username, cert_file=user_credential_path) api.set_debug_print(False, False) try: code, message, data = api.endpoint(endpoint_name) if not data["activated"]: try: print "==Try autoactivation==" code, message, data = api.endpoint_autoactivate(endpoint_name) except: print "Cannot autoactivate" except: pass try: code, message, data = api.endpoint(endpoint_name) except: data={'activated': "Unavailable"} if not data["activated"]: # and data["activated"] == "Unavailable": try: if myproxy_username != "none": print "==Try myproxy for %s ==" % myproxy_username status, message, data = api.endpoint_autoactivate(endpoint_name) data.set_requirement_value("myproxy", "username", myproxy_username) from getpass import getpass passphrase = getpass() data.set_requirement_value("myproxy", "passphrase", passphrase) api.endpoint_activate(endpoint_name, data) #activer=[username,"-c",os.getcwd()+"/credential.pem"] #api, _ = create_client_from_args(activer) #conditional_activation(endpoint_name,myproxy_username) code, message, data = api.endpoint(endpoint_name) else: raise except: print "==Local proxy activation==" _, _, reqs = api.endpoint_activation_requirements(endpoint_name, type="delegate_proxy") #print "endpoint_name",endpoint_name #print reqs public_key = reqs.get_requirement_value("delegate_proxy", "public_key") #print public_key proxy = x509_proxy.create_proxy_from_file(user_credential_path, public_key) #print "proxy" #print proxy reqs.set_requirement_value("delegate_proxy", "proxy_chain", proxy) #print reqs result = api.endpoint_activate(endpoint_name, reqs)
def getGlobusTaskData(task): authToken = config['globus']['valid_users'][task['user']]['auth_token'] api = TransferAPIClient(username=task['user'], goauth=authToken) try: logger.debug("%s requesting task data from Globus" % task['globus_id']) status_code, status_message, task_data = api.task(task['globus_id']) except (APIError, ClientError) as e: try: # Refreshing auth tokens and retry generateAuthTokens() authToken = config['globus']['valid_users'][ task['user']]['auth_token'] api = TransferAPIClient(username=task['user'], goauth=authToken) status_code, status_message, task_data = api.task( task['globus_id']) except (APIError, ClientError) as e: logger.error("%s error checking with Globus for transfer status" % task['globus_id']) status_code = 503 if status_code == 200: return task_data else: return None
def __init__(self, auth=['', ''], resource_file_path='', debug=False): """ Initialize the Globus session: login with Globus username and password Credentials can also be passed through a resource json file like this: { "globus_username":"", "globus_password":"", "myproxy_username": "", "myproxy_password": "", "PEM_passphrase": "" } :param auth: a list containing username[0], password[1], certfile[2] and keyfile[3] :param resource_file_path: path to a json file containing the credentials :param debug: set True to enable debug messages """ if debug: LOGGER.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) LOGGER.addHandler(ch) # Read resources from file: self.globus_init = None if resource_file_path and os.path.isfile(resource_file_path): self.globus_init = json.load(open(resource_file_path)) if not auth[0]: auth[0] = self.globus_init['globus_username'] if not auth[1]: auth[1] = self.globus_init['globus_password'] if not auth[0] or not auth[1]: print "Can not init session without a username and password.." return self.api = None self.proxy_name = 'credential-' + auth[0] + '.pem' try: result = get_access_token(username=auth[0], password=auth[1]) goauth = result.token self.api = TransferAPIClient(username=auth[0], goauth=goauth) self.api.set_debug_print(False, False) LOGGER.info("Successfully logged in with Globus Online!") except Exception as e: print "Globus Online authentication failed: {0}".format(e)
def activateEndpoints(): src = config['globus']["source_endpoint_id"] dest = config['globus']["destination_endpoint_id"] generateAuthToken() api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) # TODO: Can't use autoactivate; must populate credentials """try: actList = api.endpoint_activation_requirements(src)[2] actList.set_requirement_value('myproxy', 'username', 'data_mover') actList.set_requirement_value('myproxy', 'passphrase', 'terraref2016') actList.set_requirement_value('delegate_proxy', 'proxy_chain', 'some PEM cert w public key') except:""" api.endpoint_autoactivate(src) api.endpoint_autoactivate(dest)
def run(self): self.debug("thread for <%i>: %s" % (self.id, self.task)) from globusonline.transfer.api_client \ import TransferAPIClient token = self.get_token() api = TransferAPIClient(self.user, goauth=token) while True: code, reason, data = api.task(self.task, fields="status") status = data["status"] print(status) if status in ("SUCCEEDED", "FAILED"): break self.debug("Globus: done " + status) self.q.put(status)
def submit(request): '''View to submit a Globus transfer request. The access token and files to download are retrieved from the session. ''' openid = request.user.profile.openid() # get a password if authoactivation failed and a user was asked for a password password = request.POST.get(ESGF_PASSWORD) # retrieve all data transfer request parameters from session username = request.session[GLOBUS_USERNAME] access_token = request.session[GLOBUS_ACCESS_TOKEN] download_map = request.session[GLOBUS_DOWNLOAD_MAP] target_endpoint = request.session[TARGET_ENDPOINT] target_folder = request.session[TARGET_FOLDER] print 'Downloading files=%s' % download_map.items() print 'User selected destionation endpoint:%s, folder: %s' % ( target_endpoint, target_folder) api_client = TransferAPIClient(username, goauth=access_token) # loop over source endpoints and autoactivate them # if the autoactivation fails, redirect to a form asking for a password activateEndpoint(api_client, target_endpoint) for source_endpoint, source_files in download_map.items(): status, message = activateEndpoint(api_client, source_endpoint, openid, password) if not status: return render(request, 'cog/globus/password.html', { 'openid': openid, 'message': message }) # loop over source endpoints, submit one transfer for each source endpoint task_ids = [] # list of submitted task ids for source_endpoint, source_files in download_map.items(): # submit transfer request task_id = submitTransfer(api_client, source_endpoint, source_files, target_endpoint, target_folder) task_ids.append(task_id) # display confirmation page return render(request, 'cog/globus/confirmation.html', { 'task_ids': task_ids, 'title': 'Globus Download Confirmation' })
servers = ep["DATA"] print >> output, " servers:" for s in servers: print >> output, " " + s["uri"], if s["subject"]: print >> output, " (%s)" % s["subject"] else: print >> output, "" def display_endpoints(): code, reason, endpoints = api.endpoints(limit=100) print >> output, ("Found %d endpoints for user %s:" % (endpoints["length"], api.username)) for ep in endpoints["DATA"]: _print_endpoint(ep) if __name__ == '__main__': options, args = process_args() api = TransferAPIClient(args[0], server_ca_file=options.server_ca_file, cert_file=options.cert_file, key_file=options.key_file, saml_cookie=options.saml_cookie, base_url=options.base_url) output = options.output display_endpoints() transfer(options.source_ep, options.source_path, options.destination_ep, options.destination_path, options.deadline)
def mover(username, src_site, dst_site, dst_dir): """ Do a bunch of API calls and display the results. Does a small transfer between tutorial endpoints, but otherwise does not modify user data. Uses module global API client instance. """ global api #activer=[username,"-c",os.getcwd()+"/credential.pem"] #api, _ = create_client_from_args(activer) user_credential_path=os.getcwd()+"/credential-"+username+".pem" #print "user_credential_path=",user_credential_path api = TransferAPIClient(username, cert_file=user_credential_path) api.set_debug_print(False, False) #print " Here (mover): ",api.task_list() # See what is in the account before we make any submissions. time.sleep(0.01) print "" print "" print "" print "=== Before transfer ===" #display_tasksummary(); print #display_task_list(); print #display_endpoint_list(); print print "=== Activate endpoints ===" dest_directory= dst_dir site_ep1 = src_site site_ep2 = dst_site print "Please enter your myproxy username (\'none\' if you do not have one)." myproxy_username = sys.stdin.readline().rstrip() preferred_activation(username, site_ep1, myproxy_username) preferred_activation(username, site_ep2, myproxy_username) print "=== Prepare transfer ===" #raw_input("Press Enter to continue...") # submit a transfer oldstdout=sys.stdout sys.stdout = open(os.devnull,'w') code, message, data = api.transfer_submission_id() sys.stdout = oldstdout # enable output #code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(minutes=10) t = Transfer(submission_id, site_ep1, site_ep2)#, deadline) f=open('json_file','r') json_results=f.read() f.close #print json_results,type(json_results) results=json.loads(json_results) #print results,type(results) #print results[0],type(results[0]) for result in results: #print "Result: ",result if result[-1]=="/": #print "Result: it is a directory" t.add_item(result, dest_directory, recursive=True) else: #print "Result: it is a file" file_name=re.split("/",result)[-1] #print "Result: "+file_name t.add_item(result, dest_directory+"/"+file_name) print "=== Submit transfer ===" oldstdout=sys.stdout sys.stdout = open(os.devnull,'w') code, reason, data = api.transfer(t) sys.stdout = oldstdout # enable output #code, reason, data = api.transfer(t) task_id = data["task_id"] #print " Task ID is %s " % (task_id) # see the new transfer show up #print "=== After submit ===" #display_tasksummary(); print #display_task(task_id); print #raw_input("Press Enter to continue...") # wait for the task to complete, and see the summary and lists # update print "=== Checking completion ===" # To save the task_id for further check could be useful. #wait_for_task(task_id) max_wait = 10*1 waiting = True if waiting: print "Task %s is still under process..." % (task_id) oldstdout=sys.stdout sys.stdout = open(os.devnull,'w') display_tasksummary(); print sys.stdout = oldstdout # enable output #display_task(task_id); print #display_ls("cin0641a#GSI-PLX"); print waiting=wait_for_task(task_id,max_wait) print "=== Exiting ===" #display_tasksummary(); print print "The task ID for this operation is: "+task_id; print oldstdout=sys.stdout sys.stdout = open(os.devnull,'w') status, reason, result = api.task(task_id) sys.stdout = oldstdout # enable output print "Its status is "+result["status"]; print
def transfer(request): """ takes a transfer specification parsed from json: { "globus_username": "******", "x509_proxy": "/tmp/...", "src_endpoint": "rynge#obelix", "dst_endpoint": "rynge#workflow", "files":[ {"src":"/etc/hosts","dst":"/tmp/foobar.txt"}, {"src":"/etc/hosts","dst":"/tmp/foobar-2.txt"} ], } """ # global so that we can use it in signal handlers global api global task_id # connect to the service api = TransferAPIClient(request["globus_username"], cert_file=request["x509_proxy"]) # make sure we can auto-activate the endpoints src_ep = activate_ep(api, request["src_endpoint"]) dst_ep = activate_ep(api, request["dst_endpoint"]) label = None if "PEGASUS_WF_UUID" in os.environ and "PEGASUS_DAG_JOB_ID" in os.environ: label = os.environ["PEGASUS_WF_UUID"] + " - " + os.environ[ "PEGASUS_DAG_JOB_ID"] # set up a new transfer code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(hours=24) t = Transfer(submission_id, request["src_endpoint"], request["dst_endpoint"], deadline=deadline, label=label, notify_on_succeeded=False, notify_on_failed=False, notify_on_inactive=False) for pair in request["files"]: t.add_item(pair["src"], pair["dst"]) # finalize and submit the transfer code, reason, data = api.transfer(t) task_id = data["task_id"] # how many faults will we accept before giving up? acceptable_faults = min(100, len(request["files"]) * 3) # wait for the task to complete, and see the tasks and # endpoint ls change try: status = wait_for_task(api, task_id, acceptable_faults) except Exception, err: logger.error(err) cancel_task(api, task_id) sys.exit(1)
def initializeGlobusTransfer(globus_batch_obj): globus_batch = globus_batch_obj['contents'] globus_batch_id = globus_batch_obj['id'] api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) submissionID = generateGlobusSubmissionID() if submissionID: # Prepare transfer object transferObj = Transfer(submissionID, config['globus']['source_endpoint_id'], config['globus']['destination_endpoint_id'], verify_checksum=True) queue_length = 0 for ds in globus_batch: if 'files' in globus_batch[ds]: for fkey in globus_batch[ds]['files']: fobj = globus_batch[ds]['files'][fkey] if 'src_path' not in fobj: srcpath = fobj['orig_path'] if srcpath.startswith('/LemnaTec'): srcpath = "/gantry_data" + srcpath else: srcpath = fobj['src_path'] transferObj.add_item(srcpath, fobj['path']) queue_length += 1 # Send transfer to Globus try: logger.debug("- attempting to send new transfer") status_code, status_message, transfer_data = api.transfer( transferObj) except: try: # Try refreshing endpoints and retrying activateEndpoints() api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) status_code, status_message, transfer_data = api.transfer( transferObj) except (APIError, ClientError) as e: logger.error("- problem initializing Globus transfer") status_code = 503 status_message = e except: logger.error( "- unexpected problem initializing Globus transfer") status_code = 503 status_message = "" if status_code == 200 or status_code == 202: # Notify NCSA monitor of new task, and add to activeTasks for logging globusID = transfer_data['task_id'] logger.info("%s new Globus transfer task started (%s files)" % (globusID, queue_length), extra={ "globus_id": globusID, "action": "TRANSFER STARTED", "contents": globus_batch }) created_task = { "globus_id": globusID, "contents": globus_batch, "started": str(datetime.datetime.now()), "status": "CREATED" } writeTaskToDatabase(created_task) removePendingTask(globus_batch_id) return True else: # If failed, leave pending list as-is and try again on next iteration (e.g. in 180 seconds) logger.error( "- Globus transfer initialization failed for %s (%s: %s)" % (ds, status_code, status_message)) return False
auth_result = get_access_token(username=username, password=password, ca_certs=nexus_cert) if auth_result is not None: break except ssl.SSLError, e: if "timed out" not in str(e): raise e time.sleep(30) except GOCredentialsError, e: print "Globus Online Username: "******"Globus Online Password: "******"timed out" not in str(e): raise e
local_ep = options['LOCALENDPOINT'] local_dir = options['LOCALPATH'] remote_ep = options['REMOTEENDPOINT'] remote_dir = options['REMOTEPATH'] oauthtokenfile = options['OAUTHTOKEN'] if os.path.exists(oauthtokenfile): authfile = open(oauthtokenfile, 'r') auth = authfile.readlines() goauth = auth[0].rstrip('\n') logging.debug('Running gosync.py with the following options:') logging.debug(options) api = TransferAPIClient(username, goauth=goauth, httplib_debuglevel=1, max_attempts=5) logging.debug('Client created:') logging.debug(api) logging.debug('Starting initial transfer.') do_transfer(local_ep, local_dir, remote_ep, remote_dir) logging.debug('Watching for file system changes.') event_handler = MyEventHandler(local_ep, local_dir, remote_ep, remote_dir) observer = Observer() logging.debug('Created observer to watch {}.'.format(local_dir)) observer.schedule(event_handler, local_dir, recursive=True) observer.start() logging.debug('Observer started.')