def get_api(conf): username = conf.get_go_username() if username is None: print("Globus Username: "******"Globus Password: "******"Test": globusonline.transfer.api_client.goauth.HOST = \ "graph.api.test.globuscs.info" nexus_cert = os.path.join( os.path.dirname( globus.connect.security.__file__), "graph.api.test.globuscs.info.pem") api_ca = os.path.join( os.path.dirname( globus.connect.security.__file__), "go-ca-cert.pem") globusonline.transfer.api_client.verified_https.match_hostname = \ lambda cert, hostname: True base_url = "https://transfer.test.api.globusonline.org/" + globusonline.transfer.api_client.API_VERSION socket.setdefaulttimeout(300) for tries in range(0,10): try: auth_result = get_access_token( username=username, password=password, ca_certs=nexus_cert) if auth_result is not None: break except ssl.SSLError as e: if "timed out" not in str(e): raise e time.sleep(30) except GOCredentialsError as e: print("Globus Username: "******"Globus Password: ") api = TransferAPIClient( username=auth_result.username, goauth=auth_result.token, base_url=base_url, server_ca_file=api_ca, max_attempts=10) api.password = password return api
def get_api(conf): username = conf.get_go_username() if username is None: print("Globus Id: ", end=' ') username = sys.stdin.readline().strip() atglobusidorg = username.rfind("@globusid.org") if atglobusidorg != -1: username = username[:atglobusidorg] password = conf.get_go_password() if password is None: password = getpass.getpass("Password: "******"Production": globusonline.transfer.api_client.goauth.HOST = \ "nexus.api.%s.globuscs.info" % (go_instance) base_url = \ "https://transfer.%s.api.globusonline.org/%s" \ % (go_instance, globusonline.transfer.api_client.API_VERSION) nexus_cert = os.path.join( os.path.dirname(globus.connect.security.__file__), "lets_encrypt_fullchain.pem") api_cert = nexus_cert globusonline.transfer.api_client.verified_https.match_hostname = \ lambda cert, hostname: True socket.setdefaulttimeout(300) for tries in range(0, 10): try: auth_result = get_access_token(username=username, password=password, ca_certs=nexus_cert) if auth_result is not None: break except ssl.SSLError as e: if "timed out" not in str(e): raise e time.sleep(30) except GOCredentialsError as e: print("Globus Id: ", end=' ') username = sys.stdin.readline().strip() password = getpass.getpass("Globus Password: ") api = TransferAPIClient(username=auth_result.username, goauth=auth_result.token, base_url=base_url, server_ca_file=api_ca, timeout=300.0, max_attempts=10) api.password = password return api
def main(): """ Main function to test the library """ pass username = '******' cert_file = '/home/fiameni/Documents/Personal/reserved/certificates/infn/expires_2015/usercert.pem' key_file = '/home/fiameni/Documents/Personal/reserved/certificates/infn/expires_2015/userkey.pem' api = TransferAPIClient(username, cert_file, key_file) print api.endpoint_list()
def remove(request): """ removes files on a remote Globus Online endpoint - API is not complete, so transfer 0 byte files instead of actually deleting anything """ # global so that we can use it in signal handlers global api global task_id # connect to the service api = TransferAPIClient(request["globus_username"], cert_file=request["x509_proxy"]) # make sure we can auto-activate the endpoints ep = activate_ep(api, request["endpoint"]) label = None if "PEGASUS_WF_UUID" in os.environ and "PEGASUS_DAG_JOB_ID" in os.environ: label = os.environ["PEGASUS_WF_UUID"] + " - " + os.environ[ "PEGASUS_DAG_JOB_ID"] # set up a new transfer code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(hours=24) t = Transfer(submission_id, request["endpoint"], request["endpoint"], deadline=deadline, label=label, notify_on_succeeded=False, notify_on_failed=False, notify_on_inactive=False) for f in request["files"]: t.add_item("/dev/null", f) # finalize and submit the transfer code, reason, data = api.transfer(t) task_id = data["task_id"] # how many faults will we accept before giving up? acceptable_faults = min(100, len(request["files"]) * 3) # wait for the task to complete, and see the tasks and # endpoint ls change try: status = wait_for_task(api, task_id, acceptable_faults) except Exception, err: logger.error(err) cancel_task(api, task_id) sys.exit(1)
def main(): # Derive command line options from the method args, making # the endpoint name positional and the rest options. argspec = inspect.getargspec(TransferAPIClient.endpoint_create) fargs = [ arg.replace("_", "-") for arg in argspec.args if arg not in ("self", "endpoint_name") ] parser = OptionParser(usage="Usage: %prog [options] endpoint_name") for i, arg in enumerate(fargs): default = argspec.defaults[i] action = "store" type_ = "string" if arg in "public is-globus-connect".split(): action = "store_true" type_ = None elif arg == "port": type_ = "int" if default not in (None, ""): doc = "[DEFAULT: %s]" % default else: doc = None parser.add_option("--%s" % arg, action=action, type=type_, default=default, help=doc) options, args = parser.parse_args(sys.argv) if len(args) != 2: parser.error("requires one positional argument with endpoint name") # Convert options object into a keyword dict we can pass to the method kw = {} for arg in fargs: name = arg.replace("-", "_") value = getattr(options, name) if value is not None: kw[name] = value # will prompt for username and password auth_result = get_access_token() api = TransferAPIClient(username=auth_result.username, goauth=auth_result.token) _, _, data = api.endpoint_create(args[1], **kw) setup_key = data.get("globus_connect_setup_key") if setup_key: print "GC Setup Key: %s" % setup_key print "Endpoint Name: %s" % data["canonical_name"] print data["message"]
def activateEndpoints(): src = config['globus']["source_endpoint_id"] dest = config['globus']["destination_endpoint_id"] generateAuthToken() api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) # TODO: Can't use autoactivate; must populate credentials """try: actList = api.endpoint_activation_requirements(src)[2] actList.set_requirement_value('myproxy', 'username', 'data_mover') actList.set_requirement_value('myproxy', 'passphrase', 'terraref2016') actList.set_requirement_value('delegate_proxy', 'proxy_chain', 'some PEM cert w public key') except:""" api.endpoint_autoactivate(src) api.endpoint_autoactivate(dest)
def __init__(self, auth=['', ''], resource_file_path='', debug=False): """ Initialize the Globus session: login with Globus username and password Credentials can also be passed through a resource json file like this: { "globus_username":"", "globus_password":"", "myproxy_username": "", "myproxy_password": "", "PEM_passphrase": "" } :param auth: a list containing username[0], password[1], certfile[2] and keyfile[3] :param resource_file_path: path to a json file containing the credentials :param debug: set True to enable debug messages """ if debug: LOGGER.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) LOGGER.addHandler(ch) # Read resources from file: self.globus_init = None if resource_file_path and os.path.isfile(resource_file_path): self.globus_init = json.load(open(resource_file_path)) if not auth[0]: auth[0] = self.globus_init['globus_username'] if not auth[1]: auth[1] = self.globus_init['globus_password'] if not auth[0] or not auth[1]: print "Can not init session without a username and password.." return self.api = None self.proxy_name = 'credential-' + auth[0] + '.pem' try: result = get_access_token(username=auth[0], password=auth[1]) goauth = result.token self.api = TransferAPIClient(username=auth[0], goauth=goauth) self.api.set_debug_print(False, False) LOGGER.info("Successfully logged in with Globus Online!") except Exception as e: print "Globus Online authentication failed: {0}".format(e)
def __init__(self, auth, http_session=None): """ Initialize the Globus session: login with username and certificates :param auth: a list containing username[0], certfile[1] and keyfile[2] :param http_session: """ self.auth = auth if len(self.auth) < 4: #or !self.auth[0] or !self.auth[2] or !self.auth[3]: print "Can not init session: some parameters are missing.." self.auth = auth self.http_session = http_session self.myproxy_username = '' self.api = None if len(self.auth) > 0: self.proxy_name = 'credential-' + self.auth[0] + '.pem' try: self.api = TransferAPIClient(username=self.auth[0], cert_file=self.auth[2], key_file=self.auth[3]) self.api.set_debug_print(False, False) LOGGER.info("Successfully logged in with Globus online!") except Exception as e: raise Exception("GSI authentication failed: {0}".format(e))
def run(self): self.debug("thread for <%i>: %s" % (self.id, self.task)) from globusonline.transfer.api_client \ import TransferAPIClient token = self.get_token() api = TransferAPIClient(self.user, goauth=token) while True: code, reason, data = api.task(self.task, fields="status") status = data["status"] print(status) if status in ("SUCCEEDED", "FAILED"): break self.debug("Globus: done " + status) self.q.put(status)
def main(): # Derive command line options from the method args, making # the endpoint name positional and the rest options. argspec = inspect.getargspec(TransferAPIClient.endpoint_create) fargs = [arg.replace("_", "-") for arg in argspec.args if arg not in ("self", "endpoint_name")] parser = OptionParser(usage="Usage: %prog [options] endpoint_name") for i, arg in enumerate(fargs): default = argspec.defaults[i] action = "store" type_ = "string" if arg in "public is-globus-connect".split(): action = "store_true" type_ = None elif arg == "port": type_ = "int" if default not in (None, ""): doc = "[DEFAULT: %s]" % default else: doc = None parser.add_option("--%s" % arg, action=action, type=type_, default=default, help=doc) options, args = parser.parse_args(sys.argv) if len(args) != 2: parser.error("requires one positional argument with endpoint name") # Convert options object into a keyword dict we can pass to the method kw = {} for arg in fargs: name = arg.replace("-", "_") value = getattr(options, name) if value is not None: kw[name] = value # will prompt for username and password auth_result = get_access_token() api = TransferAPIClient(username=auth_result.username, goauth=auth_result.token) _, _, data = api.endpoint_create(args[1], **kw) setup_key = data.get("globus_connect_setup_key") if setup_key: print "GC Setup Key: %s" % setup_key print "Endpoint Name: %s" % data["canonical_name"] print data["message"]
def transfer(options): # Map legacy endpoint names to UUIDs srcendpoint = options.srcendpoint dstendpoint = options.dstendpoint if '#' in srcendpoint: srcendpoint = config['endpoint'][srcendpoint] if '#' in dstendpoint: dstendpoint = config['endpoint'][dstendpoint] # Get access token (This method of getting an acces token is deprecated and should be replaced by OAuth2 calls). auth_result = get_access_token(config['globus']['username'], config['globus']['password']) # Create a transfer submission api_client = TransferAPIClient(config['globus']['username'], goauth=auth_result.token) activate_endpoint(api_client, srcendpoint) activate_endpoint(api_client, dstendpoint) code, message, data = api_client.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(days=10) transfer_task = Transfer(submission_id, srcendpoint, dstendpoint, deadline) # Add srcpath to the transfer task if options.srcpath: transfer_task.add_item(options.srcpath, get_destination_path(options.srcpath, options.dstpath, options.recursive), recursive=options.recursive) # Add srclist to the transfer task if options.srclist: with open(options.srclist) as f: srcpath = f.readline().rstrip('\n') transfer_task.add_item(srcpath, get_destination_path(srcpath, options.dstpath, options.recursive), recursive=options.recursive) # Start the transfer task_id = None try: code, reason, data = api_client.transfer(transfer_task) task_id = data["task_id"] print 'task_id %s' % task_id except Exception as e: msg = "Could not submit the transfer. Error: %s" % str(e) sys.exit(1) # Check a status of the transfer every minute (60 secs) while True: code, reason, data = api_client.task(task_id) if data['status'] == 'SUCCEEDED': print 'progress %d/%d' % (data['files_transferred'], data['files']) return ('success', '') elif data['status'] == 'FAILED': return ('error', data['nice_status_details']) elif data['status'] == 'ACTIVE': print 'progress %d/%d' % (data['files_transferred'], data['files']) time.sleep(10)
def generateGlobusSubmissionID(): try: api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) status_code, status_message, submission_id = api.submission_id() except: try: # Try activating endpoints and retrying activateEndpoints() api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) status_code, status_message, submission_id = api.submission_id() except: logger.error("- exception generating submission ID for Globus transfer") return None if status_code == 200: logger.debug("- generated new Globus submission ID %s" % submission_id['value']) return submission_id['value'] else: logger.error("- could not generate new Globus submission ID (%s: %s)" % (status_code, status_message)) return None
def submit(request): '''View to submit a Globus transfer request. The access token and files to download are retrieved from the session. ''' openid = request.user.profile.openid() openid_parsed = urlparse(openid) hostname = openid_parsed.hostname # get a username and password if autoactivation failed and a user was asked for a password if hostname == "ceda.ac.uk": myproxy_server = discover_myproxy(openid) esgf_username = request.POST.get(ESGF_USERNAME) else: myproxy_server = hostname esgf_username = os.path.basename(openid_parsed.path) esgf_password = request.POST.get(ESGF_PASSWORD) # retrieve all data transfer request parameters from session username = request.session.get(GLOBUS_USERNAME) access_token = request.session.get(GLOBUS_ACCESS_TOKEN) download_map = request.session.get(GLOBUS_DOWNLOAD_MAP) target_endpoint = request.session.get(TARGET_ENDPOINT) target_folder = request.session.get(TARGET_FOLDER) #print 'Downloading files=%s' % download_map.items() print 'User selected destionation endpoint:%s, folder: %s' % (target_endpoint, target_folder) api_client = TransferAPIClient(username, goauth=access_token) # loop over source endpoints and autoactivate them # if the autoactivation fails, redirect to a form asking for a password activateEndpoint(api_client, target_endpoint) for source_endpoint, source_files in download_map.items(): status, message = activateEndpoint( api_client, source_endpoint, myproxy_server=myproxy_server, username=esgf_username, password=esgf_password) if not status: print hostname return render(request, 'cog/globus/password.html', { 'openid': openid, 'username': hostname=='ceda.ac.uk', 'message': message }) # loop over source endpoints, submit one transfer for each source endpoint task_ids = [] # list of submitted task ids for source_endpoint, source_files in download_map.items(): # submit transfer request task_id = submitTransfer( api_client, source_endpoint, source_files, target_endpoint, target_folder) task_ids.append(task_id) # display confirmation page return render(request, 'cog/globus/confirmation.html', { 'task_ids':task_ids, 'title':'Globus Download Confirmation' })
def getGlobusTaskData(task): authToken = config['globus']['valid_users'][task['user']]['auth_token'] api = TransferAPIClient(username=task['user'], goauth=authToken) try: logger.debug("%s requesting task data from Globus" % task['globus_id']) status_code, status_message, task_data = api.task(task['globus_id']) except (APIError, ClientError) as e: try: # Refreshing auth tokens and retry generateAuthTokens() authToken = config['globus']['valid_users'][task['user']]['auth_token'] api = TransferAPIClient(username=task['user'], goauth=authToken) status_code, status_message, task_data = api.task(task['globus_id']) except (APIError, ClientError) as e: logger.error("%s error checking with Globus for transfer status" % task['globus_id']) status_code = 503 if status_code == 200: return task_data else: return None
def transfer(src_endpoint, src_path, dst_endpoint, dst_path, access_token, openid): (server, username) = decompose_openid(openid) client = TransferAPIClient(username, goauth=access_token) client.connect() globus_submission_id = client.submission_id() transfer = Transfer(globus_submission_id, src_endpoint, dst_endpoint) transfer.add_item(src_path, dst_path, recursive=True) client.transfer(transfer) return {'status': 'Success', 'message': ''}
def generateGlobusSubmissionID(): try: api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) status_code, status_message, submission_id = api.submission_id() except: try: # Try activating endpoints and retrying activateEndpoints() api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) status_code, status_message, submission_id = api.submission_id() except: logger.error( "- exception generating submission ID for Globus transfer") return None if status_code == 200: logger.debug("- generated new Globus submission ID %s" % submission_id['value']) return submission_id['value'] else: logger.error("- could not generate new Globus submission ID (%s: %s)" % (status_code, status_message)) return None
def getGlobusTaskData(task): authToken = config['globus']['auth_token'] if len(task['user']) == 0: guser = config['globus']['username'] else: guser = task['user'] api = TransferAPIClient(username=guser, goauth=authToken) try: logger.debug("%s requesting task data from Globus as %s" % (task['globus_id'], guser)) status_code, status_message, task_data = api.task(task['globus_id']) except: try: # Refreshing auth tokens and retry generateAuthToken() authToken = config['globus']['auth_token'] api = TransferAPIClient(username=guser, goauth=authToken) status_code, status_message, task_data = api.task( task['globus_id']) except: logger.error("%s error checking with Globus for transfer status" % task['globus_id']) status_code = 503 if status_code == 200: return task_data else: return None
def submitTransfer(openid, username, access_token, source_endpoint, source_files, target_endpoint, target_directory): ''' Method to submit a data transfer request to Globus. ''' # instantiate Globus Transfer API client api_client = TransferAPIClient(username, goauth=access_token) # must activate the endpoints using cached credentials activateEndpoint(api_client, source_endpoint, openid) activateEndpoint(api_client, target_endpoint) # obtain a submission id from Globus code, message, data = api_client.transfer_submission_id() submission_id = data["value"] print "Obtained transfer submission id: %s" % submission_id # maximum time for completing the transfer deadline = datetime.utcnow() + timedelta(days=10) # create a transfer request transfer_task = Transfer(submission_id, source_endpoint, target_endpoint, deadline) for source_file in source_files: source_directory, filename = os.path.split(source_file) target_file = os.path.join(target_directory, filename) transfer_task.add_item(source_file, target_file) # submit the transfer request try: code, reason, data = api_client.transfer(transfer_task) task_id = data["task_id"] print "Submitted transfer task with id: %s" % task_id except Exception as e: print "Could not submit the transfer. Error: %s" % str(e) task_id = "Could not submit the transfer. Please contact the ESGF node admin to investigate the issue." return task_id
def submit(request): '''View to submit a Globus transfer request. The access token and files to download are retrieved from the session. ''' openid = request.user.profile.openid() # get a password if authoactivation failed and a user was asked for a password password = request.POST.get(ESGF_PASSWORD) # retrieve all data transfer request parameters from session username = request.session[GLOBUS_USERNAME] access_token = request.session[GLOBUS_ACCESS_TOKEN] download_map = request.session[GLOBUS_DOWNLOAD_MAP] target_endpoint = request.session[TARGET_ENDPOINT] target_folder = request.session[TARGET_FOLDER] print 'Downloading files=%s' % download_map.items() print 'User selected destionation endpoint:%s, folder: %s' % ( target_endpoint, target_folder) api_client = TransferAPIClient(username, goauth=access_token) # loop over source endpoints and autoactivate them # if the autoactivation fails, redirect to a form asking for a password activateEndpoint(api_client, target_endpoint) for source_endpoint, source_files in download_map.items(): status, message = activateEndpoint(api_client, source_endpoint, openid, password) if not status: return render(request, 'cog/globus/password.html', { 'openid': openid, 'message': message }) # loop over source endpoints, submit one transfer for each source endpoint task_ids = [] # list of submitted task ids for source_endpoint, source_files in download_map.items(): # submit transfer request task_id = submitTransfer(api_client, source_endpoint, source_files, target_endpoint, target_folder) task_ids.append(task_id) # display confirmation page return render(request, 'cog/globus/confirmation.html', { 'task_ids': task_ids, 'title': 'Globus Download Confirmation' })
def getGlobusTaskData(task): authToken = config['globus']['auth_token'] if len(task['user']) == 0: guser = config['globus']['username'] else: guser = task['user'] api = TransferAPIClient(username=guser, goauth=authToken) try: logger.debug("%s requesting task data from Globus as %s" % (task['globus_id'], guser)) status_code, status_message, task_data = api.task(task['globus_id']) except gaierror as e: logger.error( "%s gaierror checking with Globus for transfer status: %s" % (task['globus_id'], e)) status_code = 404 except Exception as e: if hasattr(e, 'status_code') and e.status_code == 404: return {"status": "NOT FOUND"} try: # Refreshing auth tokens and retry generateAuthToken() authToken = config['globus']['destinations'][end_id]['auth_token'] api = TransferAPIClient(username=guser, goauth=authToken) status_code, status_message, task_data = api.task( task['globus_id']) except gaierror as e: logger.error( "%s gaierror checking with Globus for transfer status: %s" % (task['globus_id'], e)) status_code = 404 except Exception as e: if hasattr(e, 'status_code') and e.status_code == 404: return {"status": "NOT FOUND"} logger.error("%s error checking with Globus for transfer status" % task['globus_id']) status_code = 503 if status_code == 200: return task_data else: return None
def getGlobusTaskData(task): authToken = config['globus']['valid_users'][task['user']]['auth_token'] api = TransferAPIClient(username=task['user'], goauth=authToken) try: logger.debug("%s requesting task data from Globus" % task['globus_id']) status_code, status_message, task_data = api.task(task['globus_id']) except (APIError, ClientError) as e: try: # Refreshing auth tokens and retry generateAuthTokens() authToken = config['globus']['valid_users'][ task['user']]['auth_token'] api = TransferAPIClient(username=task['user'], goauth=authToken) status_code, status_message, task_data = api.task( task['globus_id']) except (APIError, ClientError) as e: logger.error("%s error checking with Globus for transfer status" % task['globus_id']) status_code = 503 if status_code == 200: return task_data else: return None
def initializeGlobusTransfer(globus_batch_obj): globus_batch = globus_batch_obj['contents'] globus_batch_id = globus_batch_obj['id'] api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) submissionID = generateGlobusSubmissionID() if submissionID: # Prepare transfer object transferObj = Transfer(submissionID, config['globus']['source_endpoint_id'], config['globus']['destination_endpoint_id'], verify_checksum=True) queue_length = 0 for ds in globus_batch: if 'files' in globus_batch[ds]: for fkey in globus_batch[ds]['files']: fobj = globus_batch[ds]['files'][fkey] if 'src_path' not in fobj: srcpath = fobj['orig_path'] if srcpath.startswith('/LemnaTec'): srcpath = "/gantry_data" + srcpath else: srcpath = fobj['src_path'] transferObj.add_item(srcpath, fobj['path']) queue_length += 1 # Send transfer to Globus try: logger.debug("- attempting to send new transfer") status_code, status_message, transfer_data = api.transfer( transferObj) except: try: # Try refreshing endpoints and retrying activateEndpoints() api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) status_code, status_message, transfer_data = api.transfer( transferObj) except (APIError, ClientError) as e: logger.error("- problem initializing Globus transfer") status_code = 503 status_message = e except: logger.error( "- unexpected problem initializing Globus transfer") status_code = 503 status_message = "" if status_code == 200 or status_code == 202: # Notify NCSA monitor of new task, and add to activeTasks for logging globusID = transfer_data['task_id'] logger.info("%s new Globus transfer task started (%s files)" % (globusID, queue_length), extra={ "globus_id": globusID, "action": "TRANSFER STARTED", "contents": globus_batch }) created_task = { "globus_id": globusID, "contents": globus_batch, "started": str(datetime.datetime.now()), "status": "CREATED" } writeTaskToDatabase(created_task) removePendingTask(globus_batch_id) return True else: # If failed, leave pending list as-is and try again on next iteration (e.g. in 180 seconds) logger.error( "- Globus transfer initialization failed for %s (%s: %s)" % (ds, status_code, status_message)) return False
def transfer(request): """ takes a transfer specification parsed from json: { "globus_username": "******", "x509_proxy": "/tmp/...", "src_endpoint": "rynge#obelix", "dst_endpoint": "rynge#workflow", "files":[ {"src":"/etc/hosts","dst":"/tmp/foobar.txt"}, {"src":"/etc/hosts","dst":"/tmp/foobar-2.txt"} ], } """ # global so that we can use it in signal handlers global api global task_id # connect to the service api = TransferAPIClient(request["globus_username"], cert_file=request["x509_proxy"]) # make sure we can auto-activate the endpoints src_ep = activate_ep(api, request["src_endpoint"]) dst_ep = activate_ep(api, request["dst_endpoint"]) label = None if "PEGASUS_WF_UUID" in os.environ and "PEGASUS_DAG_JOB_ID" in os.environ: label = os.environ["PEGASUS_WF_UUID"] + " - " + os.environ[ "PEGASUS_DAG_JOB_ID"] # set up a new transfer code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(hours=24) t = Transfer(submission_id, request["src_endpoint"], request["dst_endpoint"], deadline=deadline, label=label, notify_on_succeeded=False, notify_on_failed=False, notify_on_inactive=False) for pair in request["files"]: t.add_item(pair["src"], pair["dst"]) # finalize and submit the transfer code, reason, data = api.transfer(t) task_id = data["task_id"] # how many faults will we accept before giving up? acceptable_faults = min(100, len(request["files"]) * 3) # wait for the task to complete, and see the tasks and # endpoint ls change try: status = wait_for_task(api, task_id, acceptable_faults) except Exception, err: logger.error(err) cancel_task(api, task_id) sys.exit(1)
def initializeGlobusTransfer(): global pendingTransfers global activeTasks global status_numActive global status_numPending maxQueue = config['globus']['max_transfer_file_count'] api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) submissionID = generateGlobusSubmissionID() if submissionID: # Prepare transfer object transferObj = Transfer(submissionID, config['globus']['source_endpoint_id'], config['globus']['destination_endpoint_id'], verify_checksum=True) queueLength = 0 # Loop over a copy of the list instead of actual thing - other thread will be appending to actual thing loopingTransfers = safeCopy(pendingTransfers) currentTransferBatch = {} logger.debug("- building transfer %s from pending queue" % submissionID) for ds in loopingTransfers: if "files" in loopingTransfers[ds]: if ds not in currentTransferBatch: currentTransferBatch[ds] = {} currentTransferBatch[ds]['files'] = {} # Add files from each dataset for f in loopingTransfers[ds]['files']: if queueLength < maxQueue: fobj = loopingTransfers[ds]['files'][f] if fobj["path"].find(config['globus']['source_path']) > -1: src_path = os.path.join(fobj["path"], fobj["name"]) dest_path = os.path.join(config['globus']['destination_path'], fobj["path"].replace(config['globus']['source_path'], ""), fobj["name"]) else: src_path = os.path.join(config['globus']['source_path'], fobj["path"], fobj["name"]) dest_path = os.path.join(config['globus']['destination_path'], fobj["path"], fobj["name"]) # Clean up dest path to new folder structure # ua-mac/raw_data/LemnaTec/EnvironmentLogger/2016-01-01/2016-08-03_04-05-34_environmentlogger.json # ua-mac/raw_data/LemnaTec/MovingSensor/co2Sensor/2016-01-01/2016-08-02__09-42-51-195/file.json # ua-mac/raw_data/LemnaTec/3DScannerRawDataTopTmp/scanner3DTop/2016-01-01/2016-08-02__09-42-51-195/file.json # ua-mac/raw_data/MAC/lightning/2016-01-01/weather_2016_06_29.dat dest_path = dest_path.replace("LemnaTec/", "") dest_path = dest_path.replace("MovingSensor/", "") dest_path = dest_path.replace("MAC/", "") dest_path = dest_path.replace("3DScannerRawDataTopTmp/", "") dest_path = dest_path.replace("3DScannerRawDataLowerOnEastSideTmp/", "") dest_path = dest_path.replace("3DScannerRawDataLowerOnWestSideTmp/", "") # /ua-mac/raw_data/MovingSensor.reproc2016-8-18/scanner3DTop/2016-08-22/2016-08-22__15-13-01-672/6af8d63b-b5bb-49b2-8e0e-c26e719f5d72__Top-heading-east_0.png # /ua-mac/raw_data/MovingSensor.reproc2016-8-18/scanner3DTop/2016-08-22/2016-08-22__15-13-01-672/6af8d63b-b5bb-49b2-8e0e-c26e719f5d72__Top-heading-east_0.ply if dest_path.endswith(".ply") and (dest_path.find("scanner3DTop") or dest_path.find("scanner3DLowerOnEastSide") or dest_path.find("scanner3DLowerOnWestSide")) > -1: dest_path = dest_path.replace("raw_data", "Level_1") if dest_path.find("MovingSensor.reproc") > -1: new_dest_path = "" dirs = dest_path.split("/") for dir_part in dirs: if dir_part.find("MovingSensor.reproc") == -1: new_dest_path = os.path.join(new_dest_path, dir_part) dest_path = new_dest_path # danforth/raw_data/sorghum_pilot_dataset/snapshot123456/file.png transferObj.add_item(src_path, dest_path) # remainingTransfers will have leftover data once max Globus transfer size is met queueLength += 1 fobj["orig_path"] = fobj["path"] fobj["path"] = dest_path currentTransferBatch[ds]['files'][f] = fobj else: break # Clean up placeholder entries once queue length is exceeded if currentTransferBatch[ds]['files'] == {}: del currentTransferBatch[ds]['files'] if currentTransferBatch[ds] == {}: del currentTransferBatch[ds] if queueLength >= maxQueue: break if queueLength > 0: # Send transfer to Globus try: logger.debug("- attempting to send new transfer") status_code, status_message, transfer_data = api.transfer(transferObj) except (APIError, ClientError) as e: try: # Try refreshing endpoints and retrying activateEndpoints() api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) status_code, status_message, transfer_data = api.transfer(transferObj) except (APIError, ClientError) as e: logger.error("- problem initializing Globus transfer") status_code = 503 status_message = e except: logger.error("- unexpected problem initializing Globus transfer") status_code = 503 status_message = e if status_code == 200 or status_code == 202: # Notify NCSA monitor of new task, and add to activeTasks for logging globusID = transfer_data['task_id'] logger.info("%s new Globus transfer task started (%s files)" % (globusID, queueLength), extra={ "globus_id": globusID, "action": "TRANSFER STARTED", "contents": currentTransferBatch }) activeTasks[globusID] = { "globus_id": globusID, "contents": currentTransferBatch, "started": str(datetime.datetime.now()), "status": "IN PROGRESS" } writeTasksToDisk(os.path.join(config['log_path'], "active_tasks.json"), activeTasks) # Now that we've safely sent the pending transfers, remove them for ds in currentTransferBatch: if ds in pendingTransfers: for f in currentTransferBatch[ds]['files']: if f in pendingTransfers[ds]['files']: del pendingTransfers[ds]['files'][f] notifyMonitorOfNewTransfer(globusID, currentTransferBatch) writeTasksToDisk(os.path.join(config['log_path'], "pending_transfers.json"), pendingTransfers) else: # If failed, leave pending list as-is and try again on next iteration (e.g. in 180 seconds) logger.error("- Globus transfer initialization failed for %s (%s: %s)" % (ds, status_code, status_message)) return status_numActive = len(activeTasks) cleanPendingTransfers()
def mover(username, src_site, dst_site, dst_dir): """ Do a bunch of API calls and display the results. Does a small transfer between tutorial endpoints, but otherwise does not modify user data. Uses module global API client instance. """ global api #activer=[username,"-c",os.getcwd()+"/credential.pem"] #api, _ = create_client_from_args(activer) user_credential_path=os.getcwd()+"/credential-"+username+".pem" #print "user_credential_path=",user_credential_path api = TransferAPIClient(username, cert_file=user_credential_path) api.set_debug_print(False, False) #print " Here (mover): ",api.task_list() # See what is in the account before we make any submissions. time.sleep(0.01) print "" print "" print "" print "=== Before transfer ===" #display_tasksummary(); print #display_task_list(); print #display_endpoint_list(); print print "=== Activate endpoints ===" dest_directory= dst_dir site_ep1 = src_site site_ep2 = dst_site print "Please enter your myproxy username (\'none\' if you do not have one)." myproxy_username = sys.stdin.readline().rstrip() preferred_activation(username, site_ep1, myproxy_username) preferred_activation(username, site_ep2, myproxy_username) print "=== Prepare transfer ===" #raw_input("Press Enter to continue...") # submit a transfer oldstdout=sys.stdout sys.stdout = open(os.devnull,'w') code, message, data = api.transfer_submission_id() sys.stdout = oldstdout # enable output #code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(minutes=10) t = Transfer(submission_id, site_ep1, site_ep2)#, deadline) f=open('json_file','r') json_results=f.read() f.close #print json_results,type(json_results) results=json.loads(json_results) #print results,type(results) #print results[0],type(results[0]) for result in results: #print "Result: ",result if result[-1]=="/": #print "Result: it is a directory" t.add_item(result, dest_directory, recursive=True) else: #print "Result: it is a file" file_name=re.split("/",result)[-1] #print "Result: "+file_name t.add_item(result, dest_directory+"/"+file_name) print "=== Submit transfer ===" oldstdout=sys.stdout sys.stdout = open(os.devnull,'w') code, reason, data = api.transfer(t) sys.stdout = oldstdout # enable output #code, reason, data = api.transfer(t) task_id = data["task_id"] #print " Task ID is %s " % (task_id) # see the new transfer show up #print "=== After submit ===" #display_tasksummary(); print #display_task(task_id); print #raw_input("Press Enter to continue...") # wait for the task to complete, and see the summary and lists # update print "=== Checking completion ===" # To save the task_id for further check could be useful. #wait_for_task(task_id) max_wait = 10*1 waiting = True if waiting: print "Task %s is still under process..." % (task_id) oldstdout=sys.stdout sys.stdout = open(os.devnull,'w') display_tasksummary(); print sys.stdout = oldstdout # enable output #display_task(task_id); print #display_ls("cin0641a#GSI-PLX"); print waiting=wait_for_task(task_id,max_wait) print "=== Exiting ===" #display_tasksummary(); print print "The task ID for this operation is: "+task_id; print oldstdout=sys.stdout sys.stdout = open(os.devnull,'w') status, reason, result = api.task(task_id) sys.stdout = oldstdout # enable output print "Its status is "+result["status"]; print
def initializeGlobusTransfer(): global pendingTransfers global activeTasks global status_numActive global status_numPending maxQueue = config['globus']['max_transfer_file_count'] api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) submissionID = generateGlobusSubmissionID() if submissionID: # Prepare transfer object transferObj = Transfer(submissionID, config['globus']['source_endpoint_id'], config['globus']['destination_endpoint_id'], verify_checksum=True, preserve_timestamp=True) queueLength = 0 # Loop over a copy of the list instead of actual thing - other thread will be appending to actual thing loopingTransfers = safeCopy(pendingTransfers) currentTransferBatch = {} logger.debug("- building transfer %s from pending queue" % submissionID) for ds in loopingTransfers: if "files" in loopingTransfers[ds]: if ds not in currentTransferBatch: currentTransferBatch[ds] = {} currentTransferBatch[ds]['files'] = {} # Add files from each dataset for f in loopingTransfers[ds]['files']: if queueLength < maxQueue: fobj = loopingTransfers[ds]['files'][f] src_path = os.path.join(config['globus']['source_path'], fobj["path"], fobj["name"]) dest_path = os.path.join(config['globus']['destination_path'], fobj["path"], fobj["name"]) transferObj.add_item(src_path, dest_path) # remainingTransfers will have leftover data once max Globus transfer size is met queueLength += 1 currentTransferBatch[ds]['files'][f] = fobj else: break # Clean up placeholder entries once queue length is exceeded if currentTransferBatch[ds]['files'] == {}: del currentTransferBatch[ds]['files'] if currentTransferBatch[ds] == {}: del currentTransferBatch[ds] if queueLength >= maxQueue: break if queueLength > 0: # Send transfer to Globus try: logger.debug("- attempting to send new transfer") status_code, status_message, transfer_data = api.transfer(transferObj) except (APIError, ClientError) as e: try: # Try refreshing endpoints and retrying activateEndpoints() api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) status_code, status_message, transfer_data = api.transfer(transferObj) except (APIError, ClientError) as e: logger.error("- problem initializing Globus transfer") status_code = 503 status_message = e if status_code == 200 or status_code == 202: # Notify NCSA monitor of new task, and add to activeTasks for logging globusID = transfer_data['task_id'] logger.info("%s new Globus transfer task started (%s files)" % (globusID, queueLength), extra={ "globus_id": globusID, "action": "TRANSFER STARTED", "contents": currentTransferBatch }) activeTasks[globusID] = { "globus_id": globusID, "contents": currentTransferBatch, "started": str(datetime.datetime.now()), "status": "IN PROGRESS" } writeTasksToDisk(config['active_tasks_path'], activeTasks) # Now that we've safely sent the pending transfers, remove them for ds in currentTransferBatch: if ds in pendingTransfers: for f in currentTransferBatch[ds]['files']: if f in pendingTransfers[ds]['files']: del pendingTransfers[ds]['files'][f] notifyMonitorOfNewTransfer(globusID, currentTransferBatch) writeTasksToDisk(config['pending_transfers_path'], pendingTransfers) else: # If failed, leave pending list as-is and try again on next iteration (e.g. in 180 seconds) logger.error("- Globus transfer initialization failed for %s (%s: %s)" % (ds, status_code, status_message)) return status_numActive = len(activeTasks) cleanPendingTransfers()
class ClientGlobus(): def __init__(self, auth=['', ''], resource_file_path='', debug=False): """ Initialize the Globus session: login with Globus username and password Credentials can also be passed through a resource json file like this: { "globus_username":"", "globus_password":"", "myproxy_username": "", "myproxy_password": "", "PEM_passphrase": "" } :param auth: a list containing username[0], password[1], certfile[2] and keyfile[3] :param resource_file_path: path to a json file containing the credentials :param debug: set True to enable debug messages """ if debug: LOGGER.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) LOGGER.addHandler(ch) # Read resources from file: self.globus_init = None if resource_file_path and os.path.isfile(resource_file_path): self.globus_init = json.load(open(resource_file_path)) if not auth[0]: auth[0] = self.globus_init['globus_username'] if not auth[1]: auth[1] = self.globus_init['globus_password'] if not auth[0] or not auth[1]: print "Can not init session without a username and password.." return self.api = None self.proxy_name = 'credential-' + auth[0] + '.pem' try: result = get_access_token(username=auth[0], password=auth[1]) goauth = result.token self.api = TransferAPIClient(username=auth[0], goauth=goauth) self.api.set_debug_print(False, False) LOGGER.info("Successfully logged in with Globus Online!") except Exception as e: print "Globus Online authentication failed: {0}".format(e) def display_endpoint_list(self): code, reason, endpoint_list = self.api.endpoint_list(limit=100) LOGGER.info("Found %d endpoints for user %s:" \ % (endpoint_list["length"], self.api.username)) for ep in endpoint_list["DATA"]: self._print_endpoint(ep) def _print_endpoint(self, ep): name = ep["canonical_name"] print name if ep["activated"]: print " activated (expires: %s)" % ep["expire_time"] else: print " not activated" if ep["public"]: print " public" else: print " not public" if ep["myproxy_server"]: print " default myproxy server: %s" % ep["myproxy_server"] else: print " no default myproxy server" servers = ep.get("DATA", ()) print " servers:" for s in servers: uri = s["uri"] if not uri: uri = "GC endpoint, no uri available" print " " + uri, if s["subject"]: print " (%s)" % s["subject"] else: print def endpoint_activation(self, endpoint_name, myproxy_username='', myproxy_password=''): """ Activate a GridFTP endpoint. Different attempts will be made: autocativation, using myproxy and using a delegate proxy (in the latter case \'grid-proxy-init\' is needed to generate a X.509 proxy). Myproxy username and password will be used to try the authentication with myproxy: they will be prompted if not passed as parameter or not found in the resource json file. The passphrase of the user certificate (needed for generating an X.509 proxy certificate) will be prompted if not found in the resource json file. :param endpoint_name: name of the endpoint to activate :param myproxy_username: myproxy user name (optional, if empty end needed it will be prompted) :param myproxy_password: myproxy user password (optional, if empty end needed it will be prompted) """ if self.api is None: print "You need to be authenticated with Globus Online to perform " \ "this operation. " return False LOGGER.debug("Checking if endpoint {0} is already activated..".format( endpoint_name)) _, _, data = self.api.endpoint(endpoint_name) if data["activated"]: LOGGER.info( "Endpoint {0} is already active!".format(endpoint_name)) return True # Trying with autoactivation LOGGER.debug("Trying autoactivation..") code, message, data = self.api.endpoint_autoactivate(endpoint_name) if not data["code"].startswith("AutoActivationFailed"): LOGGER.info("Endpoint {0} activated!".format(endpoint_name)) LOGGER.debug("result: {0} ({1})".format(data["code"], data["message"])) return True LOGGER.debug("Trying activating with myproxy..") if not myproxy_username and self.globus_init: myproxy_username = self.globus_init['myproxy_username'] if not myproxy_username and self.globus_init: myproxy_username = raw_input( "Please insert your \'myproxy\' username:"******"Timeout must be multiple of poll_interval" timeout_left = timeout while no_timeout or timeout_left >= 0: code, reason, data = self.api.task(task_id, fields="status") status = data["status"] LOGGER.debug("Checking task {0} status: {1}".format( task_id, status)) if status in ("SUCCEEDED", "FAILED"): return status if timeout_left > 0 or no_timeout: time.sleep(poll_interval) timeout_left -= poll_interval return None def display_task_summary(self): code, reason, data = self.api.tasksummary() LOGGER.info("Task Summary for %s:" % self.api.username) for k, v in data.iteritems(): if k == "DATA_TYPE": continue LOGGER.info("%3d %s" % (int(v), k.upper().ljust(9))) def _print_task(self, data, indent_level=0): indent = " " * indent_level indent += " " * 2 for k, v in data.iteritems(): if k in ("DATA_TYPE", "LINKS"): continue print indent + "%s: %s" % (k, v) def display_task(self, task_id, show_successful_transfers=True): """ Prints the status of a transfer. :param task_id: id of the transfer :param show_successful_transfers: """ code, reason, data = self.api.task(task_id) print 'Task {0}'.format(task_id) self._print_task(data, 0) if show_successful_transfers: try: code, reason, data = self.api.task_successful_transfers( task_id) transfer_list = data["DATA"] print "Successful Transfers (src -> dst)" for t in transfer_list: print " %s -> %s" % (t[u'source_path'], t[u'destination_path']) except Exception as e: "Error verifying successful transfer: {0}".format(e) def display_successful_transfer(self, task_id): """ Print successful transfers given the task id :param task_id: """ try: code, reason, data = self.api.task_successful_transfers(task_id) transfer_list = data["DATA"] print "Successful Transfers (src -> dst)" for t in transfer_list: print " %s -> %s" % (t[u'source_path'], t[u'destination_path']) except Exception as e: "Error verifying successful transfer: {0}".format(e) def check_proxy(self): """ Check for a local x509 prox. If not found or expired it creates a new one.""" grid_proxy_init_options = ' -out ' + self.proxy_name passphrase = '' if self.globus_init: passphrase = self.globus_init['PEM_passphrase'] LOGGER.debug("Checking for a valid proxy") if os.path.exists(self.proxy_name): LOGGER.debug(self.proxy_name + ' exists..') try: ret_val = os.system('grid-proxy-info -exists -f ' + self.proxy_name) if ret_val == 0: LOGGER.info("Proxy found!") else: # Remove interactive message: add arguments to manage the # request for GRID pass phrase LOGGER.debug( "Proxy expired. I will try to create a new one..") os.system('grid-proxy-init' + grid_proxy_init_options) except: print "Proxy invalid. New one, please!" if passphrase: os.system('echo ' + passphrase + ' | ' + 'grid-proxy-init' + ' -pwstdin' + grid_proxy_init_options) else: os.system('grid-proxy-init' + grid_proxy_init_options) else: LOGGER.debug(self.proxy_name + " does not exist. I'll try to create it..") if passphrase: os.system('echo \'' + passphrase + '\' | ' + 'grid-proxy-init' + ' -pwstdin' + grid_proxy_init_options) else: os.system('grid-proxy-init' + grid_proxy_init_options) # def get(self): # pass def get_endpoint_from_url(self, url): """ Read the endpoints.cfg configuration file to discover which endpoint is associated to the url. At the moment it is not possible to discover which is the Globus endpoint that can be used to access a resource. Resolving the PID it is possible to get the physical URL of the resource (iRODS URL), but the information about the endpoint is missing. For now a configuration file can be used to set the association between iRODS URL and Globus endpoint. :param url: physical URL of the resource :return: Globus endpoint name """ # get minimal url o = urlparse(url) simple_url = o.netloc.rsplit(':')[0] config = ConfigParser.SafeConfigParser() config.read(os.path.join(os.getcwd(), "endpoints.cfg")) res = '' try: res = config.get('irods_endpoints', simple_url) LOGGER.info("Found endpoint {0} associated to URL {1}".format( res, simple_url)) except ConfigParser.NoOptionError: print "Endpoint not found for URL {0}. You can add more endpoints " \ "editing the config file 'endpoints.cfg'.".format( simple_url) return res
local_ep = options['LOCALENDPOINT'] local_dir = options['LOCALPATH'] remote_ep = options['REMOTEENDPOINT'] remote_dir = options['REMOTEPATH'] oauthtokenfile = options['OAUTHTOKEN'] if os.path.exists(oauthtokenfile): authfile = open(oauthtokenfile, 'r') auth = authfile.readlines() goauth = auth[0].rstrip('\n') logging.debug('Running gosync.py with the following options:') logging.debug(options) api = TransferAPIClient(username, goauth=goauth, httplib_debuglevel=1, max_attempts=5) logging.debug('Client created:') logging.debug(api) logging.debug('Starting initial transfer.') do_transfer(local_ep, local_dir, remote_ep, remote_dir) logging.debug('Watching for file system changes.') event_handler = MyEventHandler(local_ep, local_dir, remote_ep, remote_dir) observer = Observer() logging.debug('Created observer to watch {}.'.format(local_dir)) observer.schedule(event_handler, local_dir, recursive=True) observer.start() logging.debug('Observer started.')
class GlobusOnlineAPIHelper(GlobusOnlineHelper): def __init__(self, inst, go_cert_file, go_key_file, go_server_ca): self.inst = inst self.go_cert_file = go_cert_file self.go_key_file = go_key_file self.go_server_ca = go_server_ca def connect(self, username): self.api = TransferAPIClient(username, self.go_server_ca, self.go_cert_file, self.go_key_file) def endpoint_exists(self, ep): try: (code, msg, data) = self.api.endpoint(ep.name) if code == 200: return True elif code == 404: return False else: raise GlobusOnlineException, "Unexpected error %i: %s" % (code, msg) except ClientError as ce: if ce.status_code == 404: return False else: raise GlobusOnlineException, "Unexpected GO API exception: %s" % ce def disconnect(self): self.api.close() def endpoint_remove(self, ep): try: (code, msg, data) = self.api.endpoint_delete(ep.name) if code >= 400: raise GlobusOnlineException, "Unexpected error %i: %s" % (code, msg) except ClientError as ce: raise GlobusOnlineException, "Unexpected GO API exception: %s" % ce def endpoint_gc_create(self, ep, replace): if self.endpoint_exists(ep): if not replace: raise EndpointExistsException, "An endpoint called '%s' already exists. Please choose a different name." % ep.name else: self.endpoint_remove(ep) try: (code, msg, data) = self.api.endpoint_create(ep.name, None, description="Globus Provision endpoint", is_globus_connect=True) if code >= 400: raise GlobusOnlineException, "Unexpected error %i: %s" % (code, msg) except ClientError as ce: raise GlobusOnlineException, "Unexpected GO API exception: %s" % ce gc_setupkey = data["globus_connect_setup_key"] return gc_setupkey def endpoint_gc_create_finalize(self, ep): gridftp_hostname, gridftp_subject, myproxy_hostname, myproxy_subject = self._get_hostnames_subjects(ep) # This is a kludge until http://jira.globus.org/browse/KOA-1624 gets fixed username = self.inst.config.get("ec2-username") keyfile = os.path.expanduser(self.inst.config.get("ec2-keyfile")) gc_setupkey = self.inst.topology.get_node_by_id(ep.gridftp[5:]).gc_setupkey ssh = SSH(username, gridftp_hostname, keyfile, default_outf = None, default_errf = None) ssh.open() outf = StringIO.StringIO() errf = StringIO.StringIO() rc = ssh.run("grid-cert-info -subject -file /etc/grid-security/gc-cert-%s.pem" % (gc_setupkey), outf=outf, errf=errf, exception_on_error=False) if rc != 0: raise GlobusOnlineException, "Could not create endpoint %s" % ep.name ssh.close() gridftp_subject = outf.getvalue().strip() self.endpoint_remove(ep) self._endpoint_create(ep.name, gridftp_hostname, gridftp_subject, myproxy_hostname, ep.public) def _endpoint_create(self, ep_name, gridftp_hostname, gridftp_subject, myproxy_hostname, public): try: (code, msg, data) = self.api.endpoint_create(ep_name, gridftp_hostname, description="Globus Provision endpoint", scheme="gsiftp", port=2811, subject=gridftp_subject, myproxy_server=myproxy_hostname) if code >= 400: raise GlobusOnlineException, "Unexpected error %i: %s" % (code, msg) except ClientError as ce: raise GlobusOnlineException, "Unexpected GO API exception: %s" % ce if public: try: (code, msg, data) = self.api.endpoint(ep_name) if code >= 400: raise GlobusOnlineException, "Unexpected error %i: %s" % (code, msg) except ClientError as ce: raise GlobusOnlineException, "Unexpected GO API exception: %s" % ce data["public"] = True try: (code, msg, data) = self.api.endpoint_update(ep_name, data) if code >= 400: raise GlobusOnlineException, "Unexpected error %i: %s" % (code, msg) except ClientError as ce: raise GlobusOnlineException, "Unexpected GO API exception: %s" % ce
def preferred_activation(username, endpoint_name, myproxy_username): user_credential_path=os.getcwd()+"/credential-"+username+".pem" print "==Activating %s ==" % endpoint_name api = TransferAPIClient(username, cert_file=user_credential_path) api.set_debug_print(False, False) try: code, message, data = api.endpoint(endpoint_name) if not data["activated"]: try: print "==Try autoactivation==" code, message, data = api.endpoint_autoactivate(endpoint_name) except: print "Cannot autoactivate" except: pass try: code, message, data = api.endpoint(endpoint_name) except: data={'activated': "Unavailable"} if not data["activated"]: # and data["activated"] == "Unavailable": try: if myproxy_username != "none": print "==Try myproxy for %s ==" % myproxy_username status, message, data = api.endpoint_autoactivate(endpoint_name) data.set_requirement_value("myproxy", "username", myproxy_username) from getpass import getpass passphrase = getpass() data.set_requirement_value("myproxy", "passphrase", passphrase) api.endpoint_activate(endpoint_name, data) #activer=[username,"-c",os.getcwd()+"/credential.pem"] #api, _ = create_client_from_args(activer) #conditional_activation(endpoint_name,myproxy_username) code, message, data = api.endpoint(endpoint_name) else: raise except: print "==Local proxy activation==" _, _, reqs = api.endpoint_activation_requirements(endpoint_name, type="delegate_proxy") #print "endpoint_name",endpoint_name #print reqs public_key = reqs.get_requirement_value("delegate_proxy", "public_key") #print public_key proxy = x509_proxy.create_proxy_from_file(user_credential_path, public_key) #print "proxy" #print proxy reqs.set_requirement_value("delegate_proxy", "proxy_chain", proxy) #print reqs result = api.endpoint_activate(endpoint_name, reqs)
servers = ep["DATA"] print >> output, " servers:" for s in servers: print >> output, " " + s["uri"], if s["subject"]: print >> output, " (%s)" % s["subject"] else: print >> output, "" def display_endpoints(): code, reason, endpoints = api.endpoints(limit=100) print >> output, ("Found %d endpoints for user %s:" % (endpoints["length"], api.username)) for ep in endpoints["DATA"]: _print_endpoint(ep) if __name__ == '__main__': options, args = process_args() api = TransferAPIClient(args[0], server_ca_file=options.server_ca_file, cert_file=options.cert_file, key_file=options.key_file, saml_cookie=options.saml_cookie, base_url=options.base_url) output = options.output display_endpoints() transfer(options.source_ep, options.source_path, options.destination_ep, options.destination_path, options.deadline)
def connect(self, username): self.api = TransferAPIClient(username, self.go_server_ca, self.go_cert_file, self.go_key_file)
class ClientGlobus(): def __init__(self, auth, http_session=None): """ Initialize the Globus session: login with username and certificates :param auth: a list containing username[0], certfile[1] and keyfile[2] :param http_session: """ self.auth = auth if len(self.auth) < 4: #or !self.auth[0] or !self.auth[2] or !self.auth[3]: print "Can not init session: some parameters are missing.." self.auth = auth self.http_session = http_session self.myproxy_username = '' self.api = None if len(self.auth) > 0: self.proxy_name = 'credential-' + self.auth[0] + '.pem' try: self.api = TransferAPIClient(username=self.auth[0], cert_file=self.auth[2], key_file=self.auth[3]) self.api.set_debug_print(False, False) LOGGER.info("Successfully logged in with Globus online!") except Exception as e: raise Exception("GSI authentication failed: {0}".format(e)) def display_endpoint_list(self): code, reason, endpoint_list = self.api.endpoint_list(limit=100) LOGGER.info("Found %d endpoints for user %s:" \ % (endpoint_list["length"], self.api.username)) for ep in endpoint_list["DATA"]: self._print_endpoint(ep) def _print_endpoint(self, ep): name = ep["canonical_name"] print name if ep["activated"]: print " activated (expires: %s)" % ep["expire_time"] else: print " not activated" if ep["public"]: print " public" else: print " not public" if ep["myproxy_server"]: print " default myproxy server: %s" % ep["myproxy_server"] else: print " no default myproxy server" servers = ep.get("DATA", ()) print " servers:" for s in servers: uri = s["uri"] if not uri: uri = "GC endpoint, no uri available" print " " + uri, if s["subject"]: print " (%s)" % s["subject"] else: print def endpoint_activation(self, endpoint_name, myproxy_username=''): """ Activate a GridFTP endpoint :param endpoint_name: name of the endpoint to activate :param myproxy_username: myproxy user name """ #code, reason, data = self.api.endpoint(endpoint_name) #self._print_endpoint(data) #self.display_endpoint_list(); LOGGER.debug("Checking if endpoint {0} is already activated".format( endpoint_name)) _, _, data = self.api.endpoint(endpoint_name) if data["activated"]: LOGGER.info("Endpoint {0} is already active!".format(endpoint_name)) return True # Trying with autoactivation LOGGER.debug("Trying autoactivation") code, message, data = self.api.endpoint_autoactivate( endpoint_name) if not data["code"].startswith("AutoActivationFailed"): LOGGER.info("Endpoint {0} activated!".format(endpoint_name)) LOGGER.debug("result: {0} ({1})".format(data["code"], data["message"])) return True LOGGER.debug("Trying activating with myproxy") data.set_requirement_value("myproxy", "username", myproxy_username) # Remove interactive messages: add arguments to manage this part from getpass import getpass passphrase = getpass() data.set_requirement_value("myproxy", "passphrase", passphrase) try: status, message, data = self.api.endpoint_activate(endpoint_name, data, if_expires_in=600) if not data["code"].startswith("AutoActivationFailed"): LOGGER.info("Endpoint {0} activated!\n".format(endpoint_name)) LOGGER.debug("result: {0} ({1})\n".format(data["code"], data["message"])) return True except Exception as e: print "Error: {0}".format(e) # Trying activating a delegate proxy self.check_proxy() user_credential_path = os.path.join(os.getcwd(), self.proxy_name) LOGGER.debug("Trying delegate proxy activation") _, _, reqs = self.api.endpoint_activation_requirements( endpoint_name, type="delegate_proxy") public_key = reqs.get_requirement_value("delegate_proxy", "public_key") #print public_key proxy = x509_proxy.create_proxy_from_file(user_credential_path, public_key, lifetime_hours=3) #print proxy reqs.set_requirement_value("delegate_proxy", "proxy_chain", proxy) try: code, message, data = self.api.endpoint_activate(endpoint_name, reqs) if not data["code"].startswith("AutoActivationFailed"): LOGGER.info("Endpoint {0} activated!\n".format(endpoint_name)) LOGGER.debug("result: {0} ({1})\n".format(data["code"], data["message"])) return True except Exception as e: print "Error: {0}".format(e) print "Can not active the endpoint {0}\n".format(endpoint_name) if "proxy is not valid until" in str(e): print "This error may be related to clock time skew. " \ "Please, check if your client clock is server " \ "synchronized and not ahead (you could check with " \ "\"www.time.is\")" return False def transfer(self, src_endpoint, dst_endpoint, item, dst_dir): """ Transfer a file from one endpoint to another. Return the Globus task_id. :param src_endpoint: source endpoint name (i.e. user#endpoint) :param dst_endpoint: destination endpoint name (i.e. user#endpoint) :param item: object to be transferred :param dst_dir: destination directory :return: Globus task_id """ if src_endpoint: self.endpoint_activation(src_endpoint) if dst_endpoint: self.endpoint_activation(dst_endpoint) # submit a transfer #oldstdout=sys.stdout #sys.stdout = open(os.devnull,'w') code, message, data = self.api.transfer_submission_id() #sys.stdout = oldstdout # enable output submission_id = data["value"] #deadline = datetime.utcnow() + timedelta(minutes=10) t = Transfer(submission_id, src_endpoint, dst_endpoint)#, deadline) LOGGER.info("Transferring {0} from endpoint {1} to endpoint {2}".format( item, src_endpoint, dst_endpoint)) t.add_item(item, os.path.join(dst_dir, os.path.basename(item))) code, reason, data = self.api.transfer(t) task_id = data["task_id"] self.display_task(task_id) return task_id def display_tasksummary(self): code, reason, data = self.api.tasksummary() LOGGER.info("Task Summary for %s:" % self.api.username) for k, v in data.iteritems(): if k == "DATA_TYPE": continue LOGGER.info("%3d %s" % (int(v), k.upper().ljust(9))) def _print_task(self, data, indent_level=0): indent = " " * indent_level indent += " " * 2 for k, v in data.iteritems(): if k in ("DATA_TYPE", "LINKS"): continue LOGGER.info(indent + "%s: %s" % (k, v)) def display_task(self, task_id, show_successful_transfers=True): code, reason, data = self.api.task(task_id) LOGGER.info("Task %s:" % task_id) self._print_task(data, 0) if show_successful_transfers: try: code, reason, data = self.api.task_successful_transfers( task_id) transfer_list = data["DATA"] LOGGER.info("Successful Transfers (src -> dst)") for t in transfer_list: LOGGER.info(" %s -> %s" % (t[u'source_path'], t[u'destination_path'])) except Exception as e: "Error verifying successful transfer: {0}".format(e) def check_proxy(self): """ Check for a local x509 prox. If not found or expired it creates a new one.""" grid_proxy_init_options = ' -out ' + self.proxy_name LOGGER.debug("Checking for a valid proxy") if os.path.exists(self.proxy_name): LOGGER.debug(self.proxy_name + ' exists..') try: ret_val = os.system('grid-proxy-info -exists -f ' + self.proxy_name) if ret_val == 0: LOGGER.info("Proxy found!") else: # Remove interactive message: add arguments to manage the # request for GRID pass phrase LOGGER.debug("Proxy expired. I will try to create a new one..") os.system('grid-proxy-init' + grid_proxy_init_options) except: print "Proxy invalid. New one, please!" os.system('grid-proxy-init' + grid_proxy_init_options) else: LOGGER.debug(self.proxy_name + " does not exist. I'll try to create it..") os.system('grid-proxy-init' + grid_proxy_init_options) #def get(self): # pass def get_endpoint_from_url(self, url): """ Read the endpoints.cfg configuration file to discover which endpoint is associated to the url. At the moment it is not possible to discover which is the Globus endpoint that can be used to access a resource. Resolving the PID it is possible to get the physical URL of the resource (iRODS URL), but the information about the endpoint is missing. For now a configuration file can be used to set the association between iRODS URL and Globus endpoint. :param url: physical URL of the resource :return: Globus endpoint name """ # get minimal url o = urlparse(url) simple_url = o.netloc.rsplit(':')[0] config = ConfigParser.SafeConfigParser() config.read(os.path.join(os.getcwd(), "endpoints.cfg")) res = '' try: res = config.get('irods_endpoints', simple_url) LOGGER.info("Found endpoint {0} associated to URL {1}".format(res, simple_url)) except ConfigParser.NoOptionError: print "Endpoint not found for URL {0}. You can add more endpoints " \ "editing the config file 'endpoints.cfg'.".format( simple_url) return res
password=password, ca_certs=nexus_cert) if auth_result is not None: break except ssl.SSLError, e: if "timed out" not in str(e): raise e time.sleep(30) except GOCredentialsError, e: print "Globus Online Username: "******"Globus Online Password: "******"timed out" not in str(e):
class ClientGlobus(): def __init__(self, auth=['', ''], resource_file_path='', debug=False): """ Initialize the Globus session: login with Globus username and password Credentials can also be passed through a resource json file like this: { "globus_username":"", "globus_password":"", "myproxy_username": "", "myproxy_password": "", "PEM_passphrase": "" } :param auth: a list containing username[0], password[1], certfile[2] and keyfile[3] :param resource_file_path: path to a json file containing the credentials :param debug: set True to enable debug messages """ if debug: LOGGER.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) LOGGER.addHandler(ch) # Read resources from file: self.globus_init = None if resource_file_path and os.path.isfile(resource_file_path): self.globus_init = json.load(open(resource_file_path)) if not auth[0]: auth[0] = self.globus_init['globus_username'] if not auth[1]: auth[1] = self.globus_init['globus_password'] if not auth[0] or not auth[1]: print "Can not init session without a username and password.." return self.api = None self.proxy_name = 'credential-' + auth[0] + '.pem' try: result = get_access_token(username=auth[0], password=auth[1]) goauth = result.token self.api = TransferAPIClient(username=auth[0], goauth=goauth) self.api.set_debug_print(False, False) LOGGER.info("Successfully logged in with Globus Online!") except Exception as e: print "Globus Online authentication failed: {0}".format(e) def display_endpoint_list(self): code, reason, endpoint_list = self.api.endpoint_list(limit=100) LOGGER.info("Found %d endpoints for user %s:" \ % (endpoint_list["length"], self.api.username)) for ep in endpoint_list["DATA"]: self._print_endpoint(ep) def _print_endpoint(self, ep): name = ep["canonical_name"] print name if ep["activated"]: print " activated (expires: %s)" % ep["expire_time"] else: print " not activated" if ep["public"]: print " public" else: print " not public" if ep["myproxy_server"]: print " default myproxy server: %s" % ep["myproxy_server"] else: print " no default myproxy server" servers = ep.get("DATA", ()) print " servers:" for s in servers: uri = s["uri"] if not uri: uri = "GC endpoint, no uri available" print " " + uri, if s["subject"]: print " (%s)" % s["subject"] else: print def endpoint_activation(self, endpoint_name, myproxy_username='', myproxy_password=''): """ Activate a GridFTP endpoint. Different attempts will be made: autocativation, using myproxy and using a delegate proxy (in the latter case \'grid-proxy-init\' is needed to generate a X.509 proxy). Myproxy username and password will be used to try the authentication with myproxy: they will be prompted if not passed as parameter or not found in the resource json file. The passphrase of the user certificate (needed for generating an X.509 proxy certificate) will be prompted if not found in the resource json file. :param endpoint_name: name of the endpoint to activate :param myproxy_username: myproxy user name (optional, if empty end needed it will be prompted) :param myproxy_password: myproxy user password (optional, if empty end needed it will be prompted) """ if self.api is None: print "You need to be authenticated with Globus Online to perform " \ "this operation. " return False LOGGER.debug("Checking if endpoint {0} is already activated..".format( endpoint_name)) _, _, data = self.api.endpoint(endpoint_name) if data["activated"]: LOGGER.info( "Endpoint {0} is already active!".format(endpoint_name)) return True # Trying with autoactivation LOGGER.debug("Trying autoactivation..") code, message, data = self.api.endpoint_autoactivate( endpoint_name) if not data["code"].startswith("AutoActivationFailed"): LOGGER.info("Endpoint {0} activated!".format(endpoint_name)) LOGGER.debug("result: {0} ({1})".format(data["code"], data["message"])) return True LOGGER.debug("Trying activating with myproxy..") if not myproxy_username and self.globus_init: myproxy_username = self.globus_init['myproxy_username'] if not myproxy_username and self.globus_init: myproxy_username = raw_input( "Please insert your \'myproxy\' username:"******"Timeout must be multiple of poll_interval" timeout_left = timeout while no_timeout or timeout_left >= 0: code, reason, data = self.api.task(task_id, fields="status") status = data["status"] LOGGER.debug("Checking task {0} status: {1}".format(task_id, status)) if status in ("SUCCEEDED", "FAILED"): return status if timeout_left > 0 or no_timeout: time.sleep(poll_interval) timeout_left -= poll_interval return None def display_task_summary(self): code, reason, data = self.api.tasksummary() LOGGER.info("Task Summary for %s:" % self.api.username) for k, v in data.iteritems(): if k == "DATA_TYPE": continue LOGGER.info("%3d %s" % (int(v), k.upper().ljust(9))) def _print_task(self, data, indent_level=0): indent = " " * indent_level indent += " " * 2 for k, v in data.iteritems(): if k in ("DATA_TYPE", "LINKS"): continue print indent + "%s: %s" % (k, v) def display_task(self, task_id, show_successful_transfers=True): """ Prints the status of a transfer. :param task_id: id of the transfer :param show_successful_transfers: """ code, reason, data = self.api.task(task_id) print 'Task {0}'.format(task_id) self._print_task(data, 0) if show_successful_transfers: try: code, reason, data = self.api.task_successful_transfers( task_id) transfer_list = data["DATA"] print "Successful Transfers (src -> dst)" for t in transfer_list: print " %s -> %s" % (t[u'source_path'], t[u'destination_path']) except Exception as e: "Error verifying successful transfer: {0}".format(e) def display_successful_transfer(self, task_id): """ Print successful transfers given the task id :param task_id: """ try: code, reason, data = self.api.task_successful_transfers( task_id) transfer_list = data["DATA"] print "Successful Transfers (src -> dst)" for t in transfer_list: print " %s -> %s" % (t[u'source_path'], t[u'destination_path']) except Exception as e: "Error verifying successful transfer: {0}".format(e) def check_proxy(self): """ Check for a local x509 prox. If not found or expired it creates a new one.""" grid_proxy_init_options = ' -out ' + self.proxy_name passphrase = '' if self.globus_init: passphrase = self.globus_init['PEM_passphrase'] LOGGER.debug("Checking for a valid proxy") if os.path.exists(self.proxy_name): LOGGER.debug(self.proxy_name + ' exists..') try: ret_val = os.system( 'grid-proxy-info -exists -f ' + self.proxy_name) if ret_val == 0: LOGGER.info("Proxy found!") else: # Remove interactive message: add arguments to manage the # request for GRID pass phrase LOGGER.debug( "Proxy expired. I will try to create a new one..") os.system('grid-proxy-init' + grid_proxy_init_options) except: print "Proxy invalid. New one, please!" if passphrase: os.system('echo ' + passphrase + ' | ' + 'grid-proxy-init' + ' -pwstdin' + grid_proxy_init_options) else: os.system('grid-proxy-init' + grid_proxy_init_options) else: LOGGER.debug( self.proxy_name + " does not exist. I'll try to create it..") if passphrase: os.system('echo \'' + passphrase + '\' | ' + 'grid-proxy-init' + ' -pwstdin' +grid_proxy_init_options) else: os.system('grid-proxy-init' + grid_proxy_init_options) # def get(self): # pass def get_endpoint_from_url(self, url): """ Read the endpoints.cfg configuration file to discover which endpoint is associated to the url. At the moment it is not possible to discover which is the Globus endpoint that can be used to access a resource. Resolving the PID it is possible to get the physical URL of the resource (iRODS URL), but the information about the endpoint is missing. For now a configuration file can be used to set the association between iRODS URL and Globus endpoint. :param url: physical URL of the resource :return: Globus endpoint name """ # get minimal url o = urlparse(url) simple_url = o.netloc.rsplit(':')[0] config = ConfigParser.SafeConfigParser() config.read(os.path.join(os.getcwd(), "endpoints.cfg")) res = '' try: res = config.get('irods_endpoints', simple_url) LOGGER.info("Found endpoint {0} associated to URL {1}".format(res, simple_url)) except ConfigParser.NoOptionError: print "Endpoint not found for URL {0}. You can add more endpoints " \ "editing the config file 'endpoints.cfg'.".format( simple_url) return res
auth_result = get_access_token(username=username, password=password, ca_certs=nexus_cert) if auth_result is not None: break except ssl.SSLError, e: if "timed out" not in str(e): raise e time.sleep(30) except GOCredentialsError, e: print "Globus Online Username: "******"Globus Online Password: "******"timed out" not in str(e): raise e