def GO_transfer(ep1, dir1, ep2, dir2, files, api): print ep1, dir1, ep2, dir2, files status_code, status_message, data = api.submission_id() sid = data['value'] print 'SID', sid t = Transfer(sid, ep1, ep2) for f in files: path1 = dir1+f path2 = dir2+f print path1, path2 t.add_item(path1, path2) status_code, status_message, data = api.transfer(t) tid = data['task_id']# print 'task No.', tid task_status='ACTIVE' while task_status=='ACTIVE': status_code, status_message, data = api.task(tid) task_status=data['status'] # print 'data:', data print 'task:', task_status if task_status!='ACTIVE':break time.sleep(5)
def GO_transfer(ep1, dir1, ep2, dir2, files, api): print ep1, dir1, ep2, dir2, files status_code, status_message, data = api.submission_id() sid = data['value'] print 'SID', sid t = Transfer(sid, ep1, ep2) for f in files: path1 = dir1 + f path2 = dir2 + f print path1, path2 t.add_item(path1, path2) status_code, status_message, data = api.transfer(t) tid = data['task_id'] # print 'task No.', tid task_status = 'ACTIVE' while task_status == 'ACTIVE': status_code, status_message, data = api.task(tid) task_status = data['status'] # print 'data:', data print 'task:', task_status if task_status != 'ACTIVE': break time.sleep(5)
def start_transfer(self, src, dest): """ Starts a transfer """ if self.file_list == None or len(self.file_list) == 0: raise TransferException("No files specified for transfer") api = self._get_api() # activate endpoints _, _, data = api.endpoint_autoactivate(src) _, _, data = api.endpoint_autoactivate(dest) # create transfer and populate files _, _, data = api.transfer_submission_id() submission_id = data["value"] t = Transfer(submission_id, src, dest) for file in self.file_list: t.add_item(os.path.join(self.src_dir, file), os.path.join(self.dest_dir, file)) # transfer _, _, data = api.transfer(t) self.go_task_id = data['task_id']
def submitTransfer(api_client, source_endpoint, source_files, target_endpoint, target_directory): ''' Method to submit a data transfer request to Globus. ''' # obtain a submission id from Globus code, message, data = api_client.transfer_submission_id() submission_id = data["value"] print "Obtained transfer submission id: %s" % submission_id # maximum time for completing the transfer deadline = datetime.utcnow() + timedelta(days=10) # create a transfer request transfer_task = Transfer(submission_id, source_endpoint, target_endpoint, deadline) for source_file in source_files: source_directory, filename = os.path.split(source_file) target_file = os.path.join(target_directory, filename) transfer_task.add_item(source_file, target_file) # submit the transfer request try: code, reason, data = api_client.transfer(transfer_task) task_id = data["task_id"] print "Submitted transfer task with id: %s" % task_id except Exception as e: print "Could not submit the transfer. Error: %s" % str(e) task_id = "Could not submit the transfer. Please contact the ESGF node admin to investigate the issue." return task_id
def transfer(self, src_endpoint, dst_endpoint, src_item, dst_dir, recursive=False): """ Transfer a file from one endpoint to another and return the Globus task_id. Set @recursive to True to transfer a folder. :param src_endpoint: source endpoint name (i.e. user#endpoint) :param dst_endpoint: destination endpoint name (i.e. user#endpoint) :param src_item: object to be transferred :param dst_dir: destination directory :param recursive: flag to enable folder transfer :return: Globus task_id """ if src_endpoint: self.endpoint_activation(src_endpoint) if dst_endpoint: self.endpoint_activation(dst_endpoint) # submit a transfer code, message, data = self.api.transfer_submission_id() submission_id = data["value"] t = Transfer(submission_id, src_endpoint, dst_endpoint) # , deadline) LOGGER.info( "Transferring {0} from endpoint {1} to endpoint {2}".format( src_item, src_endpoint, dst_endpoint)) t.add_item(src_item, os.path.join(dst_dir, os.path.basename(src_item)), recursive=recursive) code, reason, data = self.api.transfer(t) task_id = data["task_id"] return task_id
def transfer(self, src_endpoint, dst_endpoint, item, dst_dir): """ Transfer a file from one endpoint to another. Return the Globus task_id. :param src_endpoint: source endpoint name (i.e. user#endpoint) :param dst_endpoint: destination endpoint name (i.e. user#endpoint) :param item: object to be transferred :param dst_dir: destination directory :return: Globus task_id """ if src_endpoint: self.endpoint_activation(src_endpoint) if dst_endpoint: self.endpoint_activation(dst_endpoint) # submit a transfer #oldstdout=sys.stdout #sys.stdout = open(os.devnull,'w') code, message, data = self.api.transfer_submission_id() #sys.stdout = oldstdout # enable output submission_id = data["value"] #deadline = datetime.utcnow() + timedelta(minutes=10) t = Transfer(submission_id, src_endpoint, dst_endpoint)#, deadline) LOGGER.info("Transferring {0} from endpoint {1} to endpoint {2}".format( item, src_endpoint, dst_endpoint)) t.add_item(item, os.path.join(dst_dir, os.path.basename(item))) code, reason, data = self.api.transfer(t) task_id = data["task_id"] self.display_task(task_id) return task_id
def transfer(options): # Map legacy endpoint names to UUIDs srcendpoint = options.srcendpoint dstendpoint = options.dstendpoint if '#' in srcendpoint: srcendpoint = config['endpoint'][srcendpoint] if '#' in dstendpoint: dstendpoint = config['endpoint'][dstendpoint] # Get access token (This method of getting an acces token is deprecated and should be replaced by OAuth2 calls). auth_result = get_access_token(config['globus']['username'], config['globus']['password']) # Create a transfer submission api_client = TransferAPIClient(config['globus']['username'], goauth=auth_result.token) activate_endpoint(api_client, srcendpoint) activate_endpoint(api_client, dstendpoint) code, message, data = api_client.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(days=10) transfer_task = Transfer(submission_id, srcendpoint, dstendpoint, deadline) # Add srcpath to the transfer task if options.srcpath: transfer_task.add_item(options.srcpath, get_destination_path(options.srcpath, options.dstpath, options.recursive), recursive=options.recursive) # Add srclist to the transfer task if options.srclist: with open(options.srclist) as f: srcpath = f.readline().rstrip('\n') transfer_task.add_item(srcpath, get_destination_path(srcpath, options.dstpath, options.recursive), recursive=options.recursive) # Start the transfer task_id = None try: code, reason, data = api_client.transfer(transfer_task) task_id = data["task_id"] print 'task_id %s' % task_id except Exception as e: msg = "Could not submit the transfer. Error: %s" % str(e) sys.exit(1) # Check a status of the transfer every minute (60 secs) while True: code, reason, data = api_client.task(task_id) print data['status'] if data['status'] == 'SUCCEEDED': print 'progress %d/%d' % (data['files_transferred'], data['files']) return ('success', '') elif data['status'] == 'FAILED': return ('error', data['nice_status_details']) elif data['status'] == 'ACTIVE': print 'progress %d/%d' % (data['files_transferred'], data['files']) time.sleep(10)
def transfer(src_endpoint, src_path, dst_endpoint, dst_path, access_token, openid): (server, username) = decompose_openid(openid) client = TransferAPIClient(username, goauth=access_token) client.connect() globus_submission_id = client.submission_id() transfer = Transfer(globus_submission_id, src_endpoint, dst_endpoint) transfer.add_item(src_path, dst_path, recursive=True) client.transfer(transfer) return {'status': 'Success', 'message': ''}
def tutorial(): """ Do a bunch of API calls and display the results. Does a small transfer between tutorial endpoints, but otherwise does not modify user data. Uses module global API client instance. """ # See what is in the account before we make any submissions. print "=== Before tutorial ===" display_tasksummary(); print display_task_list(); print display_endpoint_list(); print # auto activate the endpoint, and display before/after. display_activation("go#ep1") display_activation("go#ep2") print "=== Before transfer ===" display_ls("go#ep1"); print display_ls("go#ep2"); print # submit a transfer code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(minutes=10) t = Transfer(submission_id, "go#ep1", "go#ep2", deadline) t.add_item("/~/.bashrc", "/~/api-example-bashrc-copy") code, reason, data = api.transfer(t) task_id = data["task_id"] # see the new transfer show up print "=== After submit ===" display_tasksummary(); print display_task(task_id, False); print # wait for the task to complete, and see the tasks and # endpoint ls change status = wait_for_task(task_id) if status is None: # Task didn't complete before the timeout. # Since the example transfers a single small file and the # timeout is 2 mintues, this shouldn't happen unless one of the # endpoints is having problems or the user already has a bunch # of other active tasks (there is a limit to the # number of concurrent active tasks a user can have). print "WARNING: task did not complete before timeout!" else: print "Task %s complete with status %s" % (task_id, status) print "=== After completion ===" display_tasksummary(); print display_task(task_id); print display_ls("go#ep2"); print
def remove(request): """ removes files on a remote Globus Online endpoint - API is not complete, so transfer 0 byte files instead of actually deleting anything """ # global so that we can use it in signal handlers global api global task_id # connect to the service api = TransferAPIClient(request["globus_username"], cert_file=request["x509_proxy"]) # make sure we can auto-activate the endpoints ep = activate_ep(api, request["endpoint"]) label = None if "PEGASUS_WF_UUID" in os.environ and "PEGASUS_DAG_JOB_ID" in os.environ: label = os.environ["PEGASUS_WF_UUID"] + " - " + os.environ[ "PEGASUS_DAG_JOB_ID"] # set up a new transfer code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(hours=24) t = Transfer(submission_id, request["endpoint"], request["endpoint"], deadline=deadline, label=label, notify_on_succeeded=False, notify_on_failed=False, notify_on_inactive=False) for f in request["files"]: t.add_item("/dev/null", f) # finalize and submit the transfer code, reason, data = api.transfer(t) task_id = data["task_id"] # how many faults will we accept before giving up? acceptable_faults = min(100, len(request["files"]) * 3) # wait for the task to complete, and see the tasks and # endpoint ls change try: status = wait_for_task(api, task_id, acceptable_faults) except Exception, err: logger.error(err) cancel_task(api, task_id) sys.exit(1)
def tutorial(): """ Do a bunch of API calls and display the results. Does a small transfer between tutorial endpoints, but otherwise does not modify user data. Uses module global API client instance. """ # See what is in the account before we make any submissions. print "=== Before tutorial ===" display_tasksummary() print display_task_list() print display_endpoint_list() print # auto activate the endpoint, and display before/after. display_activation("go#ep1") display_activation("go#ep2") print "=== Before transfer ===" display_ls("go#ep1") print display_ls("go#ep2") print # submit a transfer code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(minutes=10) t = Transfer(submission_id, "go#ep1", "go#ep2", deadline) t.add_item("/~/.bashrc", "/~/api-example-bashrc-copy") code, reason, data = api.transfer(t) task_id = data["task_id"] # see the new transfer show up print "=== After submit ===" display_tasksummary() print display_task(task_id) print # wait for the task to complete, and see the summary and lists # update if wait_for_task(task_id): print "=== After completion ===" display_tasksummary() print display_task(task_id) print display_ls("go#ep2") print
def tutorial(): """ Do a bunch of API calls and display the results. Does a small transfer between tutorial endpoints, but otherwise does not modify user data. Uses module global API client instance. """ # See what is in the account before we make any submissions. print "=== Before tutorial ===" display_task_list(); print display_endpoint_list(); print # auto activate the endpoint, and display before/after. display_activation("go#ep1") display_activation("go#ep2") print "=== Before transfer ===" display_ls("go#ep1"); print display_ls("go#ep2"); print # submit a transfer code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(minutes=10) t = Transfer(submission_id, "go#ep1", "go#ep2", deadline) t.add_item("/~/.bashrc", "/~/api-example-bashrc-copy") code, reason, data = api.transfer(t) task_id = data["task_id"] # see the new transfer show up print "=== After submit ===" display_task(task_id, False); print # wait for the task to complete, and see the tasks and # endpoint ls change status = wait_for_task(task_id) if status is None: # Task didn't complete before the timeout. # Since the example transfers a single small file and the # timeout is 2 mintues, this shouldn't happen unless one of the # endpoints is having problems or the user already has a bunch # of other active tasks (there is a limit to the # number of concurrent active tasks a user can have). print "WARNING: task did not complete before timeout!" else: print "Task %s complete with status %s" % (task_id, status) print "=== After completion ===" display_task(task_id); print display_ls("go#ep2"); print
def do_transfer(local_ep, local_dir, remote_ep, remote_dir): logging.debug('Initiating transfer with:') logging.debug('local endpoint: {0}, local dir: {1}, remote endpoint: {2}, remote dir: {3}'.format(local_ep, local_dir, remote_ep, remote_dir)) submission_id = get_submission_id() transfer = Transfer(submission_id, local_ep, remote_ep, sync_level=3, encrypt_data=True) logging.debug('Adding transfer item: {}, {}'.format(local_dir, remote_dir)) transfer.add_item(local_dir, remote_dir, recursive=True) logging.debug('Transfer object as JSON:') logging.debug(transfer.as_json()) logging.debug('API Client instance: {}'.format(api)) api.transfer(transfer) api.close()
def do_transfer(local_ep, local_dir, remote_ep, remote_dir): logging.debug('Initiating transfer with:') logging.debug( 'local endpoint: {0}, local dir: {1}, remote endpoint: {2}, remote dir: {3}' .format(local_ep, local_dir, remote_ep, remote_dir)) submission_id = get_submission_id() transfer = Transfer(submission_id, local_ep, remote_ep, sync_level=3, encrypt_data=True) logging.debug('Adding transfer item: {}, {}'.format(local_dir, remote_dir)) transfer.add_item(local_dir, remote_dir, recursive=True) logging.debug('Transfer object as JSON:') logging.debug(transfer.as_json()) logging.debug('API Client instance: {}'.format(api)) api.transfer(transfer) api.close()
def tutorial(): """ Do a bunch of API calls and display the results. Does a small transfer between tutorial endpoints, but otherwise does not modify user data. Uses module global API client instance. """ # See what is in the account before we make any submissions. print "=== Before tutorial ===" display_tasksummary(); print display_task_list(); print display_endpoint_list(); print # auto activate the endpoint, and display before/after. display_activation("go#ep1") display_activation("go#ep2") print "=== Before transfer ===" display_ls("go#ep1"); print display_ls("go#ep2"); print # submit a transfer code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(minutes=10) t = Transfer(submission_id, "go#ep1", "go#ep2", deadline) t.add_item("/~/.bashrc", "/~/api-example-bashrc-copy") code, reason, data = api.transfer(t) task_id = data["task_id"] # see the new transfer show up print "=== After submit ===" display_tasksummary(); print display_task(task_id); print # wait for the task to complete, and see the summary and lists # update if wait_for_task(task_id): print "=== After completion ===" display_tasksummary(); print display_task(task_id); print display_ls("go#ep2"); print
def start_transfer(self, filelist): """ activate src endpoint Parameters ---------- filelist : list List of files to transfer Returns ------- str of the globus task id """ src_endpoint = self.srcinfo['endpoint'] result = self.endpoint_activate(src_endpoint) # activate dst endpoint dst_endpoint = self.dstinfo['endpoint'] result = self.endpoint_activate(dst_endpoint) # create dst directories self.makedirs([finfo['dst'] for finfo in filelist.values()], dst_endpoint) ## Get a submission id: _, _, result = self.goclient.transfer_submission_id() self.submission_id = result["value"] miscutils.fwdebug(1, 'GLOBUS_ONLINE_DEBUG', "\tsubmission id = %s" % self.submission_id) ## Create a transfer object: #t = Transfer(submission_id, src_endpoint, dst_endpoint, notify_on_succeeded = False, # notify_on_failed = False, notify_on_inactive= False, deadline='2m') deadline = datetime.utcnow() + timedelta(minutes=30) t = Transfer(self.submission_id, src_endpoint, dst_endpoint, notify_on_succeeded=False, notify_on_failed=False, notify_on_inactive=False, deadline=deadline) #print t.as_data() # add files to transfer for _, finfo in filelist.items(): sfile = finfo['src'] dfile = finfo['dst'] miscutils.fwdebug(2, 'GLOBUS_ONLINE_DEBUG', "\tadding to transfer %s = %s" % (sfile, dfile)) if sfile.endswith('/'): t.add_item(sfile, dfile, recursive=True) # error if true for file else: t.add_item(sfile, dfile) # start transfer _, _, result = self.goclient.transfer(t) task_id = result["task_id"] miscutils.fwdebug(1, 'GLOBUS_ONLINE_DEBUG', "\ttask id = %s" % task_id) return task_id
def transfer(options): # Map legacy endpoint names to UUIDs srcendpoint = options.srcendpoint dstendpoint = options.dstendpoint if '#' in srcendpoint: srcendpoint = config['endpoint'][srcendpoint] if '#' in dstendpoint: dstendpoint = config['endpoint'][dstendpoint] # Get access token (This method of getting an acces token is deprecated and should be replaced by OAuth2 calls). auth_result = get_access_token(config['globus']['username'], config['globus']['password']) # Create a transfer submission api_client = TransferAPIClient(config['globus']['username'], goauth=auth_result.token) activate_endpoint(api_client, srcendpoint) activate_endpoint(api_client, dstendpoint) code, message, data = api_client.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(days=10) transfer_task = Transfer(submission_id, srcendpoint, dstendpoint, deadline) # Add srcpath to the transfer task if options.srcpath: transfer_task.add_item(options.srcpath, get_destination_path(options.srcpath, options.dstpath, options.recursive), recursive=options.recursive) # Add srclist to the transfer task if options.srclist: with open(options.srclist) as f: srcpath = f.readline().rstrip('\n') transfer_task.add_item(srcpath, get_destination_path(srcpath, options.dstpath, options.recursive), recursive=options.recursive) # Start the transfer task_id = None try: code, reason, data = api_client.transfer(transfer_task) task_id = data["task_id"] print 'task_id %s' % task_id except Exception as e: msg = "Could not submit the transfer. Error: %s" % str(e) sys.exit(1) # Check a status of the transfer every minute (60 secs) while True: code, reason, data = api_client.task(task_id) if data['status'] == 'SUCCEEDED': print 'progress %d/%d' % (data['files_transferred'], data['files']) return ('success', '') elif data['status'] == 'FAILED': return ('error', data['nice_status_details']) elif data['status'] == 'ACTIVE': print 'progress %d/%d' % (data['files_transferred'], data['files']) time.sleep(10)
def transfer(request): """ takes a transfer specification parsed from json: { "globus_username": "******", "x509_proxy": "/tmp/...", "src_endpoint": "rynge#obelix", "dst_endpoint": "rynge#workflow", "files":[ {"src":"/etc/hosts","dst":"/tmp/foobar.txt"}, {"src":"/etc/hosts","dst":"/tmp/foobar-2.txt"} ], } """ # global so that we can use it in signal handlers global api global task_id # connect to the service api = TransferAPIClient(request["globus_username"], cert_file=request["x509_proxy"]) # make sure we can auto-activate the endpoints src_ep = activate_ep(api, request["src_endpoint"]) dst_ep = activate_ep(api, request["dst_endpoint"]) label = None if "PEGASUS_WF_UUID" in os.environ and "PEGASUS_DAG_JOB_ID" in os.environ: label = os.environ["PEGASUS_WF_UUID"] + " - " + os.environ[ "PEGASUS_DAG_JOB_ID"] # set up a new transfer code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(hours=24) t = Transfer(submission_id, request["src_endpoint"], request["dst_endpoint"], deadline=deadline, label=label, notify_on_succeeded=False, notify_on_failed=False, notify_on_inactive=False) for pair in request["files"]: t.add_item(pair["src"], pair["dst"]) # finalize and submit the transfer code, reason, data = api.transfer(t) task_id = data["task_id"] # how many faults will we accept before giving up? acceptable_faults = min(100, len(request["files"]) * 3) # wait for the task to complete, and see the tasks and # endpoint ls change try: status = wait_for_task(api, task_id, acceptable_faults) except Exception, err: logger.error(err) cancel_task(api, task_id) sys.exit(1)
def mover(username, src_site, dst_site, dst_dir): """ Do a bunch of API calls and display the results. Does a small transfer between tutorial endpoints, but otherwise does not modify user data. Uses module global API client instance. """ global api #activer=[username,"-c",os.getcwd()+"/credential.pem"] #api, _ = create_client_from_args(activer) user_credential_path=os.getcwd()+"/credential-"+username+".pem" #print "user_credential_path=",user_credential_path api = TransferAPIClient(username, cert_file=user_credential_path) api.set_debug_print(False, False) #print " Here (mover): ",api.task_list() # See what is in the account before we make any submissions. time.sleep(0.01) print "" print "" print "" print "=== Before transfer ===" #display_tasksummary(); print #display_task_list(); print #display_endpoint_list(); print print "=== Activate endpoints ===" dest_directory= dst_dir site_ep1 = src_site site_ep2 = dst_site print "Please enter your myproxy username (\'none\' if you do not have one)." myproxy_username = sys.stdin.readline().rstrip() preferred_activation(username, site_ep1, myproxy_username) preferred_activation(username, site_ep2, myproxy_username) print "=== Prepare transfer ===" #raw_input("Press Enter to continue...") # submit a transfer oldstdout=sys.stdout sys.stdout = open(os.devnull,'w') code, message, data = api.transfer_submission_id() sys.stdout = oldstdout # enable output #code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(minutes=10) t = Transfer(submission_id, site_ep1, site_ep2)#, deadline) f=open('json_file','r') json_results=f.read() f.close #print json_results,type(json_results) results=json.loads(json_results) #print results,type(results) #print results[0],type(results[0]) for result in results: #print "Result: ",result if result[-1]=="/": #print "Result: it is a directory" t.add_item(result, dest_directory, recursive=True) else: #print "Result: it is a file" file_name=re.split("/",result)[-1] #print "Result: "+file_name t.add_item(result, dest_directory+"/"+file_name) print "=== Submit transfer ===" oldstdout=sys.stdout sys.stdout = open(os.devnull,'w') code, reason, data = api.transfer(t) sys.stdout = oldstdout # enable output #code, reason, data = api.transfer(t) task_id = data["task_id"] #print " Task ID is %s " % (task_id) # see the new transfer show up #print "=== After submit ===" #display_tasksummary(); print #display_task(task_id); print #raw_input("Press Enter to continue...") # wait for the task to complete, and see the summary and lists # update print "=== Checking completion ===" # To save the task_id for further check could be useful. #wait_for_task(task_id) max_wait = 10*1 waiting = True if waiting: print "Task %s is still under process..." % (task_id) oldstdout=sys.stdout sys.stdout = open(os.devnull,'w') display_tasksummary(); print sys.stdout = oldstdout # enable output #display_task(task_id); print #display_ls("cin0641a#GSI-PLX"); print waiting=wait_for_task(task_id,max_wait) print "=== Exiting ===" #display_tasksummary(); print print "The task ID for this operation is: "+task_id; print oldstdout=sys.stdout sys.stdout = open(os.devnull,'w') status, reason, result = api.task(task_id) sys.stdout = oldstdout # enable output print "Its status is "+result["status"]; print
def transfer(id, source, dest): t = Transfer(id, source, dest) t.add_item(file_path, file_path) status, reason, result = api.transfer(t) os._exit(1)
src_proxy = x509_proxy.create_proxy_from_file(cred_file, src_pubkey) src_reqs.set_requirement_value("delegate_proxy", "proxy_chain", src_proxy) src_result = api.endpoint_activate(src, src_reqs) #The manual activation process, repeated for the destination endpoint. #We found that both endpoints need to be activated manually due to differences #in configurations across the different endpoints. dst_reqs = api.endpoint_activation_requirements(dst, type="delegate_proxy")[2] dst_pubkey = dst_reqs.get_requirement_value("delegate_proxy", "public_key") dst_proxy = x509_proxy.create_proxy_from_file(cred_file, dst_pubkey) dst_reqs.set_requirement_value("delegate_proxy", "proxy_chain", dst_proxy) dst_result = api.endpoint_activate(dst, dst_reqs) #Transfer setup and execution. sub_code, sub_reason, sub_data = api.submission_id() mytransfer = Transfer(sub_data['value'], src, dst) mytransfer.add_item(src_path, dest_path) code, reason, data = api.transfer(mytransfer) #Getting the task_id so we can poll for information on the transfer task_id = data['task_id'] #Loop for polling to see if the transfer has completed. while not api.task(task_id)[2]['completion_time']: #wait for the transfer to complete sleep(1) print "The transfer has " + api.task(task_id)[2]['status'] #We can deactivate the endpoints now that the transfer is done. #Deactivating endpoints before the transfer is complete will cause problems! api.endpoint_deactivate(src)
_print_task(t, 4) if __name__ == '__main__': api, _ = create_client_from_args() username=raw_input('Enter Myproxy username:'******'Enter MyProxy pass phrase:') activate(src_machine) activate(dest_machine) code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.datetime.utcnow() + datetime.timedelta(hours=max_hours) sync_level=None label=None t = Transfer(submission_id, src_machine, dest_machine, deadline,sync_level,label,verify_checksum=False) f=open(file,'r') list=f.readlines() f.close for line in list: mtch=m.match(line) if mtch==None: sys.stderr.write('Invalid input in input file.\n') sys.exit(1) paths=mtch.groups() if len(paths)!=2: sys.stderr.write('Invalid input in input file.\n') sys.exit(1)
def initializeGlobusTransfer(): global pendingTransfers global activeTasks global status_numActive global status_numPending maxQueue = config['globus']['max_transfer_file_count'] api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) submissionID = generateGlobusSubmissionID() if submissionID: # Prepare transfer object transferObj = Transfer(submissionID, config['globus']['source_endpoint_id'], config['globus']['destination_endpoint_id'], verify_checksum=True) queueLength = 0 # Loop over a copy of the list instead of actual thing - other thread will be appending to actual thing loopingTransfers = safeCopy(pendingTransfers) currentTransferBatch = {} logger.debug("- building transfer %s from pending queue" % submissionID) for ds in loopingTransfers: if "files" in loopingTransfers[ds]: if ds not in currentTransferBatch: currentTransferBatch[ds] = {} currentTransferBatch[ds]['files'] = {} # Add files from each dataset for f in loopingTransfers[ds]['files']: if queueLength < maxQueue: fobj = loopingTransfers[ds]['files'][f] if fobj["path"].find(config['globus']['source_path']) > -1: src_path = os.path.join(fobj["path"], fobj["name"]) dest_path = os.path.join(config['globus']['destination_path'], fobj["path"].replace(config['globus']['source_path'], ""), fobj["name"]) else: src_path = os.path.join(config['globus']['source_path'], fobj["path"], fobj["name"]) dest_path = os.path.join(config['globus']['destination_path'], fobj["path"], fobj["name"]) # Clean up dest path to new folder structure # ua-mac/raw_data/LemnaTec/EnvironmentLogger/2016-01-01/2016-08-03_04-05-34_environmentlogger.json # ua-mac/raw_data/LemnaTec/MovingSensor/co2Sensor/2016-01-01/2016-08-02__09-42-51-195/file.json # ua-mac/raw_data/LemnaTec/3DScannerRawDataTopTmp/scanner3DTop/2016-01-01/2016-08-02__09-42-51-195/file.json # ua-mac/raw_data/MAC/lightning/2016-01-01/weather_2016_06_29.dat dest_path = dest_path.replace("LemnaTec/", "") dest_path = dest_path.replace("MovingSensor/", "") dest_path = dest_path.replace("MAC/", "") dest_path = dest_path.replace("3DScannerRawDataTopTmp/", "") dest_path = dest_path.replace("3DScannerRawDataLowerOnEastSideTmp/", "") dest_path = dest_path.replace("3DScannerRawDataLowerOnWestSideTmp/", "") # /ua-mac/raw_data/MovingSensor.reproc2016-8-18/scanner3DTop/2016-08-22/2016-08-22__15-13-01-672/6af8d63b-b5bb-49b2-8e0e-c26e719f5d72__Top-heading-east_0.png # /ua-mac/raw_data/MovingSensor.reproc2016-8-18/scanner3DTop/2016-08-22/2016-08-22__15-13-01-672/6af8d63b-b5bb-49b2-8e0e-c26e719f5d72__Top-heading-east_0.ply if dest_path.endswith(".ply") and (dest_path.find("scanner3DTop") or dest_path.find("scanner3DLowerOnEastSide") or dest_path.find("scanner3DLowerOnWestSide")) > -1: dest_path = dest_path.replace("raw_data", "Level_1") if dest_path.find("MovingSensor.reproc") > -1: new_dest_path = "" dirs = dest_path.split("/") for dir_part in dirs: if dir_part.find("MovingSensor.reproc") == -1: new_dest_path = os.path.join(new_dest_path, dir_part) dest_path = new_dest_path # danforth/raw_data/sorghum_pilot_dataset/snapshot123456/file.png transferObj.add_item(src_path, dest_path) # remainingTransfers will have leftover data once max Globus transfer size is met queueLength += 1 fobj["orig_path"] = fobj["path"] fobj["path"] = dest_path currentTransferBatch[ds]['files'][f] = fobj else: break # Clean up placeholder entries once queue length is exceeded if currentTransferBatch[ds]['files'] == {}: del currentTransferBatch[ds]['files'] if currentTransferBatch[ds] == {}: del currentTransferBatch[ds] if queueLength >= maxQueue: break if queueLength > 0: # Send transfer to Globus try: logger.debug("- attempting to send new transfer") status_code, status_message, transfer_data = api.transfer(transferObj) except (APIError, ClientError) as e: try: # Try refreshing endpoints and retrying activateEndpoints() api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) status_code, status_message, transfer_data = api.transfer(transferObj) except (APIError, ClientError) as e: logger.error("- problem initializing Globus transfer") status_code = 503 status_message = e except: logger.error("- unexpected problem initializing Globus transfer") status_code = 503 status_message = e if status_code == 200 or status_code == 202: # Notify NCSA monitor of new task, and add to activeTasks for logging globusID = transfer_data['task_id'] logger.info("%s new Globus transfer task started (%s files)" % (globusID, queueLength), extra={ "globus_id": globusID, "action": "TRANSFER STARTED", "contents": currentTransferBatch }) activeTasks[globusID] = { "globus_id": globusID, "contents": currentTransferBatch, "started": str(datetime.datetime.now()), "status": "IN PROGRESS" } writeTasksToDisk(os.path.join(config['log_path'], "active_tasks.json"), activeTasks) # Now that we've safely sent the pending transfers, remove them for ds in currentTransferBatch: if ds in pendingTransfers: for f in currentTransferBatch[ds]['files']: if f in pendingTransfers[ds]['files']: del pendingTransfers[ds]['files'][f] notifyMonitorOfNewTransfer(globusID, currentTransferBatch) writeTasksToDisk(os.path.join(config['log_path'], "pending_transfers.json"), pendingTransfers) else: # If failed, leave pending list as-is and try again on next iteration (e.g. in 180 seconds) logger.error("- Globus transfer initialization failed for %s (%s: %s)" % (ds, status_code, status_message)) return status_numActive = len(activeTasks) cleanPendingTransfers()
def globus_transfer(image_id, cfg): packages_directory = os.path.join(os.path.expanduser("~"), ".gdclient","packages") package_directory = os.path.join(packages_directory, image_id) if not os.path.isdir(package_directory): print "Cannot find image "+image_id+" for transfer; please run --package level collaboration" return if not cfg.config['GLOBUS']['local-folder']: print "Cannot obtain from config file: local folder shared in GLOBUS " return if not cfg.config['GLOBUS']['local-endpoint']: print "Cannot obtain from config file: GLOBUS local endpoint" return if not cfg.config['GLOBUS']['remote-endpoint']: print "Cannot obtain from config file: GLOBUS remote endpoint" return if not cfg.config['GLOBUS']['globus-remote-folder']: print "Cannot obtain from config file: globus folder shared in remote GLOBUS endpoint" return if not cfg.config['GLOBUS']['globus-local-folder']: print "Cannot obtain from config file: globus folder shared in local GLOBUS endpoint" return print "Please wait. Saving image ..." short_image_name = image_id.strip()+".tgz" image_file = cfg.config['GLOBUS']['local-folder']+"/"+short_image_name command = "cd "+packages_directory+";tar czf "+image_file+" "+ image_id # print "will run: ",command run_command(command) source_endpoint = cfg.config['GLOBUS']['local-endpoint'] # 'globuspublish#cirlab' source_folder= cfg.config['GLOBUS']['globus-local-folder'] # '/globuspublication_52/' remote_endpoint = cfg.config['GLOBUS']['remote-endpoint'] # 'globuspublish#cirlab' remote_folder= cfg.config['GLOBUS']['globus-remote-folder'] # '/globuspublication_52/' # # command = "ssh [email protected] scp cvlaescx#test_2:/docker_image/dockeraf8db02049ae25d3e64436769411206258b3b003.tar cvlaescx#test:/home/cristian/Installs/new3.tar" # source = "%s%s%s" %(source_endpoint,source_folder, short_image_name) # destination = "%s%s%s" %(remote_endpoint,remote_folder, short_image_name) # command = "ssh %[email protected] transfer -- %s %s" % (cfg.config['Default']['uname'], source, destination) # # print "will run: ",command # run_command(command) # #print "USAGE: transfer [destination_endpoint:destination_folder]" transfer_args = [cfg.config['Default']['uname'], '-g', cfg.config['Default']['goauth-token']] TransferAPIClient, _ = create_client_from_args(transfer_args) activate_endpoint(TransferAPIClient, source_endpoint) activate_endpoint(TransferAPIClient, remote_endpoint) # submit the transfer code, message, data = TransferAPIClient.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(minutes=10) t = Transfer(submission_id, source_endpoint, remote_endpoint, deadline) source_image_name = "%s%s"%(source_folder,short_image_name) destination_image_name = "%s%s"%(remote_folder,short_image_name) t.add_item(source_image_name, destination_image_name) code, reason, data = TransferAPIClient.transfer(t) task_id = data["task_id"] print "Submitted transfer id: ", task_id return "globus://%s%s%s"%(remote_endpoint,remote_folder,short_image_name)
def initializeGlobusTransfer(globus_batch_obj): globus_batch = globus_batch_obj['contents'] globus_batch_id = globus_batch_obj['id'] api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) submissionID = generateGlobusSubmissionID() if submissionID: # Prepare transfer object transferObj = Transfer(submissionID, config['globus']['source_endpoint_id'], config['globus']['destination_endpoint_id'], verify_checksum=True) queue_length = 0 for ds in globus_batch: if 'files' in globus_batch[ds]: for fkey in globus_batch[ds]['files']: fobj = globus_batch[ds]['files'][fkey] if 'src_path' not in fobj: srcpath = fobj['orig_path'] if srcpath.startswith('/LemnaTec'): srcpath = "/gantry_data" + srcpath else: srcpath = fobj['src_path'] transferObj.add_item(srcpath, fobj['path']) queue_length += 1 # Send transfer to Globus try: logger.debug("- attempting to send new transfer") status_code, status_message, transfer_data = api.transfer( transferObj) except: try: # Try refreshing endpoints and retrying activateEndpoints() api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) status_code, status_message, transfer_data = api.transfer( transferObj) except (APIError, ClientError) as e: logger.error("- problem initializing Globus transfer") status_code = 503 status_message = e except: logger.error( "- unexpected problem initializing Globus transfer") status_code = 503 status_message = "" if status_code == 200 or status_code == 202: # Notify NCSA monitor of new task, and add to activeTasks for logging globusID = transfer_data['task_id'] logger.info("%s new Globus transfer task started (%s files)" % (globusID, queue_length), extra={ "globus_id": globusID, "action": "TRANSFER STARTED", "contents": globus_batch }) created_task = { "globus_id": globusID, "contents": globus_batch, "started": str(datetime.datetime.now()), "status": "CREATED" } writeTaskToDatabase(created_task) removePendingTask(globus_batch_id) return True else: # If failed, leave pending list as-is and try again on next iteration (e.g. in 180 seconds) logger.error( "- Globus transfer initialization failed for %s (%s: %s)" % (ds, status_code, status_message)) return False
def initializeGlobusTransfer(): global pendingTransfers global activeTasks global status_numActive global status_numPending maxQueue = config['globus']['max_transfer_file_count'] api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) submissionID = generateGlobusSubmissionID() if submissionID: # Prepare transfer object transferObj = Transfer(submissionID, config['globus']['source_endpoint_id'], config['globus']['destination_endpoint_id'], verify_checksum=True, preserve_timestamp=True) queueLength = 0 # Loop over a copy of the list instead of actual thing - other thread will be appending to actual thing loopingTransfers = safeCopy(pendingTransfers) currentTransferBatch = {} logger.debug("- building transfer %s from pending queue" % submissionID) for ds in loopingTransfers: if "files" in loopingTransfers[ds]: if ds not in currentTransferBatch: currentTransferBatch[ds] = {} currentTransferBatch[ds]['files'] = {} # Add files from each dataset for f in loopingTransfers[ds]['files']: if queueLength < maxQueue: fobj = loopingTransfers[ds]['files'][f] src_path = os.path.join(config['globus']['source_path'], fobj["path"], fobj["name"]) dest_path = os.path.join(config['globus']['destination_path'], fobj["path"], fobj["name"]) transferObj.add_item(src_path, dest_path) # remainingTransfers will have leftover data once max Globus transfer size is met queueLength += 1 currentTransferBatch[ds]['files'][f] = fobj else: break # Clean up placeholder entries once queue length is exceeded if currentTransferBatch[ds]['files'] == {}: del currentTransferBatch[ds]['files'] if currentTransferBatch[ds] == {}: del currentTransferBatch[ds] if queueLength >= maxQueue: break if queueLength > 0: # Send transfer to Globus try: logger.debug("- attempting to send new transfer") status_code, status_message, transfer_data = api.transfer(transferObj) except (APIError, ClientError) as e: try: # Try refreshing endpoints and retrying activateEndpoints() api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token']) status_code, status_message, transfer_data = api.transfer(transferObj) except (APIError, ClientError) as e: logger.error("- problem initializing Globus transfer") status_code = 503 status_message = e if status_code == 200 or status_code == 202: # Notify NCSA monitor of new task, and add to activeTasks for logging globusID = transfer_data['task_id'] logger.info("%s new Globus transfer task started (%s files)" % (globusID, queueLength), extra={ "globus_id": globusID, "action": "TRANSFER STARTED", "contents": currentTransferBatch }) activeTasks[globusID] = { "globus_id": globusID, "contents": currentTransferBatch, "started": str(datetime.datetime.now()), "status": "IN PROGRESS" } writeTasksToDisk(config['active_tasks_path'], activeTasks) # Now that we've safely sent the pending transfers, remove them for ds in currentTransferBatch: if ds in pendingTransfers: for f in currentTransferBatch[ds]['files']: if f in pendingTransfers[ds]['files']: del pendingTransfers[ds]['files'][f] notifyMonitorOfNewTransfer(globusID, currentTransferBatch) writeTasksToDisk(config['pending_transfers_path'], pendingTransfers) else: # If failed, leave pending list as-is and try again on next iteration (e.g. in 180 seconds) logger.error("- Globus transfer initialization failed for %s (%s: %s)" % (ds, status_code, status_message)) return status_numActive = len(activeTasks) cleanPendingTransfers()
def tutorial(): """ Do a bunch of API calls and display the results. Does a small transfer between tutorial endpoints, but otherwise does not modify user data. Uses module global API client instance. """ # See what is in the account before we make any submissions. #print "=== Before tutorial ===" #display_tasksummary(); print #display_task_list(); print #display_endpoint_list(); print print "=== Before transfer ===" # auto activate the endpoint, and display before/after. #display_activation("go#ep1") #display_activation("go#ep2") site_ep1 = "cin0641a#irods-dev" site_ep2 = "cin0641a#GSI-PLX" site_username = "******" api.set_debug_print(False, True) #status, message, data = api.endpoint_autoactivate("go#ep1") status, message, data = api.endpoint_autoactivate(site_ep1) #data = { #"DATA_TYPE": "activation_requirements", #"type": "myproxy", #"name": "username", #"description": "null", #"canonical_name": "null", #"public": "null", #"is_globus_connect": "null", #} #data = ActivationRequirementList(data) print "=== Activate endpoints ===" data.set_requirement_value("myproxy", "username", site_username) from getpass import getpass passphrase = getpass() data.set_requirement_value("myproxy", "passphrase", passphrase) #status, message, data = api.endpoint_activate(site_ep, data) #data["code"] api.endpoint_activate(site_ep1, data) api.endpoint_activate(site_ep2, data) #status, message, data = api.endpoint_activate(site_ep2, data) #data["code"] #display_activation("cin0641a#irods-dev") #display_ls("cin0641a#irods-dev"); print #display_activation("cin0641a#vzSARA") #display_ls("go#ep1"); print #display_ls("go#ep2"); print #display_ls("cin0641a#vzSARA"); print #display_activation("cin0641a#GSI-PLX") #display_ls("cin0641a#GSI-PLX"); print print "=== Prepare transfer ===" #raw_input("Press Enter to continue...") # submit a transfer code, message, data = api.transfer_submission_id() submission_id = data["value"] deadline = datetime.utcnow() + timedelta(minutes=10) t = Transfer(submission_id, site_ep1, site_ep2, deadline) t.add_item("/CINECA/home/gmariani/go-test/some-file/1", "/plx/userprace/pr1is019/tmp/1") t.add_item("/CINECA/home/gmariani/go-test/some-file/2", "/plx/userprace/pr1is019/tmp/2") t.add_item("/CINECA/home/gmariani/go-test/some-file/3", "/plx/userprace/pr1is019/tmp/3") t.add_item("/CINECA/home/gmariani/go-test/some-file/4", "/plx/userprace/pr1is019/tmp/4") print "=== Submit transfer ===" code, reason, data = api.transfer(t) task_id = data["task_id"] # see the new transfer show up #print "=== After submit ===" #display_tasksummary(); print #display_task(task_id); print #raw_input("Press Enter to continue...") # wait for the task to complete, and see the summary and lists # update print "=== Checking completion ===" wait_for_task(task_id) #if wait_for_task(task_id): #display_tasksummary(); print #display_task(task_id); print #display_ls("cin0641a#GSI-PLX"); print print "=== Deactivate endpoint ===" api.endpoint_deactivate(site_ep1) api.endpoint_deactivate(site_ep2)
def transfer(id, source, dest): t = Transfer(id, source, dest) t.add_item(src_file_path, dest_file_path) status, reason, result = api.transfer(t) os._exit(1)