def GO_transfer(ep1, dir1, ep2, dir2, files, api):
print ep1, dir1, ep2, dir2, files
status_code, status_message, data = api.submission_id()
sid = data['value']
print 'SID', sid
t = Transfer(sid, ep1, ep2)
for f in files:
path1 = dir1+f
path2 = dir2+f
print path1, path2
t.add_item(path1, path2)
status_code, status_message, data = api.transfer(t)
tid = data['task_id']# print 'task No.', tid
task_status='ACTIVE'
while task_status=='ACTIVE':
status_code, status_message, data = api.task(tid)
task_status=data['status']
# print 'data:', data
print 'task:', task_status
if task_status!='ACTIVE':break
time.sleep(5)
def GO_transfer(ep1, dir1, ep2, dir2, files, api):
print ep1, dir1, ep2, dir2, files
status_code, status_message, data = api.submission_id()
sid = data['value']
print 'SID', sid
t = Transfer(sid, ep1, ep2)
for f in files:
path1 = dir1 + f
path2 = dir2 + f
print path1, path2
t.add_item(path1, path2)
status_code, status_message, data = api.transfer(t)
tid = data['task_id'] # print 'task No.', tid
task_status = 'ACTIVE'
while task_status == 'ACTIVE':
status_code, status_message, data = api.task(tid)
task_status = data['status']
# print 'data:', data
print 'task:', task_status
if task_status != 'ACTIVE': break
time.sleep(5)
def start_transfer(self, src, dest):
"""
Starts a transfer
"""
if self.file_list == None or len(self.file_list) == 0:
raise TransferException("No files specified for transfer")
api = self._get_api()
# activate endpoints
_, _, data = api.endpoint_autoactivate(src)
_, _, data = api.endpoint_autoactivate(dest)
# create transfer and populate files
_, _, data = api.transfer_submission_id()
submission_id = data["value"]
t = Transfer(submission_id, src, dest)
for file in self.file_list:
t.add_item(os.path.join(self.src_dir, file), os.path.join(self.dest_dir, file))
# transfer
_, _, data = api.transfer(t)
self.go_task_id = data['task_id']
def submitTransfer(api_client, source_endpoint, source_files, target_endpoint, target_directory):
'''
Method to submit a data transfer request to Globus.
'''
# obtain a submission id from Globus
code, message, data = api_client.transfer_submission_id()
submission_id = data["value"]
print "Obtained transfer submission id: %s" % submission_id
# maximum time for completing the transfer
deadline = datetime.utcnow() + timedelta(days=10)
# create a transfer request
transfer_task = Transfer(submission_id, source_endpoint, target_endpoint, deadline)
for source_file in source_files:
source_directory, filename = os.path.split(source_file)
target_file = os.path.join(target_directory, filename)
transfer_task.add_item(source_file, target_file)
# submit the transfer request
try:
code, reason, data = api_client.transfer(transfer_task)
task_id = data["task_id"]
print "Submitted transfer task with id: %s" % task_id
except Exception as e:
print "Could not submit the transfer. Error: %s" % str(e)
task_id = "Could not submit the transfer. Please contact the ESGF node admin to investigate the issue."
return task_id
def transfer(self, src_endpoint, dst_endpoint, src_item, dst_dir, recursive=False):
"""
Transfer a file from one endpoint to another and return the Globus
task_id.
Set @recursive to True to transfer a folder.
:param src_endpoint: source endpoint name (i.e. user#endpoint)
:param dst_endpoint: destination endpoint name (i.e. user#endpoint)
:param src_item: object to be transferred
:param dst_dir: destination directory
:param recursive: flag to enable folder transfer
:return: Globus task_id
"""
if src_endpoint:
self.endpoint_activation(src_endpoint)
if dst_endpoint:
self.endpoint_activation(dst_endpoint)
# submit a transfer
code, message, data = self.api.transfer_submission_id()
submission_id = data["value"]
t = Transfer(submission_id, src_endpoint, dst_endpoint) # , deadline)
LOGGER.info(
"Transferring {0} from endpoint {1} to endpoint {2}".format(
src_item, src_endpoint, dst_endpoint))
t.add_item(src_item, os.path.join(dst_dir, os.path.basename(src_item)),
recursive=recursive)
code, reason, data = self.api.transfer(t)
task_id = data["task_id"]
return task_id
def transfer(self, src_endpoint, dst_endpoint, item, dst_dir):
"""
Transfer a file from one endpoint to another. Return the Globus
task_id.
:param src_endpoint: source endpoint name (i.e. user#endpoint)
:param dst_endpoint: destination endpoint name (i.e. user#endpoint)
:param item: object to be transferred
:param dst_dir: destination directory
:return: Globus task_id
"""
if src_endpoint:
self.endpoint_activation(src_endpoint)
if dst_endpoint:
self.endpoint_activation(dst_endpoint)
# submit a transfer
#oldstdout=sys.stdout
#sys.stdout = open(os.devnull,'w')
code, message, data = self.api.transfer_submission_id()
#sys.stdout = oldstdout # enable output
submission_id = data["value"]
#deadline = datetime.utcnow() + timedelta(minutes=10)
t = Transfer(submission_id, src_endpoint, dst_endpoint)#, deadline)
LOGGER.info("Transferring {0} from endpoint {1} to endpoint {2}".format(
item, src_endpoint, dst_endpoint))
t.add_item(item, os.path.join(dst_dir, os.path.basename(item)))
code, reason, data = self.api.transfer(t)
task_id = data["task_id"]
self.display_task(task_id)
return task_id
def transfer(options):
# Map legacy endpoint names to UUIDs
srcendpoint = options.srcendpoint
dstendpoint = options.dstendpoint
if '#' in srcendpoint:
srcendpoint = config['endpoint'][srcendpoint]
if '#' in dstendpoint:
dstendpoint = config['endpoint'][dstendpoint]
# Get access token (This method of getting an acces token is deprecated and should be replaced by OAuth2 calls).
auth_result = get_access_token(config['globus']['username'], config['globus']['password'])
# Create a transfer submission
api_client = TransferAPIClient(config['globus']['username'], goauth=auth_result.token)
activate_endpoint(api_client, srcendpoint)
activate_endpoint(api_client, dstendpoint)
code, message, data = api_client.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(days=10)
transfer_task = Transfer(submission_id, srcendpoint, dstendpoint, deadline)
# Add srcpath to the transfer task
if options.srcpath:
transfer_task.add_item(options.srcpath,
get_destination_path(options.srcpath, options.dstpath, options.recursive),
recursive=options.recursive)
# Add srclist to the transfer task
if options.srclist:
with open(options.srclist) as f:
srcpath = f.readline().rstrip('\n')
transfer_task.add_item(srcpath,
get_destination_path(srcpath, options.dstpath, options.recursive),
recursive=options.recursive)
# Start the transfer
task_id = None
try:
code, reason, data = api_client.transfer(transfer_task)
task_id = data["task_id"]
print 'task_id %s' % task_id
except Exception as e:
msg = "Could not submit the transfer. Error: %s" % str(e)
sys.exit(1)
# Check a status of the transfer every minute (60 secs)
while True:
code, reason, data = api_client.task(task_id)
print data['status']
if data['status'] == 'SUCCEEDED':
print 'progress %d/%d' % (data['files_transferred'], data['files'])
return ('success', '')
elif data['status'] == 'FAILED':
return ('error', data['nice_status_details'])
elif data['status'] == 'ACTIVE':
print 'progress %d/%d' % (data['files_transferred'], data['files'])
time.sleep(10)
def transfer(src_endpoint, src_path, dst_endpoint, dst_path, access_token,
openid):
(server, username) = decompose_openid(openid)
client = TransferAPIClient(username, goauth=access_token)
client.connect()
globus_submission_id = client.submission_id()
transfer = Transfer(globus_submission_id, src_endpoint, dst_endpoint)
transfer.add_item(src_path, dst_path, recursive=True)
client.transfer(transfer)
return {'status': 'Success', 'message': ''}
def tutorial():
"""
Do a bunch of API calls and display the results. Does a small transfer
between tutorial endpoints, but otherwise does not modify user data.
Uses module global API client instance.
"""
# See what is in the account before we make any submissions.
print "=== Before tutorial ==="
display_tasksummary(); print
display_task_list(); print
display_endpoint_list(); print
# auto activate the endpoint, and display before/after.
display_activation("go#ep1")
display_activation("go#ep2")
print "=== Before transfer ==="
display_ls("go#ep1"); print
display_ls("go#ep2"); print
# submit a transfer
code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(minutes=10)
t = Transfer(submission_id, "go#ep1", "go#ep2", deadline)
t.add_item("/~/.bashrc", "/~/api-example-bashrc-copy")
code, reason, data = api.transfer(t)
task_id = data["task_id"]
# see the new transfer show up
print "=== After submit ==="
display_tasksummary(); print
display_task(task_id, False); print
# wait for the task to complete, and see the tasks and
# endpoint ls change
status = wait_for_task(task_id)
if status is None:
# Task didn't complete before the timeout.
# Since the example transfers a single small file and the
# timeout is 2 mintues, this shouldn't happen unless one of the
# endpoints is having problems or the user already has a bunch
# of other active tasks (there is a limit to the
# number of concurrent active tasks a user can have).
print "WARNING: task did not complete before timeout!"
else:
print "Task %s complete with status %s" % (task_id, status)
print "=== After completion ==="
display_tasksummary(); print
display_task(task_id); print
display_ls("go#ep2"); print
def remove(request):
"""
removes files on a remote Globus Online endpoint - API is not complete, so transfer
0 byte files instead of actually deleting anything
"""
# global so that we can use it in signal handlers
global api
global task_id
# connect to the service
api = TransferAPIClient(request["globus_username"],
cert_file=request["x509_proxy"])
# make sure we can auto-activate the endpoints
ep = activate_ep(api, request["endpoint"])
label = None
if "PEGASUS_WF_UUID" in os.environ and "PEGASUS_DAG_JOB_ID" in os.environ:
label = os.environ["PEGASUS_WF_UUID"] + " - " + os.environ[
"PEGASUS_DAG_JOB_ID"]
# set up a new transfer
code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(hours=24)
t = Transfer(submission_id,
request["endpoint"],
request["endpoint"],
deadline=deadline,
label=label,
notify_on_succeeded=False,
notify_on_failed=False,
notify_on_inactive=False)
for f in request["files"]:
t.add_item("/dev/null", f)
# finalize and submit the transfer
code, reason, data = api.transfer(t)
task_id = data["task_id"]
# how many faults will we accept before giving up?
acceptable_faults = min(100, len(request["files"]) * 3)
# wait for the task to complete, and see the tasks and
# endpoint ls change
try:
status = wait_for_task(api, task_id, acceptable_faults)
except Exception, err:
logger.error(err)
cancel_task(api, task_id)
sys.exit(1)
def tutorial():
"""
Do a bunch of API calls and display the results. Does a small transfer
between tutorial endpoints, but otherwise does not modify user data.
Uses module global API client instance.
"""
# See what is in the account before we make any submissions.
print "=== Before tutorial ==="
display_tasksummary()
print
display_task_list()
print
display_endpoint_list()
print
# auto activate the endpoint, and display before/after.
display_activation("go#ep1")
display_activation("go#ep2")
print "=== Before transfer ==="
display_ls("go#ep1")
print
display_ls("go#ep2")
print
# submit a transfer
code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(minutes=10)
t = Transfer(submission_id, "go#ep1", "go#ep2", deadline)
t.add_item("/~/.bashrc", "/~/api-example-bashrc-copy")
code, reason, data = api.transfer(t)
task_id = data["task_id"]
# see the new transfer show up
print "=== After submit ==="
display_tasksummary()
print
display_task(task_id)
print
# wait for the task to complete, and see the summary and lists
# update
if wait_for_task(task_id):
print "=== After completion ==="
display_tasksummary()
print
display_task(task_id)
print
display_ls("go#ep2")
print
def tutorial():
"""
Do a bunch of API calls and display the results. Does a small transfer
between tutorial endpoints, but otherwise does not modify user data.
Uses module global API client instance.
"""
# See what is in the account before we make any submissions.
print "=== Before tutorial ==="
display_task_list(); print
display_endpoint_list(); print
# auto activate the endpoint, and display before/after.
display_activation("go#ep1")
display_activation("go#ep2")
print "=== Before transfer ==="
display_ls("go#ep1"); print
display_ls("go#ep2"); print
# submit a transfer
code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(minutes=10)
t = Transfer(submission_id, "go#ep1", "go#ep2", deadline)
t.add_item("/~/.bashrc", "/~/api-example-bashrc-copy")
code, reason, data = api.transfer(t)
task_id = data["task_id"]
# see the new transfer show up
print "=== After submit ==="
display_task(task_id, False); print
# wait for the task to complete, and see the tasks and
# endpoint ls change
status = wait_for_task(task_id)
if status is None:
# Task didn't complete before the timeout.
# Since the example transfers a single small file and the
# timeout is 2 mintues, this shouldn't happen unless one of the
# endpoints is having problems or the user already has a bunch
# of other active tasks (there is a limit to the
# number of concurrent active tasks a user can have).
print "WARNING: task did not complete before timeout!"
else:
print "Task %s complete with status %s" % (task_id, status)
print "=== After completion ==="
display_task(task_id); print
display_ls("go#ep2"); print
def do_transfer(local_ep, local_dir, remote_ep, remote_dir):
logging.debug('Initiating transfer with:')
logging.debug('local endpoint: {0}, local dir: {1}, remote endpoint: {2}, remote dir: {3}'.format(local_ep, local_dir,
remote_ep, remote_dir))
submission_id = get_submission_id()
transfer = Transfer(submission_id,
local_ep, remote_ep,
sync_level=3, encrypt_data=True)
logging.debug('Adding transfer item: {}, {}'.format(local_dir, remote_dir))
transfer.add_item(local_dir, remote_dir, recursive=True)
logging.debug('Transfer object as JSON:')
logging.debug(transfer.as_json())
logging.debug('API Client instance: {}'.format(api))
api.transfer(transfer)
api.close()
def do_transfer(local_ep, local_dir, remote_ep, remote_dir):
logging.debug('Initiating transfer with:')
logging.debug(
'local endpoint: {0}, local dir: {1}, remote endpoint: {2}, remote dir: {3}'
.format(local_ep, local_dir, remote_ep, remote_dir))
submission_id = get_submission_id()
transfer = Transfer(submission_id,
local_ep,
remote_ep,
sync_level=3,
encrypt_data=True)
logging.debug('Adding transfer item: {}, {}'.format(local_dir, remote_dir))
transfer.add_item(local_dir, remote_dir, recursive=True)
logging.debug('Transfer object as JSON:')
logging.debug(transfer.as_json())
logging.debug('API Client instance: {}'.format(api))
api.transfer(transfer)
api.close()
def tutorial():
"""
Do a bunch of API calls and display the results. Does a small transfer
between tutorial endpoints, but otherwise does not modify user data.
Uses module global API client instance.
"""
# See what is in the account before we make any submissions.
print "=== Before tutorial ==="
display_tasksummary(); print
display_task_list(); print
display_endpoint_list(); print
# auto activate the endpoint, and display before/after.
display_activation("go#ep1")
display_activation("go#ep2")
print "=== Before transfer ==="
display_ls("go#ep1"); print
display_ls("go#ep2"); print
# submit a transfer
code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(minutes=10)
t = Transfer(submission_id, "go#ep1", "go#ep2", deadline)
t.add_item("/~/.bashrc", "/~/api-example-bashrc-copy")
code, reason, data = api.transfer(t)
task_id = data["task_id"]
# see the new transfer show up
print "=== After submit ==="
display_tasksummary(); print
display_task(task_id); print
# wait for the task to complete, and see the summary and lists
# update
if wait_for_task(task_id):
print "=== After completion ==="
display_tasksummary(); print
display_task(task_id); print
display_ls("go#ep2"); print
def start_transfer(self, filelist):
""" activate src endpoint
Parameters
----------
filelist : list
List of files to transfer
Returns
-------
str of the globus task id
"""
src_endpoint = self.srcinfo['endpoint']
result = self.endpoint_activate(src_endpoint)
# activate dst endpoint
dst_endpoint = self.dstinfo['endpoint']
result = self.endpoint_activate(dst_endpoint)
# create dst directories
self.makedirs([finfo['dst'] for finfo in filelist.values()],
dst_endpoint)
## Get a submission id:
_, _, result = self.goclient.transfer_submission_id()
self.submission_id = result["value"]
miscutils.fwdebug(1, 'GLOBUS_ONLINE_DEBUG',
"\tsubmission id = %s" % self.submission_id)
## Create a transfer object:
#t = Transfer(submission_id, src_endpoint, dst_endpoint, notify_on_succeeded = False,
# notify_on_failed = False, notify_on_inactive= False, deadline='2m')
deadline = datetime.utcnow() + timedelta(minutes=30)
t = Transfer(self.submission_id,
src_endpoint,
dst_endpoint,
notify_on_succeeded=False,
notify_on_failed=False,
notify_on_inactive=False,
deadline=deadline)
#print t.as_data()
# add files to transfer
for _, finfo in filelist.items():
sfile = finfo['src']
dfile = finfo['dst']
miscutils.fwdebug(2, 'GLOBUS_ONLINE_DEBUG',
"\tadding to transfer %s = %s" % (sfile, dfile))
if sfile.endswith('/'):
t.add_item(sfile, dfile,
recursive=True) # error if true for file
else:
t.add_item(sfile, dfile)
# start transfer
_, _, result = self.goclient.transfer(t)
task_id = result["task_id"]
miscutils.fwdebug(1, 'GLOBUS_ONLINE_DEBUG', "\ttask id = %s" % task_id)
return task_id
def transfer(self,
src_endpoint,
dst_endpoint,
src_item,
dst_dir,
recursive=False):
"""
Transfer a file from one endpoint to another and return the Globus
task_id.
Set @recursive to True to transfer a folder.
:param src_endpoint: source endpoint name (i.e. user#endpoint)
:param dst_endpoint: destination endpoint name (i.e. user#endpoint)
:param src_item: object to be transferred
:param dst_dir: destination directory
:param recursive: flag to enable folder transfer
:return: Globus task_id
"""
if src_endpoint:
self.endpoint_activation(src_endpoint)
if dst_endpoint:
self.endpoint_activation(dst_endpoint)
# submit a transfer
code, message, data = self.api.transfer_submission_id()
submission_id = data["value"]
t = Transfer(submission_id, src_endpoint, dst_endpoint) # , deadline)
LOGGER.info(
"Transferring {0} from endpoint {1} to endpoint {2}".format(
src_item, src_endpoint, dst_endpoint))
t.add_item(src_item,
os.path.join(dst_dir, os.path.basename(src_item)),
recursive=recursive)
code, reason, data = self.api.transfer(t)
task_id = data["task_id"]
return task_id
def transfer(options):
# Map legacy endpoint names to UUIDs
srcendpoint = options.srcendpoint
dstendpoint = options.dstendpoint
if '#' in srcendpoint:
srcendpoint = config['endpoint'][srcendpoint]
if '#' in dstendpoint:
dstendpoint = config['endpoint'][dstendpoint]
# Get access token (This method of getting an acces token is deprecated and should be replaced by OAuth2 calls).
auth_result = get_access_token(config['globus']['username'], config['globus']['password'])
# Create a transfer submission
api_client = TransferAPIClient(config['globus']['username'], goauth=auth_result.token)
activate_endpoint(api_client, srcendpoint)
activate_endpoint(api_client, dstendpoint)
code, message, data = api_client.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(days=10)
transfer_task = Transfer(submission_id, srcendpoint, dstendpoint, deadline)
# Add srcpath to the transfer task
if options.srcpath:
transfer_task.add_item(options.srcpath,
get_destination_path(options.srcpath, options.dstpath, options.recursive),
recursive=options.recursive)
# Add srclist to the transfer task
if options.srclist:
with open(options.srclist) as f:
srcpath = f.readline().rstrip('\n')
transfer_task.add_item(srcpath,
get_destination_path(srcpath, options.dstpath, options.recursive),
recursive=options.recursive)
# Start the transfer
task_id = None
try:
code, reason, data = api_client.transfer(transfer_task)
task_id = data["task_id"]
print 'task_id %s' % task_id
except Exception as e:
msg = "Could not submit the transfer. Error: %s" % str(e)
sys.exit(1)
# Check a status of the transfer every minute (60 secs)
while True:
code, reason, data = api_client.task(task_id)
if data['status'] == 'SUCCEEDED':
print 'progress %d/%d' % (data['files_transferred'], data['files'])
return ('success', '')
elif data['status'] == 'FAILED':
return ('error', data['nice_status_details'])
elif data['status'] == 'ACTIVE':
print 'progress %d/%d' % (data['files_transferred'], data['files'])
time.sleep(10)
def transfer(request):
"""
takes a transfer specification parsed from json:
{
"globus_username": "******",
"x509_proxy": "/tmp/...",
"src_endpoint": "rynge#obelix",
"dst_endpoint": "rynge#workflow",
"files":[
{"src":"/etc/hosts","dst":"/tmp/foobar.txt"},
{"src":"/etc/hosts","dst":"/tmp/foobar-2.txt"}
],
}
"""
# global so that we can use it in signal handlers
global api
global task_id
# connect to the service
api = TransferAPIClient(request["globus_username"],
cert_file=request["x509_proxy"])
# make sure we can auto-activate the endpoints
src_ep = activate_ep(api, request["src_endpoint"])
dst_ep = activate_ep(api, request["dst_endpoint"])
label = None
if "PEGASUS_WF_UUID" in os.environ and "PEGASUS_DAG_JOB_ID" in os.environ:
label = os.environ["PEGASUS_WF_UUID"] + " - " + os.environ[
"PEGASUS_DAG_JOB_ID"]
# set up a new transfer
code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(hours=24)
t = Transfer(submission_id,
request["src_endpoint"],
request["dst_endpoint"],
deadline=deadline,
label=label,
notify_on_succeeded=False,
notify_on_failed=False,
notify_on_inactive=False)
for pair in request["files"]:
t.add_item(pair["src"], pair["dst"])
# finalize and submit the transfer
code, reason, data = api.transfer(t)
task_id = data["task_id"]
# how many faults will we accept before giving up?
acceptable_faults = min(100, len(request["files"]) * 3)
# wait for the task to complete, and see the tasks and
# endpoint ls change
try:
status = wait_for_task(api, task_id, acceptable_faults)
except Exception, err:
logger.error(err)
cancel_task(api, task_id)
sys.exit(1)
def mover(username, src_site, dst_site, dst_dir):
"""
Do a bunch of API calls and display the results. Does a small transfer
between tutorial endpoints, but otherwise does not modify user data.
Uses module global API client instance.
"""
global api
#activer=[username,"-c",os.getcwd()+"/credential.pem"]
#api, _ = create_client_from_args(activer)
user_credential_path=os.getcwd()+"/credential-"+username+".pem"
#print "user_credential_path=",user_credential_path
api = TransferAPIClient(username, cert_file=user_credential_path)
api.set_debug_print(False, False)
#print " Here (mover): ",api.task_list()
# See what is in the account before we make any submissions.
time.sleep(0.01)
print ""
print ""
print ""
print "=== Before transfer ==="
#display_tasksummary(); print
#display_task_list(); print
#display_endpoint_list(); print
print "=== Activate endpoints ==="
dest_directory= dst_dir
site_ep1 = src_site
site_ep2 = dst_site
print "Please enter your myproxy username (\'none\' if you do not have one)."
myproxy_username = sys.stdin.readline().rstrip()
preferred_activation(username, site_ep1, myproxy_username)
preferred_activation(username, site_ep2, myproxy_username)
print "=== Prepare transfer ==="
#raw_input("Press Enter to continue...")
# submit a transfer
oldstdout=sys.stdout
sys.stdout = open(os.devnull,'w')
code, message, data = api.transfer_submission_id()
sys.stdout = oldstdout # enable output
#code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(minutes=10)
t = Transfer(submission_id, site_ep1, site_ep2)#, deadline)
f=open('json_file','r')
json_results=f.read()
f.close
#print json_results,type(json_results)
results=json.loads(json_results)
#print results,type(results)
#print results[0],type(results[0])
for result in results:
#print "Result: ",result
if result[-1]=="/":
#print "Result: it is a directory"
t.add_item(result, dest_directory, recursive=True)
else:
#print "Result: it is a file"
file_name=re.split("/",result)[-1]
#print "Result: "+file_name
t.add_item(result, dest_directory+"/"+file_name)
print "=== Submit transfer ==="
oldstdout=sys.stdout
sys.stdout = open(os.devnull,'w')
code, reason, data = api.transfer(t)
sys.stdout = oldstdout # enable output
#code, reason, data = api.transfer(t)
task_id = data["task_id"]
#print " Task ID is %s " % (task_id)
# see the new transfer show up
#print "=== After submit ==="
#display_tasksummary(); print
#display_task(task_id); print
#raw_input("Press Enter to continue...")
# wait for the task to complete, and see the summary and lists
# update
print "=== Checking completion ==="
# To save the task_id for further check could be useful.
#wait_for_task(task_id)
max_wait = 10*1
waiting = True
if waiting:
print "Task %s is still under process..." % (task_id)
oldstdout=sys.stdout
sys.stdout = open(os.devnull,'w')
display_tasksummary(); print
sys.stdout = oldstdout # enable output
#display_task(task_id); print
#display_ls("cin0641a#GSI-PLX"); print
waiting=wait_for_task(task_id,max_wait)
print "=== Exiting ==="
#display_tasksummary(); print
print "The task ID for this operation is: "+task_id; print
oldstdout=sys.stdout
sys.stdout = open(os.devnull,'w')
status, reason, result = api.task(task_id)
sys.stdout = oldstdout # enable output
print "Its status is "+result["status"]; print
def transfer(id, source, dest):
t = Transfer(id, source, dest)
t.add_item(file_path, file_path)
status, reason, result = api.transfer(t)
os._exit(1)
src_proxy = x509_proxy.create_proxy_from_file(cred_file, src_pubkey)
src_reqs.set_requirement_value("delegate_proxy", "proxy_chain", src_proxy)
src_result = api.endpoint_activate(src, src_reqs)
#The manual activation process, repeated for the destination endpoint.
#We found that both endpoints need to be activated manually due to differences
#in configurations across the different endpoints.
dst_reqs = api.endpoint_activation_requirements(dst, type="delegate_proxy")[2]
dst_pubkey = dst_reqs.get_requirement_value("delegate_proxy", "public_key")
dst_proxy = x509_proxy.create_proxy_from_file(cred_file, dst_pubkey)
dst_reqs.set_requirement_value("delegate_proxy", "proxy_chain", dst_proxy)
dst_result = api.endpoint_activate(dst, dst_reqs)
#Transfer setup and execution.
sub_code, sub_reason, sub_data = api.submission_id()
mytransfer = Transfer(sub_data['value'], src, dst)
mytransfer.add_item(src_path, dest_path)
code, reason, data = api.transfer(mytransfer)
#Getting the task_id so we can poll for information on the transfer
task_id = data['task_id']
#Loop for polling to see if the transfer has completed.
while not api.task(task_id)[2]['completion_time']:
#wait for the transfer to complete
sleep(1)
print "The transfer has " + api.task(task_id)[2]['status']
#We can deactivate the endpoints now that the transfer is done.
#Deactivating endpoints before the transfer is complete will cause problems!
api.endpoint_deactivate(src)
_print_task(t, 4)
if __name__ == '__main__':
api, _ = create_client_from_args()
username=raw_input('Enter Myproxy username:'******'Enter MyProxy pass phrase:')
activate(src_machine)
activate(dest_machine)
code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.datetime.utcnow() + datetime.timedelta(hours=max_hours)
sync_level=None
label=None
t = Transfer(submission_id, src_machine, dest_machine, deadline,sync_level,label,verify_checksum=False)
f=open(file,'r')
list=f.readlines()
f.close
for line in list:
mtch=m.match(line)
if mtch==None:
sys.stderr.write('Invalid input in input file.\n')
sys.exit(1)
paths=mtch.groups()
if len(paths)!=2:
sys.stderr.write('Invalid input in input file.\n')
sys.exit(1)
def initializeGlobusTransfer():
global pendingTransfers
global activeTasks
global status_numActive
global status_numPending
maxQueue = config['globus']['max_transfer_file_count']
api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token'])
submissionID = generateGlobusSubmissionID()
if submissionID:
# Prepare transfer object
transferObj = Transfer(submissionID,
config['globus']['source_endpoint_id'],
config['globus']['destination_endpoint_id'],
verify_checksum=True)
queueLength = 0
# Loop over a copy of the list instead of actual thing - other thread will be appending to actual thing
loopingTransfers = safeCopy(pendingTransfers)
currentTransferBatch = {}
logger.debug("- building transfer %s from pending queue" % submissionID)
for ds in loopingTransfers:
if "files" in loopingTransfers[ds]:
if ds not in currentTransferBatch:
currentTransferBatch[ds] = {}
currentTransferBatch[ds]['files'] = {}
# Add files from each dataset
for f in loopingTransfers[ds]['files']:
if queueLength < maxQueue:
fobj = loopingTransfers[ds]['files'][f]
if fobj["path"].find(config['globus']['source_path']) > -1:
src_path = os.path.join(fobj["path"], fobj["name"])
dest_path = os.path.join(config['globus']['destination_path'],
fobj["path"].replace(config['globus']['source_path'], ""),
fobj["name"])
else:
src_path = os.path.join(config['globus']['source_path'], fobj["path"], fobj["name"])
dest_path = os.path.join(config['globus']['destination_path'], fobj["path"], fobj["name"])
# Clean up dest path to new folder structure
# ua-mac/raw_data/LemnaTec/EnvironmentLogger/2016-01-01/2016-08-03_04-05-34_environmentlogger.json
# ua-mac/raw_data/LemnaTec/MovingSensor/co2Sensor/2016-01-01/2016-08-02__09-42-51-195/file.json
# ua-mac/raw_data/LemnaTec/3DScannerRawDataTopTmp/scanner3DTop/2016-01-01/2016-08-02__09-42-51-195/file.json
# ua-mac/raw_data/MAC/lightning/2016-01-01/weather_2016_06_29.dat
dest_path = dest_path.replace("LemnaTec/", "")
dest_path = dest_path.replace("MovingSensor/", "")
dest_path = dest_path.replace("MAC/", "")
dest_path = dest_path.replace("3DScannerRawDataTopTmp/", "")
dest_path = dest_path.replace("3DScannerRawDataLowerOnEastSideTmp/", "")
dest_path = dest_path.replace("3DScannerRawDataLowerOnWestSideTmp/", "")
# /ua-mac/raw_data/MovingSensor.reproc2016-8-18/scanner3DTop/2016-08-22/2016-08-22__15-13-01-672/6af8d63b-b5bb-49b2-8e0e-c26e719f5d72__Top-heading-east_0.png
# /ua-mac/raw_data/MovingSensor.reproc2016-8-18/scanner3DTop/2016-08-22/2016-08-22__15-13-01-672/6af8d63b-b5bb-49b2-8e0e-c26e719f5d72__Top-heading-east_0.ply
if dest_path.endswith(".ply") and (dest_path.find("scanner3DTop") or
dest_path.find("scanner3DLowerOnEastSide") or
dest_path.find("scanner3DLowerOnWestSide")) > -1:
dest_path = dest_path.replace("raw_data", "Level_1")
if dest_path.find("MovingSensor.reproc") > -1:
new_dest_path = ""
dirs = dest_path.split("/")
for dir_part in dirs:
if dir_part.find("MovingSensor.reproc") == -1:
new_dest_path = os.path.join(new_dest_path, dir_part)
dest_path = new_dest_path
# danforth/raw_data/sorghum_pilot_dataset/snapshot123456/file.png
transferObj.add_item(src_path, dest_path)
# remainingTransfers will have leftover data once max Globus transfer size is met
queueLength += 1
fobj["orig_path"] = fobj["path"]
fobj["path"] = dest_path
currentTransferBatch[ds]['files'][f] = fobj
else:
break
# Clean up placeholder entries once queue length is exceeded
if currentTransferBatch[ds]['files'] == {}:
del currentTransferBatch[ds]['files']
if currentTransferBatch[ds] == {}:
del currentTransferBatch[ds]
if queueLength >= maxQueue:
break
if queueLength > 0:
# Send transfer to Globus
try:
logger.debug("- attempting to send new transfer")
status_code, status_message, transfer_data = api.transfer(transferObj)
except (APIError, ClientError) as e:
try:
# Try refreshing endpoints and retrying
activateEndpoints()
api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token'])
status_code, status_message, transfer_data = api.transfer(transferObj)
except (APIError, ClientError) as e:
logger.error("- problem initializing Globus transfer")
status_code = 503
status_message = e
except:
logger.error("- unexpected problem initializing Globus transfer")
status_code = 503
status_message = e
if status_code == 200 or status_code == 202:
# Notify NCSA monitor of new task, and add to activeTasks for logging
globusID = transfer_data['task_id']
logger.info("%s new Globus transfer task started (%s files)" % (globusID, queueLength), extra={
"globus_id": globusID,
"action": "TRANSFER STARTED",
"contents": currentTransferBatch
})
activeTasks[globusID] = {
"globus_id": globusID,
"contents": currentTransferBatch,
"started": str(datetime.datetime.now()),
"status": "IN PROGRESS"
}
writeTasksToDisk(os.path.join(config['log_path'], "active_tasks.json"), activeTasks)
# Now that we've safely sent the pending transfers, remove them
for ds in currentTransferBatch:
if ds in pendingTransfers:
for f in currentTransferBatch[ds]['files']:
if f in pendingTransfers[ds]['files']:
del pendingTransfers[ds]['files'][f]
notifyMonitorOfNewTransfer(globusID, currentTransferBatch)
writeTasksToDisk(os.path.join(config['log_path'], "pending_transfers.json"), pendingTransfers)
else:
# If failed, leave pending list as-is and try again on next iteration (e.g. in 180 seconds)
logger.error("- Globus transfer initialization failed for %s (%s: %s)" % (ds, status_code, status_message))
return
status_numActive = len(activeTasks)
cleanPendingTransfers()
def globus_transfer(image_id, cfg):
packages_directory = os.path.join(os.path.expanduser("~"), ".gdclient","packages")
package_directory = os.path.join(packages_directory, image_id)
if not os.path.isdir(package_directory):
print "Cannot find image "+image_id+" for transfer; please run --package level collaboration"
return
if not cfg.config['GLOBUS']['local-folder']:
print "Cannot obtain from config file: local folder shared in GLOBUS "
return
if not cfg.config['GLOBUS']['local-endpoint']:
print "Cannot obtain from config file: GLOBUS local endpoint"
return
if not cfg.config['GLOBUS']['remote-endpoint']:
print "Cannot obtain from config file: GLOBUS remote endpoint"
return
if not cfg.config['GLOBUS']['globus-remote-folder']:
print "Cannot obtain from config file: globus folder shared in remote GLOBUS endpoint"
return
if not cfg.config['GLOBUS']['globus-local-folder']:
print "Cannot obtain from config file: globus folder shared in local GLOBUS endpoint"
return
print "Please wait. Saving image ..."
short_image_name = image_id.strip()+".tgz"
image_file = cfg.config['GLOBUS']['local-folder']+"/"+short_image_name
command = "cd "+packages_directory+";tar czf "+image_file+" "+ image_id
# print "will run: ",command
run_command(command)
source_endpoint = cfg.config['GLOBUS']['local-endpoint'] # 'globuspublish#cirlab'
source_folder= cfg.config['GLOBUS']['globus-local-folder'] # '/globuspublication_52/'
remote_endpoint = cfg.config['GLOBUS']['remote-endpoint'] # 'globuspublish#cirlab'
remote_folder= cfg.config['GLOBUS']['globus-remote-folder'] # '/globuspublication_52/'
# # command = "ssh [email protected] scp cvlaescx#test_2:/docker_image/dockeraf8db02049ae25d3e64436769411206258b3b003.tar cvlaescx#test:/home/cristian/Installs/new3.tar"
# source = "%s%s%s" %(source_endpoint,source_folder, short_image_name)
# destination = "%s%s%s" %(remote_endpoint,remote_folder, short_image_name)
# command = "ssh %[email protected] transfer -- %s %s" % (cfg.config['Default']['uname'], source, destination)
# # print "will run: ",command
# run_command(command)
# #print "USAGE: transfer [destination_endpoint:destination_folder]"
transfer_args = [cfg.config['Default']['uname'], '-g', cfg.config['Default']['goauth-token']]
TransferAPIClient, _ = create_client_from_args(transfer_args)
activate_endpoint(TransferAPIClient, source_endpoint)
activate_endpoint(TransferAPIClient, remote_endpoint)
# submit the transfer
code, message, data = TransferAPIClient.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(minutes=10)
t = Transfer(submission_id, source_endpoint, remote_endpoint, deadline)
source_image_name = "%s%s"%(source_folder,short_image_name)
destination_image_name = "%s%s"%(remote_folder,short_image_name)
t.add_item(source_image_name, destination_image_name)
code, reason, data = TransferAPIClient.transfer(t)
task_id = data["task_id"]
print "Submitted transfer id: ", task_id
return "globus://%s%s%s"%(remote_endpoint,remote_folder,short_image_name)
def initializeGlobusTransfer(globus_batch_obj):
globus_batch = globus_batch_obj['contents']
globus_batch_id = globus_batch_obj['id']
api = TransferAPIClient(username=config['globus']['username'],
goauth=config['globus']['auth_token'])
submissionID = generateGlobusSubmissionID()
if submissionID:
# Prepare transfer object
transferObj = Transfer(submissionID,
config['globus']['source_endpoint_id'],
config['globus']['destination_endpoint_id'],
verify_checksum=True)
queue_length = 0
for ds in globus_batch:
if 'files' in globus_batch[ds]:
for fkey in globus_batch[ds]['files']:
fobj = globus_batch[ds]['files'][fkey]
if 'src_path' not in fobj:
srcpath = fobj['orig_path']
if srcpath.startswith('/LemnaTec'):
srcpath = "/gantry_data" + srcpath
else:
srcpath = fobj['src_path']
transferObj.add_item(srcpath, fobj['path'])
queue_length += 1
# Send transfer to Globus
try:
logger.debug("- attempting to send new transfer")
status_code, status_message, transfer_data = api.transfer(
transferObj)
except:
try:
# Try refreshing endpoints and retrying
activateEndpoints()
api = TransferAPIClient(username=config['globus']['username'],
goauth=config['globus']['auth_token'])
status_code, status_message, transfer_data = api.transfer(
transferObj)
except (APIError, ClientError) as e:
logger.error("- problem initializing Globus transfer")
status_code = 503
status_message = e
except:
logger.error(
"- unexpected problem initializing Globus transfer")
status_code = 503
status_message = ""
if status_code == 200 or status_code == 202:
# Notify NCSA monitor of new task, and add to activeTasks for logging
globusID = transfer_data['task_id']
logger.info("%s new Globus transfer task started (%s files)" %
(globusID, queue_length),
extra={
"globus_id": globusID,
"action": "TRANSFER STARTED",
"contents": globus_batch
})
created_task = {
"globus_id": globusID,
"contents": globus_batch,
"started": str(datetime.datetime.now()),
"status": "CREATED"
}
writeTaskToDatabase(created_task)
removePendingTask(globus_batch_id)
return True
else:
# If failed, leave pending list as-is and try again on next iteration (e.g. in 180 seconds)
logger.error(
"- Globus transfer initialization failed for %s (%s: %s)" %
(ds, status_code, status_message))
return False
def initializeGlobusTransfer():
global pendingTransfers
global activeTasks
global status_numActive
global status_numPending
maxQueue = config['globus']['max_transfer_file_count']
api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token'])
submissionID = generateGlobusSubmissionID()
if submissionID:
# Prepare transfer object
transferObj = Transfer(submissionID,
config['globus']['source_endpoint_id'],
config['globus']['destination_endpoint_id'],
verify_checksum=True,
preserve_timestamp=True)
queueLength = 0
# Loop over a copy of the list instead of actual thing - other thread will be appending to actual thing
loopingTransfers = safeCopy(pendingTransfers)
currentTransferBatch = {}
logger.debug("- building transfer %s from pending queue" % submissionID)
for ds in loopingTransfers:
if "files" in loopingTransfers[ds]:
if ds not in currentTransferBatch:
currentTransferBatch[ds] = {}
currentTransferBatch[ds]['files'] = {}
# Add files from each dataset
for f in loopingTransfers[ds]['files']:
if queueLength < maxQueue:
fobj = loopingTransfers[ds]['files'][f]
src_path = os.path.join(config['globus']['source_path'], fobj["path"], fobj["name"])
dest_path = os.path.join(config['globus']['destination_path'], fobj["path"], fobj["name"])
transferObj.add_item(src_path, dest_path)
# remainingTransfers will have leftover data once max Globus transfer size is met
queueLength += 1
currentTransferBatch[ds]['files'][f] = fobj
else:
break
# Clean up placeholder entries once queue length is exceeded
if currentTransferBatch[ds]['files'] == {}:
del currentTransferBatch[ds]['files']
if currentTransferBatch[ds] == {}:
del currentTransferBatch[ds]
if queueLength >= maxQueue:
break
if queueLength > 0:
# Send transfer to Globus
try:
logger.debug("- attempting to send new transfer")
status_code, status_message, transfer_data = api.transfer(transferObj)
except (APIError, ClientError) as e:
try:
# Try refreshing endpoints and retrying
activateEndpoints()
api = TransferAPIClient(username=config['globus']['username'], goauth=config['globus']['auth_token'])
status_code, status_message, transfer_data = api.transfer(transferObj)
except (APIError, ClientError) as e:
logger.error("- problem initializing Globus transfer")
status_code = 503
status_message = e
if status_code == 200 or status_code == 202:
# Notify NCSA monitor of new task, and add to activeTasks for logging
globusID = transfer_data['task_id']
logger.info("%s new Globus transfer task started (%s files)" % (globusID, queueLength), extra={
"globus_id": globusID,
"action": "TRANSFER STARTED",
"contents": currentTransferBatch
})
activeTasks[globusID] = {
"globus_id": globusID,
"contents": currentTransferBatch,
"started": str(datetime.datetime.now()),
"status": "IN PROGRESS"
}
writeTasksToDisk(config['active_tasks_path'], activeTasks)
# Now that we've safely sent the pending transfers, remove them
for ds in currentTransferBatch:
if ds in pendingTransfers:
for f in currentTransferBatch[ds]['files']:
if f in pendingTransfers[ds]['files']:
del pendingTransfers[ds]['files'][f]
notifyMonitorOfNewTransfer(globusID, currentTransferBatch)
writeTasksToDisk(config['pending_transfers_path'], pendingTransfers)
else:
# If failed, leave pending list as-is and try again on next iteration (e.g. in 180 seconds)
logger.error("- Globus transfer initialization failed for %s (%s: %s)" % (ds, status_code, status_message))
return
status_numActive = len(activeTasks)
cleanPendingTransfers()
def tutorial():
"""
Do a bunch of API calls and display the results. Does a small transfer
between tutorial endpoints, but otherwise does not modify user data.
Uses module global API client instance.
"""
# See what is in the account before we make any submissions.
#print "=== Before tutorial ==="
#display_tasksummary(); print
#display_task_list(); print
#display_endpoint_list(); print
print "=== Before transfer ==="
# auto activate the endpoint, and display before/after.
#display_activation("go#ep1")
#display_activation("go#ep2")
site_ep1 = "cin0641a#irods-dev"
site_ep2 = "cin0641a#GSI-PLX"
site_username = "******"
api.set_debug_print(False, True)
#status, message, data = api.endpoint_autoactivate("go#ep1")
status, message, data = api.endpoint_autoactivate(site_ep1)
#data = {
#"DATA_TYPE": "activation_requirements",
#"type": "myproxy",
#"name": "username",
#"description": "null",
#"canonical_name": "null",
#"public": "null",
#"is_globus_connect": "null",
#}
#data = ActivationRequirementList(data)
print "=== Activate endpoints ==="
data.set_requirement_value("myproxy", "username", site_username)
from getpass import getpass
passphrase = getpass()
data.set_requirement_value("myproxy", "passphrase", passphrase)
#status, message, data = api.endpoint_activate(site_ep, data)
#data["code"]
api.endpoint_activate(site_ep1, data)
api.endpoint_activate(site_ep2, data)
#status, message, data = api.endpoint_activate(site_ep2, data)
#data["code"]
#display_activation("cin0641a#irods-dev")
#display_ls("cin0641a#irods-dev"); print
#display_activation("cin0641a#vzSARA")
#display_ls("go#ep1"); print
#display_ls("go#ep2"); print
#display_ls("cin0641a#vzSARA"); print
#display_activation("cin0641a#GSI-PLX")
#display_ls("cin0641a#GSI-PLX"); print
print "=== Prepare transfer ==="
#raw_input("Press Enter to continue...")
# submit a transfer
code, message, data = api.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(minutes=10)
t = Transfer(submission_id, site_ep1, site_ep2, deadline)
t.add_item("/CINECA/home/gmariani/go-test/some-file/1", "/plx/userprace/pr1is019/tmp/1")
t.add_item("/CINECA/home/gmariani/go-test/some-file/2", "/plx/userprace/pr1is019/tmp/2")
t.add_item("/CINECA/home/gmariani/go-test/some-file/3", "/plx/userprace/pr1is019/tmp/3")
t.add_item("/CINECA/home/gmariani/go-test/some-file/4", "/plx/userprace/pr1is019/tmp/4")
print "=== Submit transfer ==="
code, reason, data = api.transfer(t)
task_id = data["task_id"]
# see the new transfer show up
#print "=== After submit ==="
#display_tasksummary(); print
#display_task(task_id); print
#raw_input("Press Enter to continue...")
# wait for the task to complete, and see the summary and lists
# update
print "=== Checking completion ==="
wait_for_task(task_id)
#if wait_for_task(task_id):
#display_tasksummary(); print
#display_task(task_id); print
#display_ls("cin0641a#GSI-PLX"); print
print "=== Deactivate endpoint ==="
api.endpoint_deactivate(site_ep1)
api.endpoint_deactivate(site_ep2)
def globus_transfer(image_id, cfg):
packages_directory = os.path.join(os.path.expanduser("~"), ".gdclient","packages")
package_directory = os.path.join(packages_directory, image_id)
if not os.path.isdir(package_directory):
print "Cannot find image "+image_id+" for transfer; please run --package level collaboration"
return
if not cfg.config['GLOBUS']['local-folder']:
print "Cannot obtain from config file: local folder shared in GLOBUS "
return
if not cfg.config['GLOBUS']['local-endpoint']:
print "Cannot obtain from config file: GLOBUS local endpoint"
return
if not cfg.config['GLOBUS']['remote-endpoint']:
print "Cannot obtain from config file: GLOBUS remote endpoint"
return
if not cfg.config['GLOBUS']['globus-remote-folder']:
print "Cannot obtain from config file: globus folder shared in remote GLOBUS endpoint"
return
if not cfg.config['GLOBUS']['globus-local-folder']:
print "Cannot obtain from config file: globus folder shared in local GLOBUS endpoint"
return
print "Please wait. Saving image ..."
short_image_name = image_id.strip()+".tgz"
image_file = cfg.config['GLOBUS']['local-folder']+"/"+short_image_name
command = "cd "+packages_directory+";tar czf "+image_file+" "+ image_id
# print "will run: ",command
run_command(command)
source_endpoint = cfg.config['GLOBUS']['local-endpoint'] # 'globuspublish#cirlab'
source_folder= cfg.config['GLOBUS']['globus-local-folder'] # '/globuspublication_52/'
remote_endpoint = cfg.config['GLOBUS']['remote-endpoint'] # 'globuspublish#cirlab'
remote_folder= cfg.config['GLOBUS']['globus-remote-folder'] # '/globuspublication_52/'
# # command = "ssh [email protected] scp cvlaescx#test_2:/docker_image/dockeraf8db02049ae25d3e64436769411206258b3b003.tar cvlaescx#test:/home/cristian/Installs/new3.tar"
# source = "%s%s%s" %(source_endpoint,source_folder, short_image_name)
# destination = "%s%s%s" %(remote_endpoint,remote_folder, short_image_name)
# command = "ssh %[email protected] transfer -- %s %s" % (cfg.config['Default']['uname'], source, destination)
# # print "will run: ",command
# run_command(command)
# #print "USAGE: transfer [destination_endpoint:destination_folder]"
transfer_args = [cfg.config['Default']['uname'], '-g', cfg.config['Default']['goauth-token']]
TransferAPIClient, _ = create_client_from_args(transfer_args)
activate_endpoint(TransferAPIClient, source_endpoint)
activate_endpoint(TransferAPIClient, remote_endpoint)
# submit the transfer
code, message, data = TransferAPIClient.transfer_submission_id()
submission_id = data["value"]
deadline = datetime.utcnow() + timedelta(minutes=10)
t = Transfer(submission_id, source_endpoint, remote_endpoint, deadline)
source_image_name = "%s%s"%(source_folder,short_image_name)
destination_image_name = "%s%s"%(remote_folder,short_image_name)
t.add_item(source_image_name, destination_image_name)
code, reason, data = TransferAPIClient.transfer(t)
task_id = data["task_id"]
print "Submitted transfer id: ", task_id
return "globus://%s%s%s"%(remote_endpoint,remote_folder,short_image_name)
def transfer(id, source, dest):
t = Transfer(id, source, dest)
t.add_item(src_file_path, dest_file_path)
status, reason, result = api.transfer(t)
os._exit(1)
