def GO_create_api(globus_user, proxy, host_cert):
try:
return api_client.TransferAPIClient(username=globus_user,
server_ca_file=host_cert,
cert_file=proxy,
key_file=proxy)
except:
return None
def __init__(self, local_endpoint, remote_endpoint):
"""Create a wrapper around the Globus API Client."""
# Get credentials.
auth_result = api_client.goauth.get_access_token()
self.api = api_client.TransferAPIClient(username=auth_result.username,
goauth=auth_result.token)
# Activate endpoints.
self.local_endpoint, self.remote_endpoint = local_endpoint, remote_endpoint
status, msg, data = self.api.endpoint_autoactivate(local_endpoint)
print data['message']
assert status == 200
status, msg, data = self.api.endpoint_autoactivate(remote_endpoint)
print data['message']
assert status == 200
# Setup asynchronous task queue.
self.task_queue = AsyncTaskQueue(self)
def __init__(self, service_url):
self.service_url = service_url
result = urlparse.urlparse(service_url)
self.host = result.netloc
self.query = result.path
self.ep = self.__get_ep(self.query)
self.path = self.__get_path(self.query)
self.user = result.username
self.password = result.password
result = get_go_auth(ca_certs=None, username=self.user, password=self.password)
saml_cookie = result.cookie
self.api = api_client.TransferAPIClient(username=self.user,
saml_cookie=saml_cookie
)
status_code, status_message, data = self.api.task_list()
# initialize ssh client
self.__state=State.New
def GO_create_gc_endpoint(ep, globus_user, proxy, host_cert):
api = None
try:
api = api_client.TransferAPIClient(username=globus_user,
server_ca_file=host_cert,
cert_file=proxy,
key_file=proxy)
except:
return 1, None
data = {}
try:
status_code, status_message, data = api.endpoint_create(
ep, is_globus_connect=True)
print status_code, status_message, data
except:
return 2, None
st, out = commands.getstatusoutput('sh globusconnect -stop')
print 'Result of preemptive gc stop', out
setup_key = data['globus_connect_setup_key']
print 'Activating', setup_key
st, out = commands.getstatusoutput('sh globusconnect -setup ' + setup_key)
print 'Activation result', out
arg = ['sh', 'globusconnect', '-start']
subprocess.Popen(arg)
print 'Started'
time.sleep(1)
try:
status_code, status_message, data = api.endpoint_autoactivate(ep)
time.sleep(1)
status_code, status_message, data = api.endpoint(ep) # print data
except:
return 3, None
return 0, api
def transferfile(results):
src_file_path = results['file_abs_path']
dest_file_path = "~/example-copy"
save_json(results, src_file_path)
auth = get_access_token()
src = "mattbest#NICHO-LENO5" # source endpoint
dst = "mattbest#Oba-Nicho5-Dell" # destination endpoint
# authenticate using access token
api = api_client.TransferAPIClient(
username=auth.username,
goauth=auth.token
)
# activate endpoints
status, message, data = api.endpoint_autoactivate(src)
status, message, data = api.endpoint_autoactivate(dst)
# get submission id
code, reason, result = api.transfer_submission_id()
submission_id = result["value"]
# designate endpoints(1) and items(2) for transfer(3)
def transfer(id, source, dest):
t = Transfer(id, source, dest)
t.add_item(src_file_path, dest_file_path)
status, reason, result = api.transfer(t)
os._exit(1)
### SCHEDULE TRANSFER JOB ###
scheduler = BlockingScheduler()
now = datetime.now()
scheduler.add_job(lambda:
transfer(submission_id, src, dst),
'date',
run_date=datetime(now.year, now.month, now.day, now.hour, now.minute+1, 0)
)
# returns 0 in the child, pid of the child in the parent
if os.fork():
sys.exit()
scheduler.start()
scheduler.shutdown()
def transfers_end():
_, _, access_token = api_client.goauth.get_access_token(
username=globus_username, password=globus_password)
api = api_client.TransferAPIClient(username=globus_username,
goauth=access_token)
for task_id in globus_tasks:
code, reason, data = api.task(task_id, fields="status")
status = data['status']
sdlog.debug(
"SDDMGLOB-016",
"Checking the status of Globus transfer tasks, id: %s, status: %s"
% (task_id, status))
for item in globus_tasks[task_id]['items']:
tr = item['tr']
if status == "SUCCEEDED":
assert tr.size is not None
if int(tr.size) != os.path.getsize(tr.get_full_local_path()):
sdlog.error(
"SDDMGLOB-002",
"size don't match (remote_size=%i,local_size=%i,local_path=%s)"
% (int(tr.size),
os.path.getsize(tr.get_full_local_path()),
tr.get_full_local_path()))
# retrieve local and remote checksum
checksum_type = tr.checksum_type if tr.checksum_type is not None else sdconst.CHECKSUM_TYPE_MD5
local_checksum = sdutils.compute_checksum(
tr.get_full_local_path(), checksum_type)
remote_checksum = tr.checksum # retrieve remote checksum
if remote_checksum != None:
# remote checksum exists
# compare local and remote checksum
if remote_checksum == local_checksum:
# checksum is ok
tr.status = sdconst.TRANSFER_STATUS_DONE
else:
# checksum is not ok
if incorrect_checksum_action == "remove":
tr.status = sdconst.TRANSFER_STATUS_ERROR
tr.priority -= 1
tr.error_msg = "File corruption detected: local checksum doesn't match remote checksum"
# remove file from local repository
sdlog.error(
"SDDMGLOB-155",
"checksum don't match: remove local file (local_checksum=%s,remote_checksum=%s,local_path=%s)"
% (local_checksum, remote_checksum,
tr.get_full_local_path()))
try:
os.remove(tr.get_full_local_path())
except Exception, e:
sdlog.error(
"SDDMGLOB-158",
"error occurs while removing local file (%s)"
% tr.get_full_local_path())
elif incorrect_checksum_action == "keep":
sdlog.info(
"SDDMGLOB-157",
"local checksum doesn't match remote checksum (%s)"
% tr.get_full_local_path())
tr.status = sdconst.TRANSFER_STATUS_DONE
else:
raise FatalException(
"SDDMGLOB-507", "incorrect value (%s)" %
incorrect_checksum_action)
else:
# remote checksum is missing
# NOTE: we DON'T store the local checksum ('file' table contains only the REMOTE checksum)
tr.status = sdconst.TRANSFER_STATUS_DONE
if tr.status == sdconst.TRANSFER_STATUS_DONE:
tr.end_date = sdtime.now(
) # WARNING: this is not the real end of transfer date but the date when we ask the globus scheduler if the transfer is done.
tr.error_msg = ""
sdlog.info("SDDMGLOB-101", "Transfer done (%s)" % str(tr))
elif status == "FAILED":
tr.status = sdconst.TRANSFER_STATUS_ERROR
tr.priority -= 1
tr.error_msg = "Error occurs during download."
sdlog.info("SDDMGLOB-101", "Transfer failed (%s)" % str(tr))
# Remove local file if exists
if os.path.isfile(tr.get_full_local_path()):
try:
os.remove(tr.get_full_local_path())
except Exception, e:
sdlog.error(
"SDDMGLOB-528",
"Error occurs during file suppression (%s,%s)" %
(tr.get_full_local_path(), str(e)))
def transfers_begin(transfers):
# Activate the destination endpoint
_, _, access_token = api_client.goauth.get_access_token(
username=globus_username, password=globus_password)
api = api_client.TransferAPIClient(username=globus_username,
goauth=access_token)
activate_endpoint(api)
# Divide all files that are to be transferred into groups based on the source globus endpoint
globus_transfers = {}
for tr in transfers:
src_endpoint, src_path, path = map_to_globus(tr.url)
local_path = tr.get_full_local_path()
if not src_endpoint in globus_transfers:
globus_transfers[src_endpoint] = {
'src_endpoint': src_endpoint,
'items': []
}
globus_transfers[src_endpoint]['items'].append({
'src_path': src_path,
'dst_path': local_path,
'tr': tr
})
sdlog.info(
"SDDMGLOB-001", "src_endpoint: %s, src_path: %s, local_path: %s" %
(src_endpoint, src_path, local_path))
# Submit transfers
for src_endpoint in globus_transfers:
# Activate the source endpoint
activate_endpoint(api, src_endpoint)
# Create a transfer and add files to the transfer
code, message, data = api.transfer_submission_id()
if code != 200:
raise FatalException()
submission_id = data['value']
t = api_client.Transfer(submission_id, src_endpoint, dst_endpoint)
sdlog.info(
"SDDMGLOB-004",
"Globus transfer, source endpoint: %s, destination endpoint: %s" %
(src_endpoint, dst_endpoint))
for item in globus_transfers[src_endpoint]['items']:
t.add_item(item['src_path'], item['dst_path'])
sdlog.info(
"SDDMGLOB-005",
"Globus transfer item, source path: %s, destination path: %s" %
(item['src_path'], item['dst_path']))
# Submit the transfer
code, message, data = api.transfer(t)
if code != 202:
sdlog.error(
"SDDMGLOB-006",
"Error: Cannot add a transfer: (%s, %s)" % (code, message))
raise FatalException()
task_id = data['task_id']
sdlog.info("SDDMGLOB-007", "Submitted Globus task, id: %s" % task_id)
globus_tasks[task_id] = globus_transfers[src_endpoint]
help="Destination endpoint filepath to transfer")
parser.add_option("-d", "--deadline", dest="deadline",
help="Deadline for transfer in minutes.")
parser.add_option("-g", "--galaxy-dataset-id", dest="galaxy_dataset_id",
help="Galaxy Dataset Id For This Transfer")
parser.set_defaults(base_url=transfer_api.DEFAULT_BASE_URL)
parser.set_defaults(output=sys.stdout)
options, args = parser.parse_args(args)
if len(args) != 1:
parser.error("username arguments is required")
if options.output != sys.stdout:
options.output = open(options.output, "w")
return options, args
if __name__ == '__main__':
options, args = process_args()
api = transfer_api.TransferAPIClient(args[0], server_ca_file=options.server_ca_file,
cert_file=options.cert_file,
key_file=options.key_file,
saml_cookie=options.saml_cookie,
base_url=options.base_url)
output = options.output
display_endpoints()
transfer(options.source_ep, options.source_path,
options.destination_ep, options.destination_path,
options.deadline)