def bulk_submit_xfer(submitjob, recursive=False):
cfg = load_config()
client_id = cfg['globus']['apps'][GLOBUS_AUTH_APP]['client_id']
auth_client = NativeAppAuthClient(client_id)
refresh_token = cfg['globus']['apps'][GLOBUS_AUTH_APP]['refresh_token']
source_endpoint_id = submitjob[0].get('metadata').get(
'source_globus_endpoint_id')
destination_endpoint_id = submitjob[0].get('metadata').get(
'dest_globus_endpoint_id')
authorizer = RefreshTokenAuthorizer(refresh_token=refresh_token,
auth_client=auth_client)
tc = TransferClient(authorizer=authorizer)
# as both endpoints are expected to be Globus Server endpoints, send auto-activate commands for both globus endpoints
a = auto_activate_endpoint(tc, source_endpoint_id)
logging.debug('a: %s' % a)
if a != 'AlreadyActivated':
return None
b = auto_activate_endpoint(tc, destination_endpoint_id)
logging.debug('b: %s' % b)
if b != 'AlreadyActivated':
return None
# make job_label for task a timestamp
x = datetime.now()
job_label = x.strftime('%Y%m%d%H%M%s')
# from Globus... sync_level=checksum means that before files are transferred, Globus will compute checksums on the source
# and destination files, and only transfer files that have different checksums are transferred. verify_checksum=True means
# that after a file is transferred, Globus will compute checksums on the source and destination files to verify that the
# file was transferred correctly. If the checksums do not match, it will redo the transfer of that file.
tdata = TransferData(tc,
source_endpoint_id,
destination_endpoint_id,
label=job_label,
sync_level="checksum")
for file in submitjob:
source_path = file.get('sources')[0]
dest_path = file.get('destinations')[0]
filesize = file['metadata']['filesize']
# TODO: support passing a recursive parameter to Globus
# md5 = file['metadata']['md5']
# tdata.add_item(source_path, dest_path, recursive=False, external_checksum=md5)
tdata.add_item(source_path, dest_path, recursive=False)
record_counter(
'daemons.conveyor.transfer_submitter.globus.transfers.submit.filesize',
filesize)
# logging.info('submitting transfer...')
transfer_result = tc.submit_transfer(tdata)
# logging.info("task_id =", transfer_result["task_id"])
return transfer_result["task_id"]
def globus_download_files(client: globus_sdk.TransferClient, endpoint_id: str,
files: tuple) -> None:
"""Gets the details of the files in the list
Arguments:
client: the Globus transfer client to use
endpoint_id: the ID of the endpoint to access
files: the list of files to fetch
Return:
Returns an updated list of file details
"""
# Fetch metadata and pull information out of it
file_transfers = {}
for one_file in files:
globus_save_path = os.path.join(LOCAL_SAVE_PATH,
os.path.basename(one_file))
if not os.path.exists(globus_save_path):
globus_remote_path = one_file
file_transfers[globus_remote_path] = globus_save_path
if file_transfers:
have_exception = False
cnt = 1
for remote_path, save_path in file_transfers.items():
try:
logging.info("Trying transfer %s: %s", str(cnt),
str(remote_path))
cnt += 1
transfer_setup = globus_sdk.TransferData(
client,
endpoint_id,
GLOBUS_LOCAL_ENDPOINT_ID,
label="Get image file",
sync_level="checksum")
transfer_setup.add_item(remote_path, save_path)
transfer_request = client.submit_transfer(transfer_setup)
task_result = client.task_wait(transfer_request['task_id'],
timeout=600,
polling_interval=5)
if not task_result:
raise RuntimeError("Unable to retrieve JSON metadata: %s" %
remote_path)
if not os.path.exists(save_path):
raise RuntimeError(
"Unable to find downloaded file at: %s" % save_path)
except RuntimeError as ex:
have_exception = True
logging.warning("Failed to get image: %s", str(ex))
if have_exception:
raise RuntimeError("Unable to retrieve all files individually")
del file_transfers
def submit_transfer():
"""
- Take the data returned by the Browse Endpoint helper page
and make a Globus transfer request.
- Send the user to the transfer status page with the task id
from the transfer.
"""
browse_endpoint_form = request.form
selected = session['form']['datasets']
filtered_datasets = [ds for ds in datasets if ds['id'] in selected]
transfer_tokens = session['tokens']['transfer.api.globus.org']
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
load_portal_client(),
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'])
transfer = TransferClient(authorizer=authorizer)
source_endpoint_id = app.config['DATASET_ENDPOINT_ID']
source_endpoint_base = app.config['DATASET_ENDPOINT_BASE']
destination_endpoint_id = browse_endpoint_form['endpoint_id']
destination_folder = browse_endpoint_form.get('folder[0]')
transfer_data = TransferData(transfer_client=transfer,
source_endpoint=source_endpoint_id,
destination_endpoint=destination_endpoint_id,
label=browse_endpoint_form.get('label'))
for ds in filtered_datasets:
source_path = source_endpoint_base + ds['path']
dest_path = browse_endpoint_form['path']
if destination_folder:
dest_path += destination_folder + '/'
dest_path += ds['name'] + '/'
transfer_data.add_item(source_path=source_path,
destination_path=dest_path,
recursive=True)
transfer.endpoint_autoactivate(source_endpoint_id)
transfer.endpoint_autoactivate(destination_endpoint_id)
task_id = transfer.submit_transfer(transfer_data)['task_id']
flash('Transfer request submitted successfully. Task ID: ' + task_id)
return (redirect(url_for('transfer_status', task_id=task_id)))
def bulk_submit_xfer(submitjob, recursive=False, logger=logging.log):
cfg = load_config(logger=logger)
client_id = cfg['globus']['apps'][GLOBUS_AUTH_APP]['client_id']
auth_client = NativeAppAuthClient(client_id)
refresh_token = cfg['globus']['apps'][GLOBUS_AUTH_APP]['refresh_token']
source_endpoint_id = submitjob[0].get('metadata').get(
'source_globus_endpoint_id')
destination_endpoint_id = submitjob[0].get('metadata').get(
'dest_globus_endpoint_id')
authorizer = RefreshTokenAuthorizer(refresh_token=refresh_token,
auth_client=auth_client)
tc = TransferClient(authorizer=authorizer)
# make job_label for task a timestamp
now = datetime.datetime.now()
job_label = now.strftime('%Y%m%d%H%M%s')
# retrieve globus_task_deadline value to enforce time window to complete transfers
# default is 2880 minutes or 48 hours
globus_task_deadline = config_get_int('conveyor', 'globus_task_deadline',
False, 2880)
deadline = now + datetime.timedelta(minutes=globus_task_deadline)
# from Globus... sync_level=checksum means that before files are transferred, Globus will compute checksums on the source
# and destination files, and only transfer files that have different checksums are transferred. verify_checksum=True means
# that after a file is transferred, Globus will compute checksums on the source and destination files to verify that the
# file was transferred correctly. If the checksums do not match, it will redo the transfer of that file.
tdata = TransferData(tc,
source_endpoint_id,
destination_endpoint_id,
label=job_label,
sync_level="checksum",
deadline=str(deadline))
for file in submitjob:
source_path = file.get('sources')[0]
dest_path = file.get('destinations')[0]
filesize = file['metadata']['filesize']
# TODO: support passing a recursive parameter to Globus
# md5 = file['metadata']['md5']
# tdata.add_item(source_path, dest_path, recursive=False, external_checksum=md5)
tdata.add_item(source_path, dest_path, recursive=False)
record_counter(
'daemons.conveyor.transfer_submitter.globus.transfers.submit.filesize',
filesize)
# logging.info('submitting transfer...')
transfer_result = tc.submit_transfer(tdata)
logger(logging.INFO, "transfer_result: %s" % transfer_result)
return transfer_result["task_id"]
class UOCloudTransferClient:
def __init__(self, config: UOCloudSyncConfig):
confidential_client = ConfidentialAppAuthClient(
client_id=config.get_client_id(), client_secret=config.get_client_secret())
scopes = "urn:globus:auth:scope:transfer.api.globus.org:all"
cc_authorizer = ClientCredentialsAuthorizer(confidential_client, scopes)
# create a new client
self._transfer_client = TransferClient(authorizer=cc_authorizer)
self._src_endpoint = None
self._dest_endpoint = None
def get_endpoint_id(self, endpoint_name: str):
endpoints = self._transfer_client.endpoint_search(filter_fulltext=endpoint_name)
# Just return the first result. Hope it is right!
for ep in endpoints:
return ep['id']
def transfer_data(self, src_endpoint: str, src_path: Union[str, Path, PathLike],
dest_endpoint: str, dest_path: Union[str, Path, PathLike]):
self._src_endpoint = src_endpoint
self._dest_endpoint = dest_endpoint
src_endpoint_id = self.get_endpoint_id(src_endpoint)
if not src_endpoint_id:
print(f'ERROR: Unable to find source endpoint id for: "{self._src_endpoint}"')
return
dest_endpoint_id = self.get_endpoint_id(dest_endpoint)
if not dest_endpoint_id:
print(f'ERROR: Unable to find destination endpoint id for: "{self._dest_endpoint}"')
return
transfer_data = TransferData(self._transfer_client,
src_endpoint_id,
dest_endpoint_id,
encrypt_data=True)
transfer_data.add_item(src_path, dest_path, recursive=True)
try:
print(
f'Submitting a transfer task from {self._src_endpoint}:{src_path} to {self._dest_endpoint}:{dest_path}')
task = self._transfer_client.submit_transfer(transfer_data)
except TransferAPIError as e:
print(str(e))
sys.exit(1)
task_id = task['task_id']
print(f'\tWaiting for transfer to complete with task_id: {task_id}')
while not self._transfer_client.task_wait(task_id=task_id, timeout=3600, polling_interval=60):
print('.', end='')
print('Transferred files:')
for info in self._transfer_client.task_successful_transfers(task_id=task_id, num_results=None):
print("\t{} -> {}".format(info["source_path"], info["destination_path"]))
def globus_download_files(client: globus_sdk.TransferClient, endpoint_id: str,
files: tuple) -> None:
"""Gets the details of the files in the list
Arguments:
client: the Globus transfer client to use
endpoint_id: the ID of the endpoint to access
files: the list of files to fetch
Return:
Returns an updated list of file details
"""
# Fetch metadata and pull information out of it
file_transfers = {}
for one_file in files:
globus_save_path = os.path.join(LOCAL_SAVE_PATH,
os.path.basename(one_file))
if not os.path.exists(globus_save_path):
globus_remote_path = one_file
file_transfers[globus_remote_path] = globus_save_path
if file_transfers:
have_exception = False
cnt = 1
resp = subprocess.run(['icd', IRODS_LOCATION], stdout=subprocess.PIPE)
if resp.returncode != 0:
raise RuntimeError("Unable to change to iRODS location %s" %
IRODS_LOCATION)
for remote_path, save_path in file_transfers.items():
try:
logging.info("Trying transfer %s: %s", str(cnt),
str(remote_path))
cnt += 1
transfer_setup = globus_sdk.TransferData(
client,
endpoint_id,
GLOBUS_LOCAL_ENDPOINT_ID,
label="Get image file",
sync_level="checksum")
transfer_setup.add_item(remote_path, save_path)
transfer_request = client.submit_transfer(transfer_setup)
task_result = client.task_wait(transfer_request['task_id'],
timeout=600,
polling_interval=5)
if not task_result:
raise RuntimeError("Unable to retrieve file: %s" %
remote_path)
if not os.path.exists(save_path):
raise RuntimeError(
"Unable to find downloaded file at: %s" % save_path)
local_dir = os.getcwd()
os.chdir(os.path.dirname(save_path))
print("Uploading file to irods: %s", save_path)
resp = subprocess.run(
['iput', '-K', '-f',
os.path.basename(save_path)],
stdout=subprocess.PIPE)
if resp.returncode != 0:
os.chdir(local_dir)
raise RuntimeError("Unable to load file to iRODS %s" %
save_path)
os.chdir(local_dir)
print(" removing uploaded file")
os.remove(save_path)
except RuntimeError as ex:
have_exception = True
logging.warning("Failed to get image: %s", str(ex))
if have_exception:
raise RuntimeError("Unable to retrieve all files individually")
del file_transfers
try:
tc = TransferClient(authorizer=authorizer)
except:
print(
"ERROR: TransferClient() call failed! Unable to call the Globus transfer interface with the provided auth info!"
)
sys.exit(-1)
# print(transfer)
# Now we should have auth, try setting up a transfer.
tdata = TransferData(tc,
source_endpoint_id,
destination_endpoint_id,
label="DCDE Relion transfer",
sync_level="size")
tdata.add_item(source_dir, dest_dir, recursive=True)
transfer_result = tc.submit_transfer(tdata)
print("task_id =", transfer_result["task_id"])
while not tc.task_wait(
transfer_result['task_id'], timeout=1200, polling_interval=10):
print(".", end="")
print("\n{} completed!".format(transfer_result['task_id']))
os.listdir(path=dest_dir)
def do_job(self,tokens, task_color,stage_in_source,stage_in_dest,stage_out_dest,stage_in_source_path,stage_in_dest_path,stage_out_dest_path):
def post_refresh_message(token_data):
print("I got called")
requests.post('http://localhost:8081/api/messenger', headers={'content-type': 'application/json'},data=json.dumps({'key_message':token_data.by_resource_server['transfer.api.globus.org']['access_token'], 'task_id':task_id,'step':'1','task_color':task_color}))
#socketio.emit('message_log', {'message_body':'Testing for emit'})
auth_client = dill.loads(redis_store.get('auth_client'))
#send json message with key special_message that include new access token
#requests.post('http://localhost:8081/api/messenger', headers={'content-type': 'application/json'},data=json.dumps({'key_message':token_data.by_resource_server['transfer.api.globus.org']['access_token'], 'task_id':task_id}
authorizer = globus_sdk.RefreshTokenAuthorizer(tokens['transfer.api.globus.org']['refresh_token'], auth_client,tokens['transfer.api.globus.org']['access_token'], expires_at=tokens['transfer.api.globus.org']['expires_at_seconds'],on_refresh=post_refresh_message)
#stage_in_source = stage_in_source
stage_in_destination= stage_in_dest
stage_out_destination = stage_out_dest
#stage_in_source_path = redis_store.get('stage_in_source_path').decode('utf-8')
stage_in_destination_path = stage_in_dest_path
stage_out_destination_path = stage_out_dest_path
task_id = do_job.request.id
tc = TransferClient(authorizer=authorizer)
#auth_client=load_auth_client()
data = globus_sdk.TransferData(tc,stage_in_source, stage_in_destination,label="stagein")
data.add_item(stage_in_source_path, stage_in_destination_path, True)
status = tc.submit_transfer(data)
requests.post('http://localhost:8081/api/messenger', headers={'content-type': 'application/json'},data=json.dumps({'message':'['+task_id+']Queue wait is done, now initiating Stage in....','task_id':task_id,'step':'2','task_color':task_color}))
tc.task_wait(status["task_id"])#task id of the stage_in
result_in=tc.get_task(status["task_id"])
#print("The response for task is :")
#print(result_in)
complete_status = result_in['status']
print("The complete status is :")
print(complete_status)
if complete_status == "SUCCEEDED":
requests.post('http://localhost:8081/api/messenger', headers={'content-type': 'application/json'},data=json.dumps({'message':'['+task_id+'] Stage In succeeded', 'task_id':task_id,'step':'2','task_color':task_color}))
else:
requests.post('http://localhost:8081/api/messenger', headers={'content-type': 'application/json'},data=json.dumps({'message':'['+task_id+'] Stage In failed, canceling the job..... ','task_id':task_id,'step':'2','task_color':task_color}))
# stop and delete the job
raise Reject("Stage in Failed",requeue=False)
#print to the log that job informations, with id, running the fake job
requests.post('http://localhost:8081/api/messenger', headers={'content-type': 'application/json'},data=json.dumps({'message':'['+task_id+']Running the job','task_id':task_id,'step':'3','task_color':task_color}))
time.sleep(3)
#fetching new token
#validate now active
#fake job is done
requests.post('http://localhost:8081/api/messenger', headers={'content-type': 'application/json'},data=json.dumps({'message':'['+task_id+']Job is done','task_id':task_id,'step':'3','task_color':task_color}))
requests.post('http://localhost:8081/api/messenger', headers={'content-type': 'application/json'},data=json.dumps({'message':'['+task_id+'] Initiating Stage out.... ','task_id':task_id,'step':'4','task_color':task_color}))
#tc = TransferClient(authorizer=authorizer)
data = globus_sdk.TransferData(tc, stage_in_destination, stage_out_destination,label="stageout")
data.add_item(stage_in_destination_path, stage_out_destination_path, True)
#hopefully refresh token lambda called here or after here supposed to log refreshed ok
status = tc.submit_transfer(data)
tc.task_wait(status["task_id"])
result_in=tc.get_task(status["task_id"])
complete_status = result_in['status']
if complete_status == "SUCCEEDED":
requests.post('http://localhost:8081/api/messenger', headers={'content-type': 'application/json'},data=json.dumps({'message':'['+task_id+'] Stage Out succeeded ','task_id':task_id,'step':'4','task_color':task_color}))
else:
requests.post('http://localhost:8081/api/messenger', headers={'content-type': 'application/json'},data=json.dumps({'message':'['+task_id+'] Stage Out failed, canceling the job.....','task_id':task_id,'step':'4','task_color':task_color}))
raise Reject("Stage out Failed",requeue=False)
def globus_transfer( # noqa: C901
remote_endpoint,
remote_path,
name,
transfer_type,
non_blocking=False):
"""
Read the local globus endpoint UUID from ~/.zstash.ini.
If the ini file does not exist, create an ini file with empty values,
and try to find the local endpoint UUID based on the FQDN
"""
ini_path = os.path.expanduser("~/.zstash.ini")
ini = configparser.ConfigParser()
local_endpoint = None
if ini.read(ini_path):
if "local" in ini.sections():
local_endpoint = ini["local"].get("globus_endpoint_uuid")
else:
ini["local"] = {"globus_endpoint_uuid": ""}
try:
with open(ini_path, "w") as f:
ini.write(f)
except Exception as e:
logger.error(e)
sys.exit(1)
if not local_endpoint:
fqdn = socket.getfqdn()
for pattern in regex_endpoint_map.keys():
if re.fullmatch(pattern, fqdn):
local_endpoint = regex_endpoint_map.get(pattern)
break
if not local_endpoint:
logger.error(
"{} does not have the local Globus endpoint set".format(ini_path))
sys.exit(1)
if remote_endpoint.upper() in hpss_endpoint_map.keys():
remote_endpoint = hpss_endpoint_map.get(remote_endpoint.upper())
if transfer_type == "get":
src_ep = remote_endpoint
src_path = os.path.join(remote_path, name)
dst_ep = local_endpoint
dst_path = os.path.join(os.getcwd(), name)
else:
src_ep = local_endpoint
src_path = os.path.join(os.getcwd(), name)
dst_ep = remote_endpoint
dst_path = os.path.join(remote_path, name)
subdir = os.path.basename(os.path.normpath(remote_path))
subdir_label = re.sub("[^A-Za-z0-9_ -]", "", subdir)
filename = name.split(".")[0]
label = subdir_label + " " + filename
native_client = NativeClient(
client_id="6c1629cf-446c-49e7-af95-323c6412397f",
app_name="Zstash",
default_scopes=
"openid urn:globus:auth:scope:transfer.api.globus.org:all",
)
native_client.login(no_local_server=True, refresh_tokens=True)
transfer_authorizer = native_client.get_authorizers().get(
"transfer.api.globus.org")
tc = TransferClient(transfer_authorizer)
for ep_id in [src_ep, dst_ep]:
r = tc.endpoint_autoactivate(ep_id, if_expires_in=600)
if r.get("code") == "AutoActivationFailed":
logger.error(
"The {} endpoint is not activated or the current activation expires soon. Please go to https://app.globus.org/file-manager/collections/{} and (re)activate the endpoint."
.format(ep_id, ep_id))
sys.exit(1)
td = TransferData(
tc,
src_ep,
dst_ep,
label=label,
sync_level="checksum",
verify_checksum=True,
preserve_timestamp=True,
fail_on_quota_errors=True,
)
td.add_item(src_path, dst_path)
try:
task = tc.submit_transfer(td)
except TransferAPIError as e:
if e.code == "NoCredException":
logger.error(
"{}. Please go to https://app.globus.org/endpoints and activate the endpoint."
.format(e.message))
else:
logger.error(e)
sys.exit(1)
except Exception as e:
logger.error("Exception: {}".format(e))
sys.exit(1)
if non_blocking:
return
try:
task_id = task.get("task_id")
"""
A Globus transfer job (task) can be in one of the three states:
ACTIVE, SUCCEEDED, FAILED. The script every 20 seconds polls a
status of the transfer job (task) from the Globus Transfer service,
with 20 second timeout limit. If the task is ACTIVE after time runs
out 'task_wait' returns False, and True otherwise.
"""
while not tc.task_wait(task_id, 20, 20):
pass
"""
The Globus transfer job (task) has been finished (SUCCEEDED or FAILED).
Check if the transfer SUCCEEDED or FAILED.
"""
task = tc.get_task(task_id)
if task["status"] == "SUCCEEDED":
logger.info(
"Globus transfer {}, from {}{} to {}{} succeeded".format(
task_id, src_ep, src_path, dst_ep, dst_path))
else:
logger.error("Transfer FAILED")
except TransferAPIError as e:
if e.code == "NoCredException":
logger.error(
"{}. Please go to https://app.globus.org/endpoints and activate the endpoint."
.format(e.message))
else:
logger.error(e)
sys.exit(1)
except Exception as e:
logger.error("Exception: {}".format(e))
sys.exit(1)
async def create_transfer_globus(transferObject: TransferBase,
transfer_client: TransferClient,
isFolder: bool = False):
"""This function verifies if globus authentication is present in session"""
#transfer_client = await get_transfer_client(request)
source = transferObject.source
target = transferObject.target
source_name = ''
source_path = ''
# example source: globus://fd9c190c-b824-11e9-98d7-0a63aa6b37da:/gridftp/pub/databases/eva/PRJEB6057/MOCH.population_sites.CHIR1_0.20140307_EVA_ss_IDs.fixed.vcf.gz
if source:
source_endpoint_id = source.split(':')[1].replace('/', '')
source_path = source.split(':')[2]
source_path_array = source_path.split('/')
source_name = source_path_array[len(source_path_array) - 1]
if target:
target_endpoint_id = target.split(':')[1].replace('/', '')
target_path = target.split(':')[2]
if target_path.endswith('/'):
if source_name:
target_path = target_path + source_name
transfer_response = None
# source path ends with '/'
if source_name == '':
isFolder = True
if transferObject.options:
if 'recursive' in transferObject.options:
if transferObject.options['recursive'] == "True":
isFolder = True
time = datetime.now().strftime("%d-%m-%Y %H-%M-%S")
try:
tdata = TransferData(transfer_client,
source_endpoint_id,
target_endpoint_id,
label='RDSDS ' + time,
sync_level="checksum")
if isFolder:
tdata.add_item(source_path, target_path, recursive=True)
else:
tdata.add_item(source_path, target_path)
transfer_result = transfer_client.submit_transfer(tdata)
transfer_result_json = json.loads(str(transfer_result))
transfer_response = {'globus_response': transfer_result_json}
transfer_response['status'] = 200
rdsds_tracking_id = 'globus-' + transfer_result["task_id"]
transfer_response['rdsds_tracking_id'] = rdsds_tracking_id
transfer_response[
'globus_status_url'] = 'https://app.globus.org/activity/' + transfer_result[
"task_id"] + '/overview'
return transfer_response
except GlobusAPIError as e:
# Error response from the REST service, check the code and message for
# details.
return handle_globus_api_error(e)
except NetworkError:
logging.error(("Network Failure. "
"Possibly a firewall or connectivity issue"))
raise
except GlobusError:
logging.exception("Totally unexpected GlobusError!")
raise
def transfer(sp,destination_endpoint_id,one_endpoint):
tokens = None
try:
# if we already have tokens, load and use them
tokens = load_tokens_from_file(TOKEN_FILE)
except:
pass
if not tokens:
# if we need to get tokens, start the Native App authentication process
tokens = do_native_app_authentication(CLIENT_ID, REDIRECT_URI, SCOPES)
try:
save_tokens_to_file(TOKEN_FILE, tokens)
except:
pass
transfer_tokens = tokens['transfer.api.globus.org']
auth_client = NativeAppAuthClient(client_id=CLIENT_ID,environment='sandbox')
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
auth_client,
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=update_tokens_file_on_refresh)
#transfer = TransferClient(authorizer=authorizer,environment='sandbox')
tc = TransferClient(authorizer=authorizer, environment="sandbox")
##################---ENDPOINTS---###########################
source_endpoint_id = '5a2e5704-b028-11e7-bdad-22000bdb2406' #sb vmtb4
#source_endpoint_id = '55705028-aa15-11e7-bdad-22000bdb2406' #sb yulie7t
#source_endpoint_id = 'b0b16296-88e7-11e7-a971-22000a92523b' #bare chameleon
#source_endpoint_id = 'e5762bc2-8466-11e7-a8ed-22000a92523b' #large_chameleon
#source_endpoint_id = '8b26cc0e-877b-11e7-a949-22000a92523b'#ubuntu-vm
#source_endpoint_id = 'ad19b012-77cf-11e7-8b98-22000b9923ef'#chameleon
# source_endpoint_id = raw_input('Input source endpoint UUID: ')
#destination path
##############---SOURCE PATH---######################
#source_path = '/home/parallels/stream_transfer/test_files/'
#source_path = '/home/parallels/stream_transfer/zero_globus/test_files/'
source_path = sp
#source_path ='/home/cc/streaming/zero_globus/test_files/test.txt'
#source_path = '/home/parallels/stream_transfer/zero_globus/test_files/test.txt'
#destination path
destination_path = '/~/'
#destination_path = '/~/'+ sp.split("/")[-1] #use for one file
##if one_endpoint:
## destination_path = '/projects/BrainImagingADSP/yzamora/'
##else:
## destination_path = '/projects/BrainImagingADSP/yzamora/'+ sp.split("/")[-1] #use for one file
#Using my sample UUID from globus tutorial
#destination_endpoint_id = 'ddb59aef-6d04-11e5-ba46-22000b92c6ec' #globus
#destination_endpoint_id = '5d1da0fe-3c07-11e7-bcfc-22000b9a448b' #laptop
#tc.endpoint_autoactivate(source_endpoint_id)
#tc.endpoint_autoactive(destination_endpoint_id)
ep1 = tc.get_endpoint(destination_endpoint_id)
tc.endpoint_autoactivate(destination_endpoint_id)
#ep1 is setting the activated endpoint to be a variable to work with
tc.endpoint_autoactivate(source_endpoint_id)
label = "medium data transfer"
#tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label, sync_level='0')
tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label, perf_cc=3, sync_level=None, verify_checksum=False)
#tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label)
if one_endpoint:
tdata.add_item(source_path,destination_path,recursive=True)
else:
tdata.add_item(source_path,destination_path,recursive=False)
submit_result = tc.submit_transfer(tdata)
print("Task ID:", submit_result["task_id"])
"""
Checking for time completion using globus calls
"""
#print("Completion time:", submit_result["completion_time"])
#setup of the transfer, submits as a https post request
#transfer_data = TransferData(transfer_client=tc,
# source_endpoint=source_endpoint_id,
# destination_endpoint=destination_endpoint_id,
# label='Transfer',
# sync_level='checksum')
#transfer_data.add_item(source_path=source_path,destination_path=destination_path, recursive=False)
#task_id=transfer.submit_transfer(transfer_data)['task_id']
#waiting for file to transfer
status = tc.get_task(submit_result["task_id"],fields="status")["status"]
poll_interval = 2
max_wait = 90
wait_time = 0
while not (status in ["SUCCEEDED", "FAILED"]):
if (wait_time >= max_wait): break
print("Task not yet complete (status {}), sleeping for {} seconds..." \
.format(status, poll_interval))
time.sleep(poll_interval)
wait_time += poll_interval
status = tc.get_task(submit_result["task_id"], fields="status")["status"]
if status == "FAILED":
print("WARNING! File transfer FAILED!")
#deleting file after transfer
if status == "SUCCEEDED":
end_time = datetime.datetime.utcnow()
start_time = end_time - datetime.timedelta(minutes=200)
#limit = response objects
# data = tc.task_list(filter="type:TRANSFER,DELETE/request_time:%s,%s"
# % (start_time, end_time), limit=5)
#print("File transfer SUCCEEDED, will delete file from local directory now")
""" r = tc.task_list(num_results=1, filter="type:TRANSFER,DELETE")
def validate():
params = request.json
crawl_id = params["crawl_id"]
globus_eid = params["globus_eid"]
transfer_token = params["transfer_token"]
source_destination = params["source_destination"]
dataset_info = params["dataset_info"] # To be implemented later
client = boto3.client('sqs',
aws_access_key_id=os.environ["aws_access"],
aws_secret_access_key=os.environ["aws_secret"],
region_name='us-east-1')
try:
response = client.get_queue_url(
QueueName=f'validate_{crawl_id}',
QueueOwnerAWSAccountId=os.environ["aws_account_id"])
except: # Add SQS.Client.exceptions.QueueDoesNotExist error
abort(400, "Invalid crawl ID")
try:
authorizer = AccessTokenAuthorizer(transfer_token)
tc = TransferClient(authorizer=authorizer)
except: # Add exception
abort(400, "Invalid transfer token")
crawl_queue = response["QueueUrl"]
date = datetime.datetime.now()
file_name = date.strftime("%m_%d_%Y-%H_%M_%S") + ".txt"
try:
with open(file_name, "w") as f:
while True:
sqs_response = client.receive_message(
QueueUrl=crawl_queue,
MaxNumberOfMessages=1, # To be toggled
WaitTimeSeconds=1)
if "Messages" not in sqs_response:
# xtract_status = requests.get(f"{eb_url}/get_extract_status", json={"crawl_id": crawl_id})
# print("HERE")
# print(xtract_status.content)
# xtract_content = json.loads(xtract_status.content)
# # print(xtract_content)
#
# if xtract_content["IDLE"] == 0 and xtract_content["PENDING"] == 0:
break
del_list = []
for message in sqs_response["Messages"]:
message_body = message["Body"]
# PROCESS MESSAGE_BODY
f.write(message_body)
# print(message_body)
del_list.append({
'ReceiptHandle': message["ReceiptHandle"],
'Id': message["MessageId"]
})
if len(del_list) > 0:
client.delete_message_batch(QueueUrl=crawl_queue,
Entries=del_list)
tdata = TransferData(
tc,
"5ecf6444-affc-11e9-98d4-0a63aa6b37da", #TODO: Add source endpoint
globus_eid,
label=f"{crawl_id}")
tdata.add_item(os.path.abspath(file_name),
os.path.join(source_destination, file_name))
tc.endpoint_autoactivate(
"5ecf6444-affc-11e9-98d4-0a63aa6b37da") #TODO: Add source endpoint
tc.endpoint_autoactivate(globus_eid)
submit_result = tc.submit_transfer(tdata)
while True:
result = tc.get_task(submit_result['task_id'])
if result.data["status"] == "SUCCEEDED":
break
elif result.data["status"] == "FAILED":
raise RuntimeError # TODO: Change this
else:
time.sleep(0.5)
except Exception as e:
print(e)
abort(400, "Failed to validate")
finally:
os.remove(file_name)
return "[200] Submitted"
class Transfer:
'''
Modified Transfer, add an option to pass refresh token to avoid the web login
'''
def __init__(self,
src_endpoint_name,
dst_endpoint_name,
transfer_rt=None,
log_lv=logging.INFO):
log_format = '%(asctime)-15s %(levelname)s:\t class:%(name)s %(message)s'
logging.basicConfig(format=log_format)
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.setLevel(log_lv)
self.logger.debug('CLIENT_ID: {0}'.format(CLIENT_ID))
self.client = NativeAppAuthClient(CLIENT_ID)
self.client.oauth2_start_flow(refresh_tokens=True)
if transfer_rt is not None:
self.authorizer = RefreshTokenAuthorizer(transfer_rt, self.client)
else:
authorize_url = self.client.oauth2_get_authorize_url()
print('Please go to this URL and login: {0}'.format(authorize_url))
get_input = getattr(__builtins__, 'raw_input', input)
auth_code = get_input(
'Please enter the code you get after login here: ').strip()
token_response = self.client.oauth2_exchange_code_for_tokens(
auth_code)
self.globus_auth_data = token_response.by_resource_server[
'auth.globus.org']
self.globus_transfer_data = token_response.by_resource_server[
'transfer.api.globus.org']
auth_token = self.globus_auth_data['access_token']
transfer_token = self.globus_transfer_data['access_token']
transfer_rt = self.globus_transfer_data['refresh_token']
transfer_at = self.globus_transfer_data['access_token']
expires_at_s = self.globus_transfer_data['expires_at_seconds']
self.authorizer = RefreshTokenAuthorizer(transfer_rt,
self.client,
access_token=transfer_at,
expires_at=expires_at_s)
self.transferClient = TransferClient(authorizer=self.authorizer)
self.src_endpoint = None
self.dst_endpoint = None
for ep in self.transferClient.endpoint_search(
filter_scope="shared-with-me"):
if ep["display_name"] == src_endpoint_name:
self.src_endpoint = ep
self.logger.info('Source endpoint: [{0}] {1}'.format(
self.src_endpoint['id'],
self.src_endpoint['display_name']))
if self.src_endpoint is None:
self.logger.error(
'No endpoint shared with you with name: {0}'.format(
src_endpoint_name))
raise LookupError
for ep in self.transferClient.endpoint_search(
filter_scope="my-endpoints"):
if ep['display_name'] == dst_endpoint_name:
self.dst_endpoint = ep
self.logger.info('Destination endpoint: [{0}] {1}'.format(
self.dst_endpoint['id'],
self.dst_endpoint['display_name']))
if self.dst_endpoint is None:
self.logger.error('You don\'t have endpoint named: {0}'.format(
dst_endpoint_name))
raise LookupError
def transfer_dir(self, src_dir, dst_dir):
transfer_data = TransferData(self.transferClient,
self.src_endpoint['id'],
self.dst_endpoint['id'])
transfer_data.add_item(src_dir, dst_dir, recursive=True)
result = self.transferClient.submit_transfer(transfer_data)
self.logger.info('task [{0}] {1}'.format(result['task_id'],
result['code']))
return result
def transfer_file(self, src_file, dst_file):
transfer_data = TransferData(self.transferClient,
self.src_endpoint['id'],
self.dst_endpoint['id'])
transfer_data.add_item(src_file, dst_file)
result = self.transferClient.submit_transfer(transfer_data)
self.logger.info('task_id [{0}] {1}'.format(result['task_id'],
result['code']))
return result
def ls_src_dir(self, path, ls_filter=''):
# using iteration to get every entry from result
# an entry contain two keys: 'name' and 'type'
# type define the entry is a file or folder
result = self.transferClient.operation_ls(self.src_endpoint['id'],
path=path,
filter=ls_filter)
for entry in result:
self.logger.debug('name: {0}\ttype: {1}'.format(
entry["name"], entry["type"]))
return result
def task_list(self, num_results=10):
result = self.transferClient.task_list(num_results=num_results)
for task in result:
self.logger.debug('task_id: [{0}]\t status: {1}'.format(
task['task_id'], task['status']))
result = self.transferClient.task_list(num_results=num_results)
return result
def get_task(self, task_id):
return self.transferClient.get_task(task_id)
