def send_delete_task(endpoint_id=None, path=None):
tc = getTransferClient()
ddata = DeleteData(tc, endpoint_id, recursive=True)
ddata.add_item(path)
delete_result = tc.submit_delete(ddata)
return delete_result
def send_delete_task(endpoint_id=None, path=None, logger=logging.log):
tc = get_transfer_client(logger=logger)
ddata = DeleteData(tc, endpoint_id, recursive=True)
ddata.add_item(path)
delete_result = tc.submit_delete(ddata)
return delete_result
def rm_command(ignore_missing, star_silent, recursive, enable_globs,
endpoint_plus_path, label, submission_id, dry_run, deadline,
skip_activation_check, notify, meow, heartbeat,
polling_interval, timeout):
"""
Executor for `globus rm`
"""
endpoint_id, path = endpoint_plus_path
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify)
if not star_silent and enable_globs and path.endswith('*'):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (err_is_terminal() and term_is_interactive() and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.
format(path),
err=True)):
safeprint('Aborted.', write_to_stderr=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data,
response_key='DATA',
fields=[('Path', 'path')])
# exit safely
return
# Print task submission to stderr so that `-Fjson` is still correctly
# respected, as it will be by `task wait`
res = client.submit_delete(delete_data)
task_id = res['task_id']
safeprint('Delete task submitted under ID "{}"'.format(task_id),
write_to_stderr=True)
# do a `task wait` equivalent, including printing and correct exit status
task_wait_with_io(meow,
heartbeat,
polling_interval,
timeout,
task_id,
client=client)
def rmdir(self, ep_path, recursive=False):
''' remove a directory at ep_path '''
tc = self.xfer_client
ep, path = self.ep_parts(ep_path)
ddata = DeleteData(tc, ep, recursive=recursive)
ddata.add_item(path)
task_id = tc.submit_delete(ddata)['task_id']
return self._wait(task_id)
def send_bulk_delete_task(endpoint_id=None, pfns=None, logger=logging.log):
tc = get_transfer_client(logger=logger)
ddata = DeleteData(tc, endpoint_id, recursive=True)
for pfn in pfns:
logger(logging.DEBUG, 'pfn: %s' % pfn)
ddata.add_item(pfn)
bulk_delete_result = tc.submit_delete(ddata)
return bulk_delete_result
def delete_files_globus(self):
ep_id = hpss_globus_endpoint
r = self.transfer_client.endpoint_autoactivate(ep_id, if_expires_in=60)
if r.get("code") == "AutoActivationFailed":
self.fail(
"The {} endpoint is not activated. Please go to https://app.globus.org/file-manager/collections/{} and activate the endpoint."
.format(ep_id, ep_id))
ddata = DeleteData(self.transfer_client,
hpss_globus_endpoint,
recursive=True)
ddata.add_item("/~/zstash_test/")
try:
task = self.transfer_client.submit_delete(ddata)
task_id = task.get("task_id")
"""
A Globus transfer job (task) can be in one of the three states:
ACTIVE, SUCCEEDED, FAILED. The script every 5 seconds polls a
status of the transfer job (task) from the Globus Transfer service,
with 5 second timeout limit. If the task is ACTIVE after time runs
out 'task_wait' returns False, and True otherwise.
"""
while not self.transfer_client.task_wait(task_id, 5, 5):
task = self.transfer_client.get_task(task_id)
if task.get("is_paused"):
break
"""
The Globus transfer job (task) has been finished (SUCCEEDED or FAILED),
or is still active (ACTIVE). Check if the transfer SUCCEEDED or FAILED.
"""
task = self.transfer_client.get_task(task_id)
if task["status"] == "SUCCEEDED":
pass
elif task.get("status") == "ACTIVE":
if task.get("is_paused"):
pause_info = self.transfer_client.task_pause_info(task_id)
paused_rules = pause_info.get("pause_rules")
reason = paused_rules[0].get("message")
message = "The task was paused. Reason: {}".format(reason)
print(message)
else:
message = "The task reached a {} second deadline\n".format(
24 * 3600)
print(message)
self.transfer_client.cancel_task(task_id)
else:
print("Globus delete FAILED")
except TransferAPIError as e:
if e.code == "NoCredException":
self.fail(
"{}. Please go to https://app.globus.org/endpoints and activate the endpoint."
.format(e.message))
else:
self.fail(e)
except Exception as e:
self.fail("{} - exception: {}".format(self, e))
def cleanup():
user_identity_name = request.form.get('user_identity_name')
dependent_tokens = get_dependent_tokens(g.req_token)
transfer_token = dependent_tokens.by_resource_server[
'transfer.api.globus.org']['access_token']
dest_ep = app.config['GRAPH_ENDPOINT_ID']
dest_base = app.config['GRAPH_ENDPOINT_BASE']
dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name)
transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token))
transfer.endpoint_autoactivate(dest_ep)
try:
acl = next(acl for acl in transfer.endpoint_acl_list(dest_ep)
if dest_path == acl['path'])
except StopIteration:
pass
except TransferAPIError as ex:
# PermissionDenied can happen if a new Portal client is swapped
# in and it doesn't have endpoint manager on the dest_ep.
# The /portal/processed directory has been set to to writeable
# for all users so the delete task will succeed even if an ACL
# can't be set.
if ex.code == 'PermissionDenied':
pass
else:
transfer.delete_endpoint_acl_rule(dest_ep, acl['id'])
delete_request = DeleteData(transfer_client=transfer,
endpoint=dest_ep,
label="Delete Graphs from the Service Demo",
recursive=True)
delete_request.add_item(dest_path)
try:
task = transfer.submit_delete(delete_request)
except TransferAPIError as ex:
raise InternalServerError(message=ex.message)
else:
return jsonify(dict(task_id=task['task_id']))
def cleanup():
user_identity_name = request.form.get('user_identity_name')
dependent_tokens = get_dependent_tokens(g.req_token)
transfer_token = dependent_tokens.by_resource_server[
'transfer.api.globus.org']['access_token']
dest_ep = app.config['GRAPH_ENDPOINT_ID']
dest_base = app.config['GRAPH_ENDPOINT_BASE']
dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name)
transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token))
transfer.endpoint_autoactivate(dest_ep)
try:
acl = next(acl for acl in transfer.endpoint_acl_list(dest_ep)
if dest_path == acl['path'])
except StopIteration:
pass
except TransferAPIError as ex:
# PermissionDenied can happen if a new Portal client is swapped
# in and it doesn't have endpoint manager on the dest_ep.
# The /portal/processed directory has been set to to writeable
# for all users so the delete task will succeed even if an ACL
# can't be set.
if ex.code == 'PermissionDenied':
pass
else:
transfer.delete_endpoint_acl_rule(dest_ep, acl['id'])
delete_request = DeleteData(transfer_client=transfer, endpoint=dest_ep,
label="Delete Graphs from the Service Demo",
recursive=True)
delete_request.add_item(dest_path)
try:
task = transfer.submit_delete(delete_request)
except TransferAPIError as ex:
raise InternalServerError(message=ex.message)
else:
return jsonify(dict(task_id=task['task_id']))
def delete_command(
batch,
ignore_missing,
star_silent,
recursive,
enable_globs,
endpoint_plus_path,
label,
submission_id,
dry_run,
deadline,
skip_activation_check,
notify,
):
"""
Executor for `globus delete`
"""
endpoint_id, path = endpoint_plus_path
if path is None and (not batch):
raise click.UsageError("delete requires either a PATH OR --batch")
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify)
if batch:
# although this sophisticated structure (like that in transfer)
# isn't strictly necessary, it gives us the ability to add options in
# the future to these lines with trivial modifications
@click.command()
@click.argument("path", type=TaskPath(base_dir=path))
def process_batch_line(path):
"""
Parse a line of batch input and add it to the delete submission
item.
"""
delete_data.add_item(str(path))
shlex_process_stdin(process_batch_line,
"Enter paths to delete, line by line.")
else:
if not star_silent and enable_globs and path.endswith("*"):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (err_is_terminal() and term_is_interactive(
) and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.
format(path),
err=True,
)):
click.echo("Aborted.", err=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data,
response_key="DATA",
fields=[("Path", "path")])
# exit safely
return
res = client.submit_delete(delete_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)
def rm_command(
ignore_missing,
star_silent,
recursive,
enable_globs,
endpoint_plus_path,
label,
submission_id,
dry_run,
deadline,
skip_activation_check,
notify,
meow,
heartbeat,
polling_interval,
timeout,
timeout_exit_code,
):
"""
Executor for `globus rm`
"""
endpoint_id, path = endpoint_plus_path
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(
client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify
)
if not star_silent and enable_globs and path.endswith("*"):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (
err_is_terminal()
and term_is_interactive()
and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.format(path),
err=True,
)
):
safeprint("Aborted.", write_to_stderr=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data, response_key="DATA", fields=[("Path", "path")])
# exit safely
return
# Print task submission to stderr so that `-Fjson` is still correctly
# respected, as it will be by `task wait`
res = client.submit_delete(delete_data)
task_id = res["task_id"]
safeprint(
'Delete task submitted under ID "{}"'.format(task_id), write_to_stderr=True
)
# do a `task wait` equivalent, including printing and correct exit status
task_wait_with_io(
meow,
heartbeat,
polling_interval,
timeout,
task_id,
timeout_exit_code,
client=client,
)
def remove_path(path_to_remove, transfer_client=None):
transfer = transfer_client if transfer_client else init()
ddata = DeleteData(transfer, config.GWAS_ENDPOINT_ID, recursive=True)
ddata.add_item(path_to_remove)
delete_result = transfer.submit_delete(ddata)
return delete_result
def delete_command(
batch,
ignore_missing,
star_silent,
recursive,
enable_globs,
endpoint_plus_path,
label,
submission_id,
dry_run,
deadline,
skip_activation_check,
notify,
):
"""
Submits an asynchronous task that deletes files and/or directories on the target
endpoint.
*globus delete* has two modes. Single target, which deletes one
file or one directory, and batch, which takes in several lines to delete
multiple files or directories. See "Batch Input" below for more information.
Symbolic links are never followed - only unlinked (deleted).
=== Batch Input
If you give a SOURCE_PATH without the --batch flag, you will submit a
single-file or single-directory delete task. This has
behavior similar to `rm` and `rm -r`, across endpoints.
Using `--batch`, *globus delete* can submit a task which deletes
multiple files or directories. Lines are taken from stdin, respecting quotes,
and every line is treated as a path to a file or directory to delete.
\b
Lines are of the form
PATH
Note that unlike 'globus transfer' --recursive is not an option at the per line
level, instead, if given with the original command, all paths that point to
directories will be recursively deleted.
Empty lines and comments beginning with '#' are ignored.
Batch only requires an ENDPOINT before passing lines, on stdin, but if you pass
an ENPDOINT:PATH on the original command, this path will be used as a prefixes
to all paths on stdin.
{AUTOMATIC_ACTIVATION}
"""
endpoint_id, path = endpoint_plus_path
if path is None and (not batch):
raise click.UsageError("delete requires either a PATH OR --batch")
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify)
if batch:
# although this sophisticated structure (like that in transfer)
# isn't strictly necessary, it gives us the ability to add options in
# the future to these lines with trivial modifications
@click.command()
@click.argument("path", type=TaskPath(base_dir=path))
def process_batch_line(path):
"""
Parse a line of batch input and add it to the delete submission
item.
"""
delete_data.add_item(str(path))
shlex_process_stdin(process_batch_line,
"Enter paths to delete, line by line.")
else:
if not star_silent and enable_globs and path.endswith("*"):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (err_is_terminal() and term_is_interactive(
) and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.
format(path),
err=True,
)):
click.echo("Aborted.", err=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data,
response_key="DATA",
fields=[("Path", "path")])
# exit safely
return
res = client.submit_delete(delete_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)
def delete_command(
batch,
ignore_missing,
star_silent,
recursive,
enable_globs,
endpoint_plus_path,
label,
submission_id,
dry_run,
deadline,
skip_activation_check,
notify,
):
"""
Executor for `globus delete`
"""
endpoint_id, path = endpoint_plus_path
if path is None and (not batch):
raise click.UsageError("delete requires either a PATH OR --batch")
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(
client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify
)
if batch:
# although this sophisticated structure (like that in transfer)
# isn't strictly necessary, it gives us the ability to add options in
# the future to these lines with trivial modifications
@click.command()
@click.argument("path", type=TaskPath(base_dir=path))
def process_batch_line(path):
"""
Parse a line of batch input and add it to the delete submission
item.
"""
delete_data.add_item(str(path))
shlex_process_stdin(process_batch_line, "Enter paths to delete, line by line.")
else:
if not star_silent and enable_globs and path.endswith("*"):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (
err_is_terminal()
and term_is_interactive()
and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.format(
path
),
err=True,
)
):
safeprint("Aborted.", write_to_stderr=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data, response_key="DATA", fields=[("Path", "path")])
# exit safely
return
res = client.submit_delete(delete_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)