def transfer_command(
batch,
sync_level,
recursive,
destination,
source,
label,
preserve_mtime,
verify_checksum,
encrypt,
submission_id,
dry_run,
delete,
deadline,
skip_activation_check,
notify,
perf_cc,
perf_p,
perf_pp,
perf_udt,
):
"""
Executor for `globus transfer`
"""
source_endpoint, cmd_source_path = source
dest_endpoint, cmd_dest_path = destination
if recursive and batch:
raise click.UsageError(
("You cannot use --recursive in addition to --batch. "
"Instead, use --recursive on lines of --batch input "
"which need it"))
if (cmd_source_path is None or cmd_dest_path is None) and (not batch):
raise click.UsageError(
("transfer requires either SOURCE_PATH and DEST_PATH or "
"--batch"))
# because python can't handle multiple **kwargs expansions in a single
# call, we need to get a little bit clever
# both the performance options (of which there are a few), and the
# notification options (also there are a few) have elements which should be
# omitted in some cases
# notify comes to us clean, perf opts need more care
# put them together into a dict before passing to TransferData
kwargs = {}
perf_opts = dict((k, v) for (k, v) in dict(
perf_cc=perf_cc, perf_p=perf_p, perf_pp=perf_pp,
perf_udt=perf_udt).items() if v is not None)
kwargs.update(perf_opts)
kwargs.update(notify)
client = get_client()
transfer_data = TransferData(client,
source_endpoint,
dest_endpoint,
label=label,
sync_level=sync_level,
verify_checksum=verify_checksum,
preserve_timestamp=preserve_mtime,
encrypt_data=encrypt,
submission_id=submission_id,
delete_destination_extra=delete,
deadline=deadline,
skip_activation_check=skip_activation_check,
**kwargs)
if batch:
@click.command()
@click.option("--recursive", "-r", is_flag=True)
@click.argument("source_path", type=TaskPath(base_dir=cmd_source_path))
@click.argument("dest_path", type=TaskPath(base_dir=cmd_dest_path))
def process_batch_line(dest_path, source_path, recursive):
"""
Parse a line of batch input and turn it into a transfer submission
item.
"""
transfer_data.add_item(str(source_path),
str(dest_path),
recursive=recursive)
shlex_process_stdin(
process_batch_line,
("Enter transfers, line by line, as\n\n"
" [--recursive] SOURCE_PATH DEST_PATH\n"),
)
else:
transfer_data.add_item(cmd_source_path,
cmd_dest_path,
recursive=recursive)
if dry_run:
formatted_print(
transfer_data,
response_key="DATA",
fields=(
("Source Path", "source_path"),
("Dest Path", "destination_path"),
("Recursive", "recursive"),
),
)
# exit safely
return
# autoactivate after parsing all args and putting things together
# skip this if skip-activation-check is given
if not skip_activation_check:
autoactivate(client, source_endpoint, if_expires_in=60)
autoactivate(client, dest_endpoint, if_expires_in=60)
res = client.submit_transfer(transfer_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)
def delete_command(
batch,
ignore_missing,
star_silent,
recursive,
enable_globs,
endpoint_plus_path,
label,
submission_id,
dry_run,
deadline,
skip_activation_check,
notify,
):
"""
Executor for `globus delete`
"""
endpoint_id, path = endpoint_plus_path
if path is None and (not batch):
raise click.UsageError("delete requires either a PATH OR --batch")
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify)
if batch:
# although this sophisticated structure (like that in transfer)
# isn't strictly necessary, it gives us the ability to add options in
# the future to these lines with trivial modifications
@click.command()
@click.argument("path", type=TaskPath(base_dir=path))
def process_batch_line(path):
"""
Parse a line of batch input and add it to the delete submission
item.
"""
delete_data.add_item(str(path))
shlex_process_stdin(process_batch_line,
"Enter paths to delete, line by line.")
else:
if not star_silent and enable_globs and path.endswith("*"):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (err_is_terminal() and term_is_interactive(
) and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.
format(path),
err=True,
)):
click.echo("Aborted.", err=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data,
response_key="DATA",
fields=[("Path", "path")])
# exit safely
return
res = client.submit_delete(delete_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)
def transfer_command(
batch,
sync_level,
recursive,
destination,
source,
label,
preserve_mtime,
verify_checksum,
encrypt,
submission_id,
dry_run,
delete,
deadline,
skip_activation_check,
notify,
perf_cc,
perf_p,
perf_pp,
perf_udt,
):
"""
Executor for `globus transfer`
"""
source_endpoint, cmd_source_path = source
dest_endpoint, cmd_dest_path = destination
if recursive and batch:
raise click.UsageError(
(
"You cannot use --recursive in addition to --batch. "
"Instead, use --recursive on lines of --batch input "
"which need it"
)
)
if (cmd_source_path is None or cmd_dest_path is None) and (not batch):
raise click.UsageError(
("transfer requires either SOURCE_PATH and DEST_PATH or " "--batch")
)
# because python can't handle multiple **kwargs expansions in a single
# call, we need to get a little bit clever
# both the performance options (of which there are a few), and the
# notification options (also there are a few) have elements which should be
# omitted in some cases
# notify comes to us clean, perf opts need more care
# put them together into a dict before passing to TransferData
kwargs = {}
perf_opts = dict(
(k, v)
for (k, v) in dict(
perf_cc=perf_cc, perf_p=perf_p, perf_pp=perf_pp, perf_udt=perf_udt
).items()
if v is not None
)
kwargs.update(perf_opts)
kwargs.update(notify)
client = get_client()
transfer_data = TransferData(
client,
source_endpoint,
dest_endpoint,
label=label,
sync_level=sync_level,
verify_checksum=verify_checksum,
preserve_timestamp=preserve_mtime,
encrypt_data=encrypt,
submission_id=submission_id,
delete_destination_extra=delete,
deadline=deadline,
skip_activation_check=skip_activation_check,
**kwargs
)
if batch:
@click.command()
@click.option("--recursive", "-r", is_flag=True)
@click.argument("source_path", type=TaskPath(base_dir=cmd_source_path))
@click.argument("dest_path", type=TaskPath(base_dir=cmd_dest_path))
def process_batch_line(dest_path, source_path, recursive):
"""
Parse a line of batch input and turn it into a transfer submission
item.
"""
transfer_data.add_item(
str(source_path), str(dest_path), recursive=recursive
)
shlex_process_stdin(
process_batch_line,
(
"Enter transfers, line by line, as\n\n"
" [--recursive] SOURCE_PATH DEST_PATH\n"
),
)
else:
transfer_data.add_item(cmd_source_path, cmd_dest_path, recursive=recursive)
if dry_run:
formatted_print(
transfer_data,
response_key="DATA",
fields=(
("Source Path", "source_path"),
("Dest Path", "destination_path"),
("Recursive", "recursive"),
),
)
# exit safely
return
# autoactivate after parsing all args and putting things together
# skip this if skip-activation-check is given
if not skip_activation_check:
autoactivate(client, source_endpoint, if_expires_in=60)
autoactivate(client, dest_endpoint, if_expires_in=60)
res = client.submit_transfer(transfer_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)
def delete_command(
batch,
ignore_missing,
star_silent,
recursive,
enable_globs,
endpoint_plus_path,
label,
submission_id,
dry_run,
deadline,
skip_activation_check,
notify,
):
"""
Submits an asynchronous task that deletes files and/or directories on the target
endpoint.
*globus delete* has two modes. Single target, which deletes one
file or one directory, and batch, which takes in several lines to delete
multiple files or directories. See "Batch Input" below for more information.
Symbolic links are never followed - only unlinked (deleted).
=== Batch Input
If you give a SOURCE_PATH without the --batch flag, you will submit a
single-file or single-directory delete task. This has
behavior similar to `rm` and `rm -r`, across endpoints.
Using `--batch`, *globus delete* can submit a task which deletes
multiple files or directories. Lines are taken from stdin, respecting quotes,
and every line is treated as a path to a file or directory to delete.
\b
Lines are of the form
PATH
Note that unlike 'globus transfer' --recursive is not an option at the per line
level, instead, if given with the original command, all paths that point to
directories will be recursively deleted.
Empty lines and comments beginning with '#' are ignored.
Batch only requires an ENDPOINT before passing lines, on stdin, but if you pass
an ENPDOINT:PATH on the original command, this path will be used as a prefixes
to all paths on stdin.
{AUTOMATIC_ACTIVATION}
"""
endpoint_id, path = endpoint_plus_path
if path is None and (not batch):
raise click.UsageError("delete requires either a PATH OR --batch")
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify)
if batch:
# although this sophisticated structure (like that in transfer)
# isn't strictly necessary, it gives us the ability to add options in
# the future to these lines with trivial modifications
@click.command()
@click.argument("path", type=TaskPath(base_dir=path))
def process_batch_line(path):
"""
Parse a line of batch input and add it to the delete submission
item.
"""
delete_data.add_item(str(path))
shlex_process_stdin(process_batch_line,
"Enter paths to delete, line by line.")
else:
if not star_silent and enable_globs and path.endswith("*"):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (err_is_terminal() and term_is_interactive(
) and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.
format(path),
err=True,
)):
click.echo("Aborted.", err=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data,
response_key="DATA",
fields=[("Path", "path")])
# exit safely
return
res = client.submit_delete(delete_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)
def delete_command(
batch,
ignore_missing,
star_silent,
recursive,
enable_globs,
endpoint_plus_path,
label,
submission_id,
dry_run,
deadline,
skip_activation_check,
notify,
):
"""
Executor for `globus delete`
"""
endpoint_id, path = endpoint_plus_path
if path is None and (not batch):
raise click.UsageError("delete requires either a PATH OR --batch")
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(
client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify
)
if batch:
# although this sophisticated structure (like that in transfer)
# isn't strictly necessary, it gives us the ability to add options in
# the future to these lines with trivial modifications
@click.command()
@click.argument("path", type=TaskPath(base_dir=path))
def process_batch_line(path):
"""
Parse a line of batch input and add it to the delete submission
item.
"""
delete_data.add_item(str(path))
shlex_process_stdin(process_batch_line, "Enter paths to delete, line by line.")
else:
if not star_silent and enable_globs and path.endswith("*"):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (
err_is_terminal()
and term_is_interactive()
and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.format(
path
),
err=True,
)
):
safeprint("Aborted.", write_to_stderr=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data, response_key="DATA", fields=[("Path", "path")])
# exit safely
return
res = client.submit_delete(delete_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)
def transfer_command(
batch,
sync_level,
recursive,
destination,
source,
checksum_algorithm,
external_checksum,
label,
preserve_mtime,
verify_checksum,
encrypt,
submission_id,
dry_run,
delete,
deadline,
skip_activation_check,
notify,
perf_cc,
perf_p,
perf_pp,
perf_udt,
):
"""
Copy a file or directory from one endpoint to another as an asynchronous
task.
'globus transfer' has two modes. Single target, which transfers one
file or one directory, and batch, which takes in several lines to transfer
multiple files or directories. See "Batch Input" below for more information.
'globus transfer' will always place the dest files in a
consistent, deterministic location. The contents of a source directory will
be placed inside the dest directory. A source file will be copied to
the dest file path, which must not be an existing directory. All
intermediate / parent directories on the dest will be automatically
created if they don't exist.
If the files or directories given as input are symbolic links, they are
followed. However, no other symbolic links are followed and no symbolic links
are ever created on the dest.
\b
=== Batched Input
If you use `SOURCE_PATH` and `DEST_PATH` without the `--batch` flag, you
will submit a single-file or single-directory transfer task.
This has behavior similar to `cp` and `cp -r` across endpoints.
Using `--batch`, `globus transfer` can submit a task which transfers
multiple files or directories. Paths to transfer are taken from stdin.
Lines are split on spaces, respecting quotes, and every line is treated as
a file or directory to transfer.
\b
Lines are of the form
[--recursive] [--external-checksum TEXT] SOURCE_PATH DEST_PATH\n
Skips empty lines and allows comments beginning with "#".
\b
If you use `--batch` and a commandline SOURCE_PATH and/or DEST_PATH, these
paths will be used as dir prefixes to any paths on stdin.
\b
=== Sync Levels
Sync Levels are ways to decide whether or not files are copied, with the
following definitions:
EXISTS: Determine whether or not to transfer based on file existence.
If the destination file is absent, do the transfer.
SIZE: Determine whether or not to transfer based on the size of the file.
If destination file size does not match the source, do the transfer.
MTIME: Determine whether or not to transfer based on modification times.
If source has a newer modififed time than the destination, do the transfer.
CHECKSUM: Determine whether or not to transfer based on checksums of file
contents.
If source and destination contents differ, as determined by a checksum of
their contents, do the transfer.
If a transfer fails, CHECKSUM must be used to restart the transfer.
All other levels can lead to data corruption.
{AUTOMATIC_ACTIVATION}
"""
source_endpoint, cmd_source_path = source
dest_endpoint, cmd_dest_path = destination
if recursive and batch:
raise click.UsageError(
(
"You cannot use --recursive in addition to --batch. "
"Instead, use --recursive on lines of --batch input "
"which need it"
)
)
if external_checksum and batch:
raise click.UsageError(
(
"You cannot use --external-checksum in addition to --batch. "
"Instead, use --external-checksum on lines of --batch input "
"which need it"
)
)
if recursive and external_checksum:
raise click.UsageError(
"--recursive and --external-checksum are mutually exclusive"
)
if (cmd_source_path is None or cmd_dest_path is None) and (not batch):
raise click.UsageError(
"transfer requires either SOURCE_PATH and DEST_PATH or --batch"
)
# because python can't handle multiple **kwargs expansions in a single
# call, we need to get a little bit clever
# both the performance options (of which there are a few), and the
# notification options (also there are a few) have elements which should be
# omitted in some cases
# notify comes to us clean, perf opts need more care
# put them together into a dict before passing to TransferData
kwargs = {}
perf_opts = dict(
(k, v)
for (k, v) in dict(
perf_cc=perf_cc, perf_p=perf_p, perf_pp=perf_pp, perf_udt=perf_udt
).items()
if v is not None
)
kwargs.update(perf_opts)
kwargs.update(notify)
client = get_client()
transfer_data = TransferData(
client,
source_endpoint,
dest_endpoint,
label=label,
sync_level=sync_level,
verify_checksum=verify_checksum,
preserve_timestamp=preserve_mtime,
encrypt_data=encrypt,
submission_id=submission_id,
delete_destination_extra=delete,
deadline=deadline,
skip_activation_check=skip_activation_check,
**kwargs
)
if batch:
@click.command()
@click.option("--external-checksum")
@click.option("--recursive", "-r", is_flag=True)
@click.argument("source_path", type=TaskPath(base_dir=cmd_source_path))
@click.argument("dest_path", type=TaskPath(base_dir=cmd_dest_path))
def process_batch_line(dest_path, source_path, recursive, external_checksum):
"""
Parse a line of batch input and turn it into a transfer submission
item.
"""
if recursive and external_checksum:
raise click.UsageError(
"--recursive and --external-checksum are mutually exclusive"
)
transfer_data.add_item(
str(source_path),
str(dest_path),
external_checksum=external_checksum,
checksum_algorithm=checksum_algorithm,
recursive=recursive,
)
shlex_process_stdin(
process_batch_line,
(
"Enter transfers, line by line, as\n\n"
" [--recursive] [--external-checksum TEXT] SOURCE_PATH DEST_PATH\n"
),
)
else:
transfer_data.add_item(
cmd_source_path,
cmd_dest_path,
external_checksum=external_checksum,
checksum_algorithm=checksum_algorithm,
recursive=recursive,
)
if dry_run:
formatted_print(
transfer_data,
response_key="DATA",
fields=(
("Source Path", "source_path"),
("Dest Path", "destination_path"),
("Recursive", "recursive"),
("External Checksum", "external_checksum"),
),
)
# exit safely
return
# autoactivate after parsing all args and putting things together
# skip this if skip-activation-check is given
if not skip_activation_check:
autoactivate(client, source_endpoint, if_expires_in=60)
autoactivate(client, dest_endpoint, if_expires_in=60)
res = client.submit_transfer(transfer_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)