def rename_command(source, destination):
"""Rename a file or directory on an endpoint.
The old path must be an existing file or directory. The new path must not yet
exist.
The new path does not have to be in the same directory as the old path, but
most endpoints will require it to stay on the same filesystem.
The endpoint must be entered twice for the sake of path syntax consistency.
"""
source_ep, source_path = source
dest_ep, dest_path = destination
if source_ep != dest_ep:
raise click.UsageError(
("rename requires that the source and dest "
"endpoints are the same, {} != {}").format(source_ep, dest_ep))
endpoint_id = source_ep
client = get_client()
autoactivate(client, endpoint_id, if_expires_in=60)
res = client.operation_rename(endpoint_id,
oldpath=source_path,
newpath=dest_path)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def rm_command(ignore_missing, star_silent, recursive, enable_globs,
endpoint_plus_path, label, submission_id, dry_run, deadline,
skip_activation_check, notify, meow, heartbeat,
polling_interval, timeout):
"""
Executor for `globus rm`
"""
endpoint_id, path = endpoint_plus_path
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify)
if not star_silent and enable_globs and path.endswith('*'):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (err_is_terminal() and term_is_interactive() and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.
format(path),
err=True)):
safeprint('Aborted.', write_to_stderr=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data,
response_key='DATA',
fields=[('Path', 'path')])
# exit safely
return
# Print task submission to stderr so that `-Fjson` is still correctly
# respected, as it will be by `task wait`
res = client.submit_delete(delete_data)
task_id = res['task_id']
safeprint('Delete task submitted under ID "{}"'.format(task_id),
write_to_stderr=True)
# do a `task wait` equivalent, including printing and correct exit status
task_wait_with_io(meow,
heartbeat,
polling_interval,
timeout,
task_id,
client=client)
def mkdir_command(endpoint_plus_path):
"""
Executor for `globus mkdir`
"""
endpoint_id, path = endpoint_plus_path
client = get_client()
autoactivate(client, endpoint_id, if_expires_in=60)
res = client.operation_mkdir(endpoint_id, path=path)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def mkdir_command(endpoint_plus_path):
"""
Executor for `globus mkdir`
"""
endpoint_id, path = endpoint_plus_path
client = get_client()
autoactivate(client, endpoint_id, if_expires_in=60)
res = client.operation_mkdir(endpoint_id, path=path)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def mkdir_command(endpoint_plus_path):
"""Make a directory on an endpoint at the given path.
{AUTOMATIC_ACTIVATION}
"""
endpoint_id, path = endpoint_plus_path
client = get_client()
autoactivate(client, endpoint_id, if_expires_in=60)
res = client.operation_mkdir(endpoint_id, path=path)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def ls_command(endpoint_plus_path, recursive_depth_limit,
recursive, long_output, show_hidden, filter_val):
"""
Executor for `globus ls`
"""
endpoint_id, path = endpoint_plus_path
# do autoactivation before the `ls` call so that recursive invocations
# won't do this repeatedly, and won't have to instantiate new clients
client = get_client()
autoactivate(client, endpoint_id, if_expires_in=60)
# create the query paramaters to send to operation_ls
ls_params = {"show_hidden": int(show_hidden)}
if path:
ls_params["path"] = path
if filter_val:
# this char has special meaning in the LS API's filter clause
# can't be part of the pattern (but we don't support globbing across
# dir structures anyway)
if '/' in filter_val:
raise click.UsageError('--filter cannot contain "/"')
# format into a simple filter clause which operates on filenames
ls_params['filter'] = 'name:{}'.format(filter_val)
# get the `ls` result
if recursive:
# NOTE:
# --recursive and --filter have an interplay that some users may find
# surprising
# if we're asked to change or "improve" the behavior in the future, we
# could do so with "type:dir" or "type:file" filters added in, and
# potentially work out some viable behavior based on what people want
res = client.recursive_operation_ls(
endpoint_id, depth=recursive_depth_limit, **ls_params)
else:
res = client.operation_ls(endpoint_id, **ls_params)
def cleaned_item_name(item):
return item['name'] + ('/' if item['type'] == 'dir' else '')
# and then print it, per formatting rules
formatted_print(
res, fields=[('Permissions', 'permissions'), ('User', 'user'),
('Group', 'group'), ('Size', 'size'),
('Last Modified', 'last_modified'), ('File Type', 'type'),
('Filename', cleaned_item_name)],
simple_text=(
None
if long_output or is_verbose() or not outformat_is_text() else
"\n".join(cleaned_item_name(x) for x in res)),
json_converter=iterable_response_to_dict)
def endpoint_create(**kwargs):
"""
Create a new endpoint.
Requires a display name and exactly one of --personal, --server, or --shared to make
a Globus Connect Personal, Globus Connect Server, or Shared endpoint respectively.
"""
client = get_client()
# get endpoint type, ensure unambiguous.
personal = kwargs.pop("personal")
server = kwargs.pop("server")
shared = kwargs.pop("shared")
if personal and (not server) and (not shared):
endpoint_type = "personal"
elif server and (not personal) and (not shared):
endpoint_type = "server"
elif shared and (not personal) and (not server):
endpoint_type = "shared"
else:
raise click.UsageError(
"Exactly one of --personal, --server, or --shared is required.")
# validate options
kwargs["is_globus_connect"] = personal or None
validate_endpoint_create_and_update_params(endpoint_type, False, kwargs)
# shared endpoint creation
if shared:
endpoint_id, host_path = shared
kwargs["host_endpoint"] = endpoint_id
kwargs["host_path"] = host_path
ep_doc = assemble_generic_doc("shared_endpoint", **kwargs)
autoactivate(client, endpoint_id, if_expires_in=60)
res = client.create_shared_endpoint(ep_doc)
# non shared endpoint creation
else:
# omit `is_globus_connect` key if not GCP, otherwise include as `True`
ep_doc = assemble_generic_doc("endpoint", **kwargs)
res = client.create_endpoint(ep_doc)
# output
formatted_print(
res,
fields=(COMMON_FIELDS + GCP_FIELDS if personal else COMMON_FIELDS),
text_format=FORMAT_TEXT_RECORD,
)
def rename_command(source, destination):
"""
Executor for `globus rename`
"""
source_ep, source_path = source
dest_ep, dest_path = destination
if source_ep != dest_ep:
raise click.UsageError(('rename requires that the source and dest '
'endpoints are the same, {} != {}')
.format(source_ep, dest_ep))
endpoint_id = source_ep
client = get_client()
autoactivate(client, endpoint_id, if_expires_in=60)
res = client.operation_rename(endpoint_id, oldpath=source_path,
newpath=dest_path)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key='message')
def ls_command(
endpoint_plus_path,
recursive_depth_limit,
recursive,
long_output,
show_hidden,
filter_val,
):
"""
List the contents of a directory on an endpoint. If no path is given, the default
directory on that endpoint will be used.
If using text output files and directories are printed with one entry per line in
alphabetical order. Directories are always displayed with a trailing '/'.
\b
=== Filtering
--filter takes "filter patterns" subject to the following rules.
\b
Filter patterns must start with "=", "~", "!", or "!~"
If none of these are given, "=" will be used
\b
"=" does exact matching
"~" does regex matching, supporting globs (*)
"!" does inverse "=" matching
"!~" does inverse "~" matching
\b
"~*.txt" matches all .txt files, for example
{AUTOMATIC_ACTIVATION}
"""
endpoint_id, path = endpoint_plus_path
# do autoactivation before the `ls` call so that recursive invocations
# won't do this repeatedly, and won't have to instantiate new clients
client = get_client()
autoactivate(client, endpoint_id, if_expires_in=60)
# create the query paramaters to send to operation_ls
ls_params = {"show_hidden": int(show_hidden)}
if path:
ls_params["path"] = path
if filter_val:
# this char has special meaning in the LS API's filter clause
# can't be part of the pattern (but we don't support globbing across
# dir structures anyway)
if "/" in filter_val:
raise click.UsageError('--filter cannot contain "/"')
# format into a simple filter clause which operates on filenames
ls_params["filter"] = "name:{}".format(filter_val)
# get the `ls` result
if recursive:
# NOTE:
# --recursive and --filter have an interplay that some users may find
# surprising
# if we're asked to change or "improve" the behavior in the future, we
# could do so with "type:dir" or "type:file" filters added in, and
# potentially work out some viable behavior based on what people want
res = client.recursive_operation_ls(endpoint_id,
depth=recursive_depth_limit,
**ls_params)
else:
res = client.operation_ls(endpoint_id, **ls_params)
def cleaned_item_name(item):
return item["name"] + ("/" if item["type"] == "dir" else "")
# and then print it, per formatting rules
formatted_print(
res,
fields=[
("Permissions", "permissions"),
("User", "user"),
("Group", "group"),
("Size", "size"),
("Last Modified", "last_modified"),
("File Type", "type"),
("Filename", cleaned_item_name),
],
simple_text=(None if long_output or is_verbose()
or not outformat_is_text() else "\n".join(
cleaned_item_name(x) for x in res)),
json_converter=iterable_response_to_dict,
)
def rm_command(
ignore_missing,
star_silent,
recursive,
enable_globs,
endpoint_plus_path,
label,
submission_id,
dry_run,
deadline,
skip_activation_check,
notify,
meow,
heartbeat,
polling_interval,
timeout,
timeout_exit_code,
):
"""
Executor for `globus rm`
"""
endpoint_id, path = endpoint_plus_path
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(
client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify
)
if not star_silent and enable_globs and path.endswith("*"):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (
err_is_terminal()
and term_is_interactive()
and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.format(path),
err=True,
)
):
safeprint("Aborted.", write_to_stderr=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data, response_key="DATA", fields=[("Path", "path")])
# exit safely
return
# Print task submission to stderr so that `-Fjson` is still correctly
# respected, as it will be by `task wait`
res = client.submit_delete(delete_data)
task_id = res["task_id"]
safeprint(
'Delete task submitted under ID "{}"'.format(task_id), write_to_stderr=True
)
# do a `task wait` equivalent, including printing and correct exit status
task_wait_with_io(
meow,
heartbeat,
polling_interval,
timeout,
task_id,
timeout_exit_code,
client=client,
)
def ls_command(
endpoint_plus_path,
recursive_depth_limit,
recursive,
long_output,
show_hidden,
filter_val,
):
"""
Executor for `globus ls`
"""
endpoint_id, path = endpoint_plus_path
# do autoactivation before the `ls` call so that recursive invocations
# won't do this repeatedly, and won't have to instantiate new clients
client = get_client()
autoactivate(client, endpoint_id, if_expires_in=60)
# create the query paramaters to send to operation_ls
ls_params = {"show_hidden": int(show_hidden)}
if path:
ls_params["path"] = path
if filter_val:
# this char has special meaning in the LS API's filter clause
# can't be part of the pattern (but we don't support globbing across
# dir structures anyway)
if "/" in filter_val:
raise click.UsageError('--filter cannot contain "/"')
# format into a simple filter clause which operates on filenames
ls_params["filter"] = "name:{}".format(filter_val)
# get the `ls` result
if recursive:
# NOTE:
# --recursive and --filter have an interplay that some users may find
# surprising
# if we're asked to change or "improve" the behavior in the future, we
# could do so with "type:dir" or "type:file" filters added in, and
# potentially work out some viable behavior based on what people want
res = client.recursive_operation_ls(
endpoint_id, depth=recursive_depth_limit, **ls_params
)
else:
res = client.operation_ls(endpoint_id, **ls_params)
def cleaned_item_name(item):
return item["name"] + ("/" if item["type"] == "dir" else "")
# and then print it, per formatting rules
formatted_print(
res,
fields=[
("Permissions", "permissions"),
("User", "user"),
("Group", "group"),
("Size", "size"),
("Last Modified", "last_modified"),
("File Type", "type"),
("Filename", cleaned_item_name),
],
simple_text=(
None
if long_output or is_verbose() or not outformat_is_text()
else "\n".join(cleaned_item_name(x) for x in res)
),
json_converter=iterable_response_to_dict,
)
def transfer_command(
batch,
sync_level,
recursive,
destination,
source,
label,
preserve_mtime,
verify_checksum,
encrypt,
submission_id,
dry_run,
delete,
deadline,
skip_activation_check,
notify,
perf_cc,
perf_p,
perf_pp,
perf_udt,
):
"""
Executor for `globus transfer`
"""
source_endpoint, cmd_source_path = source
dest_endpoint, cmd_dest_path = destination
if recursive and batch:
raise click.UsageError(
(
"You cannot use --recursive in addition to --batch. "
"Instead, use --recursive on lines of --batch input "
"which need it"
)
)
if (cmd_source_path is None or cmd_dest_path is None) and (not batch):
raise click.UsageError(
("transfer requires either SOURCE_PATH and DEST_PATH or " "--batch")
)
# because python can't handle multiple **kwargs expansions in a single
# call, we need to get a little bit clever
# both the performance options (of which there are a few), and the
# notification options (also there are a few) have elements which should be
# omitted in some cases
# notify comes to us clean, perf opts need more care
# put them together into a dict before passing to TransferData
kwargs = {}
perf_opts = dict(
(k, v)
for (k, v) in dict(
perf_cc=perf_cc, perf_p=perf_p, perf_pp=perf_pp, perf_udt=perf_udt
).items()
if v is not None
)
kwargs.update(perf_opts)
kwargs.update(notify)
client = get_client()
transfer_data = TransferData(
client,
source_endpoint,
dest_endpoint,
label=label,
sync_level=sync_level,
verify_checksum=verify_checksum,
preserve_timestamp=preserve_mtime,
encrypt_data=encrypt,
submission_id=submission_id,
delete_destination_extra=delete,
deadline=deadline,
skip_activation_check=skip_activation_check,
**kwargs
)
if batch:
@click.command()
@click.option("--recursive", "-r", is_flag=True)
@click.argument("source_path", type=TaskPath(base_dir=cmd_source_path))
@click.argument("dest_path", type=TaskPath(base_dir=cmd_dest_path))
def process_batch_line(dest_path, source_path, recursive):
"""
Parse a line of batch input and turn it into a transfer submission
item.
"""
transfer_data.add_item(
str(source_path), str(dest_path), recursive=recursive
)
shlex_process_stdin(
process_batch_line,
(
"Enter transfers, line by line, as\n\n"
" [--recursive] SOURCE_PATH DEST_PATH\n"
),
)
else:
transfer_data.add_item(cmd_source_path, cmd_dest_path, recursive=recursive)
if dry_run:
formatted_print(
transfer_data,
response_key="DATA",
fields=(
("Source Path", "source_path"),
("Dest Path", "destination_path"),
("Recursive", "recursive"),
),
)
# exit safely
return
# autoactivate after parsing all args and putting things together
# skip this if skip-activation-check is given
if not skip_activation_check:
autoactivate(client, source_endpoint, if_expires_in=60)
autoactivate(client, dest_endpoint, if_expires_in=60)
res = client.submit_transfer(transfer_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)
def delete_command(
batch,
ignore_missing,
star_silent,
recursive,
enable_globs,
endpoint_plus_path,
label,
submission_id,
dry_run,
deadline,
skip_activation_check,
notify,
):
"""
Submits an asynchronous task that deletes files and/or directories on the target
endpoint.
*globus delete* has two modes. Single target, which deletes one
file or one directory, and batch, which takes in several lines to delete
multiple files or directories. See "Batch Input" below for more information.
Symbolic links are never followed - only unlinked (deleted).
=== Batch Input
If you give a SOURCE_PATH without the --batch flag, you will submit a
single-file or single-directory delete task. This has
behavior similar to `rm` and `rm -r`, across endpoints.
Using `--batch`, *globus delete* can submit a task which deletes
multiple files or directories. Lines are taken from stdin, respecting quotes,
and every line is treated as a path to a file or directory to delete.
\b
Lines are of the form
PATH
Note that unlike 'globus transfer' --recursive is not an option at the per line
level, instead, if given with the original command, all paths that point to
directories will be recursively deleted.
Empty lines and comments beginning with '#' are ignored.
Batch only requires an ENDPOINT before passing lines, on stdin, but if you pass
an ENPDOINT:PATH on the original command, this path will be used as a prefixes
to all paths on stdin.
{AUTOMATIC_ACTIVATION}
"""
endpoint_id, path = endpoint_plus_path
if path is None and (not batch):
raise click.UsageError("delete requires either a PATH OR --batch")
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify)
if batch:
# although this sophisticated structure (like that in transfer)
# isn't strictly necessary, it gives us the ability to add options in
# the future to these lines with trivial modifications
@click.command()
@click.argument("path", type=TaskPath(base_dir=path))
def process_batch_line(path):
"""
Parse a line of batch input and add it to the delete submission
item.
"""
delete_data.add_item(str(path))
shlex_process_stdin(process_batch_line,
"Enter paths to delete, line by line.")
else:
if not star_silent and enable_globs and path.endswith("*"):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (err_is_terminal() and term_is_interactive(
) and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.
format(path),
err=True,
)):
click.echo("Aborted.", err=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data,
response_key="DATA",
fields=[("Path", "path")])
# exit safely
return
res = client.submit_delete(delete_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)
def delete_command(
batch,
ignore_missing,
star_silent,
recursive,
enable_globs,
endpoint_plus_path,
label,
submission_id,
dry_run,
deadline,
skip_activation_check,
notify,
):
"""
Executor for `globus delete`
"""
endpoint_id, path = endpoint_plus_path
if path is None and (not batch):
raise click.UsageError("delete requires either a PATH OR --batch")
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(
client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify
)
if batch:
# although this sophisticated structure (like that in transfer)
# isn't strictly necessary, it gives us the ability to add options in
# the future to these lines with trivial modifications
@click.command()
@click.argument("path", type=TaskPath(base_dir=path))
def process_batch_line(path):
"""
Parse a line of batch input and add it to the delete submission
item.
"""
delete_data.add_item(str(path))
shlex_process_stdin(process_batch_line, "Enter paths to delete, line by line.")
else:
if not star_silent and enable_globs and path.endswith("*"):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (
err_is_terminal()
and term_is_interactive()
and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.format(
path
),
err=True,
)
):
safeprint("Aborted.", write_to_stderr=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data, response_key="DATA", fields=[("Path", "path")])
# exit safely
return
res = client.submit_delete(delete_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)
def delete_command(
batch,
ignore_missing,
star_silent,
recursive,
enable_globs,
endpoint_plus_path,
label,
submission_id,
dry_run,
deadline,
skip_activation_check,
notify,
):
"""
Executor for `globus delete`
"""
endpoint_id, path = endpoint_plus_path
if path is None and (not batch):
raise click.UsageError("delete requires either a PATH OR --batch")
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify)
if batch:
# although this sophisticated structure (like that in transfer)
# isn't strictly necessary, it gives us the ability to add options in
# the future to these lines with trivial modifications
@click.command()
@click.argument("path", type=TaskPath(base_dir=path))
def process_batch_line(path):
"""
Parse a line of batch input and add it to the delete submission
item.
"""
delete_data.add_item(str(path))
shlex_process_stdin(process_batch_line,
"Enter paths to delete, line by line.")
else:
if not star_silent and enable_globs and path.endswith("*"):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (err_is_terminal() and term_is_interactive(
) and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.
format(path),
err=True,
)):
click.echo("Aborted.", err=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data,
response_key="DATA",
fields=[("Path", "path")])
# exit safely
return
res = client.submit_delete(delete_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)
def transfer_command(
batch,
sync_level,
recursive,
destination,
source,
label,
preserve_mtime,
verify_checksum,
encrypt,
submission_id,
dry_run,
delete,
deadline,
skip_activation_check,
notify,
perf_cc,
perf_p,
perf_pp,
perf_udt,
):
"""
Executor for `globus transfer`
"""
source_endpoint, cmd_source_path = source
dest_endpoint, cmd_dest_path = destination
if recursive and batch:
raise click.UsageError(
("You cannot use --recursive in addition to --batch. "
"Instead, use --recursive on lines of --batch input "
"which need it"))
if (cmd_source_path is None or cmd_dest_path is None) and (not batch):
raise click.UsageError(
("transfer requires either SOURCE_PATH and DEST_PATH or "
"--batch"))
# because python can't handle multiple **kwargs expansions in a single
# call, we need to get a little bit clever
# both the performance options (of which there are a few), and the
# notification options (also there are a few) have elements which should be
# omitted in some cases
# notify comes to us clean, perf opts need more care
# put them together into a dict before passing to TransferData
kwargs = {}
perf_opts = dict((k, v) for (k, v) in dict(
perf_cc=perf_cc, perf_p=perf_p, perf_pp=perf_pp,
perf_udt=perf_udt).items() if v is not None)
kwargs.update(perf_opts)
kwargs.update(notify)
client = get_client()
transfer_data = TransferData(client,
source_endpoint,
dest_endpoint,
label=label,
sync_level=sync_level,
verify_checksum=verify_checksum,
preserve_timestamp=preserve_mtime,
encrypt_data=encrypt,
submission_id=submission_id,
delete_destination_extra=delete,
deadline=deadline,
skip_activation_check=skip_activation_check,
**kwargs)
if batch:
@click.command()
@click.option("--recursive", "-r", is_flag=True)
@click.argument("source_path", type=TaskPath(base_dir=cmd_source_path))
@click.argument("dest_path", type=TaskPath(base_dir=cmd_dest_path))
def process_batch_line(dest_path, source_path, recursive):
"""
Parse a line of batch input and turn it into a transfer submission
item.
"""
transfer_data.add_item(str(source_path),
str(dest_path),
recursive=recursive)
shlex_process_stdin(
process_batch_line,
("Enter transfers, line by line, as\n\n"
" [--recursive] SOURCE_PATH DEST_PATH\n"),
)
else:
transfer_data.add_item(cmd_source_path,
cmd_dest_path,
recursive=recursive)
if dry_run:
formatted_print(
transfer_data,
response_key="DATA",
fields=(
("Source Path", "source_path"),
("Dest Path", "destination_path"),
("Recursive", "recursive"),
),
)
# exit safely
return
# autoactivate after parsing all args and putting things together
# skip this if skip-activation-check is given
if not skip_activation_check:
autoactivate(client, source_endpoint, if_expires_in=60)
autoactivate(client, dest_endpoint, if_expires_in=60)
res = client.submit_transfer(transfer_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)
def transfer_command(
batch,
sync_level,
recursive,
destination,
source,
checksum_algorithm,
external_checksum,
label,
preserve_mtime,
verify_checksum,
encrypt,
submission_id,
dry_run,
delete,
deadline,
skip_activation_check,
notify,
perf_cc,
perf_p,
perf_pp,
perf_udt,
):
"""
Copy a file or directory from one endpoint to another as an asynchronous
task.
'globus transfer' has two modes. Single target, which transfers one
file or one directory, and batch, which takes in several lines to transfer
multiple files or directories. See "Batch Input" below for more information.
'globus transfer' will always place the dest files in a
consistent, deterministic location. The contents of a source directory will
be placed inside the dest directory. A source file will be copied to
the dest file path, which must not be an existing directory. All
intermediate / parent directories on the dest will be automatically
created if they don't exist.
If the files or directories given as input are symbolic links, they are
followed. However, no other symbolic links are followed and no symbolic links
are ever created on the dest.
\b
=== Batched Input
If you use `SOURCE_PATH` and `DEST_PATH` without the `--batch` flag, you
will submit a single-file or single-directory transfer task.
This has behavior similar to `cp` and `cp -r` across endpoints.
Using `--batch`, `globus transfer` can submit a task which transfers
multiple files or directories. Paths to transfer are taken from stdin.
Lines are split on spaces, respecting quotes, and every line is treated as
a file or directory to transfer.
\b
Lines are of the form
[--recursive] [--external-checksum TEXT] SOURCE_PATH DEST_PATH\n
Skips empty lines and allows comments beginning with "#".
\b
If you use `--batch` and a commandline SOURCE_PATH and/or DEST_PATH, these
paths will be used as dir prefixes to any paths on stdin.
\b
=== Sync Levels
Sync Levels are ways to decide whether or not files are copied, with the
following definitions:
EXISTS: Determine whether or not to transfer based on file existence.
If the destination file is absent, do the transfer.
SIZE: Determine whether or not to transfer based on the size of the file.
If destination file size does not match the source, do the transfer.
MTIME: Determine whether or not to transfer based on modification times.
If source has a newer modififed time than the destination, do the transfer.
CHECKSUM: Determine whether or not to transfer based on checksums of file
contents.
If source and destination contents differ, as determined by a checksum of
their contents, do the transfer.
If a transfer fails, CHECKSUM must be used to restart the transfer.
All other levels can lead to data corruption.
{AUTOMATIC_ACTIVATION}
"""
source_endpoint, cmd_source_path = source
dest_endpoint, cmd_dest_path = destination
if recursive and batch:
raise click.UsageError(
(
"You cannot use --recursive in addition to --batch. "
"Instead, use --recursive on lines of --batch input "
"which need it"
)
)
if external_checksum and batch:
raise click.UsageError(
(
"You cannot use --external-checksum in addition to --batch. "
"Instead, use --external-checksum on lines of --batch input "
"which need it"
)
)
if recursive and external_checksum:
raise click.UsageError(
"--recursive and --external-checksum are mutually exclusive"
)
if (cmd_source_path is None or cmd_dest_path is None) and (not batch):
raise click.UsageError(
"transfer requires either SOURCE_PATH and DEST_PATH or --batch"
)
# because python can't handle multiple **kwargs expansions in a single
# call, we need to get a little bit clever
# both the performance options (of which there are a few), and the
# notification options (also there are a few) have elements which should be
# omitted in some cases
# notify comes to us clean, perf opts need more care
# put them together into a dict before passing to TransferData
kwargs = {}
perf_opts = dict(
(k, v)
for (k, v) in dict(
perf_cc=perf_cc, perf_p=perf_p, perf_pp=perf_pp, perf_udt=perf_udt
).items()
if v is not None
)
kwargs.update(perf_opts)
kwargs.update(notify)
client = get_client()
transfer_data = TransferData(
client,
source_endpoint,
dest_endpoint,
label=label,
sync_level=sync_level,
verify_checksum=verify_checksum,
preserve_timestamp=preserve_mtime,
encrypt_data=encrypt,
submission_id=submission_id,
delete_destination_extra=delete,
deadline=deadline,
skip_activation_check=skip_activation_check,
**kwargs
)
if batch:
@click.command()
@click.option("--external-checksum")
@click.option("--recursive", "-r", is_flag=True)
@click.argument("source_path", type=TaskPath(base_dir=cmd_source_path))
@click.argument("dest_path", type=TaskPath(base_dir=cmd_dest_path))
def process_batch_line(dest_path, source_path, recursive, external_checksum):
"""
Parse a line of batch input and turn it into a transfer submission
item.
"""
if recursive and external_checksum:
raise click.UsageError(
"--recursive and --external-checksum are mutually exclusive"
)
transfer_data.add_item(
str(source_path),
str(dest_path),
external_checksum=external_checksum,
checksum_algorithm=checksum_algorithm,
recursive=recursive,
)
shlex_process_stdin(
process_batch_line,
(
"Enter transfers, line by line, as\n\n"
" [--recursive] [--external-checksum TEXT] SOURCE_PATH DEST_PATH\n"
),
)
else:
transfer_data.add_item(
cmd_source_path,
cmd_dest_path,
external_checksum=external_checksum,
checksum_algorithm=checksum_algorithm,
recursive=recursive,
)
if dry_run:
formatted_print(
transfer_data,
response_key="DATA",
fields=(
("Source Path", "source_path"),
("Dest Path", "destination_path"),
("Recursive", "recursive"),
("External Checksum", "external_checksum"),
),
)
# exit safely
return
# autoactivate after parsing all args and putting things together
# skip this if skip-activation-check is given
if not skip_activation_check:
autoactivate(client, source_endpoint, if_expires_in=60)
autoactivate(client, dest_endpoint, if_expires_in=60)
res = client.submit_transfer(transfer_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Message", "message"), ("Task ID", "task_id")),
)