def rename_command(source, destination): """Rename a file or directory on an endpoint. The old path must be an existing file or directory. The new path must not yet exist. The new path does not have to be in the same directory as the old path, but most endpoints will require it to stay on the same filesystem. The endpoint must be entered twice for the sake of path syntax consistency. """ source_ep, source_path = source dest_ep, dest_path = destination if source_ep != dest_ep: raise click.UsageError( ("rename requires that the source and dest " "endpoints are the same, {} != {}").format(source_ep, dest_ep)) endpoint_id = source_ep client = get_client() autoactivate(client, endpoint_id, if_expires_in=60) res = client.operation_rename(endpoint_id, oldpath=source_path, newpath=dest_path) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def rm_command(ignore_missing, star_silent, recursive, enable_globs, endpoint_plus_path, label, submission_id, dry_run, deadline, skip_activation_check, notify, meow, heartbeat, polling_interval, timeout): """ Executor for `globus rm` """ endpoint_id, path = endpoint_plus_path client = get_client() # attempt to activate unless --skip-activation-check is given if not skip_activation_check: autoactivate(client, endpoint_id, if_expires_in=60) delete_data = DeleteData(client, endpoint_id, label=label, recursive=recursive, ignore_missing=ignore_missing, submission_id=submission_id, deadline=deadline, skip_activation_check=skip_activation_check, interpret_globs=enable_globs, **notify) if not star_silent and enable_globs and path.endswith('*'): # not intuitive, but `click.confirm(abort=True)` prints to stdout # unnecessarily, which we don't really want... # only do this check if stderr is a pty if (err_is_terminal() and term_is_interactive() and not click.confirm( 'Are you sure you want to delete all files matching "{}"?'. format(path), err=True)): safeprint('Aborted.', write_to_stderr=True) click.get_current_context().exit(1) delete_data.add_item(path) if dry_run: formatted_print(delete_data, response_key='DATA', fields=[('Path', 'path')]) # exit safely return # Print task submission to stderr so that `-Fjson` is still correctly # respected, as it will be by `task wait` res = client.submit_delete(delete_data) task_id = res['task_id'] safeprint('Delete task submitted under ID "{}"'.format(task_id), write_to_stderr=True) # do a `task wait` equivalent, including printing and correct exit status task_wait_with_io(meow, heartbeat, polling_interval, timeout, task_id, client=client)
def mkdir_command(endpoint_plus_path): """ Executor for `globus mkdir` """ endpoint_id, path = endpoint_plus_path client = get_client() autoactivate(client, endpoint_id, if_expires_in=60) res = client.operation_mkdir(endpoint_id, path=path) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def mkdir_command(endpoint_plus_path): """ Executor for `globus mkdir` """ endpoint_id, path = endpoint_plus_path client = get_client() autoactivate(client, endpoint_id, if_expires_in=60) res = client.operation_mkdir(endpoint_id, path=path) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def mkdir_command(endpoint_plus_path): """Make a directory on an endpoint at the given path. {AUTOMATIC_ACTIVATION} """ endpoint_id, path = endpoint_plus_path client = get_client() autoactivate(client, endpoint_id, if_expires_in=60) res = client.operation_mkdir(endpoint_id, path=path) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def ls_command(endpoint_plus_path, recursive_depth_limit, recursive, long_output, show_hidden, filter_val): """ Executor for `globus ls` """ endpoint_id, path = endpoint_plus_path # do autoactivation before the `ls` call so that recursive invocations # won't do this repeatedly, and won't have to instantiate new clients client = get_client() autoactivate(client, endpoint_id, if_expires_in=60) # create the query paramaters to send to operation_ls ls_params = {"show_hidden": int(show_hidden)} if path: ls_params["path"] = path if filter_val: # this char has special meaning in the LS API's filter clause # can't be part of the pattern (but we don't support globbing across # dir structures anyway) if '/' in filter_val: raise click.UsageError('--filter cannot contain "/"') # format into a simple filter clause which operates on filenames ls_params['filter'] = 'name:{}'.format(filter_val) # get the `ls` result if recursive: # NOTE: # --recursive and --filter have an interplay that some users may find # surprising # if we're asked to change or "improve" the behavior in the future, we # could do so with "type:dir" or "type:file" filters added in, and # potentially work out some viable behavior based on what people want res = client.recursive_operation_ls( endpoint_id, depth=recursive_depth_limit, **ls_params) else: res = client.operation_ls(endpoint_id, **ls_params) def cleaned_item_name(item): return item['name'] + ('/' if item['type'] == 'dir' else '') # and then print it, per formatting rules formatted_print( res, fields=[('Permissions', 'permissions'), ('User', 'user'), ('Group', 'group'), ('Size', 'size'), ('Last Modified', 'last_modified'), ('File Type', 'type'), ('Filename', cleaned_item_name)], simple_text=( None if long_output or is_verbose() or not outformat_is_text() else "\n".join(cleaned_item_name(x) for x in res)), json_converter=iterable_response_to_dict)
def endpoint_create(**kwargs): """ Create a new endpoint. Requires a display name and exactly one of --personal, --server, or --shared to make a Globus Connect Personal, Globus Connect Server, or Shared endpoint respectively. """ client = get_client() # get endpoint type, ensure unambiguous. personal = kwargs.pop("personal") server = kwargs.pop("server") shared = kwargs.pop("shared") if personal and (not server) and (not shared): endpoint_type = "personal" elif server and (not personal) and (not shared): endpoint_type = "server" elif shared and (not personal) and (not server): endpoint_type = "shared" else: raise click.UsageError( "Exactly one of --personal, --server, or --shared is required.") # validate options kwargs["is_globus_connect"] = personal or None validate_endpoint_create_and_update_params(endpoint_type, False, kwargs) # shared endpoint creation if shared: endpoint_id, host_path = shared kwargs["host_endpoint"] = endpoint_id kwargs["host_path"] = host_path ep_doc = assemble_generic_doc("shared_endpoint", **kwargs) autoactivate(client, endpoint_id, if_expires_in=60) res = client.create_shared_endpoint(ep_doc) # non shared endpoint creation else: # omit `is_globus_connect` key if not GCP, otherwise include as `True` ep_doc = assemble_generic_doc("endpoint", **kwargs) res = client.create_endpoint(ep_doc) # output formatted_print( res, fields=(COMMON_FIELDS + GCP_FIELDS if personal else COMMON_FIELDS), text_format=FORMAT_TEXT_RECORD, )
def rename_command(source, destination): """ Executor for `globus rename` """ source_ep, source_path = source dest_ep, dest_path = destination if source_ep != dest_ep: raise click.UsageError(('rename requires that the source and dest ' 'endpoints are the same, {} != {}') .format(source_ep, dest_ep)) endpoint_id = source_ep client = get_client() autoactivate(client, endpoint_id, if_expires_in=60) res = client.operation_rename(endpoint_id, oldpath=source_path, newpath=dest_path) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key='message')
def ls_command( endpoint_plus_path, recursive_depth_limit, recursive, long_output, show_hidden, filter_val, ): """ List the contents of a directory on an endpoint. If no path is given, the default directory on that endpoint will be used. If using text output files and directories are printed with one entry per line in alphabetical order. Directories are always displayed with a trailing '/'. \b === Filtering --filter takes "filter patterns" subject to the following rules. \b Filter patterns must start with "=", "~", "!", or "!~" If none of these are given, "=" will be used \b "=" does exact matching "~" does regex matching, supporting globs (*) "!" does inverse "=" matching "!~" does inverse "~" matching \b "~*.txt" matches all .txt files, for example {AUTOMATIC_ACTIVATION} """ endpoint_id, path = endpoint_plus_path # do autoactivation before the `ls` call so that recursive invocations # won't do this repeatedly, and won't have to instantiate new clients client = get_client() autoactivate(client, endpoint_id, if_expires_in=60) # create the query paramaters to send to operation_ls ls_params = {"show_hidden": int(show_hidden)} if path: ls_params["path"] = path if filter_val: # this char has special meaning in the LS API's filter clause # can't be part of the pattern (but we don't support globbing across # dir structures anyway) if "/" in filter_val: raise click.UsageError('--filter cannot contain "/"') # format into a simple filter clause which operates on filenames ls_params["filter"] = "name:{}".format(filter_val) # get the `ls` result if recursive: # NOTE: # --recursive and --filter have an interplay that some users may find # surprising # if we're asked to change or "improve" the behavior in the future, we # could do so with "type:dir" or "type:file" filters added in, and # potentially work out some viable behavior based on what people want res = client.recursive_operation_ls(endpoint_id, depth=recursive_depth_limit, **ls_params) else: res = client.operation_ls(endpoint_id, **ls_params) def cleaned_item_name(item): return item["name"] + ("/" if item["type"] == "dir" else "") # and then print it, per formatting rules formatted_print( res, fields=[ ("Permissions", "permissions"), ("User", "user"), ("Group", "group"), ("Size", "size"), ("Last Modified", "last_modified"), ("File Type", "type"), ("Filename", cleaned_item_name), ], simple_text=(None if long_output or is_verbose() or not outformat_is_text() else "\n".join( cleaned_item_name(x) for x in res)), json_converter=iterable_response_to_dict, )
def rm_command( ignore_missing, star_silent, recursive, enable_globs, endpoint_plus_path, label, submission_id, dry_run, deadline, skip_activation_check, notify, meow, heartbeat, polling_interval, timeout, timeout_exit_code, ): """ Executor for `globus rm` """ endpoint_id, path = endpoint_plus_path client = get_client() # attempt to activate unless --skip-activation-check is given if not skip_activation_check: autoactivate(client, endpoint_id, if_expires_in=60) delete_data = DeleteData( client, endpoint_id, label=label, recursive=recursive, ignore_missing=ignore_missing, submission_id=submission_id, deadline=deadline, skip_activation_check=skip_activation_check, interpret_globs=enable_globs, **notify ) if not star_silent and enable_globs and path.endswith("*"): # not intuitive, but `click.confirm(abort=True)` prints to stdout # unnecessarily, which we don't really want... # only do this check if stderr is a pty if ( err_is_terminal() and term_is_interactive() and not click.confirm( 'Are you sure you want to delete all files matching "{}"?'.format(path), err=True, ) ): safeprint("Aborted.", write_to_stderr=True) click.get_current_context().exit(1) delete_data.add_item(path) if dry_run: formatted_print(delete_data, response_key="DATA", fields=[("Path", "path")]) # exit safely return # Print task submission to stderr so that `-Fjson` is still correctly # respected, as it will be by `task wait` res = client.submit_delete(delete_data) task_id = res["task_id"] safeprint( 'Delete task submitted under ID "{}"'.format(task_id), write_to_stderr=True ) # do a `task wait` equivalent, including printing and correct exit status task_wait_with_io( meow, heartbeat, polling_interval, timeout, task_id, timeout_exit_code, client=client, )
def ls_command( endpoint_plus_path, recursive_depth_limit, recursive, long_output, show_hidden, filter_val, ): """ Executor for `globus ls` """ endpoint_id, path = endpoint_plus_path # do autoactivation before the `ls` call so that recursive invocations # won't do this repeatedly, and won't have to instantiate new clients client = get_client() autoactivate(client, endpoint_id, if_expires_in=60) # create the query paramaters to send to operation_ls ls_params = {"show_hidden": int(show_hidden)} if path: ls_params["path"] = path if filter_val: # this char has special meaning in the LS API's filter clause # can't be part of the pattern (but we don't support globbing across # dir structures anyway) if "/" in filter_val: raise click.UsageError('--filter cannot contain "/"') # format into a simple filter clause which operates on filenames ls_params["filter"] = "name:{}".format(filter_val) # get the `ls` result if recursive: # NOTE: # --recursive and --filter have an interplay that some users may find # surprising # if we're asked to change or "improve" the behavior in the future, we # could do so with "type:dir" or "type:file" filters added in, and # potentially work out some viable behavior based on what people want res = client.recursive_operation_ls( endpoint_id, depth=recursive_depth_limit, **ls_params ) else: res = client.operation_ls(endpoint_id, **ls_params) def cleaned_item_name(item): return item["name"] + ("/" if item["type"] == "dir" else "") # and then print it, per formatting rules formatted_print( res, fields=[ ("Permissions", "permissions"), ("User", "user"), ("Group", "group"), ("Size", "size"), ("Last Modified", "last_modified"), ("File Type", "type"), ("Filename", cleaned_item_name), ], simple_text=( None if long_output or is_verbose() or not outformat_is_text() else "\n".join(cleaned_item_name(x) for x in res) ), json_converter=iterable_response_to_dict, )
def transfer_command( batch, sync_level, recursive, destination, source, label, preserve_mtime, verify_checksum, encrypt, submission_id, dry_run, delete, deadline, skip_activation_check, notify, perf_cc, perf_p, perf_pp, perf_udt, ): """ Executor for `globus transfer` """ source_endpoint, cmd_source_path = source dest_endpoint, cmd_dest_path = destination if recursive and batch: raise click.UsageError( ( "You cannot use --recursive in addition to --batch. " "Instead, use --recursive on lines of --batch input " "which need it" ) ) if (cmd_source_path is None or cmd_dest_path is None) and (not batch): raise click.UsageError( ("transfer requires either SOURCE_PATH and DEST_PATH or " "--batch") ) # because python can't handle multiple **kwargs expansions in a single # call, we need to get a little bit clever # both the performance options (of which there are a few), and the # notification options (also there are a few) have elements which should be # omitted in some cases # notify comes to us clean, perf opts need more care # put them together into a dict before passing to TransferData kwargs = {} perf_opts = dict( (k, v) for (k, v) in dict( perf_cc=perf_cc, perf_p=perf_p, perf_pp=perf_pp, perf_udt=perf_udt ).items() if v is not None ) kwargs.update(perf_opts) kwargs.update(notify) client = get_client() transfer_data = TransferData( client, source_endpoint, dest_endpoint, label=label, sync_level=sync_level, verify_checksum=verify_checksum, preserve_timestamp=preserve_mtime, encrypt_data=encrypt, submission_id=submission_id, delete_destination_extra=delete, deadline=deadline, skip_activation_check=skip_activation_check, **kwargs ) if batch: @click.command() @click.option("--recursive", "-r", is_flag=True) @click.argument("source_path", type=TaskPath(base_dir=cmd_source_path)) @click.argument("dest_path", type=TaskPath(base_dir=cmd_dest_path)) def process_batch_line(dest_path, source_path, recursive): """ Parse a line of batch input and turn it into a transfer submission item. """ transfer_data.add_item( str(source_path), str(dest_path), recursive=recursive ) shlex_process_stdin( process_batch_line, ( "Enter transfers, line by line, as\n\n" " [--recursive] SOURCE_PATH DEST_PATH\n" ), ) else: transfer_data.add_item(cmd_source_path, cmd_dest_path, recursive=recursive) if dry_run: formatted_print( transfer_data, response_key="DATA", fields=( ("Source Path", "source_path"), ("Dest Path", "destination_path"), ("Recursive", "recursive"), ), ) # exit safely return # autoactivate after parsing all args and putting things together # skip this if skip-activation-check is given if not skip_activation_check: autoactivate(client, source_endpoint, if_expires_in=60) autoactivate(client, dest_endpoint, if_expires_in=60) res = client.submit_transfer(transfer_data) formatted_print( res, text_format=FORMAT_TEXT_RECORD, fields=(("Message", "message"), ("Task ID", "task_id")), )
def delete_command( batch, ignore_missing, star_silent, recursive, enable_globs, endpoint_plus_path, label, submission_id, dry_run, deadline, skip_activation_check, notify, ): """ Submits an asynchronous task that deletes files and/or directories on the target endpoint. *globus delete* has two modes. Single target, which deletes one file or one directory, and batch, which takes in several lines to delete multiple files or directories. See "Batch Input" below for more information. Symbolic links are never followed - only unlinked (deleted). === Batch Input If you give a SOURCE_PATH without the --batch flag, you will submit a single-file or single-directory delete task. This has behavior similar to `rm` and `rm -r`, across endpoints. Using `--batch`, *globus delete* can submit a task which deletes multiple files or directories. Lines are taken from stdin, respecting quotes, and every line is treated as a path to a file or directory to delete. \b Lines are of the form PATH Note that unlike 'globus transfer' --recursive is not an option at the per line level, instead, if given with the original command, all paths that point to directories will be recursively deleted. Empty lines and comments beginning with '#' are ignored. Batch only requires an ENDPOINT before passing lines, on stdin, but if you pass an ENPDOINT:PATH on the original command, this path will be used as a prefixes to all paths on stdin. {AUTOMATIC_ACTIVATION} """ endpoint_id, path = endpoint_plus_path if path is None and (not batch): raise click.UsageError("delete requires either a PATH OR --batch") client = get_client() # attempt to activate unless --skip-activation-check is given if not skip_activation_check: autoactivate(client, endpoint_id, if_expires_in=60) delete_data = DeleteData(client, endpoint_id, label=label, recursive=recursive, ignore_missing=ignore_missing, submission_id=submission_id, deadline=deadline, skip_activation_check=skip_activation_check, interpret_globs=enable_globs, **notify) if batch: # although this sophisticated structure (like that in transfer) # isn't strictly necessary, it gives us the ability to add options in # the future to these lines with trivial modifications @click.command() @click.argument("path", type=TaskPath(base_dir=path)) def process_batch_line(path): """ Parse a line of batch input and add it to the delete submission item. """ delete_data.add_item(str(path)) shlex_process_stdin(process_batch_line, "Enter paths to delete, line by line.") else: if not star_silent and enable_globs and path.endswith("*"): # not intuitive, but `click.confirm(abort=True)` prints to stdout # unnecessarily, which we don't really want... # only do this check if stderr is a pty if (err_is_terminal() and term_is_interactive( ) and not click.confirm( 'Are you sure you want to delete all files matching "{}"?'. format(path), err=True, )): click.echo("Aborted.", err=True) click.get_current_context().exit(1) delete_data.add_item(path) if dry_run: formatted_print(delete_data, response_key="DATA", fields=[("Path", "path")]) # exit safely return res = client.submit_delete(delete_data) formatted_print( res, text_format=FORMAT_TEXT_RECORD, fields=(("Message", "message"), ("Task ID", "task_id")), )
def delete_command( batch, ignore_missing, star_silent, recursive, enable_globs, endpoint_plus_path, label, submission_id, dry_run, deadline, skip_activation_check, notify, ): """ Executor for `globus delete` """ endpoint_id, path = endpoint_plus_path if path is None and (not batch): raise click.UsageError("delete requires either a PATH OR --batch") client = get_client() # attempt to activate unless --skip-activation-check is given if not skip_activation_check: autoactivate(client, endpoint_id, if_expires_in=60) delete_data = DeleteData( client, endpoint_id, label=label, recursive=recursive, ignore_missing=ignore_missing, submission_id=submission_id, deadline=deadline, skip_activation_check=skip_activation_check, interpret_globs=enable_globs, **notify ) if batch: # although this sophisticated structure (like that in transfer) # isn't strictly necessary, it gives us the ability to add options in # the future to these lines with trivial modifications @click.command() @click.argument("path", type=TaskPath(base_dir=path)) def process_batch_line(path): """ Parse a line of batch input and add it to the delete submission item. """ delete_data.add_item(str(path)) shlex_process_stdin(process_batch_line, "Enter paths to delete, line by line.") else: if not star_silent and enable_globs and path.endswith("*"): # not intuitive, but `click.confirm(abort=True)` prints to stdout # unnecessarily, which we don't really want... # only do this check if stderr is a pty if ( err_is_terminal() and term_is_interactive() and not click.confirm( 'Are you sure you want to delete all files matching "{}"?'.format( path ), err=True, ) ): safeprint("Aborted.", write_to_stderr=True) click.get_current_context().exit(1) delete_data.add_item(path) if dry_run: formatted_print(delete_data, response_key="DATA", fields=[("Path", "path")]) # exit safely return res = client.submit_delete(delete_data) formatted_print( res, text_format=FORMAT_TEXT_RECORD, fields=(("Message", "message"), ("Task ID", "task_id")), )
def delete_command( batch, ignore_missing, star_silent, recursive, enable_globs, endpoint_plus_path, label, submission_id, dry_run, deadline, skip_activation_check, notify, ): """ Executor for `globus delete` """ endpoint_id, path = endpoint_plus_path if path is None and (not batch): raise click.UsageError("delete requires either a PATH OR --batch") client = get_client() # attempt to activate unless --skip-activation-check is given if not skip_activation_check: autoactivate(client, endpoint_id, if_expires_in=60) delete_data = DeleteData(client, endpoint_id, label=label, recursive=recursive, ignore_missing=ignore_missing, submission_id=submission_id, deadline=deadline, skip_activation_check=skip_activation_check, interpret_globs=enable_globs, **notify) if batch: # although this sophisticated structure (like that in transfer) # isn't strictly necessary, it gives us the ability to add options in # the future to these lines with trivial modifications @click.command() @click.argument("path", type=TaskPath(base_dir=path)) def process_batch_line(path): """ Parse a line of batch input and add it to the delete submission item. """ delete_data.add_item(str(path)) shlex_process_stdin(process_batch_line, "Enter paths to delete, line by line.") else: if not star_silent and enable_globs and path.endswith("*"): # not intuitive, but `click.confirm(abort=True)` prints to stdout # unnecessarily, which we don't really want... # only do this check if stderr is a pty if (err_is_terminal() and term_is_interactive( ) and not click.confirm( 'Are you sure you want to delete all files matching "{}"?'. format(path), err=True, )): click.echo("Aborted.", err=True) click.get_current_context().exit(1) delete_data.add_item(path) if dry_run: formatted_print(delete_data, response_key="DATA", fields=[("Path", "path")]) # exit safely return res = client.submit_delete(delete_data) formatted_print( res, text_format=FORMAT_TEXT_RECORD, fields=(("Message", "message"), ("Task ID", "task_id")), )
def transfer_command( batch, sync_level, recursive, destination, source, label, preserve_mtime, verify_checksum, encrypt, submission_id, dry_run, delete, deadline, skip_activation_check, notify, perf_cc, perf_p, perf_pp, perf_udt, ): """ Executor for `globus transfer` """ source_endpoint, cmd_source_path = source dest_endpoint, cmd_dest_path = destination if recursive and batch: raise click.UsageError( ("You cannot use --recursive in addition to --batch. " "Instead, use --recursive on lines of --batch input " "which need it")) if (cmd_source_path is None or cmd_dest_path is None) and (not batch): raise click.UsageError( ("transfer requires either SOURCE_PATH and DEST_PATH or " "--batch")) # because python can't handle multiple **kwargs expansions in a single # call, we need to get a little bit clever # both the performance options (of which there are a few), and the # notification options (also there are a few) have elements which should be # omitted in some cases # notify comes to us clean, perf opts need more care # put them together into a dict before passing to TransferData kwargs = {} perf_opts = dict((k, v) for (k, v) in dict( perf_cc=perf_cc, perf_p=perf_p, perf_pp=perf_pp, perf_udt=perf_udt).items() if v is not None) kwargs.update(perf_opts) kwargs.update(notify) client = get_client() transfer_data = TransferData(client, source_endpoint, dest_endpoint, label=label, sync_level=sync_level, verify_checksum=verify_checksum, preserve_timestamp=preserve_mtime, encrypt_data=encrypt, submission_id=submission_id, delete_destination_extra=delete, deadline=deadline, skip_activation_check=skip_activation_check, **kwargs) if batch: @click.command() @click.option("--recursive", "-r", is_flag=True) @click.argument("source_path", type=TaskPath(base_dir=cmd_source_path)) @click.argument("dest_path", type=TaskPath(base_dir=cmd_dest_path)) def process_batch_line(dest_path, source_path, recursive): """ Parse a line of batch input and turn it into a transfer submission item. """ transfer_data.add_item(str(source_path), str(dest_path), recursive=recursive) shlex_process_stdin( process_batch_line, ("Enter transfers, line by line, as\n\n" " [--recursive] SOURCE_PATH DEST_PATH\n"), ) else: transfer_data.add_item(cmd_source_path, cmd_dest_path, recursive=recursive) if dry_run: formatted_print( transfer_data, response_key="DATA", fields=( ("Source Path", "source_path"), ("Dest Path", "destination_path"), ("Recursive", "recursive"), ), ) # exit safely return # autoactivate after parsing all args and putting things together # skip this if skip-activation-check is given if not skip_activation_check: autoactivate(client, source_endpoint, if_expires_in=60) autoactivate(client, dest_endpoint, if_expires_in=60) res = client.submit_transfer(transfer_data) formatted_print( res, text_format=FORMAT_TEXT_RECORD, fields=(("Message", "message"), ("Task ID", "task_id")), )
def transfer_command( batch, sync_level, recursive, destination, source, checksum_algorithm, external_checksum, label, preserve_mtime, verify_checksum, encrypt, submission_id, dry_run, delete, deadline, skip_activation_check, notify, perf_cc, perf_p, perf_pp, perf_udt, ): """ Copy a file or directory from one endpoint to another as an asynchronous task. 'globus transfer' has two modes. Single target, which transfers one file or one directory, and batch, which takes in several lines to transfer multiple files or directories. See "Batch Input" below for more information. 'globus transfer' will always place the dest files in a consistent, deterministic location. The contents of a source directory will be placed inside the dest directory. A source file will be copied to the dest file path, which must not be an existing directory. All intermediate / parent directories on the dest will be automatically created if they don't exist. If the files or directories given as input are symbolic links, they are followed. However, no other symbolic links are followed and no symbolic links are ever created on the dest. \b === Batched Input If you use `SOURCE_PATH` and `DEST_PATH` without the `--batch` flag, you will submit a single-file or single-directory transfer task. This has behavior similar to `cp` and `cp -r` across endpoints. Using `--batch`, `globus transfer` can submit a task which transfers multiple files or directories. Paths to transfer are taken from stdin. Lines are split on spaces, respecting quotes, and every line is treated as a file or directory to transfer. \b Lines are of the form [--recursive] [--external-checksum TEXT] SOURCE_PATH DEST_PATH\n Skips empty lines and allows comments beginning with "#". \b If you use `--batch` and a commandline SOURCE_PATH and/or DEST_PATH, these paths will be used as dir prefixes to any paths on stdin. \b === Sync Levels Sync Levels are ways to decide whether or not files are copied, with the following definitions: EXISTS: Determine whether or not to transfer based on file existence. If the destination file is absent, do the transfer. SIZE: Determine whether or not to transfer based on the size of the file. If destination file size does not match the source, do the transfer. MTIME: Determine whether or not to transfer based on modification times. If source has a newer modififed time than the destination, do the transfer. CHECKSUM: Determine whether or not to transfer based on checksums of file contents. If source and destination contents differ, as determined by a checksum of their contents, do the transfer. If a transfer fails, CHECKSUM must be used to restart the transfer. All other levels can lead to data corruption. {AUTOMATIC_ACTIVATION} """ source_endpoint, cmd_source_path = source dest_endpoint, cmd_dest_path = destination if recursive and batch: raise click.UsageError( ( "You cannot use --recursive in addition to --batch. " "Instead, use --recursive on lines of --batch input " "which need it" ) ) if external_checksum and batch: raise click.UsageError( ( "You cannot use --external-checksum in addition to --batch. " "Instead, use --external-checksum on lines of --batch input " "which need it" ) ) if recursive and external_checksum: raise click.UsageError( "--recursive and --external-checksum are mutually exclusive" ) if (cmd_source_path is None or cmd_dest_path is None) and (not batch): raise click.UsageError( "transfer requires either SOURCE_PATH and DEST_PATH or --batch" ) # because python can't handle multiple **kwargs expansions in a single # call, we need to get a little bit clever # both the performance options (of which there are a few), and the # notification options (also there are a few) have elements which should be # omitted in some cases # notify comes to us clean, perf opts need more care # put them together into a dict before passing to TransferData kwargs = {} perf_opts = dict( (k, v) for (k, v) in dict( perf_cc=perf_cc, perf_p=perf_p, perf_pp=perf_pp, perf_udt=perf_udt ).items() if v is not None ) kwargs.update(perf_opts) kwargs.update(notify) client = get_client() transfer_data = TransferData( client, source_endpoint, dest_endpoint, label=label, sync_level=sync_level, verify_checksum=verify_checksum, preserve_timestamp=preserve_mtime, encrypt_data=encrypt, submission_id=submission_id, delete_destination_extra=delete, deadline=deadline, skip_activation_check=skip_activation_check, **kwargs ) if batch: @click.command() @click.option("--external-checksum") @click.option("--recursive", "-r", is_flag=True) @click.argument("source_path", type=TaskPath(base_dir=cmd_source_path)) @click.argument("dest_path", type=TaskPath(base_dir=cmd_dest_path)) def process_batch_line(dest_path, source_path, recursive, external_checksum): """ Parse a line of batch input and turn it into a transfer submission item. """ if recursive and external_checksum: raise click.UsageError( "--recursive and --external-checksum are mutually exclusive" ) transfer_data.add_item( str(source_path), str(dest_path), external_checksum=external_checksum, checksum_algorithm=checksum_algorithm, recursive=recursive, ) shlex_process_stdin( process_batch_line, ( "Enter transfers, line by line, as\n\n" " [--recursive] [--external-checksum TEXT] SOURCE_PATH DEST_PATH\n" ), ) else: transfer_data.add_item( cmd_source_path, cmd_dest_path, external_checksum=external_checksum, checksum_algorithm=checksum_algorithm, recursive=recursive, ) if dry_run: formatted_print( transfer_data, response_key="DATA", fields=( ("Source Path", "source_path"), ("Dest Path", "destination_path"), ("Recursive", "recursive"), ("External Checksum", "external_checksum"), ), ) # exit safely return # autoactivate after parsing all args and putting things together # skip this if skip-activation-check is given if not skip_activation_check: autoactivate(client, source_endpoint, if_expires_in=60) autoactivate(client, dest_endpoint, if_expires_in=60) res = client.submit_transfer(transfer_data) formatted_print( res, text_format=FORMAT_TEXT_RECORD, fields=(("Message", "message"), ("Task ID", "task_id")), )