Esempio n. 1
0
def send_delete_task(endpoint_id=None, path=None):
    tc = getTransferClient()
    ddata = DeleteData(tc, endpoint_id, recursive=True)
    ddata.add_item(path)
    delete_result = tc.submit_delete(ddata)

    return delete_result
Esempio n. 2
0
def send_delete_task(endpoint_id=None, path=None, logger=logging.log):
    tc = get_transfer_client(logger=logger)
    ddata = DeleteData(tc, endpoint_id, recursive=True)
    ddata.add_item(path)
    delete_result = tc.submit_delete(ddata)

    return delete_result
Esempio n. 3
0
def rm_command(ignore_missing, star_silent, recursive, enable_globs,
               endpoint_plus_path, label, submission_id, dry_run, deadline,
               skip_activation_check, notify, meow, heartbeat,
               polling_interval, timeout):
    """
    Executor for `globus rm`
    """
    endpoint_id, path = endpoint_plus_path

    client = get_client()

    # attempt to activate unless --skip-activation-check is given
    if not skip_activation_check:
        autoactivate(client, endpoint_id, if_expires_in=60)

    delete_data = DeleteData(client,
                             endpoint_id,
                             label=label,
                             recursive=recursive,
                             ignore_missing=ignore_missing,
                             submission_id=submission_id,
                             deadline=deadline,
                             skip_activation_check=skip_activation_check,
                             interpret_globs=enable_globs,
                             **notify)

    if not star_silent and enable_globs and path.endswith('*'):
        # not intuitive, but `click.confirm(abort=True)` prints to stdout
        # unnecessarily, which we don't really want...
        # only do this check if stderr is a pty
        if (err_is_terminal() and term_is_interactive() and not click.confirm(
                'Are you sure you want to delete all files matching "{}"?'.
                format(path),
                err=True)):
            safeprint('Aborted.', write_to_stderr=True)
            click.get_current_context().exit(1)
    delete_data.add_item(path)

    if dry_run:
        formatted_print(delete_data,
                        response_key='DATA',
                        fields=[('Path', 'path')])
        # exit safely
        return

    # Print task submission to stderr so that `-Fjson` is still correctly
    # respected, as it will be by `task wait`
    res = client.submit_delete(delete_data)
    task_id = res['task_id']
    safeprint('Delete task submitted under ID "{}"'.format(task_id),
              write_to_stderr=True)

    # do a `task wait` equivalent, including printing and correct exit status
    task_wait_with_io(meow,
                      heartbeat,
                      polling_interval,
                      timeout,
                      task_id,
                      client=client)
Esempio n. 4
0
 def rmdir(self, ep_path, recursive=False):
     ''' remove a directory at ep_path '''
     tc = self.xfer_client
     ep, path = self.ep_parts(ep_path)
     ddata = DeleteData(tc, ep, recursive=recursive)
     ddata.add_item(path)
     task_id = tc.submit_delete(ddata)['task_id']
     return self._wait(task_id)
Esempio n. 5
0
def send_bulk_delete_task(endpoint_id=None, pfns=None, logger=logging.log):
    tc = get_transfer_client(logger=logger)
    ddata = DeleteData(tc, endpoint_id, recursive=True)
    for pfn in pfns:
        logger(logging.DEBUG, 'pfn: %s' % pfn)
        ddata.add_item(pfn)
    bulk_delete_result = tc.submit_delete(ddata)

    return bulk_delete_result
Esempio n. 6
0
    def delete_files_globus(self):
        ep_id = hpss_globus_endpoint
        r = self.transfer_client.endpoint_autoactivate(ep_id, if_expires_in=60)
        if r.get("code") == "AutoActivationFailed":
            self.fail(
                "The {} endpoint is not activated. Please go to https://app.globus.org/file-manager/collections/{} and activate the endpoint."
                .format(ep_id, ep_id))

        ddata = DeleteData(self.transfer_client,
                           hpss_globus_endpoint,
                           recursive=True)
        ddata.add_item("/~/zstash_test/")
        try:
            task = self.transfer_client.submit_delete(ddata)
            task_id = task.get("task_id")
            """
            A Globus transfer job (task) can be in one of the three states:
            ACTIVE, SUCCEEDED, FAILED. The script every 5 seconds polls a
            status of the transfer job (task) from the Globus Transfer service,
            with 5 second timeout limit. If the task is ACTIVE after time runs
            out 'task_wait' returns False, and True otherwise.
            """
            while not self.transfer_client.task_wait(task_id, 5, 5):
                task = self.transfer_client.get_task(task_id)
                if task.get("is_paused"):
                    break
            """
            The Globus transfer job (task) has been finished (SUCCEEDED or FAILED),
            or is still active (ACTIVE). Check if the transfer SUCCEEDED or FAILED.
            """
            task = self.transfer_client.get_task(task_id)
            if task["status"] == "SUCCEEDED":
                pass
            elif task.get("status") == "ACTIVE":
                if task.get("is_paused"):
                    pause_info = self.transfer_client.task_pause_info(task_id)
                    paused_rules = pause_info.get("pause_rules")
                    reason = paused_rules[0].get("message")
                    message = "The task was paused. Reason: {}".format(reason)
                    print(message)
                else:
                    message = "The task reached a {} second deadline\n".format(
                        24 * 3600)
                    print(message)
                self.transfer_client.cancel_task(task_id)
            else:
                print("Globus delete FAILED")
        except TransferAPIError as e:
            if e.code == "NoCredException":
                self.fail(
                    "{}. Please go to https://app.globus.org/endpoints and activate the endpoint."
                    .format(e.message))
            else:
                self.fail(e)
        except Exception as e:
            self.fail("{} - exception: {}".format(self, e))
Esempio n. 7
0
def cleanup():
    user_identity_name = request.form.get('user_identity_name')

    dependent_tokens = get_dependent_tokens(g.req_token)

    transfer_token = dependent_tokens.by_resource_server[
        'transfer.api.globus.org']['access_token']

    dest_ep = app.config['GRAPH_ENDPOINT_ID']
    dest_base = app.config['GRAPH_ENDPOINT_BASE']
    dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name)

    transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token))

    transfer.endpoint_autoactivate(dest_ep)

    try:
        acl = next(acl for acl in transfer.endpoint_acl_list(dest_ep)
                   if dest_path == acl['path'])
    except StopIteration:
        pass
    except TransferAPIError as ex:
        # PermissionDenied can happen if a new Portal client is swapped
        # in and it doesn't have endpoint manager on the dest_ep.
        # The /portal/processed directory has been set to to writeable
        # for all users so the delete task will succeed even if an ACL
        # can't be set.
        if ex.code == 'PermissionDenied':
            pass
    else:
        transfer.delete_endpoint_acl_rule(dest_ep, acl['id'])

    delete_request = DeleteData(transfer_client=transfer,
                                endpoint=dest_ep,
                                label="Delete Graphs from the Service Demo",
                                recursive=True)
    delete_request.add_item(dest_path)

    try:
        task = transfer.submit_delete(delete_request)
    except TransferAPIError as ex:
        raise InternalServerError(message=ex.message)
    else:
        return jsonify(dict(task_id=task['task_id']))
Esempio n. 8
0
def cleanup():
    user_identity_name = request.form.get('user_identity_name')

    dependent_tokens = get_dependent_tokens(g.req_token)

    transfer_token = dependent_tokens.by_resource_server[
        'transfer.api.globus.org']['access_token']

    dest_ep = app.config['GRAPH_ENDPOINT_ID']
    dest_base = app.config['GRAPH_ENDPOINT_BASE']
    dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name)

    transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token))

    transfer.endpoint_autoactivate(dest_ep)

    try:
        acl = next(acl for acl in transfer.endpoint_acl_list(dest_ep)
                   if dest_path == acl['path'])
    except StopIteration:
        pass
    except TransferAPIError as ex:
        # PermissionDenied can happen if a new Portal client is swapped
        # in and it doesn't have endpoint manager on the dest_ep.
        # The /portal/processed directory has been set to to writeable
        # for all users so the delete task will succeed even if an ACL
        # can't be set.
        if ex.code == 'PermissionDenied':
            pass
    else:
        transfer.delete_endpoint_acl_rule(dest_ep, acl['id'])

    delete_request = DeleteData(transfer_client=transfer, endpoint=dest_ep,
                                label="Delete Graphs from the Service Demo",
                                recursive=True)
    delete_request.add_item(dest_path)

    try:
        task = transfer.submit_delete(delete_request)
    except TransferAPIError as ex:
        raise InternalServerError(message=ex.message)
    else:
        return jsonify(dict(task_id=task['task_id']))
Esempio n. 9
0
def delete_command(
    batch,
    ignore_missing,
    star_silent,
    recursive,
    enable_globs,
    endpoint_plus_path,
    label,
    submission_id,
    dry_run,
    deadline,
    skip_activation_check,
    notify,
):
    """
    Executor for `globus delete`
    """
    endpoint_id, path = endpoint_plus_path
    if path is None and (not batch):
        raise click.UsageError("delete requires either a PATH OR --batch")

    client = get_client()

    # attempt to activate unless --skip-activation-check is given
    if not skip_activation_check:
        autoactivate(client, endpoint_id, if_expires_in=60)

    delete_data = DeleteData(client,
                             endpoint_id,
                             label=label,
                             recursive=recursive,
                             ignore_missing=ignore_missing,
                             submission_id=submission_id,
                             deadline=deadline,
                             skip_activation_check=skip_activation_check,
                             interpret_globs=enable_globs,
                             **notify)

    if batch:
        # although this sophisticated structure (like that in transfer)
        # isn't strictly necessary, it gives us the ability to add options in
        # the future to these lines with trivial modifications
        @click.command()
        @click.argument("path", type=TaskPath(base_dir=path))
        def process_batch_line(path):
            """
            Parse a line of batch input and add it to the delete submission
            item.
            """
            delete_data.add_item(str(path))

        shlex_process_stdin(process_batch_line,
                            "Enter paths to delete, line by line.")
    else:
        if not star_silent and enable_globs and path.endswith("*"):
            # not intuitive, but `click.confirm(abort=True)` prints to stdout
            # unnecessarily, which we don't really want...
            # only do this check if stderr is a pty
            if (err_is_terminal() and term_is_interactive(
            ) and not click.confirm(
                    'Are you sure you want to delete all files matching "{}"?'.
                    format(path),
                    err=True,
            )):
                click.echo("Aborted.", err=True)
                click.get_current_context().exit(1)
        delete_data.add_item(path)

    if dry_run:
        formatted_print(delete_data,
                        response_key="DATA",
                        fields=[("Path", "path")])
        # exit safely
        return

    res = client.submit_delete(delete_data)
    formatted_print(
        res,
        text_format=FORMAT_TEXT_RECORD,
        fields=(("Message", "message"), ("Task ID", "task_id")),
    )
Esempio n. 10
0
def rm_command(
    ignore_missing,
    star_silent,
    recursive,
    enable_globs,
    endpoint_plus_path,
    label,
    submission_id,
    dry_run,
    deadline,
    skip_activation_check,
    notify,
    meow,
    heartbeat,
    polling_interval,
    timeout,
    timeout_exit_code,
):
    """
    Executor for `globus rm`
    """
    endpoint_id, path = endpoint_plus_path

    client = get_client()

    # attempt to activate unless --skip-activation-check is given
    if not skip_activation_check:
        autoactivate(client, endpoint_id, if_expires_in=60)

    delete_data = DeleteData(
        client,
        endpoint_id,
        label=label,
        recursive=recursive,
        ignore_missing=ignore_missing,
        submission_id=submission_id,
        deadline=deadline,
        skip_activation_check=skip_activation_check,
        interpret_globs=enable_globs,
        **notify
    )

    if not star_silent and enable_globs and path.endswith("*"):
        # not intuitive, but `click.confirm(abort=True)` prints to stdout
        # unnecessarily, which we don't really want...
        # only do this check if stderr is a pty
        if (
            err_is_terminal()
            and term_is_interactive()
            and not click.confirm(
                'Are you sure you want to delete all files matching "{}"?'.format(path),
                err=True,
            )
        ):
            safeprint("Aborted.", write_to_stderr=True)
            click.get_current_context().exit(1)
    delete_data.add_item(path)

    if dry_run:
        formatted_print(delete_data, response_key="DATA", fields=[("Path", "path")])
        # exit safely
        return

    # Print task submission to stderr so that `-Fjson` is still correctly
    # respected, as it will be by `task wait`
    res = client.submit_delete(delete_data)
    task_id = res["task_id"]
    safeprint(
        'Delete task submitted under ID "{}"'.format(task_id), write_to_stderr=True
    )

    # do a `task wait` equivalent, including printing and correct exit status
    task_wait_with_io(
        meow,
        heartbeat,
        polling_interval,
        timeout,
        task_id,
        timeout_exit_code,
        client=client,
    )
Esempio n. 11
0
def remove_path(path_to_remove, transfer_client=None):
    transfer = transfer_client if transfer_client else init()
    ddata = DeleteData(transfer, config.GWAS_ENDPOINT_ID, recursive=True)
    ddata.add_item(path_to_remove)
    delete_result = transfer.submit_delete(ddata)
    return delete_result
Esempio n. 12
0
def delete_command(
    batch,
    ignore_missing,
    star_silent,
    recursive,
    enable_globs,
    endpoint_plus_path,
    label,
    submission_id,
    dry_run,
    deadline,
    skip_activation_check,
    notify,
):
    """
    Submits an asynchronous task that deletes files and/or directories on the target
    endpoint.

    *globus delete* has two modes. Single target, which deletes one
    file or one directory, and batch, which takes in several lines to delete
    multiple files or directories. See "Batch Input" below for more information.

    Symbolic links are never followed - only unlinked (deleted).

    === Batch Input

    If you give a SOURCE_PATH without the --batch flag, you will submit a
    single-file or single-directory delete task. This has
    behavior similar to `rm` and `rm -r`, across endpoints.

    Using `--batch`, *globus delete* can submit a task which deletes
    multiple files or directories. Lines are taken from stdin, respecting quotes,
    and every line is treated as a path to a file or directory to delete.

    \b
    Lines are of the form
      PATH

    Note that unlike 'globus transfer' --recursive is not an option at the per line
    level, instead, if given with the original command, all paths that point to
    directories will be recursively deleted.

    Empty lines and comments beginning with '#' are ignored.

    Batch only requires an ENDPOINT before passing lines, on stdin, but if you pass
    an ENPDOINT:PATH on the original command, this path will be used as a prefixes
    to all paths on stdin.

    {AUTOMATIC_ACTIVATION}
    """
    endpoint_id, path = endpoint_plus_path
    if path is None and (not batch):
        raise click.UsageError("delete requires either a PATH OR --batch")

    client = get_client()

    # attempt to activate unless --skip-activation-check is given
    if not skip_activation_check:
        autoactivate(client, endpoint_id, if_expires_in=60)

    delete_data = DeleteData(client,
                             endpoint_id,
                             label=label,
                             recursive=recursive,
                             ignore_missing=ignore_missing,
                             submission_id=submission_id,
                             deadline=deadline,
                             skip_activation_check=skip_activation_check,
                             interpret_globs=enable_globs,
                             **notify)

    if batch:
        # although this sophisticated structure (like that in transfer)
        # isn't strictly necessary, it gives us the ability to add options in
        # the future to these lines with trivial modifications
        @click.command()
        @click.argument("path", type=TaskPath(base_dir=path))
        def process_batch_line(path):
            """
            Parse a line of batch input and add it to the delete submission
            item.
            """
            delete_data.add_item(str(path))

        shlex_process_stdin(process_batch_line,
                            "Enter paths to delete, line by line.")
    else:
        if not star_silent and enable_globs and path.endswith("*"):
            # not intuitive, but `click.confirm(abort=True)` prints to stdout
            # unnecessarily, which we don't really want...
            # only do this check if stderr is a pty
            if (err_is_terminal() and term_is_interactive(
            ) and not click.confirm(
                    'Are you sure you want to delete all files matching "{}"?'.
                    format(path),
                    err=True,
            )):
                click.echo("Aborted.", err=True)
                click.get_current_context().exit(1)
        delete_data.add_item(path)

    if dry_run:
        formatted_print(delete_data,
                        response_key="DATA",
                        fields=[("Path", "path")])
        # exit safely
        return

    res = client.submit_delete(delete_data)
    formatted_print(
        res,
        text_format=FORMAT_TEXT_RECORD,
        fields=(("Message", "message"), ("Task ID", "task_id")),
    )
Esempio n. 13
0
def delete_command(
    batch,
    ignore_missing,
    star_silent,
    recursive,
    enable_globs,
    endpoint_plus_path,
    label,
    submission_id,
    dry_run,
    deadline,
    skip_activation_check,
    notify,
):
    """
    Executor for `globus delete`
    """
    endpoint_id, path = endpoint_plus_path
    if path is None and (not batch):
        raise click.UsageError("delete requires either a PATH OR --batch")

    client = get_client()

    # attempt to activate unless --skip-activation-check is given
    if not skip_activation_check:
        autoactivate(client, endpoint_id, if_expires_in=60)

    delete_data = DeleteData(
        client,
        endpoint_id,
        label=label,
        recursive=recursive,
        ignore_missing=ignore_missing,
        submission_id=submission_id,
        deadline=deadline,
        skip_activation_check=skip_activation_check,
        interpret_globs=enable_globs,
        **notify
    )

    if batch:
        # although this sophisticated structure (like that in transfer)
        # isn't strictly necessary, it gives us the ability to add options in
        # the future to these lines with trivial modifications
        @click.command()
        @click.argument("path", type=TaskPath(base_dir=path))
        def process_batch_line(path):
            """
            Parse a line of batch input and add it to the delete submission
            item.
            """
            delete_data.add_item(str(path))

        shlex_process_stdin(process_batch_line, "Enter paths to delete, line by line.")
    else:
        if not star_silent and enable_globs and path.endswith("*"):
            # not intuitive, but `click.confirm(abort=True)` prints to stdout
            # unnecessarily, which we don't really want...
            # only do this check if stderr is a pty
            if (
                err_is_terminal()
                and term_is_interactive()
                and not click.confirm(
                    'Are you sure you want to delete all files matching "{}"?'.format(
                        path
                    ),
                    err=True,
                )
            ):
                safeprint("Aborted.", write_to_stderr=True)
                click.get_current_context().exit(1)
        delete_data.add_item(path)

    if dry_run:
        formatted_print(delete_data, response_key="DATA", fields=[("Path", "path")])
        # exit safely
        return

    res = client.submit_delete(delete_data)
    formatted_print(
        res,
        text_format=FORMAT_TEXT_RECORD,
        fields=(("Message", "message"), ("Task ID", "task_id")),
    )