def local_files(ctx, role, name, compute_hash: bool, export: bool,
imported: bool, path: str):
"""Add a local file directory (not managed by git) to the workspace. Subcommand of ``add``"""
ns = ctx.obj
if role is None:
if imported:
role = ResourceRoles.SOURCE_DATA_SET
elif ns.batch:
raise BatchModeError("--role")
else:
role = click.prompt(
"Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults",
type=ROLE_PARAM,
)
path = abspath(expanduser(path))
workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir)
if export and role in (ResourceRoles.SOURCE_DATA_SET, ResourceRoles.CODE):
raise click.BadOptionUsage(
message="Cannot export a source data or code resource",
option_name="export")
if export and imported:
raise click.BadOptionUsage(
message="Cannot specify both --export and --imported",
option_name="imported")
if imported and role != ResourceRoles.SOURCE_DATA_SET:
raise click.BadOptionUsage(
message="--imported only for source-data roles",
option_name="imported")
add_command("file", role, name, workspace, path, compute_hash, export,
imported)
def restore(
ctx,
workspace_dir: str,
only: Optional[str],
leave: Optional[str],
strict: bool,
tag_or_hash: str,
):
"""Restore the workspace to a prior state"""
ns = ctx.obj
if (only is not None) and (leave is not None):
raise click.BadOptionUsage(
option_name="--only",
message="Please specify either --only or --leave, but not both"
) # type: ignore
if workspace_dir is None:
if ns.batch:
raise BatchModeError("--workspace-dir")
else:
workspace_dir = click.prompt("Please enter the workspace root dir",
type=WORKSPACE_PARAM)
workspace = find_and_load_workspace(ns.batch, ns.verbose, workspace_dir)
restore_command(
workspace,
tag_or_hash,
only=only.split(",") if only else None,
leave=leave.split(",") if leave else None,
strict=strict,
)
def pull(ctx, workspace_dir: str, only: Optional[str], skip: Optional[str],
only_workspace: bool):
"""Pull the latest state of the workspace and its resources from their origins."""
ns = ctx.obj
option_cnt = ((1 if only is not None else 0) +
(1 if skip is not None else 0) +
(1 if only_workspace else 0))
if option_cnt > 1:
raise click.BadOptionUsage(
message=
"Please specify at most one of --only, --skip, or --only-workspace",
option_name="--only") # type: ignore
if workspace_dir is None:
if ns.batch:
raise BatchModeError("--workspace-dir")
else:
workspace_dir = click.prompt("Please enter the workspace root dir",
type=WORKSPACE_PARAM)
workspace = find_and_load_workspace(ns.batch, ns.verbose, workspace_dir)
pull_command(
workspace,
only=only.split(",") if only else None,
skip=skip.split(",") if skip else None,
only_workspace=only_workspace,
)
def deploy(ctx, workspace_dir):
"""Lineage-related commands"""
ns = ctx.obj
if workspace_dir is None:
if ns.batch:
raise BatchModeError("--workspace-dir")
else:
workspace_dir = click.prompt("Please enter the workspace root dir",
type=WORKSPACE_PARAM)
ns.workspace_dir = workspace_dir
def report(ctx, workspace_dir):
"""Report generation commands"""
ns = ctx.obj
if workspace_dir is None:
if ns.batch:
raise BatchModeError("--workspace-dir")
else:
workspace_dir = click.prompt("Please enter the workspace root dir",
type=WORKSPACE_PARAM)
ns.workspace_dir = workspace_dir
def diff(ctx, workspace_dir, snapshot_or_tag1, snapshot_or_tag2):
"""List differences between two snapshots"""
ns = ctx.obj
if workspace_dir is None:
if ns.batch:
raise BatchModeError("--workspace-dir")
else:
workspace_dir = click.prompt("Please enter the workspace root dir",
type=WORKSPACE_PARAM)
workspace = find_and_load_workspace(ns.batch, ns.verbose, workspace_dir)
diff_command(workspace, snapshot_or_tag1, snapshot_or_tag2)
def status(ctx, workspace_dir, history, limit):
"""NOTE: this command is DEPRECATED. Please use ``dws report status`` and ``dws report history`` instead."""
ns = ctx.obj
if workspace_dir is None:
if ns.batch:
raise BatchModeError("--workspace-dir")
else:
workspace_dir = click.prompt("Please enter the workspace root dir",
type=WORKSPACE_PARAM)
workspace = find_and_load_workspace(ns.batch, ns.verbose, workspace_dir)
status_command(workspace, history, limit)
def snapshot(ctx, workspace_dir, message, tag):
"""Take a snapshot of the current workspace's state"""
ns = ctx.obj
if workspace_dir is None:
if ns.batch:
raise BatchModeError("--workspace-dir")
else:
workspace_dir = click.prompt("Please enter the workspace root dir",
type=WORKSPACE_PARAM)
workspace = find_and_load_workspace(ns.batch, ns.verbose, workspace_dir)
snapshot_command(workspace, tag, message)
def add(ctx, workspace_dir):
"""Add a data collection to the workspace as a resource.
Possible types of resources are ``git``, ``local-files``, or ``rclone``; these are subcommands of add."""
ns = ctx.obj
if workspace_dir is None:
if ns.batch:
raise BatchModeError("--workspace-dir")
else:
workspace_dir = click.prompt("Please enter the workspace root dir",
type=WORKSPACE_PARAM)
ns.workspace_dir = workspace_dir
def s3(ctx, role, name, bucket_name: str):
"""Add a S3 resource to the workspace. Subcommand of ``add``"""
ns = ctx.obj
if role is None:
if ns.batch:
raise BatchModeError("--role")
else:
role = click.prompt(
"Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults",
type=ROLE_PARAM,
)
workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir)
add_command("s3", role, name, workspace, bucket_name)
def api_resource(ctx, role, name):
"""Resource to represent data obtained via an API. Use this when there is
no file-based representation of your data that can be versioned and captured
more directly. Subcommand of ``add``"""
ns = ctx.obj
if role is None:
if ns.batch:
raise BatchModeError("--role")
else:
role = click.prompt(
"Please enter a role for this resource, either [s]ource-data or [i]ntermediate-data",
type=DATA_ROLE_PARAM,
)
workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir)
add_command("api-resource", role, name, workspace)
def delete_snapshot(ctx, workspace_dir: str, no_include_resources: bool,
tag_or_hash: str):
"""Delete the specified snapshot. This includes the metadata and lineage
data for the snapshot. Unless --no-include-resources is specified, this
also deletes any results data saved for the snapshot (under the
snapshots subdirectory of a results resource)."""
ns = ctx.obj
if workspace_dir is None:
if ns.batch:
raise BatchModeError("--workspace-dir")
else:
workspace_dir = click.prompt("Please enter the workspace root dir",
type=WORKSPACE_PARAM)
workspace = find_and_load_workspace(ns.batch, ns.verbose, workspace_dir)
delete_snapshot_command(workspace, tag_or_hash, no_include_resources)
def rclone(
ctx,
role,
name,
config: str,
compute_hash: bool,
export: bool,
imported: bool,
source: str,
dest: str,
):
"""Add an rclone-d repository as a resource to the workspace. Subcommand of ``add``"""
ns = ctx.obj
if role is None:
if imported:
role = ResourceRoles.SOURCE_DATA_SET
elif ns.batch:
raise BatchModeError("--role")
else:
role = click.prompt(
"Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults",
type=ROLE_PARAM,
)
rclone_re = r".*:.*"
if re.match(rclone_re, source) == None:
raise click.BadOptionUsage(
message=
"Source in rclone should be specified as remotename:filepath",
option_name="source",
)
if export and role in (ResourceRoles.SOURCE_DATA_SET, ResourceRoles.CODE):
raise click.BadOptionUsage(
message="Cannot export a source data or code resource",
option_name="export")
if export and imported:
raise click.BadOptionUsage(
message="Cannot specify both --export and --imported",
option_name="imported")
if imported and role != ResourceRoles.SOURCE_DATA_SET:
raise click.BadOptionUsage(
message="--imported only for source-data roles",
option_name="imported")
dest = abspath(expanduser(dest))
workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir)
add_command("rclone", role, name, workspace, source, dest, config,
compute_hash, export, imported)
def publish(ctx, workspace_dir, skip: str, remote_repository):
"""Add a remote Git repository as the origin for the workspace and
do the initial push of the workspace and any other resources.
"""
ns = ctx.obj
if workspace_dir is None:
if ns.batch:
raise BatchModeError("--workspace-dir")
else:
workspace_dir = click.prompt("Please enter the workspace root dir",
type=WORKSPACE_PARAM)
workspace = find_and_load_workspace(ns.batch, ns.verbose, workspace_dir)
publish_command(workspace, remote_repository)
push_command(workspace,
only=None,
skip=skip.split(",") if skip else None,
only_workspace=False)
def config(ctx, workspace_dir, resource, param_name, param_value):
"""Get or set configuration parameters. Local parameters are only for this
copy of the workspace, while global parameters are stored centrally and
affect all copies.
If neither PARAMETER_NAME nor PARAMETER_VALUE are specified, this command
prints a table of all parameters and their information (scope, value, default or not,
and help text). If just PARAMETER_NAME is specified, it prints the specified parameter's
information. Finally, if both the parameter name and value are specified, the parameter
is set to the specified value."""
ns = ctx.obj
if workspace_dir is None:
if ns.batch:
raise BatchModeError("--workspace-dir")
else:
workspace_dir = click.prompt("Please enter the workspace root dir",
type=WORKSPACE_PARAM)
workspace = find_and_load_workspace(ns.batch, ns.verbose, workspace_dir)
config_command(workspace, param_name, param_value, resource)
def git(ctx, role, name, branch, read_only, export, imported, path):
"""Add a local git repository as a resource. Subcommand of ``add``"""
ns = ctx.obj
if role is None:
if imported:
role = ResourceRoles.SOURCE_DATA_SET
elif ns.batch:
raise BatchModeError("--role")
else:
role = click.prompt(
"Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults",
type=ROLE_PARAM,
)
if export and role in (ResourceRoles.SOURCE_DATA_SET, ResourceRoles.CODE):
raise click.BadOptionUsage(
message="Cannot export a source data or code resource",
option_name="export")
if export and imported:
raise click.BadOptionUsage(
message="Cannot specify both --export and --imported",
option_name="imported")
if imported and role != ResourceRoles.SOURCE_DATA_SET:
raise click.BadOptionUsage(
message="--imported only for source-data roles",
option_name="imported")
if imported:
read_only = True
if path.startswith("git@") or path.startswith("https://"):
raise click.BadOptionUsage(
message="It looks like you tried to specify a git URL (%s)." % path
+ " Currently, git resources only accept a local path." +
" Try cloning your repository and then pasing the local path to that repository.",
option_name="path",
)
path = abspath(expanduser(path))
workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir)
add_command("git", role, name, workspace, path, branch, read_only, export,
imported)
def rclone(
ctx,
role,
name,
config: str,
compute_hash: bool,
export: bool,
imported: bool,
master: str,
sync_mode: str,
size_only: bool,
remote: str,
local_path: str,
):
"""Add an rclone-d repository as a resource to the workspace. Subcommand of ``add``.
This is designed for uni-directional synchronization between a remote and a local_path.
The remote has the form remote_name:remote_path, where remote_name is an entry in your
rclone config file.
"""
ns = ctx.obj
if role is None:
if imported:
role = ResourceRoles.SOURCE_DATA_SET
elif ns.batch:
raise BatchModeError("--role")
else:
role = click.prompt(
"Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults",
type=ROLE_PARAM,
)
rclone_re = r".*:.*"
if re.match(rclone_re, remote) == None:
raise click.BadOptionUsage(
message=
"Source in rclone should be specified as remotename:filepath",
option_name="source",
)
if export and role in (ResourceRoles.SOURCE_DATA_SET, ResourceRoles.CODE):
raise click.BadOptionUsage(
message="Cannot export a source data or code resource",
option_name="export")
if export and imported:
raise click.BadOptionUsage(
message="Cannot specify both --export and --imported",
option_name="imported")
if imported and role != ResourceRoles.SOURCE_DATA_SET:
raise click.BadOptionUsage(
message="--imported only for source-data roles",
option_name="imported")
local_path = abspath(expanduser(local_path))
workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir)
add_command(
"rclone",
role,
name,
workspace,
remote,
local_path,
config,
compute_hash,
export,
imported,
master,
sync_mode,
size_only,
)