def local_files(ctx, role, name, compute_hash: bool, export: bool,
imported: bool, path: str):
"""Add a local file directory (not managed by git) to the workspace. Subcommand of ``add``"""
ns = ctx.obj
if role is None:
if imported:
role = ResourceRoles.SOURCE_DATA_SET
elif ns.batch:
raise BatchModeError("--role")
else:
role = click.prompt(
"Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults",
type=ROLE_PARAM,
)
path = abspath(expanduser(path))
workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir)
if export and role in (ResourceRoles.SOURCE_DATA_SET, ResourceRoles.CODE):
raise click.BadOptionUsage(
message="Cannot export a source data or code resource",
option_name="export")
if export and imported:
raise click.BadOptionUsage(
message="Cannot specify both --export and --imported",
option_name="imported")
if imported and role != ResourceRoles.SOURCE_DATA_SET:
raise click.BadOptionUsage(
message="--imported only for source-data roles",
option_name="imported")
add_command("file", role, name, workspace, path, compute_hash, export,
imported)
def s3(ctx, role, name, bucket_name: str):
"""Add a S3 resource to the workspace. Subcommand of ``add``"""
ns = ctx.obj
if role is None:
if ns.batch:
raise BatchModeError("--role")
else:
role = click.prompt(
"Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults",
type=ROLE_PARAM,
)
workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir)
add_command("s3", role, name, workspace, bucket_name)
def api_resource(ctx, role, name):
"""Resource to represent data obtained via an API. Use this when there is
no file-based representation of your data that can be versioned and captured
more directly. Subcommand of ``add``"""
ns = ctx.obj
if role is None:
if ns.batch:
raise BatchModeError("--role")
else:
role = click.prompt(
"Please enter a role for this resource, either [s]ource-data or [i]ntermediate-data",
type=DATA_ROLE_PARAM,
)
workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir)
add_command("api-resource", role, name, workspace)
def rclone(
ctx,
role,
name,
config: str,
compute_hash: bool,
export: bool,
imported: bool,
source: str,
dest: str,
):
"""Add an rclone-d repository as a resource to the workspace. Subcommand of ``add``"""
ns = ctx.obj
if role is None:
if imported:
role = ResourceRoles.SOURCE_DATA_SET
elif ns.batch:
raise BatchModeError("--role")
else:
role = click.prompt(
"Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults",
type=ROLE_PARAM,
)
rclone_re = r".*:.*"
if re.match(rclone_re, source) == None:
raise click.BadOptionUsage(
message=
"Source in rclone should be specified as remotename:filepath",
option_name="source",
)
if export and role in (ResourceRoles.SOURCE_DATA_SET, ResourceRoles.CODE):
raise click.BadOptionUsage(
message="Cannot export a source data or code resource",
option_name="export")
if export and imported:
raise click.BadOptionUsage(
message="Cannot specify both --export and --imported",
option_name="imported")
if imported and role != ResourceRoles.SOURCE_DATA_SET:
raise click.BadOptionUsage(
message="--imported only for source-data roles",
option_name="imported")
dest = abspath(expanduser(dest))
workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir)
add_command("rclone", role, name, workspace, source, dest, config,
compute_hash, export, imported)
def git(ctx, role, name, branch, read_only, export, imported, path):
"""Add a local git repository as a resource. Subcommand of ``add``"""
ns = ctx.obj
if role is None:
if imported:
role = ResourceRoles.SOURCE_DATA_SET
elif ns.batch:
raise BatchModeError("--role")
else:
role = click.prompt(
"Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults",
type=ROLE_PARAM,
)
if export and role in (ResourceRoles.SOURCE_DATA_SET, ResourceRoles.CODE):
raise click.BadOptionUsage(
message="Cannot export a source data or code resource",
option_name="export")
if export and imported:
raise click.BadOptionUsage(
message="Cannot specify both --export and --imported",
option_name="imported")
if imported and role != ResourceRoles.SOURCE_DATA_SET:
raise click.BadOptionUsage(
message="--imported only for source-data roles",
option_name="imported")
if imported:
read_only = True
if path.startswith("git@") or path.startswith("https://"):
raise click.BadOptionUsage(
message="It looks like you tried to specify a git URL (%s)." % path
+ " Currently, git resources only accept a local path." +
" Try cloning your repository and then pasing the local path to that repository.",
option_name="path",
)
path = abspath(expanduser(path))
workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir)
add_command("git", role, name, workspace, path, branch, read_only, export,
imported)
def rclone(
ctx,
role,
name,
config: str,
compute_hash: bool,
export: bool,
imported: bool,
master: str,
sync_mode: str,
size_only: bool,
remote: str,
local_path: str,
):
"""Add an rclone-d repository as a resource to the workspace. Subcommand of ``add``.
This is designed for uni-directional synchronization between a remote and a local_path.
The remote has the form remote_name:remote_path, where remote_name is an entry in your
rclone config file.
"""
ns = ctx.obj
if role is None:
if imported:
role = ResourceRoles.SOURCE_DATA_SET
elif ns.batch:
raise BatchModeError("--role")
else:
role = click.prompt(
"Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults",
type=ROLE_PARAM,
)
rclone_re = r".*:.*"
if re.match(rclone_re, remote) == None:
raise click.BadOptionUsage(
message=
"Source in rclone should be specified as remotename:filepath",
option_name="source",
)
if export and role in (ResourceRoles.SOURCE_DATA_SET, ResourceRoles.CODE):
raise click.BadOptionUsage(
message="Cannot export a source data or code resource",
option_name="export")
if export and imported:
raise click.BadOptionUsage(
message="Cannot specify both --export and --imported",
option_name="imported")
if imported and role != ResourceRoles.SOURCE_DATA_SET:
raise click.BadOptionUsage(
message="--imported only for source-data roles",
option_name="imported")
local_path = abspath(expanduser(local_path))
workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir)
add_command(
"rclone",
role,
name,
workspace,
remote,
local_path,
config,
compute_hash,
export,
imported,
master,
sync_mode,
size_only,
)
