def local_files(ctx, role, name, compute_hash: bool, export: bool, imported: bool, path: str): """Add a local file directory (not managed by git) to the workspace. Subcommand of ``add``""" ns = ctx.obj if role is None: if imported: role = ResourceRoles.SOURCE_DATA_SET elif ns.batch: raise BatchModeError("--role") else: role = click.prompt( "Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults", type=ROLE_PARAM, ) path = abspath(expanduser(path)) workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir) if export and role in (ResourceRoles.SOURCE_DATA_SET, ResourceRoles.CODE): raise click.BadOptionUsage( message="Cannot export a source data or code resource", option_name="export") if export and imported: raise click.BadOptionUsage( message="Cannot specify both --export and --imported", option_name="imported") if imported and role != ResourceRoles.SOURCE_DATA_SET: raise click.BadOptionUsage( message="--imported only for source-data roles", option_name="imported") add_command("file", role, name, workspace, path, compute_hash, export, imported)
def s3(ctx, role, name, bucket_name: str): """Add a S3 resource to the workspace. Subcommand of ``add``""" ns = ctx.obj if role is None: if ns.batch: raise BatchModeError("--role") else: role = click.prompt( "Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults", type=ROLE_PARAM, ) workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir) add_command("s3", role, name, workspace, bucket_name)
def api_resource(ctx, role, name): """Resource to represent data obtained via an API. Use this when there is no file-based representation of your data that can be versioned and captured more directly. Subcommand of ``add``""" ns = ctx.obj if role is None: if ns.batch: raise BatchModeError("--role") else: role = click.prompt( "Please enter a role for this resource, either [s]ource-data or [i]ntermediate-data", type=DATA_ROLE_PARAM, ) workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir) add_command("api-resource", role, name, workspace)
def rclone( ctx, role, name, config: str, compute_hash: bool, export: bool, imported: bool, source: str, dest: str, ): """Add an rclone-d repository as a resource to the workspace. Subcommand of ``add``""" ns = ctx.obj if role is None: if imported: role = ResourceRoles.SOURCE_DATA_SET elif ns.batch: raise BatchModeError("--role") else: role = click.prompt( "Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults", type=ROLE_PARAM, ) rclone_re = r".*:.*" if re.match(rclone_re, source) == None: raise click.BadOptionUsage( message= "Source in rclone should be specified as remotename:filepath", option_name="source", ) if export and role in (ResourceRoles.SOURCE_DATA_SET, ResourceRoles.CODE): raise click.BadOptionUsage( message="Cannot export a source data or code resource", option_name="export") if export and imported: raise click.BadOptionUsage( message="Cannot specify both --export and --imported", option_name="imported") if imported and role != ResourceRoles.SOURCE_DATA_SET: raise click.BadOptionUsage( message="--imported only for source-data roles", option_name="imported") dest = abspath(expanduser(dest)) workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir) add_command("rclone", role, name, workspace, source, dest, config, compute_hash, export, imported)
def git(ctx, role, name, branch, read_only, export, imported, path): """Add a local git repository as a resource. Subcommand of ``add``""" ns = ctx.obj if role is None: if imported: role = ResourceRoles.SOURCE_DATA_SET elif ns.batch: raise BatchModeError("--role") else: role = click.prompt( "Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults", type=ROLE_PARAM, ) if export and role in (ResourceRoles.SOURCE_DATA_SET, ResourceRoles.CODE): raise click.BadOptionUsage( message="Cannot export a source data or code resource", option_name="export") if export and imported: raise click.BadOptionUsage( message="Cannot specify both --export and --imported", option_name="imported") if imported and role != ResourceRoles.SOURCE_DATA_SET: raise click.BadOptionUsage( message="--imported only for source-data roles", option_name="imported") if imported: read_only = True if path.startswith("git@") or path.startswith("https://"): raise click.BadOptionUsage( message="It looks like you tried to specify a git URL (%s)." % path + " Currently, git resources only accept a local path." + " Try cloning your repository and then pasing the local path to that repository.", option_name="path", ) path = abspath(expanduser(path)) workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir) add_command("git", role, name, workspace, path, branch, read_only, export, imported)
def rclone( ctx, role, name, config: str, compute_hash: bool, export: bool, imported: bool, master: str, sync_mode: str, size_only: bool, remote: str, local_path: str, ): """Add an rclone-d repository as a resource to the workspace. Subcommand of ``add``. This is designed for uni-directional synchronization between a remote and a local_path. The remote has the form remote_name:remote_path, where remote_name is an entry in your rclone config file. """ ns = ctx.obj if role is None: if imported: role = ResourceRoles.SOURCE_DATA_SET elif ns.batch: raise BatchModeError("--role") else: role = click.prompt( "Please enter a role for this resource, one of [s]ource-data, [i]ntermediate-data, [c]ode, or [r]esults", type=ROLE_PARAM, ) rclone_re = r".*:.*" if re.match(rclone_re, remote) == None: raise click.BadOptionUsage( message= "Source in rclone should be specified as remotename:filepath", option_name="source", ) if export and role in (ResourceRoles.SOURCE_DATA_SET, ResourceRoles.CODE): raise click.BadOptionUsage( message="Cannot export a source data or code resource", option_name="export") if export and imported: raise click.BadOptionUsage( message="Cannot specify both --export and --imported", option_name="imported") if imported and role != ResourceRoles.SOURCE_DATA_SET: raise click.BadOptionUsage( message="--imported only for source-data roles", option_name="imported") local_path = abspath(expanduser(local_path)) workspace = find_and_load_workspace(ns.batch, ns.verbose, ns.workspace_dir) add_command( "rclone", role, name, workspace, remote, local_path, config, compute_hash, export, imported, master, sync_mode, size_only, )