def pull_command(
workspace: Workspace,
only: Optional[List[str]] = None,
skip: Optional[List[str]] = None,
only_workspace: bool = False,
) -> int:
if isinstance(workspace, SyncedWorkspaceMixin):
# first, sync the workspace
click.echo("Syncing workspace")
mixin = workspace.pull_workspace()
workspace = cast(Workspace, mixin)
if not only_workspace:
rcount = _pull_and_clone_resources(workspace, only, skip)
else:
rcount = 0
elif isinstance(workspace, CentralWorkspaceMixin):
if only_workspace:
raise ConfigurationError(
"--only-workspace not valid for central workspace %s" %
workspace.name)
rcount = _pull_and_clone_resources(workspace, only, skip)
else:
raise InternalError(
"Workspace %s is neither a SyncedWorkspaceMixin nor a CentralWorkspaceMixin"
% workspace.name)
workspace.save("Pull command")
return rcount
def publish_command(workspace: Workspace, remote_repository: str) -> None:
if isinstance(workspace, SyncedWorkspaceMixin):
workspace.publish(remote_repository)
else:
raise ConfigurationError(
"Workspace %s does not support publish command; only supported for synced workspaces"
% workspace.name
)
click.echo("Set remote origin to %s" % remote_repository)
def clone(self, params: JSONDict,
workspace: Workspace) -> LocalStateResourceMixin:
"""Instantiate a local copy of the resource
that came from the remote origin. We don't yet have local params,
since this resource is not yet on the local machine. If not in batch
mode, this method can ask the user for any additional information needed
(e.g. a local path). In batch mode, should either come up with a reasonable
default or error out if not enough information is available."""
workspace._get_local_scratch_space_for_resource(
params["name"], create_if_not_present=True)
return ApiResource(params["name"], params["role"], workspace)
def from_command_line(self, role: str, name: str, workspace: Workspace,
*args, **kwargs) -> Resource:
"""Instantiate a resource object from the add command's
arguments"""
if role not in (ResourceRoles.SOURCE_DATA_SET,
ResourceRoles.INTERMEDIATE_DATA):
raise ConfigurationError(
"API resources only supported for %s and %s roles" %
(ResourceRoles.SOURCE_DATA_SET,
ResourceRoles.INTERMEDIATE_DATA))
workspace._get_local_scratch_space_for_resource(
name, create_if_not_present=True)
return ApiResource(name, role, workspace)
def build_resource_list(
workspace: Workspace, only: Optional[List[str]], skip: Optional[List[str]]
) -> List[str]:
"""Build up our resource name list for either push or pull commands.
"""
if (only is not None) and (skip is not None):
raise ConfigurationError("Cannot specify both --only and --skip")
all_resource_names_set = frozenset(workspace.get_resource_names())
local_state_names_set = frozenset(workspace.get_names_of_resources_with_local_state())
if only is not None:
only_set = frozenset(only)
invalid = only_set.difference(all_resource_names_set)
if len(invalid) > 0:
raise ConfigurationError(
"Invalid resource names were included with --only: %s" % ", ".join(sorted(invalid))
)
nonsync_rnames = only_set.difference(local_state_names_set)
if len(nonsync_rnames) > 0:
click.echo(
"Skipping the following resources, which do not have local state: %s"
% ", ".join(sorted(nonsync_rnames))
)
return [rn for rn in only if rn in local_state_names_set]
elif skip is not None:
skip_set = frozenset(skip)
invalid = skip_set.difference(all_resource_names_set)
if len(invalid) > 0:
raise ConfigurationError(
"Invalid resource names were included with --skip: %s" % ", ".join(sorted(invalid))
)
nonsync_rnames = all_resource_names_set.difference(skip_set).difference(
local_state_names_set
)
if len(nonsync_rnames) > 0:
click.echo(
"Skipping the following resources, which do not have local state: %s"
% ", ".join(sorted(nonsync_rnames))
)
skip_set = skip_set.union(nonsync_rnames)
return [rn for rn in workspace.get_resource_names() if rn not in skip_set]
else:
nonsync_rnames = all_resource_names_set.difference(local_state_names_set)
if len(nonsync_rnames) > 0:
click.echo(
"Skipping the following resources, which do not have local state: %s"
% ", ".join(sorted(nonsync_rnames))
)
return [rn for rn in workspace.get_resource_names() if rn not in nonsync_rnames]
def deploy_run_command(
workspace: Workspace, image_name: Optional[str], no_mount_ssh_keys: bool
) -> None:
target_repo_dir = "/home/%s/%s" % (os.environ["USER"], workspace.name)
if image_name is None:
image_name = workspace.name
argv = [
"--target-repo-dir",
target_repo_dir,
"--image-name",
image_name,
]
if not no_mount_ssh_keys:
dot_ssh = abspath(expanduser("~/.ssh"))
argv.append("-v")
argv.append("%s:/home/%s/.ssh" % (dot_ssh, os.environ["USER"]))
if isinstance(workspace, git_backend.Workspace):
workspace_dir = workspace.get_workspace_local_path_if_any()
assert workspace_dir is not None
argv.append("dws+" + get_remote_origin_url(workspace_dir, verbose=workspace.verbose))
else:
# need to figure out how the clone url works for a non-git workspace
assert 0, "run build not yet implemented for non-git workspaces"
if workspace.verbose:
click.echo("Command args for repo2docker: %s" % repr(argv))
r2d = make_r2d(argv=argv)
r2d.initialize()
r2d.run_image()
click.echo("Run of image %s was successful." % image_name)
def add_command(scheme: str, role: str, name: str, workspace: Workspace,
*args):
current_names = set(workspace.get_resource_names())
if workspace.batch:
if name == None:
name = workspace.suggest_resource_name(scheme, role, *args)
else:
if name in current_names:
raise ConfigurationError("Resource name '%s' already in use" %
name)
else:
suggested_name = None
while (name is None) or (name in current_names):
if suggested_name == None:
suggested_name = workspace.suggest_resource_name(
scheme, role, *args)
name = click.prompt(
"Please enter a short, unique name for this resource",
default=suggested_name)
if name in current_names:
click.echo("Resource name '%s' already in use." % name,
err=True)
workspace.add_resource(name, scheme, role, *args)
workspace.save("add of %s" % name)
click.echo("Successful added resource '%s' to workspace." % name)
def print_resource_status(workspace: Workspace):
names_by_role = {role: []
for role in RESOURCE_ROLE_CHOICES
} # type:Dict[str,List[str]]
resource_names = []
roles = []
types = []
params = []
missing_roles = []
# we are going to order resources by role
for rname in workspace.get_resource_names():
role = workspace.get_resource_role(rname)
names_by_role[role].append(rname)
for role in RESOURCE_ROLE_CHOICES:
if len(names_by_role[role]) > 0:
for rname in names_by_role[role]:
resource_names.append(rname)
roles.append(role)
types.append(workspace.get_resource_type(rname))
params.append(",\n".join([
"%s=%s" % (pname, pval) for (
pname,
pval) in workspace._get_resource_params(rname).items()
if pname not in ("resource_type", "name", "role")
]))
else:
missing_roles.append(role)
print_columns(
{
"Resource": resource_names,
"Role": roles,
"Type": types,
"Parameters": params
},
# spec={'Parameters':ColSpec(width=40)},
null_value="",
title="Resources for workspace: %s" % workspace.name,
paginate=False,
)
if len(missing_roles) > 0:
click.echo("No resources for the following roles: %s." %
", ".join(missing_roles))
def delete_snapshot_command(workspace: Workspace,
tag_or_hash: str,
no_include_resources: bool = True) -> None:
if not isinstance(workspace, SnapshotWorkspaceMixin):
raise ConfigurationError("Workspace %s does not support snapshots." %
workspace.name)
mixin = cast(SnapshotWorkspaceMixin, workspace)
md = mixin.get_snapshot_by_tag_or_hash(tag_or_hash)
snapshot_name = ("%s (Tagged as: %s)" %
(md.hashval[0:7], ", ".join(md.tags))
if md.tags is not None else md.hashval)
if not workspace.batch:
if not click.confirm(
"Should I delete snapshot %s? This is not reversible." %
snapshot_name):
raise UserAbort()
mixin.delete_snapshot(md.hashval,
include_resources=not no_include_resources)
workspace.save("Deleted snapshot %s" % snapshot_name)
click.echo("Successfully deleted snapshot %s." % snapshot_name)
def lineage_graph_command(
workspace: Workspace,
output_file: str,
resource_name: Optional[str],
snapshot: Optional[str],
format="html",
width: int = 1024,
height: int = 800,
) -> None:
if not isinstance(workspace, SnapshotWorkspaceMixin):
raise ConfigurationError(
"Workspace %s does not support snapshots and lineage" %
workspace.name)
if not workspace.supports_lineage():
raise ConfigurationError("Workspace %s does not support lineage" %
workspace.name)
store = workspace.get_lineage_store()
snapshot_hash = None # type: Optional[str]
if snapshot is not None:
md = workspace.get_snapshot_by_tag_or_hash(snapshot)
snapshot_hash = md.hashval
if resource_name is not None:
workspace.validate_resource_name(resource_name)
else:
for r in workspace.get_resource_names():
if workspace.get_resource_role(r) == ResourceRoles.RESULTS:
resource_name = r
break
if resource_name is None:
raise ConfigurationError(
"Did not find a results resource in workspace. If you want to graph the lineage of a non-results resource, use the --resource option."
)
make_simplified_lineage_graph_for_resource(
workspace.get_instance(),
store,
resource_name,
output_file,
snapshot_hash=snapshot_hash,
format=format,
width=width,
height=height,
)
if snapshot is None:
click.echo("Wrote lineage for %s to %s" % (resource_name, output_file))
else:
click.echo("Wrote lineage for %s as of snapshot %s to %s" %
(resource_name, snapshot, output_file))
def deploy_build_command(
workspace: Workspace,
image_name: Optional[str],
force_rebuild: bool,
git_user_email: Optional[str],
git_user_name: Optional[str],
) -> None:
try:
from repo2docker.__main__ import make_r2d # type: ignore
except ImportError as e:
raise ConfigurationError(R2D_IMPORT_ERROR) from e
target_repo_dir = "/home/%s/%s" % (os.environ["USER"], workspace.name)
if image_name is None:
image_name = workspace.name
argv = [
"--target-repo-dir", target_repo_dir, "--image-name", image_name,
"--no-run"
]
if isinstance(workspace, git_backend.Workspace):
workspace_dir = workspace.get_workspace_local_path_if_any()
assert workspace_dir is not None
user_email = (
git_user_email if git_user_email else get_git_config_param(
workspace_dir, "user.email", verbose=workspace.verbose))
user_name = (git_user_name if git_user_name else get_git_config_param(
workspace_dir, "user.name", verbose=workspace.verbose))
argv.append(
'--appendix=RUN git config --global user.email %s; git config --global user.name "%s"'
% (user_email, user_name))
argv.append(
"dws+" +
get_remote_origin_url(workspace_dir, verbose=workspace.verbose))
else:
# need to figure out how the clone url works for a non-git workspace
assert 0, "build not yet implemented for non-git workspaces"
if force_rebuild:
click.echo("Forcing remove of image %s." % image_name)
call_subprocess(
["docker", "image", "rm", "-f", "--no-prune", image_name],
cwd=curdir,
verbose=workspace.verbose,
)
if workspace.verbose:
click.echo("Command args for repo2docker: %s" % repr(argv))
r2d = make_r2d(argv=argv)
r2d.initialize()
r2d.start()
click.echo("Build of image %s was successful." % image_name)
def _find_resource(
workspace: Workspace,
role: str,
name_or_ref: Optional[Union[str, ResourceRef]] = None) -> ResourceRef:
resource_names = [n for n in workspace.get_resource_names()]
if isinstance(name_or_ref, str):
if ((not name_or_ref.startswith("./"))
and (not name_or_ref.startswith("/"))
and (name_or_ref in resource_names)):
return ResourceRef(name_or_ref)
elif exists(name_or_ref):
return workspace.map_local_path_to_resource(
name_or_ref, expecting_a_code_resource=False)
else:
raise LineageError(
"Could not find a resource for '" + name_or_ref +
"' with role '" + role +
"' in your workspace. Please create a resource" +
" using the 'dws add' command or correct the name. " +
"Currently defined resources are: " + ", ".join([
"%s (role %s)" % (n, workspace.get_resource_role(n))
for n in resource_names
]) + ".")
elif isinstance(name_or_ref, ResourceRef):
workspace.validate_resource_name(name_or_ref.name, name_or_ref.subpath)
return name_or_ref
else:
# no resource specified. If we have exactly one for that role,
# we will use it
resource_for_role = None
for rname in workspace.get_resource_names():
if workspace.get_resource_role(rname) == role:
if resource_for_role is None:
resource_for_role = ResourceRef(rname, subpath=None)
else:
raise LineageError(
"There is more than one resource for role " + role +
" in your workspace. Please specify the resource you want"
+
" in model wrapping function or use a wrapped data set"
)
if resource_for_role is not None:
return resource_for_role
else:
raise LineageError(
"Could not find a " + role +
" resource in your workspace. Please create a resource" +
" using the dws add command.")
def __init__(
self,
resource_type: str,
name: str,
role: str,
workspace: Workspace,
bucket_name: str,
#region: str,
):
if role == ResourceRoles.RESULTS:
raise RESULTS_ROLE_NOT_SUPPORTED
super().__init__(resource_type, name, role, workspace)
self.param_defs.define(
"bucket_name",
default_value=None,
optional=False,
is_global=True,
help="Name of the bucket",
ptype=StringType(),
)
self.bucket_name = self.param_defs.get("bucket_name",
bucket_name) # type: str
# local scratch space for resource is where we store the current snapshot and
# the cache.
self.local_scratch_dir = workspace._get_local_scratch_space_for_resource(
self.name, create_if_not_present=True)
self.current_snapshot_file = join(self.local_scratch_dir,
'current_snapshot.txt')
self.current_snapshot = None # type: Optional[str]
self.snapshot_fs = None # type: Optional[S3Snapshot]
# we cache snapshot files in a subdirectory of the scratch dir
self.snapshot_cache_dir = join(self.local_scratch_dir,
"snapshot_cache")
# Make sure it exists.
if not exists(self.snapshot_cache_dir):
os.makedirs(self.snapshot_cache_dir)
if exists(self.current_snapshot_file):
with open(self.current_snapshot_file, 'r') as f:
self.current_snapshot = f.read().strip()
self.fs = S3FileSystem(version_aware=True)
self.snapshot_fs = self._load_snapshot(self.current_snapshot)
else:
self.fs = S3FileSystem()
def push_command(
workspace: Workspace,
only: Optional[List[str]] = None,
skip: Optional[List[str]] = None,
only_workspace: bool = False,
) -> int:
"""Run the push command on the pushable resources and the workspace.
"""
if only_workspace:
if isinstance(workspace, CentralWorkspaceMixin):
raise ConfigurationError(
"--only-workspace not valid for central workspace %s" % workspace.name
)
resource_list = [] # type: List[LocalStateResourceMixin]
else:
resource_list = [
cast(LocalStateResourceMixin, workspace.get_resource(rn))
for rn in build_resource_list(workspace, only, skip)
]
if isinstance(workspace, CentralWorkspaceMixin):
if len(resource_list) == 0:
click.echo("No resources to push.")
return 0
else:
print(
"Pushing resources: %s" % ", ".join([cast(Resource, r).name for r in resource_list])
)
workspace.push_resources(resource_list)
elif isinstance(workspace, SyncedWorkspaceMixin):
if len(resource_list) > 0:
click.echo(
"Pushing workspace and resources: %s"
% ", ".join([cast(Resource, r).name for r in resource_list])
)
elif not only_workspace:
click.echo("No resources to push, will still push workspace")
else:
click.echo("Pushing workspace.")
workspace.push(resource_list)
workspace.save("push command")
return len(resource_list)
def setup_path_for_hashes(role: str, name: str, workspace: Workspace,
local_path: str):
"""When creating the resource, make sure we have a path for the hashes.
Subclasses of LocalFilesResource will need to call this in their factory
functions."""
workspace_path = workspace.get_workspace_local_path_if_any()
if isinstance(workspace, git_backend.Workspace):
assert workspace_path is not None
hash_path = join(workspace_path,
_relative_rsrc_dir_for_git_workspace(role, name))
try:
os.makedirs(hash_path)
with open(os.path.join(hash_path, "dummy.txt"), "w") as f:
f.write("Placeholder to ensure directory is added to git\n")
call_subprocess(
[
GIT_EXE_PATH,
"add",
join(_relative_rsrc_dir_for_git_workspace(role, name),
"dummy.txt"),
],
cwd=workspace_path,
)
call_subprocess(
[GIT_EXE_PATH, "commit", "-m",
"Adding resource %s" % name],
cwd=workspace_path)
except OSError as exc:
if exc.errno == EEXIST and os.path.isdir(hash_path):
pass
else:
raise
else:
non_git_hashes = join(local_path, ".hashes")
if not exists(non_git_hashes):
os.mkdir(non_git_hashes)
def config_command(
workspace: Workspace,
param_name: Optional[str],
param_value: Optional[str],
resource: Optional[str],
):
if param_name is None and param_value is None:
names = []
scopes = []
values = []
isdefaults = []
helps = []
if resource is None:
handlers = [
GlobalWorkspaceHandler(workspace),
LocalWorkspaceHandler(workspace),
] # type: List[ParamConfigHandler]
else:
if resource not in workspace.get_resource_names():
raise ConfigurationError(
"No resource in this workspace with name '%s'" % resource)
resource_obj = workspace.get_resource(resource)
handlers = [GlobalResourceHandler(resource_obj, workspace)]
if isinstance(resource_obj, LocalStateResourceMixin):
handlers.append(LocalResourceHandler(resource_obj, workspace))
for handler in handlers:
for name in handler.defs.keys():
names.append(name)
scopes.append(handler.get_scope())
helps.append(handler.defs[name].help)
values.append(handler.get_value(name))
isdefaults.append("Y" if handler.is_default(name) else "N")
print_columns(
{
"Name": names,
"Scope": scopes,
"Value": values,
"Default?": isdefaults,
"Description": helps,
},
spec={"Description": ColSpec(width=40)},
paginate=False,
)
click.echo()
else:
assert param_name is not None
if resource is None:
if param_name in PARAM_DEFS:
handler = GlobalWorkspaceHandler(workspace)
elif param_name in LOCAL_PARAM_DEFS:
handler = LocalWorkspaceHandler(workspace)
else:
raise ParamNotFoundError("No workspace parameter named '%s'" %
param_name)
else: # resource-specific
if resource not in workspace.get_resource_names():
raise ConfigurationError(
"No resource in this workspace with name '%s'" % resource)
resource_obj = workspace.get_resource(resource)
if isinstance(resource_obj, LocalStateResourceMixin) and (
param_name in resource_obj.get_local_params()):
handler = LocalResourceHandler(resource_obj, workspace)
elif param_name in resource_obj.get_params().keys():
handler = GlobalResourceHandler(resource_obj, workspace)
else:
raise ParamNotFoundError(
"Resource %s has no parameter named '%s'" %
(resource, param_name))
if param_value is None:
# just print for the specific param
title = "%s parameter '%s'" % (handler.get_what_for().capitalize(),
param_name)
click.echo(title)
click.echo("=" * len(title))
click.echo()
print_columns(
{
"Value": [handler.get_value(param_name)],
"Scope": [handler.get_scope()],
"Default?":
["Y" if handler.is_default(param_name) else "N"],
"Description": [handler.defs[param_name].help],
},
spec={"Description": ColSpec(width=60)},
paginate=False,
)
click.echo()
else: # setting the parameter
parsed_value = handler.defs[param_name].parse(param_value)
handler.set_value(param_name,
handler.defs[param_name].to_json(parsed_value))
param_for = handler.get_what_for()
workspace.save("Update of %s parameter %s" %
(param_for, param_name))
click.echo("Successfully set %s %s parameter '%s' to %s." %
(param_for, handler.get_scope(), param_name,
repr(parsed_value)))
def restore_command(
workspace: Workspace,
tag_or_hash: str,
only: Optional[List[str]] = None,
leave: Optional[List[str]] = None,
strict: bool = False,
) -> int:
"""Run the restore and return the number of resources affected.
"""
if not isinstance(workspace, SnapshotWorkspaceMixin):
raise ConfigurationError("Workspace %s does not support snapshots" %
workspace.name)
mixin = cast(SnapshotWorkspaceMixin, workspace)
# First, find the history entry
md = mixin.get_snapshot_by_tag_or_hash(tag_or_hash)
# process the lists of resources
current_names = set(workspace.get_resource_names())
# get the non-null resources in snapshot
snapshot_names = set([
rn for rn in md.restore_hashes.keys()
if md.restore_hashes[rn] is not None
])
all_names = current_names.union(snapshot_names)
if (only is not None) and (leave is not None):
raise ApiParamError(
"Cannot specify both only and leave for restore command.")
elif only is not None:
# For only, we will be a little stricter, as the user is explicitly
# specifying the resources.
restore_set = set(only)
strict = True
elif leave is not None:
restore_set = all_names.difference(leave)
else:
restore_set = all_names
# We need to remove result resources from the restore set, as we
# do not restore them to their prior state.
result_resources = {
rname
for rname in restore_set
if workspace.get_resource_role(rname) == ResourceRoles.RESULTS
}
result_resources_in_restore_set = result_resources.intersection(
restore_set)
if len(result_resources_in_restore_set) > 0:
if strict:
raise ConfigurationError(
"Restore set contains result resources, which cannot be restored. The following are result resources: %s"
% ", ".join(result_resources_in_restore_set))
else:
click.echo(
"Skipping the restore of the following result resources, which are left in their latest state: %s"
% ", ".join(result_resources_in_restore_set))
restore_set = restore_set.difference(result_resources)
# error checking
invalid = restore_set.difference(all_names)
if len(invalid) > 0:
raise ConfigurationError("Resource name(s) not found: %s" %
", ".join(sorted(invalid)))
removed_names = restore_set.difference(current_names)
if len(removed_names) > 0:
if strict:
raise ConfigurationError(
"Resources have been removed from workspace or have no restore hash and strict mode is enabled."
+ " Removed resources: %s" % ", ".join(sorted(removed_names)))
else:
click.echo(
"Skipping restore of resources that have been removed from workspace or have no restore hash: %s"
% ", ".join(sorted(removed_names)),
err=True,
)
restore_set = restore_set.difference(removed_names)
added_names = restore_set.difference(snapshot_names)
if len(added_names) > 0:
if strict:
raise ConfigurationError(
"Resources have been added to workspace since restore, and strict mode enabled."
+ " Added resources: %s" % ", ".join(sorted(added_names)))
else:
click.echo(
"Resources have been added to workspace since restore, will leave them as-is: %s"
% ", ".join(sorted(added_names)),
err=True,
)
restore_set = restore_set.difference(added_names)
# get ordered list of names and resources as well as restore hashes
restore_name_list = [
rn for rn in workspace.get_resource_names() if rn in restore_set
]
if len(restore_name_list) == 0:
click.echo("No resources to restore.")
return 0
restore_resource_list = [
workspace.get_resource(rn) for rn in restore_name_list
]
for r in restore_resource_list:
if not isinstance(r, SnapshotResourceMixin):
raise InternalError(
"Resource %s was in snapshot, but is not a SnapshotResourceMixin"
% r.name)
restore_hashes = {rn: md.restore_hashes[rn] for rn in restore_set}
tagstr = " (%s)" % ",".join(md.tags) if len(md.tags) > 0 else ""
click.echo("Restoring snapshot %s%s" % (md.hashval, tagstr))
def fmt_rlist(rnames):
if len(rnames) > 0:
return ", ".join(rnames)
else:
return "None"
click.echo(" Resources to restore: %s" % fmt_rlist(restore_name_list))
names_to_leave = sorted(current_names.difference(restore_set))
click.echo(" Resources to leave: %s" % fmt_rlist(names_to_leave))
if not workspace.batch:
# Unless in batch mode, we always want to ask for confirmation
resp = input("Should I perform this restore? [Y/n]")
if resp.lower() != "y" and resp != "":
raise UserAbort()
# do the work!
mixin.restore(md.hashval, restore_hashes,
cast(List[SnapshotResourceMixin], restore_resource_list))
workspace.save("Restore to %s" % md.hashval)
return len(restore_name_list)
def diff_command(workspace: Workspace, snapshot_or_tag1: str,
snapshot_or_tag2: str) -> None:
if not isinstance(workspace, SnapshotWorkspaceMixin):
raise ConfigurationError(
"Diff command not supported for workspace %s as backend does not support snapshots"
% workspace.name)
md1 = workspace.get_snapshot_by_tag_or_hash(snapshot_or_tag1)
manifest1 = {
r["name"]: r
for r in workspace.get_snapshot_manifest(md1.hashval)
}
snstr1 = "%s, tags %s" % (md1.hashval, ",".join(md1.tags)) if len(
md1.tags) > 0 else md1.hashval
sn1_names = frozenset([n for n in manifest1.keys()])
md2 = workspace.get_snapshot_by_tag_or_hash(snapshot_or_tag2)
manifest2 = {
r["name"]: r
for r in workspace.get_snapshot_manifest(md2.hashval)
}
snstr2 = "%s, tags %s" % (md2.hashval, ",".join(md2.tags)) if len(
md2.tags) > 0 else md2.hashval
sn2_names = frozenset([n for n in manifest2.keys()])
click.echo("Comparing:\n Snapshot %s to\n Snapshot %s" %
(snstr1, snstr2))
common_names = sn1_names.intersection(sn2_names)
same_resources = []
different_resources = []
for name in sorted(common_names):
h1 = manifest1[name]["hash"]
h2 = manifest2[name]["hash"]
if h1 == h2:
same_resources.append(name)
else:
different_resources.append(name)
if len(same_resources) > 0:
click.echo(" Resources with the same value:")
for name in same_resources:
click.echo(" " + name)
else:
click.echo(" Resources with the same value: None")
if len(different_resources) > 0:
click.echo(" Resources with different values:")
for name in different_resources:
click.echo(" " + name)
else:
click.echo(" Resources with different values: None")
added_resources = sorted(sn2_names.difference(sn1_names))
if len(added_resources) > 0:
click.echo(" Added resources:")
for name in added_resources:
click.echo(" " + name)
else:
click.echo(" Added resources: None")
removed_resources = sorted(sn1_names.difference(sn2_names))
if len(removed_resources) > 0:
click.echo(" Removed resources:")
for name in removed_resources:
click.echo(" " + name)
else:
click.echo(" Removed resources: None")
def __init__(
self,
step_name: str,
start_time: datetime.datetime,
parameters: Dict[str, Any],
inputs: List[Union[str, ResourceRef]],
code: List[Union[str, ResourceRef]],
workspace: Workspace,
command_line: Optional[List[str]] = None,
current_directory: Optional[str] = None,
):
self.workspace = workspace # type: Workspace
self.instance = workspace.get_instance()
# if not isinstance(workspace, SnapshotWorkspaceMixin) or not workspace.supports_lineage():
# raise ConfigurationError("Backend for workspace %s does not support lineage" % workspace.name)
self.store = cast(SnapshotWorkspaceMixin,
workspace).get_lineage_store()
input_resource_refs = [] # type: List[ResourceRef]
for r_or_p in inputs:
if isinstance(r_or_p, ResourceRef):
workspace.validate_resource_name(r_or_p.name, r_or_p.subpath)
input_resource_refs.append(r_or_p)
else:
ref = workspace.map_local_path_to_resource(r_or_p)
input_resource_refs.append(ref)
code_resource_refs = [] # type: List[ResourceRef]
for r_or_p in code:
if isinstance(r_or_p, ResourceRef):
self.workspace.validate_resource_name(
r_or_p.name,
r_or_p.subpath,
expected_role=ResourceRoles.CODE)
code_resource_refs.append(r_or_p)
else:
ref = workspace.map_local_path_to_resource(
r_or_p, expecting_a_code_resource=True)
# For now, we will resolve code paths at the resource level.
# We drop the subpath, unless the user provided it explicitly
# through a ResourceRef.
crr = ResourceRef(ref.name, None)
if crr not in code_resource_refs:
code_resource_refs.append(crr)
# The run_from_directory can be either a resource reference (best),
# a path on the local filesystem, or None
try:
if current_directory is not None:
if not isabs(current_directory):
current_directory = abspath(expanduser(
(current_directory)))
run_from_directory = workspace.map_local_path_to_resource(
current_directory) # type: Optional[ResourceRef]
else:
run_from_directory = None
except PathNotAResourceError:
run_from_directory = None
self.step = StepLineage.make_step_lineage(
workspace.get_instance(),
step_name,
start_time,
parameters,
input_resource_refs,
code_resource_refs,
self.store,
command_line=command_line,
run_from_directory=run_from_directory,
)
self.in_progress = True
