def _find_resource(
workspace: Workspace,
role: str,
name_or_ref: Optional[Union[str, ResourceRef]] = None) -> ResourceRef:
resource_names = [n for n in workspace.get_resource_names()]
if isinstance(name_or_ref, str):
if ((not name_or_ref.startswith("./"))
and (not name_or_ref.startswith("/"))
and (name_or_ref in resource_names)):
return ResourceRef(name_or_ref)
elif exists(name_or_ref):
return workspace.map_local_path_to_resource(
name_or_ref, expecting_a_code_resource=False)
else:
raise LineageError(
"Could not find a resource for '" + name_or_ref +
"' with role '" + role +
"' in your workspace. Please create a resource" +
" using the 'dws add' command or correct the name. " +
"Currently defined resources are: " + ", ".join([
"%s (role %s)" % (n, workspace.get_resource_role(n))
for n in resource_names
]) + ".")
elif isinstance(name_or_ref, ResourceRef):
workspace.validate_resource_name(name_or_ref.name, name_or_ref.subpath)
return name_or_ref
else:
# no resource specified. If we have exactly one for that role,
# we will use it
resource_for_role = None
for rname in workspace.get_resource_names():
if workspace.get_resource_role(rname) == role:
if resource_for_role is None:
resource_for_role = ResourceRef(rname, subpath=None)
else:
raise LineageError(
"There is more than one resource for role " + role +
" in your workspace. Please specify the resource you want"
+
" in model wrapping function or use a wrapped data set"
)
if resource_for_role is not None:
return resource_for_role
else:
raise LineageError(
"Could not find a " + role +
" resource in your workspace. Please create a resource" +
" using the dws add command.")
def __init__(
self,
step_name: str,
start_time: datetime.datetime,
parameters: Dict[str, Any],
inputs: List[Union[str, ResourceRef]],
code: List[Union[str, ResourceRef]],
workspace: Workspace,
command_line: Optional[List[str]] = None,
current_directory: Optional[str] = None,
):
self.workspace = workspace # type: Workspace
self.instance = workspace.get_instance()
# if not isinstance(workspace, SnapshotWorkspaceMixin) or not workspace.supports_lineage():
# raise ConfigurationError("Backend for workspace %s does not support lineage" % workspace.name)
self.store = cast(SnapshotWorkspaceMixin,
workspace).get_lineage_store()
input_resource_refs = [] # type: List[ResourceRef]
for r_or_p in inputs:
if isinstance(r_or_p, ResourceRef):
workspace.validate_resource_name(r_or_p.name, r_or_p.subpath)
input_resource_refs.append(r_or_p)
else:
ref = workspace.map_local_path_to_resource(r_or_p)
input_resource_refs.append(ref)
code_resource_refs = [] # type: List[ResourceRef]
for r_or_p in code:
if isinstance(r_or_p, ResourceRef):
self.workspace.validate_resource_name(
r_or_p.name,
r_or_p.subpath,
expected_role=ResourceRoles.CODE)
code_resource_refs.append(r_or_p)
else:
ref = workspace.map_local_path_to_resource(
r_or_p, expecting_a_code_resource=True)
# For now, we will resolve code paths at the resource level.
# We drop the subpath, unless the user provided it explicitly
# through a ResourceRef.
crr = ResourceRef(ref.name, None)
if crr not in code_resource_refs:
code_resource_refs.append(crr)
# The run_from_directory can be either a resource reference (best),
# a path on the local filesystem, or None
try:
if current_directory is not None:
if not isabs(current_directory):
current_directory = abspath(expanduser(
(current_directory)))
run_from_directory = workspace.map_local_path_to_resource(
current_directory) # type: Optional[ResourceRef]
else:
run_from_directory = None
except PathNotAResourceError:
run_from_directory = None
self.step = StepLineage.make_step_lineage(
workspace.get_instance(),
step_name,
start_time,
parameters,
input_resource_refs,
code_resource_refs,
self.store,
command_line=command_line,
run_from_directory=run_from_directory,
)
self.in_progress = True