def _find_resource( workspace: Workspace, role: str, name_or_ref: Optional[Union[str, ResourceRef]] = None) -> ResourceRef: resource_names = [n for n in workspace.get_resource_names()] if isinstance(name_or_ref, str): if ((not name_or_ref.startswith("./")) and (not name_or_ref.startswith("/")) and (name_or_ref in resource_names)): return ResourceRef(name_or_ref) elif exists(name_or_ref): return workspace.map_local_path_to_resource( name_or_ref, expecting_a_code_resource=False) else: raise LineageError( "Could not find a resource for '" + name_or_ref + "' with role '" + role + "' in your workspace. Please create a resource" + " using the 'dws add' command or correct the name. " + "Currently defined resources are: " + ", ".join([ "%s (role %s)" % (n, workspace.get_resource_role(n)) for n in resource_names ]) + ".") elif isinstance(name_or_ref, ResourceRef): workspace.validate_resource_name(name_or_ref.name, name_or_ref.subpath) return name_or_ref else: # no resource specified. If we have exactly one for that role, # we will use it resource_for_role = None for rname in workspace.get_resource_names(): if workspace.get_resource_role(rname) == role: if resource_for_role is None: resource_for_role = ResourceRef(rname, subpath=None) else: raise LineageError( "There is more than one resource for role " + role + " in your workspace. Please specify the resource you want" + " in model wrapping function or use a wrapped data set" ) if resource_for_role is not None: return resource_for_role else: raise LineageError( "Could not find a " + role + " resource in your workspace. Please create a resource" + " using the dws add command.")
def __init__( self, step_name: str, start_time: datetime.datetime, parameters: Dict[str, Any], inputs: List[Union[str, ResourceRef]], code: List[Union[str, ResourceRef]], workspace: Workspace, command_line: Optional[List[str]] = None, current_directory: Optional[str] = None, ): self.workspace = workspace # type: Workspace self.instance = workspace.get_instance() # if not isinstance(workspace, SnapshotWorkspaceMixin) or not workspace.supports_lineage(): # raise ConfigurationError("Backend for workspace %s does not support lineage" % workspace.name) self.store = cast(SnapshotWorkspaceMixin, workspace).get_lineage_store() input_resource_refs = [] # type: List[ResourceRef] for r_or_p in inputs: if isinstance(r_or_p, ResourceRef): workspace.validate_resource_name(r_or_p.name, r_or_p.subpath) input_resource_refs.append(r_or_p) else: ref = workspace.map_local_path_to_resource(r_or_p) input_resource_refs.append(ref) code_resource_refs = [] # type: List[ResourceRef] for r_or_p in code: if isinstance(r_or_p, ResourceRef): self.workspace.validate_resource_name( r_or_p.name, r_or_p.subpath, expected_role=ResourceRoles.CODE) code_resource_refs.append(r_or_p) else: ref = workspace.map_local_path_to_resource( r_or_p, expecting_a_code_resource=True) # For now, we will resolve code paths at the resource level. # We drop the subpath, unless the user provided it explicitly # through a ResourceRef. crr = ResourceRef(ref.name, None) if crr not in code_resource_refs: code_resource_refs.append(crr) # The run_from_directory can be either a resource reference (best), # a path on the local filesystem, or None try: if current_directory is not None: if not isabs(current_directory): current_directory = abspath(expanduser( (current_directory))) run_from_directory = workspace.map_local_path_to_resource( current_directory) # type: Optional[ResourceRef] else: run_from_directory = None except PathNotAResourceError: run_from_directory = None self.step = StepLineage.make_step_lineage( workspace.get_instance(), step_name, start_time, parameters, input_resource_refs, code_resource_refs, self.store, command_line=command_line, run_from_directory=run_from_directory, ) self.in_progress = True