def lineage_graph_command( workspace: Workspace, output_file: str, resource_name: Optional[str], snapshot: Optional[str], format="html", width: int = 1024, height: int = 800, ) -> None: if not isinstance(workspace, SnapshotWorkspaceMixin): raise ConfigurationError( "Workspace %s does not support snapshots and lineage" % workspace.name) if not workspace.supports_lineage(): raise ConfigurationError("Workspace %s does not support lineage" % workspace.name) store = workspace.get_lineage_store() snapshot_hash = None # type: Optional[str] if snapshot is not None: md = workspace.get_snapshot_by_tag_or_hash(snapshot) snapshot_hash = md.hashval if resource_name is not None: workspace.validate_resource_name(resource_name) else: for r in workspace.get_resource_names(): if workspace.get_resource_role(r) == ResourceRoles.RESULTS: resource_name = r break if resource_name is None: raise ConfigurationError( "Did not find a results resource in workspace. If you want to graph the lineage of a non-results resource, use the --resource option." ) make_simplified_lineage_graph_for_resource( workspace.get_instance(), store, resource_name, output_file, snapshot_hash=snapshot_hash, format=format, width=width, height=height, ) if snapshot is None: click.echo("Wrote lineage for %s to %s" % (resource_name, output_file)) else: click.echo("Wrote lineage for %s as of snapshot %s to %s" % (resource_name, snapshot, output_file))
def __init__( self, step_name: str, start_time: datetime.datetime, parameters: Dict[str, Any], inputs: List[Union[str, ResourceRef]], code: List[Union[str, ResourceRef]], workspace: Workspace, command_line: Optional[List[str]] = None, current_directory: Optional[str] = None, ): self.workspace = workspace # type: Workspace self.instance = workspace.get_instance() # if not isinstance(workspace, SnapshotWorkspaceMixin) or not workspace.supports_lineage(): # raise ConfigurationError("Backend for workspace %s does not support lineage" % workspace.name) self.store = cast(SnapshotWorkspaceMixin, workspace).get_lineage_store() input_resource_refs = [] # type: List[ResourceRef] for r_or_p in inputs: if isinstance(r_or_p, ResourceRef): workspace.validate_resource_name(r_or_p.name, r_or_p.subpath) input_resource_refs.append(r_or_p) else: ref = workspace.map_local_path_to_resource(r_or_p) input_resource_refs.append(ref) code_resource_refs = [] # type: List[ResourceRef] for r_or_p in code: if isinstance(r_or_p, ResourceRef): self.workspace.validate_resource_name( r_or_p.name, r_or_p.subpath, expected_role=ResourceRoles.CODE) code_resource_refs.append(r_or_p) else: ref = workspace.map_local_path_to_resource( r_or_p, expecting_a_code_resource=True) # For now, we will resolve code paths at the resource level. # We drop the subpath, unless the user provided it explicitly # through a ResourceRef. crr = ResourceRef(ref.name, None) if crr not in code_resource_refs: code_resource_refs.append(crr) # The run_from_directory can be either a resource reference (best), # a path on the local filesystem, or None try: if current_directory is not None: if not isabs(current_directory): current_directory = abspath(expanduser( (current_directory))) run_from_directory = workspace.map_local_path_to_resource( current_directory) # type: Optional[ResourceRef] else: run_from_directory = None except PathNotAResourceError: run_from_directory = None self.step = StepLineage.make_step_lineage( workspace.get_instance(), step_name, start_time, parameters, input_resource_refs, code_resource_refs, self.store, command_line=command_line, run_from_directory=run_from_directory, ) self.in_progress = True