def lineage_graph_command(
workspace: Workspace,
output_file: str,
resource_name: Optional[str],
snapshot: Optional[str],
format="html",
width: int = 1024,
height: int = 800,
) -> None:
if not isinstance(workspace, SnapshotWorkspaceMixin):
raise ConfigurationError(
"Workspace %s does not support snapshots and lineage" %
workspace.name)
if not workspace.supports_lineage():
raise ConfigurationError("Workspace %s does not support lineage" %
workspace.name)
store = workspace.get_lineage_store()
snapshot_hash = None # type: Optional[str]
if snapshot is not None:
md = workspace.get_snapshot_by_tag_or_hash(snapshot)
snapshot_hash = md.hashval
if resource_name is not None:
workspace.validate_resource_name(resource_name)
else:
for r in workspace.get_resource_names():
if workspace.get_resource_role(r) == ResourceRoles.RESULTS:
resource_name = r
break
if resource_name is None:
raise ConfigurationError(
"Did not find a results resource in workspace. If you want to graph the lineage of a non-results resource, use the --resource option."
)
make_simplified_lineage_graph_for_resource(
workspace.get_instance(),
store,
resource_name,
output_file,
snapshot_hash=snapshot_hash,
format=format,
width=width,
height=height,
)
if snapshot is None:
click.echo("Wrote lineage for %s to %s" % (resource_name, output_file))
else:
click.echo("Wrote lineage for %s as of snapshot %s to %s" %
(resource_name, snapshot, output_file))
def diff_command(workspace: Workspace, snapshot_or_tag1: str,
snapshot_or_tag2: str) -> None:
if not isinstance(workspace, SnapshotWorkspaceMixin):
raise ConfigurationError(
"Diff command not supported for workspace %s as backend does not support snapshots"
% workspace.name)
md1 = workspace.get_snapshot_by_tag_or_hash(snapshot_or_tag1)
manifest1 = {
r["name"]: r
for r in workspace.get_snapshot_manifest(md1.hashval)
}
snstr1 = "%s, tags %s" % (md1.hashval, ",".join(md1.tags)) if len(
md1.tags) > 0 else md1.hashval
sn1_names = frozenset([n for n in manifest1.keys()])
md2 = workspace.get_snapshot_by_tag_or_hash(snapshot_or_tag2)
manifest2 = {
r["name"]: r
for r in workspace.get_snapshot_manifest(md2.hashval)
}
snstr2 = "%s, tags %s" % (md2.hashval, ",".join(md2.tags)) if len(
md2.tags) > 0 else md2.hashval
sn2_names = frozenset([n for n in manifest2.keys()])
click.echo("Comparing:\n Snapshot %s to\n Snapshot %s" %
(snstr1, snstr2))
common_names = sn1_names.intersection(sn2_names)
same_resources = []
different_resources = []
for name in sorted(common_names):
h1 = manifest1[name]["hash"]
h2 = manifest2[name]["hash"]
if h1 == h2:
same_resources.append(name)
else:
different_resources.append(name)
if len(same_resources) > 0:
click.echo(" Resources with the same value:")
for name in same_resources:
click.echo(" " + name)
else:
click.echo(" Resources with the same value: None")
if len(different_resources) > 0:
click.echo(" Resources with different values:")
for name in different_resources:
click.echo(" " + name)
else:
click.echo(" Resources with different values: None")
added_resources = sorted(sn2_names.difference(sn1_names))
if len(added_resources) > 0:
click.echo(" Added resources:")
for name in added_resources:
click.echo(" " + name)
else:
click.echo(" Added resources: None")
removed_resources = sorted(sn1_names.difference(sn2_names))
if len(removed_resources) > 0:
click.echo(" Removed resources:")
for name in removed_resources:
click.echo(" " + name)
else:
click.echo(" Removed resources: None")