def main(args=None) -> None:
"""
The command line interface entry point.
:param args: the command line arguments for sqllineage command
"""
parser = argparse.ArgumentParser(prog="sqllineage",
description="SQL Lineage Parser.")
parser.add_argument("-e",
metavar="<quoted-query-string>",
help="SQL from command line")
parser.add_argument("-f", metavar="<filename>", help="SQL from files")
parser.add_argument(
"-v",
"--verbose",
help="increase output verbosity, show statement level lineage result",
action="store_true",
)
parser.add_argument(
"-g",
"--graph-visualization",
help="show graph visualization of the lineage within a webserver",
action="store_true",
)
parser.add_argument(
"-p",
help="the port visualization webserver will be listening on",
type=int,
default=DEFAULT_PORT,
metavar="<port_number>{0..65536}",
)
args = parser.parse_args(args)
if args.e and args.f:
logging.warning(
"Both -e and -f options are specified. -e option will be ignored")
if args.f or args.e:
sql = extract_sql_from_args(args)
runner = LineageRunner(
sql,
verbose=args.verbose,
draw_options={
"p": args.p,
"f": args.f if args.f else None
},
)
if args.graph_visualization:
runner.draw()
else:
print(runner)
elif args.graph_visualization:
return draw_lineage_graph(
**{
"p":
args.p,
"f":
os.path.join(os.path.dirname(__file__),
"data/tpcds/query01.sql"),
})
else:
parser.print_help()
def test_runner_dummy():
runner = LineageRunner(
"""insert into tab2 select col1, col2, col3, col4, col5, col6 from tab1;
insert overwrite table tab3 select * from tab2""",
verbose=True,
)
assert str(runner)
assert runner.to_cytoscape() is not None
def lineage():
# this is to avoid circular import
from sqllineage.runner import LineageRunner
req_args = Namespace(**request.get_json())
sql = extract_sql_from_args(req_args)
lr = LineageRunner(sql, verbose=True)
resp = {"verbose": str(lr), "dag": lr.to_cytoscape()}
return jsonify(resp)
def lineage():
# this is to avoid circular import
from sqllineage.runner import LineageRunner
req_args = Namespace(**request.get_json())
sql = extract_sql_from_args(req_args)
resp = LineageRunner(sql).to_cytoscape()
return jsonify(resp)
def helper(sql, source_tables=None, target_tables=None):
lp = LineageRunner(sql)
assert set(lp.source_tables) == (
set() if source_tables is None else {Table(t) for t in source_tables}
)
assert set(lp.target_tables) == (
set() if target_tables is None else {Table(t) for t in target_tables}
)
def test_dummy():
assert str(
LineageRunner(
"""insert into tab2 select col1, col2, col3, col4, col5, col6 from tab1;
insert overwrite table tab3 select * from tab2""",
verbose=True,
))
with pytest.raises(SystemExit) as e:
main()
assert e.value.code == 2
def get_datasource_urn(self, card_details):
platform, database_name = self.get_datasource_from_id(
card_details.get("database_id", "")
)
query_type = card_details.get("dataset_query", {}).get("type", {})
source_paths = set()
if query_type == "query":
source_table_id = (
card_details.get("dataset_query", {})
.get("query", {})
.get("source-table")
)
if source_table_id is not None:
schema_name, table_name = self.get_source_table_from_id(source_table_id)
if table_name:
source_paths.add(
f"{schema_name + '.' if schema_name else ''}{table_name}"
)
else:
try:
raw_query = (
card_details.get("dataset_query", {})
.get("native", {})
.get("query", "")
)
parser = LineageRunner(raw_query)
for table in parser.source_tables:
sources = str(table).split(".")
source_schema, source_table = sources[-2], sources[-1]
if source_schema == "<default>":
source_schema = str(self.config.default_schema)
source_paths.add(f"{source_schema}.{source_table}")
except Exception as e:
self.report.report_failure(
key="metabase-query",
reason=f"Unable to retrieve lineage from query. "
f"Query: {raw_query} "
f"Reason: {str(e)} ",
)
return None
# Create dataset URNs
dataset_urn = []
dbname = f"{database_name + '.' if database_name else ''}"
source_tables = list(map(lambda tbl: f"{dbname}{tbl}", source_paths))
dataset_urn = [
builder.make_dataset_urn(platform, name, self.config.env)
for name in source_tables
]
return dataset_urn
def test_dummy(_):
assert str(
LineageRunner(
"""insert into tab2 select col1, col2, col3, col4, col5, col6 from tab1;
insert overwrite table tab3 select * from tab2""",
verbose=True,
))
main([])
main(["-e", "select * from dual"])
main(["-f", __file__])
main(["-e", "select * from dual", "-f", __file__])
main([
"-e",
"insert overwrite table tab1 select * from tab1 union select * from tab2",
"-g",
])
def _get_sources_from_query(self, db_name: str, query: str) -> List[LineageDataset]:
sources = list()
parser = LineageRunner(query)
for table in parser.source_tables:
source_schema, source_table = str(table).split(".")
if source_schema == "<default>":
source_schema = str(self.config.default_schema)
source = LineageDataset(
platform=LineageDatasetPlatform.REDSHIFT,
path=f"{db_name}.{source_schema}.{source_table}",
)
sources.append(source)
return sources
def _get_source_from_query(self, raw_query: str) -> set:
query = self._replace_definitions(raw_query)
parser = LineageRunner(query)
source_paths = set()
try:
for table in parser.source_tables:
sources = str(table).split(".")
source_schema, source_table = sources[-2], sources[-1]
if source_schema == "<default>":
source_schema = str(self.config.default_schema)
source_paths.add(f"{source_schema}.{source_table}")
except Exception as e:
self.report.report_failure(
key="mode-query",
reason=f"Unable to retrieve lineage from query. "
f"Query: {raw_query} "
f"Reason: {str(e)} ",
)
return source_paths
def test_insert_without_table():
with pytest.raises(SQLLineageException):
LineageRunner("insert into select * from foo")
def test_split_statements_with_desc():
sql = """SELECT 1;
DESC tab1;"""
assert len(LineageRunner(sql).statements()) == 2
from sqllineage.runner import LineageRunner
'''
数据血缘关系梳理工具：
1.开源工具sqllineage
pip install sqllineage
写代码
2.在线工具
https://sqlflow.gudusoft.com/#/
'''
sql = """
"""
result = LineageRunner(sql)
print(result)
# 打印result，会产出下面的信息
# Statements(#): 2
# Source Tables:
# db1.table12
# db2.table21
# db2.table22
# Target Tables:
# db3.table3
# Intermediate Tables:
# db1.table11
# 也可以直接获取各个源表
for tbl in result.source_tables:
print(tbl)
# db1.table12
# db2.table21
def test_split_statements_with_show_create_table():
sql = """SELECT 1;
SHOW CREATE TABLE tab1;"""
assert len(LineageRunner(sql).statements()) == 2
def test_statements_trim_comment():
comment = "------------------\n"
sql = "select * from dual;"
assert LineageRunner(comment +
sql).statements(strip_comments=True)[0] == sql
def test_split_statements_with_comment():
sql = """SELECT 1;
-- SELECT 2;"""
assert len(LineageRunner(sql).statements()) == 1
def test_split_statements_with_heading_and_ending_new_line():
sql = "\nSELECT * FROM tab1;\nSELECT * FROM tab2;\n"
assert len(LineageRunner(sql).statements()) == 2
def test_with_cte_without_table():
with pytest.raises(SQLLineageException):
LineageRunner(
"with as select * from foo insert into table bar select * from foo"
)
def test_select_without_table():
with pytest.raises(SQLLineageException):
LineageRunner("select * from where foo='bar'")
def test_split_statements():
sql = "SELECT * FROM tab1; SELECT * FROM tab2;"
assert len(LineageRunner(sql).statements()) == 2