def create_demo_lineage(metastore_id, uid, session=None):
query_text = """CREATE TABLE world_happiness_ranking_2015_to_2019 AS
SELECT
w5.Country,
w5.Region,
w5.HappinessRank AS [Rank2015],
w6.HappinessRank AS [Rank2016],
w7.HappinessRank AS [Rank2017],
w8.Rank AS [Rank2018],
w9.Rank AS [Rank2019]
FROM
world_happiness_2019 w9
INNER JOIN world_happiness_2018 w8 ON w9.Country = w8.Country
INNER JOIN world_happiness_2017 w7 ON w9.Country = w7.Country
INNER JOIN world_happiness_2016 w6 ON w9.Country = w6.Country
INNER JOIN world_happiness_2015 w5 ON w9.Country = w5.Country;
"""
user = user_logic.get_user_by_id(uid, session=session)
data_job_metadata = m_logic.create_job_metadata_row(
job_name="Untitled",
metastore_id=metastore_id,
job_info={"source": "demo lineage"},
job_owner=user.username,
query_text=query_text,
is_adhoc=True,
session=session,
)
lineage_logic.create_table_lineage_from_metadata(
data_job_metadata.id, "sqlite", session=session
)
def create_lineage_from_query(query_execution,
metastore_id,
datadoc_cell=None,
session=None):
cell_title = (datadoc_cell.meta["title"] if
(datadoc_cell
and "title" in datadoc_cell.meta) else "Untitled")
job_name = "{}-{}".format(cell_title, query_execution.id)
data_job_metadata = m_logic.create_job_metadata_row(
job_name,
metastore_id,
job_info={"query_execution_id": query_execution.id},
job_owner=query_execution.owner.username,
query_text=query_execution.query,
is_adhoc=True,
session=session,
)
m_logic.create_table_lineage_from_metadata(data_job_metadata.id,
query_execution.engine.language,
session=session)
def add_data_job_metadata(job_name, job_info, job_owner, query_text, is_adhoc,
metastore_id):
with DBSession() as session:
logic.delete_job_metadata_row(job_name, metastore_id, session=session)
data_job_metadata = logic.create_job_metadata_row(
job_name,
metastore_id,
job_info,
job_owner,
query_text,
is_adhoc,
session=session,
)
table_lineage_ids = None
if query_text:
table_lineage_ids = lineage.create_table_lineage_from_metadata(
data_job_metadata.id)
return {
"data_job_metadata_ids": data_job_metadata.id,
"table_lineage_ids": table_lineage_ids,
}