Ejemplo n.º 1
0
def create_demo_lineage(metastore_id, uid, session=None):
    query_text = """CREATE TABLE world_happiness_ranking_2015_to_2019 AS
SELECT
  w5.Country,
  w5.Region,
  w5.HappinessRank AS [Rank2015],
  w6.HappinessRank AS [Rank2016],
  w7.HappinessRank AS [Rank2017],
  w8.Rank AS [Rank2018],
  w9.Rank AS [Rank2019]
FROM
  world_happiness_2019 w9
  INNER JOIN world_happiness_2018 w8 ON w9.Country = w8.Country
  INNER JOIN world_happiness_2017 w7 ON w9.Country = w7.Country
  INNER JOIN world_happiness_2016 w6 ON w9.Country = w6.Country
  INNER JOIN world_happiness_2015 w5 ON w9.Country = w5.Country;
"""
    user = user_logic.get_user_by_id(uid, session=session)
    data_job_metadata = m_logic.create_job_metadata_row(
        job_name="Untitled",
        metastore_id=metastore_id,
        job_info={"source": "demo lineage"},
        job_owner=user.username,
        query_text=query_text,
        is_adhoc=True,
        session=session,
    )
    lineage_logic.create_table_lineage_from_metadata(
        data_job_metadata.id, "sqlite", session=session
    )
def create_lineage_from_query(query_execution,
                              metastore_id,
                              datadoc_cell=None,
                              session=None):
    cell_title = (datadoc_cell.meta["title"] if
                  (datadoc_cell
                   and "title" in datadoc_cell.meta) else "Untitled")
    job_name = "{}-{}".format(cell_title, query_execution.id)
    data_job_metadata = m_logic.create_job_metadata_row(
        job_name,
        metastore_id,
        job_info={"query_execution_id": query_execution.id},
        job_owner=query_execution.owner.username,
        query_text=query_execution.query,
        is_adhoc=True,
        session=session,
    )
    m_logic.create_table_lineage_from_metadata(data_job_metadata.id,
                                               query_execution.engine.language,
                                               session=session)
Ejemplo n.º 3
0
def add_data_job_metadata(job_name, job_info, job_owner, query_text, is_adhoc,
                          metastore_id):
    with DBSession() as session:
        logic.delete_job_metadata_row(job_name, metastore_id, session=session)
        data_job_metadata = logic.create_job_metadata_row(
            job_name,
            metastore_id,
            job_info,
            job_owner,
            query_text,
            is_adhoc,
            session=session,
        )
        table_lineage_ids = None
        if query_text:
            table_lineage_ids = lineage.create_table_lineage_from_metadata(
                data_job_metadata.id)
        return {
            "data_job_metadata_ids": data_job_metadata.id,
            "table_lineage_ids": table_lineage_ids,
        }