def setup_producer():
set_producer(
'https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt'
)
from openlineage.client import OpenLineageClient, OpenLineageClientOptions, set_producer
from openlineage.client.facet import DocumentationJobFacet, SourceCodeLocationJobFacet, \
NominalTimeRunFacet, ParentRunFacet, BaseFacet
from openlineage.client.run import RunEvent, RunState, Run, Job
_DAG_DEFAULT_OWNER = 'anonymous'
_DAG_DEFAULT_NAMESPACE = 'default'
_DAG_NAMESPACE = os.getenv('OPENLINEAGE_NAMESPACE', None)
if not _DAG_NAMESPACE:
_DAG_NAMESPACE = os.getenv('MARQUEZ_NAMESPACE', _DAG_DEFAULT_NAMESPACE)
_PRODUCER = f"https://github.com/OpenLineage/OpenLineage/tree/" \
f"{OPENLINEAGE_AIRFLOW_VERSION}/integration/airflow"
set_producer(_PRODUCER)
log = logging.getLogger(__name__)
class OpenLineageAdapter:
"""
Adapter for translating Airflow metadata to OpenLineage events,
instead of directly creating them from Airflow code.
"""
_client = None
def get_or_create_openlineage_client(self) -> OpenLineageClient:
if not self._client:
# Backcomp with Marquez integration
from great_expectations_provider.operators.great_expectations import GreatExpectationsOperator
from airflow.utils.dates import days_ago
from openlineage.client import set_producer
set_producer(
"https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/airflow"
)
from airflow.version import version as AIRFLOW_VERSION
from pkg_resources import parse_version
if parse_version(AIRFLOW_VERSION) < parse_version("2.0.0"):
from openlineage.airflow import DAG
else:
from airflow import DAG
default_args = {
'owner': 'datascience',
'depends_on_past': False,
'start_date': days_ago(7),
'email_on_failure': False,
'email_on_retry': False,
'email': ['*****@*****.**']
}
data_context_dir = "/opt/data/great_expectations"
dag = DAG('great_expectations_validation',
schedule_interval='@once',
default_args=default_args,
description='Validates data.')
def setup_producer():
set_producer(
'https://github.com/OpenLineage/OpenLineage/tree/0.0.1/client/python')