def setup_producer(): set_producer( 'https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt' )
from openlineage.client import OpenLineageClient, OpenLineageClientOptions, set_producer from openlineage.client.facet import DocumentationJobFacet, SourceCodeLocationJobFacet, \ NominalTimeRunFacet, ParentRunFacet, BaseFacet from openlineage.client.run import RunEvent, RunState, Run, Job _DAG_DEFAULT_OWNER = 'anonymous' _DAG_DEFAULT_NAMESPACE = 'default' _DAG_NAMESPACE = os.getenv('OPENLINEAGE_NAMESPACE', None) if not _DAG_NAMESPACE: _DAG_NAMESPACE = os.getenv('MARQUEZ_NAMESPACE', _DAG_DEFAULT_NAMESPACE) _PRODUCER = f"https://github.com/OpenLineage/OpenLineage/tree/" \ f"{OPENLINEAGE_AIRFLOW_VERSION}/integration/airflow" set_producer(_PRODUCER) log = logging.getLogger(__name__) class OpenLineageAdapter: """ Adapter for translating Airflow metadata to OpenLineage events, instead of directly creating them from Airflow code. """ _client = None def get_or_create_openlineage_client(self) -> OpenLineageClient: if not self._client: # Backcomp with Marquez integration
from great_expectations_provider.operators.great_expectations import GreatExpectationsOperator from airflow.utils.dates import days_ago from openlineage.client import set_producer set_producer( "https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/airflow" ) from airflow.version import version as AIRFLOW_VERSION from pkg_resources import parse_version if parse_version(AIRFLOW_VERSION) < parse_version("2.0.0"): from openlineage.airflow import DAG else: from airflow import DAG default_args = { 'owner': 'datascience', 'depends_on_past': False, 'start_date': days_ago(7), 'email_on_failure': False, 'email_on_retry': False, 'email': ['*****@*****.**'] } data_context_dir = "/opt/data/great_expectations" dag = DAG('great_expectations_validation', schedule_interval='@once', default_args=default_args, description='Validates data.')
def setup_producer(): set_producer( 'https://github.com/OpenLineage/OpenLineage/tree/0.0.1/client/python')