Esempio n. 1
0
def setup_producer():
    set_producer(
        'https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt'
    )
Esempio n. 2
0
from openlineage.client import OpenLineageClient, OpenLineageClientOptions, set_producer
from openlineage.client.facet import DocumentationJobFacet, SourceCodeLocationJobFacet, \
    NominalTimeRunFacet, ParentRunFacet, BaseFacet
from openlineage.client.run import RunEvent, RunState, Run, Job

_DAG_DEFAULT_OWNER = 'anonymous'
_DAG_DEFAULT_NAMESPACE = 'default'

_DAG_NAMESPACE = os.getenv('OPENLINEAGE_NAMESPACE', None)
if not _DAG_NAMESPACE:
    _DAG_NAMESPACE = os.getenv('MARQUEZ_NAMESPACE', _DAG_DEFAULT_NAMESPACE)

_PRODUCER = f"https://github.com/OpenLineage/OpenLineage/tree/" \
            f"{OPENLINEAGE_AIRFLOW_VERSION}/integration/airflow"

set_producer(_PRODUCER)

log = logging.getLogger(__name__)


class OpenLineageAdapter:
    """
    Adapter for translating Airflow metadata to OpenLineage events,
    instead of directly creating them from Airflow code.
    """
    _client = None

    def get_or_create_openlineage_client(self) -> OpenLineageClient:
        if not self._client:

            # Backcomp with Marquez integration
Esempio n. 3
0
from great_expectations_provider.operators.great_expectations import GreatExpectationsOperator

from airflow.utils.dates import days_ago
from openlineage.client import set_producer
set_producer(
    "https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/airflow"
)

from airflow.version import version as AIRFLOW_VERSION
from pkg_resources import parse_version
if parse_version(AIRFLOW_VERSION) < parse_version("2.0.0"):
    from openlineage.airflow import DAG
else:
    from airflow import DAG

default_args = {
    'owner': 'datascience',
    'depends_on_past': False,
    'start_date': days_ago(7),
    'email_on_failure': False,
    'email_on_retry': False,
    'email': ['*****@*****.**']
}

data_context_dir = "/opt/data/great_expectations"

dag = DAG('great_expectations_validation',
          schedule_interval='@once',
          default_args=default_args,
          description='Validates data.')
Esempio n. 4
0
def setup_producer():
    set_producer(
        'https://github.com/OpenLineage/OpenLineage/tree/0.0.1/client/python')