def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._marquez_dataset_cache = {}
self._marquez_source_cache = {}
self.marquez_namespace = os.getenv('MARQUEZ_NAMESPACE',
DAG.DEFAULT_NAMESPACE)
self._job_id_mapping = JobIdMapping()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.marquez_namespace = os.environ.get('MARQUEZ_NAMESPACE') or \
DAG.DEFAULT_NAMESPACE
self.marquez_location = kwargs['default_args'].get(
'marquez_location', 'unknown')
self.marquez_input_urns = kwargs['default_args'].get(
'marquez_input_urns', [])
self.marquez_output_urns = kwargs['default_args'].get(
'marquez_output_urns', [])
self._job_id_mapping = JobIdMapping()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._job_id_mapping = JobIdMapping()
self._marquez = Marquez()
# TODO: Manually define operator->extractor mappings for now,
# but we'll want to encapsulate this logic in an 'Extractors' class
# with more convenient methods (ex: 'Extractors.extractor_for_task()')
self._extractors = {
PostgresOperator: PostgresExtractor,
BigQueryOperator: BigQueryExtractor
# Append new extractors here
}
self.log.debug(
f"DAG successfully created with extractors: {self._extractors}")
