# -*- coding: utf-8 -*- from datetime import datetime import textwrap from airflow import DAG from airflow.operators.python_operator import PythonOperator from airflow.operators.postgres_operator import PostgresOperator from airflow.operators.data_preparation import DownloadUnzipOperator from airflow.operators.dummy_operator import DummyOperator import helpers import recipes.commune_recipes as recipes from config import CONN_ID, DATA_DIR default_args = helpers.default_args({"start_date": datetime(2019, 6, 11, 5)}) with DAG("prepare_commune", default_args=default_args, schedule_interval=None) as dag: start = DummyOperator(task_id="start") download = DownloadUnzipOperator( task_id="download", url= "https://www.data.gouv.fr/fr/datasets/r/07b7c9a2-d1e2-4da6-9f20-01a7b72d4b12", dir_path="{data_dir}/communes".format(data_dir=DATA_DIR)) start >> download load = PythonOperator(task_id="load",
from airflow import DAG from airflow.operators.python_operator import PythonOperator from airflow.operators.bash_operator import BashOperator from airflow.operators.dummy_operator import DummyOperator from airflow.operators.check_operator import CheckOperator from airflow.hooks.postgres_hook import PostgresHook from operators.pg_download_operator import PgDownloadOperator from airflow.operators.dagrun_operator import TriggerDagRunOperator import helpers from secmar_dags import in_path, out_path, secmar_transform from secmar_dags import SECMAR_TABLES, secmar_transformer from secmar_checks import checks default_args = helpers.default_args( {"start_date": datetime(2018, 4, 27, 5, 40)}) dag = DAG( "extract_secmar", default_args=default_args, max_active_runs=1, concurrency=2, catchup=False, schedule_interval=None, ) dag.doc_md = __doc__ def create_tables_fn(**kwargs): PostgresHook("postgresql_local").run(helpers.read_sql_query("schema"))
# replace_secmar_database Dump SECMAR database from Airflow instance to target remote server """ from datetime import datetime from airflow import DAG from airflow.operators.bash_operator import BashOperator from airflow.hooks.postgres_hook import PostgresHook import helpers from secmar_dags import SECMAR_TABLES OUTPUT_PATH = "/tmp/secmar_dump.sql" default_args = helpers.default_args({"start_date": datetime(2018, 8, 23, 14, 40)}) dag = DAG( "replace_secmar_database", default_args=default_args, max_active_runs=1, concurrency=2, catchup=False, schedule_interval=None, ) dag.doc_md = __doc__ tables = SECMAR_TABLES + ["operations_stats", "moyens_snsm"] template = "sudo -u postgres pg_dump -c --no-owner {tables} {schema} > {output}" dump_command = template.format(
# extract_secmar_oracle Extract SECMAR data from the Oracle database and dump it to CSV. This DAG can only be executed from within the intranet of the MTES. """ from datetime import datetime from airflow import DAG from airflow.operators.dummy_operator import DummyOperator from airflow.operators.bash_operator import BashOperator from airflow.operators.dagrun_operator import TriggerDagRunOperator import helpers from secmar_dags import in_path, SECMAR_TABLES default_args = helpers.default_args( {"start_date": datetime(2018, 5, 17, 6, 40)}) dag = DAG( "extract_secmar_oracle", default_args=default_args, max_active_runs=1, concurrency=2, catchup=False, schedule_interval=None, ) dag.doc_md = __doc__ def embulk_export(dag, table): return helpers.embulk_run( dag,