Exemple #1
0
# -*- coding: utf-8 -*-

from datetime import datetime
import textwrap

from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from airflow.operators.postgres_operator import PostgresOperator
from airflow.operators.data_preparation import DownloadUnzipOperator
from airflow.operators.dummy_operator import DummyOperator

import helpers
import recipes.commune_recipes as recipes
from config import CONN_ID, DATA_DIR

default_args = helpers.default_args({"start_date": datetime(2019, 6, 11, 5)})

with DAG("prepare_commune", default_args=default_args,
         schedule_interval=None) as dag:

    start = DummyOperator(task_id="start")

    download = DownloadUnzipOperator(
        task_id="download",
        url=
        "https://www.data.gouv.fr/fr/datasets/r/07b7c9a2-d1e2-4da6-9f20-01a7b72d4b12",
        dir_path="{data_dir}/communes".format(data_dir=DATA_DIR))

    start >> download

    load = PythonOperator(task_id="load",
Exemple #2
0
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from airflow.operators.bash_operator import BashOperator
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators.check_operator import CheckOperator
from airflow.hooks.postgres_hook import PostgresHook
from operators.pg_download_operator import PgDownloadOperator
from airflow.operators.dagrun_operator import TriggerDagRunOperator

import helpers
from secmar_dags import in_path, out_path, secmar_transform
from secmar_dags import SECMAR_TABLES, secmar_transformer
from secmar_checks import checks

default_args = helpers.default_args(
    {"start_date": datetime(2018, 4, 27, 5, 40)})

dag = DAG(
    "extract_secmar",
    default_args=default_args,
    max_active_runs=1,
    concurrency=2,
    catchup=False,
    schedule_interval=None,
)
dag.doc_md = __doc__


def create_tables_fn(**kwargs):
    PostgresHook("postgresql_local").run(helpers.read_sql_query("schema"))
Exemple #3
0
# replace_secmar_database
Dump SECMAR database from Airflow instance to target
remote server
"""
from datetime import datetime

from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from airflow.hooks.postgres_hook import PostgresHook

import helpers
from secmar_dags import SECMAR_TABLES

OUTPUT_PATH = "/tmp/secmar_dump.sql"

default_args = helpers.default_args({"start_date": datetime(2018, 8, 23, 14, 40)})

dag = DAG(
    "replace_secmar_database",
    default_args=default_args,
    max_active_runs=1,
    concurrency=2,
    catchup=False,
    schedule_interval=None,
)
dag.doc_md = __doc__

tables = SECMAR_TABLES + ["operations_stats", "moyens_snsm"]

template = "sudo -u postgres pg_dump -c --no-owner {tables} {schema} > {output}"
dump_command = template.format(
Exemple #4
0
# extract_secmar_oracle
Extract SECMAR data from the Oracle database and dump it to CSV.

This DAG can only be executed from within the intranet of the MTES.
"""
from datetime import datetime

from airflow import DAG
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators.bash_operator import BashOperator
from airflow.operators.dagrun_operator import TriggerDagRunOperator

import helpers
from secmar_dags import in_path, SECMAR_TABLES

default_args = helpers.default_args(
    {"start_date": datetime(2018, 5, 17, 6, 40)})

dag = DAG(
    "extract_secmar_oracle",
    default_args=default_args,
    max_active_runs=1,
    concurrency=2,
    catchup=False,
    schedule_interval=None,
)
dag.doc_md = __doc__


def embulk_export(dag, table):
    return helpers.embulk_run(
        dag,