geo_checks.append(
GEO_CHECK.make_check(
check_id="geo_check",
params=dict(
table_name=f"{dag_id}_{dag_id}_new",
geotype=["POINT"],
),
pass_value=1,
)
)
total_checks = count_checks + geo_checks
# 8. RUN bundled CHECKS
multi_checks = PostgresMultiCheckOperator(task_id="multi_check", checks=total_checks)
# 9. Rename TABLE
rename_table = PostgresTableRenameOperator(
task_id=f"rename_table_{dag_id}",
old_table_name=f"{dag_id}_{dag_id}_new",
new_table_name=f"{dag_id}_{dag_id}",
)
# 10. Grant database permissions
grant_db_permissions = PostgresPermissionsOperator(task_id="grants", dag_name=dag_id)
(
slack_at_start
>> mkdir
>> download_data
GEO_CHECK.make_check(
check_id=f"geo_check_{splitted_tablename}",
params=dict(
table_name=f"{dag_id}_{splitted_tablename}_new",
geotype=["MULTIPOLYGON"],
),
pass_value=1,
))
total_checks = count_checks + geo_checks
check_name[splitted_tablename] = total_checks
# 11. Execute bundled checks on database (see checks definition here above)
multi_checks = [
PostgresMultiCheckOperator(
task_id=f"multi_check_{splitted_tablename}",
checks=check_name[splitted_tablename],
) for key in files_to_download.keys()
for splitted_tablename in key.split("-") if "themas" not in key
]
# 12. Dummy operator acts as an interface between parallel tasks to another parallel tasks with different number of lanes
# (without this intermediar, Airflow will give an error)
Interface2 = DummyOperator(task_id="interface2")
# 13. Rename TABLES
rename_tables = [
PostgresTableRenameOperator(
task_id=f"rename_table_{splitted_tablename}",
old_table_name=f"{dag_id}_{splitted_tablename}_new",
new_table_name=f"{dag_id}_{splitted_tablename}",
) for key in files_to_download.keys()
"POLYGON",
"MULTIPOLYGON",
"MULTILINESTRING",
"LINESTRING",
],
),
pass_value=1,
)
)
total_checks = count_checks + geo_checks
check_name[resource] = total_checks
# 15. Execute bundled checks on database
multi_checks = [
PostgresMultiCheckOperator(task_id=f"multi_check_{resource}", checks=check_name[resource])
for resources in files_to_import.values()
for resource in resources
]
# 16. Create the DB target table (as specified in the JSON data schema)
# if table not exists yet
create_tables = [
SqlAlchemyCreateObjectOperator(
task_id=f"create_{resource}_based_upon_schema",
data_schema_name=dag_id,
data_table_name=f"{dag_id}_{resource}",
ind_table=True,
# when set to false, it doesn't create indexes; only tables
ind_extra_index=False,
)
geo_checks.append(
GEO_CHECK.make_check(
check_id=f"geo_check_{key}",
params=dict(
table_name=f"{dag_id}_{key}_new",
geotype=["MULTIPOLYGON"],
),
pass_value=1,
))
total_checks = count_checks + geo_checks
check_name[key] = total_checks
# 10. Execute bundled checks on database
multi_checks = [
PostgresMultiCheckOperator(task_id=f"multi_check_{key}",
checks=check_name[key])
for key in tables_to_create.keys()
]
# 11. Rename TABLE
rename_tables = [
PostgresTableRenameOperator(
task_id=f"rename_table_{key}",
old_table_name=f"{dag_id}_{key}_new",
new_table_name=f"{dag_id}_{key}",
) for key in tables_to_create.keys()
]
# 11. Grant database permissions
grant_db_permissions = PostgresPermissionsOperator(task_id="grants",
dag_name=dag_id)
params=dict(
table_name=f"{dag_id}_{table_name}_new",
geotype=["POLYGON", "MULTIPOLYGON", "MULTILINESTRING", "POINT"],
geo_column="geometrie",
),
pass_value=1,
)
)
total_checks = count_checks + geo_checks
check_name[f"{table_name}"] = total_checks
# 18. Execute bundled checks on database
multi_checks = [
PostgresMultiCheckOperator(
task_id=f"multi_check_{table_name}", checks=check_name[f"{table_name}"]
)
for table_name in files_to_download.keys()
]
# 19. Check for changes to merge in target table by using CDC
change_data_capture = [
PgComparatorCDCOperator(
task_id=f"change_data_capture_{table_name}",
source_table=f"{dag_id}_{table_name}_new",
target_table=f"{dag_id}_{table_name}",
)
for table_name in files_to_download.keys()
]
# 20. Clean up; delete temp table
colname_check = COLNAMES_CHECK.make_check(
check_id="colname_check",
parameters=["fietspaaltjes"],
pass_value=set(["id"]),
result_checker=operator.ge,
)
geo_check = GEO_CHECK.make_check(
check_id="geo_check",
params=dict(table_name="fietspaaltjes", geotype="POINT"),
pass_value=1,
)
checks = [count_check, colname_check, geo_check]
multi = PostgresMultiCheckOperator(task_id="multi",
checks=checks,
params=make_params(checks))
# swift_task
# sqls = [
# "delete from biz_data where biz_id = {{ params.tba }}",
# "insert into biz_data (biz_id, naam) values (123456789, 'testje')",
# ]
# pgtest = PostgresOperator(task_id="pgtest", sql=sqls)
# bashtest = BashOperator(
# task_id="bashtest", bash_command=f"psql {pg_params} < /tmp/doit.sql",
# )
# failing_task = PythonOperator(
# task_id="failing_task", python_callable=create_error, provide_context=True,
