def copy_provider_sources() -> None:
"""
Copies provider sources to directory where they will be refactored.
"""
def rm_build_dir() -> None:
"""
Removes build directory.
"""
build_dir = os.path.join(dirname(__file__), "build")
if os.path.isdir(build_dir):
rmtree(build_dir)
def ignore_kubernetes_files(src: str, names: List[str]) -> List[str]:
ignored_names = []
if src.endswith(os.path.sep + "example_dags"):
for file_name in names:
if "example_kubernetes" in file_name:
ignored_names.append(file_name)
return ignored_names
def ignore_some_files(src: str, names: List[str]) -> List[str]:
ignored_list = []
ignored_list.extend(ignore_kubernetes_files(src=src, names=names))
return ignored_list
rm_build_dir()
package_providers_dir = get_target_providers_folder()
if os.path.isdir(package_providers_dir):
rmtree(package_providers_dir)
copytree(get_source_providers_folder(),
get_target_providers_folder(),
ignore=ignore_some_files)
def refactor_odbc_package(self):
"""
Fixes to "odbc" providers package.
Copies some of the classes used from core Airflow to "common.utils" package of the
the provider and renames imports to use them from there.
We copy helpers.py and change import as in example diff:
.. code-block:: diff
--- ./airflow/providers/google/cloud/example_dags/example_datacatalog.py
+++ ./airflow/providers/google/cloud/example_dags/example_datacatalog.py
@@ -37,7 +37,7 @@
CloudDataCatalogUpdateTagTemplateOperator,
)
from airflow.utils.dates import days_ago
-from airflow.utils.helpers import chain
+from airflow.providers.odbc.utils.helpers import chain
default_args = {"start_date": days_ago(1)}
"""
# noinspection PyUnusedLocal
def odbc_package_filter(node: LN, capture: Capture,
filename: Filename) -> bool:
return filename.startswith("./airflow/providers/odbc/")
os.makedirs(os.path.join(get_target_providers_folder(), "odbc",
"utils"),
exist_ok=True)
copyfile(
os.path.join(get_source_airflow_folder(), "airflow", "utils",
"__init__.py"),
os.path.join(get_target_providers_package_folder("odbc"), "utils",
"__init__.py"))
copy_helper_py_file(
os.path.join(get_target_providers_package_folder("odbc"), "utils",
"helpers.py"))
(self.qry.select_module("airflow.utils.helpers").filter(
callback=odbc_package_filter).rename(
"airflow.providers.odbc.utils.helpers"))
def copy_provider_sources() -> None:
"""
Copies provider sources to directory where they will be refactored.
"""
def rm_build_dir() -> None:
"""
Removes build directory.
"""
build_dir = os.path.join(dirname(__file__), "build")
if os.path.isdir(build_dir):
rmtree(build_dir)
def ignore_bigquery_files(src: str, names: List[str]) -> List[str]:
"""
Ignore files with bigquery
:param src: source file
:param names: Name of the file
:return:
"""
ignored_names = []
if any([
src.endswith(os.path.sep + class_type)
for class_type in CLASS_TYPES
]):
ignored_names = [
name for name in names
if is_bigquery_non_dts_module(module_name=name)
]
if src.endswith(os.path.sep + "example_dags"):
for file_name in names:
file_path = src + os.path.sep + file_name
with open(file_path, "rt") as file:
text = file.read()
if any([
f"airflow.providers.google.cloud.{class_type}.bigquery"
in text for class_type in CLASS_TYPES
]) or "_to_bigquery" in text:
print(f"Ignoring {file_path}")
ignored_names.append(file_name)
return ignored_names
def ignore_kubernetes_files(src: str, names: List[str]) -> List[str]:
ignored_names = []
if src.endswith(os.path.sep + "example_dags"):
for file_name in names:
if "example_kubernetes" in file_name:
ignored_names.append(file_name)
return ignored_names
def ignore_some_files(src: str, names: List[str]) -> List[str]:
ignored_list = ignore_bigquery_files(src=src, names=names)
ignored_list.extend(ignore_kubernetes_files(src=src, names=names))
return ignored_list
rm_build_dir()
package_providers_dir = get_target_providers_folder()
if os.path.isdir(package_providers_dir):
rmtree(package_providers_dir)
copytree(get_source_providers_folder(),
get_target_providers_folder(),
ignore=ignore_some_files)