def copy_provider_sources() -> None:
"""
Copies provider sources to directory where they will be refactored.
"""
def rm_build_dir() -> None:
"""
Removes build directory.
"""
build_dir = os.path.join(dirname(__file__), "build")
if os.path.isdir(build_dir):
rmtree(build_dir)
def ignore_google_auth_backend(src: str, names: List[str]) -> List[str]:
del names
if src.endswith("google" + os.path.sep + "common"):
return ["auth_backend"]
return []
def ignore_some_files(src: str, names: List[str]) -> List[str]:
ignored_list = []
ignored_list.extend(ignore_google_auth_backend(src=src, names=names))
return ignored_list
rm_build_dir()
package_providers_dir = get_target_providers_folder()
if os.path.isdir(package_providers_dir):
rmtree(package_providers_dir)
copytree(get_source_providers_folder(),
get_target_providers_folder(),
ignore=ignore_some_files)
def copy_provider_sources(backports: bool) -> None:
"""
Copies provider sources to directory where they will be refactored.
"""
def rm_build_dir() -> None:
"""
Removes build directory.
"""
build_dir = os.path.join(dirname(__file__), "build")
if os.path.isdir(build_dir):
print(f"\nRemoving the {build_dir} directory: ", end='')
rmtree(build_dir)
print("[green]OK[/]")
def ignore_google_auth_backend(src: str, names: List[str]) -> List[str]:
del names
if src.endswith("google" + os.path.sep + "common"):
return ["auth_backend"]
return []
def ignore_some_files(src: str, names: List[str]) -> List[str]:
ignored_list = []
ignored_list.extend(ignore_google_auth_backend(src=src, names=names))
return ignored_list
rm_build_dir()
package_providers_dir = get_target_providers_folder()
if os.path.isdir(package_providers_dir):
print(
f"\nRemoving the previous source tree from {package_providers_dir}: ",
end='')
rmtree(package_providers_dir)
print("[green]OK[/]")
print(f"\nCopying the whole source tree to {package_providers_dir}: ",
end='')
copytree(
get_source_providers_folder(),
get_target_providers_folder(),
ignore=ignore_some_files if backports else None,
)
print("[green]OK[/]\n")
def refactor_odbc_package(self):
"""
Fixes to "odbc" providers package.
Copies some of the classes used from core Airflow to "common.utils" package of the
the provider and renames imports to use them from there.
We copy helpers.py and change import as in example diff:
.. code-block:: diff
--- ./airflow/providers/google/cloud/example_dags/example_datacatalog.py
+++ ./airflow/providers/google/cloud/example_dags/example_datacatalog.py
@@ -37,7 +37,7 @@
CloudDataCatalogUpdateTagTemplateOperator,
)
from airflow.utils.dates import days_ago
-from airflow.utils.helpers import chain
+from airflow.providers.odbc.utils.helpers import chain
default_args = {"start_date": days_ago(1)}
"""
def odbc_package_filter(node: LN, capture: Capture,
filename: Filename) -> bool:
return filename.startswith("./airflow/providers/odbc/")
os.makedirs(os.path.join(get_target_providers_folder(), "odbc",
"utils"),
exist_ok=True)
copyfile(
os.path.join(get_source_airflow_folder(), "airflow", "utils",
"__init__.py"),
os.path.join(get_target_providers_package_folder("odbc"), "utils",
"__init__.py"),
)
copy_helper_py_file(
os.path.join(get_target_providers_package_folder("odbc"), "utils",
"helpers.py"))
(self.qry.select_module("airflow.utils.helpers").filter(
callback=odbc_package_filter).rename(
"airflow.providers.odbc.utils.helpers"))
"Copies sources and optionally refactors provider code to be Airflow 1.10 compatible.",
formatter_class=argparse.RawTextHelpFormatter,
)
cli_parser.add_argument(
"--backports",
action='store_true',
help=textwrap.dedent(
"Includes refactoring to prepare backport packages rather than regular ones"
),
)
cli_parser.add_argument(
"--debug",
action='store_true',
help=textwrap.dedent(
"Run bowler refactoring in single process. Makes it debuggable with regular"
" IDE debugger (much slower)"),
)
return cli_parser
if __name__ == '__main__':
parser = get_parser()
args = parser.parse_args()
copy_provider_sources(args.backports)
os.chdir(get_target_providers_folder())
if args.backports:
print("\nRefactoring code to be Airflow 1.10 - compatible\n")
RefactorBackportPackages().do_refactor(in_process=args.debug)
print("\n[green]Refactored code successfully[/]\n")
