def get_extras_from_setup() -> Dict[str, str]:
"""
Returns a dict of regular extras from setup (with value = '' for non-provider extra and '*' for
provider extra
"""
all_regular_extras = set(EXTRAS_REQUIREMENTS.keys()) - set(EXTRAS_DEPRECATED_ALIASES.keys())
setup_extra_dict = {}
for setup_regular_extra in all_regular_extras:
setup_extra_dict[setup_regular_extra] = '*' if setup_regular_extra in EXTRAS_WITH_PROVIDERS else ''
return setup_extra_dict
def checks_extra_with_providers_exist() -> None:
from setup import EXTRAS_REQUIREMENTS, EXTRAS_PROVIDERS_PACKAGES # noqa # isort:skip
message = 'Check if all extras have providers defined in: EXTRAS_PROVIDERS_PACKAGES'
local_error = False
for key in EXTRAS_REQUIREMENTS.keys(): # noqa
if key not in EXTRAS_PROVIDERS_PACKAGES.keys(): # noqa
if not local_error:
local_error = True
print(f"Extra {key} NOK")
errors.append(
f"ERROR in {message}. The {key} extras is missing there."
" If you do not want to install any providers with this extra set it to []"
)
if not local_error:
print(f"{message} is ok")
sys.path.append(str(AIRFLOW_SOURCES_DIR))
RST_HEADER = ' .. START EXTRAS HERE'
RST_FOOTER = ' .. END EXTRAS HERE'
INSTALL_HEADER = '# START EXTRAS HERE'
INSTALL_FOOTER = '# END EXTRAS HERE'
CONSTANTS_HEADER = '# START EXTRAS HERE'
CONSTANTS_FOOTER = '# END EXTRAS HERE'
DEFAULT_EXTRAS = (
"amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,"
"google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,"
"postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv")
if __name__ == '__main__':
install_file_path = AIRFLOW_SOURCES_DIR / 'INSTALL'
contributing_file_path = AIRFLOW_SOURCES_DIR / 'CONTRIBUTING.rst'
global_constants_file_path = (AIRFLOW_SOURCES_DIR / "dev" / "breeze" /
"src" / "airflow_breeze" /
"global_constants.py")
extras_list = wrap(", ".join(EXTRAS_REQUIREMENTS.keys()), 100)
extras_list = [line + "\n" for line in extras_list]
extras_code = [f" {extra}\n" for extra in EXTRAS_REQUIREMENTS.keys()]
insert_documentation(install_file_path, extras_list, INSTALL_HEADER,
INSTALL_FOOTER)
insert_documentation(contributing_file_path, extras_list, RST_HEADER,
RST_FOOTER)
def get_extras_from_setup() -> Set[str]:
"""Returns a set of regular (non-deprecated) extras from setup."""
return set(EXTRAS_REQUIREMENTS.keys()) - set(EXTRAS_DEPRECATED_ALIASES.keys())
footer: str):
with open(file_path) as documentation_file:
replacing = False
result: List[str] = []
text = documentation_file.readlines()
for line in text:
if line.startswith(header):
replacing = True
result.append(line)
result.append("\n")
result += content
result.append("\n")
if line.startswith(footer):
replacing = False
if not replacing:
result.append(line)
with open(file_path, "w") as documentation_file:
documentation_file.write("".join(result))
if __name__ == '__main__':
install_file_path = os.path.join(AIRFLOW_SOURCES_DIR, 'INSTALL')
contributing_file_path = os.path.join(AIRFLOW_SOURCES_DIR,
'CONTRIBUTING.rst')
extras = wrap(", ".join(EXTRAS_REQUIREMENTS.keys()), 100)
extras = [line + "\n" for line in extras]
insert_documentation(install_file_path, extras, INSTALL_HEADER,
INSTALL_FOOTER)
insert_documentation(contributing_file_path, extras, RST_HEADER,
RST_FOOTER)