def _on_config_inited(app, config):
del app
jinja_context = getattr(config, 'jinja_contexts', None) or {}
jinja_context['providers_ctx'] = {'providers': load_package_data()}
config.jinja_contexts = jinja_context
def _prepare_transfer_data(tags: Optional[Set[str]]):
package_data = load_package_data()
all_operators_by_integration = _prepare_resource_index(
package_data, "integrations")
# Add edge case
for name in ["SQL", "Local"]:
all_operators_by_integration[name] = {"integration-name": name}
all_transfers = [{
**transfer,
'package-name':
provider['package-name'],
'source-integration':
all_operators_by_integration[transfer['source-integration-name']],
'target-integration':
all_operators_by_integration[transfer['target-integration-name']],
} for provider in package_data
for transfer in provider.get("transfers", [])]
if tags is None:
to_display_transfers = all_transfers
else:
to_display_transfers = [
transfer for transfer in all_transfers
if tags.intersection(transfer['source-integration'].get(
'tags', set())) or tags.intersection(
transfer['target-integration'].get('tags', set()))
]
for transfer in to_display_transfers:
if 'how-to-guide' not in transfer:
continue
transfer['how-to-guide'] = _docs_path(transfer['how-to-guide'])
return to_display_transfers
def _prepare_operators_data(tags: Optional[Set[str]]):
package_data = load_package_data()
all_integrations = _prepare_resource_index(package_data, "integrations")
if tags is None:
to_display_integration = all_integrations
else:
to_display_integration = [
integration for integration in all_integrations.values() if tags.intersection(integration["tags"])
]
all_operators_by_integration = _prepare_resource_index(package_data, "operators")
all_hooks_by_integration = _prepare_resource_index(package_data, "hooks")
all_sensors_by_integration = _prepare_resource_index(package_data, "hooks")
results = []
for integration in to_display_integration:
item = {
"integration": integration,
}
operators = all_operators_by_integration.get(integration['integration-name'])
sensors = all_sensors_by_integration.get(integration['integration-name'])
hooks = all_hooks_by_integration.get(integration['integration-name'])
if 'how-to-guide' in item['integration']:
item['integration']['how-to-guide'] = [_docs_path(d) for d in item['integration']['how-to-guide']]
if operators:
item['operators'] = operators
if sensors:
item['hooks'] = sensors
if hooks:
item['hooks'] = hooks
if operators or sensors or hooks:
results.append(item)
return sorted(results, key=lambda d: d["integration"]["integration-name"].lower())
def _prepare_extra_links_data():
package_data = load_package_data()
all_extra_links = {}
for provider in package_data:
extra_link_list = provider.get("extra-links")
if extra_link_list:
package_name = provider['package-name']
all_extra_links[package_name] = {
'name': provider['name'],
'extra_links': extra_link_list,
}
return all_extra_links
def _prepare_connections_data():
package_data = load_package_data()
all_connections = {}
for provider in package_data:
connections_list = provider.get("connection-types")
if connections_list:
package_name = provider['package-name']
all_connections[package_name] = {
'name': provider['name'],
'connection_types': connections_list,
}
return all_connections
def _prepare_secrets_backend_data():
package_data = load_package_data()
all_secret_backends = {}
for provider in package_data:
secret_backends_list = provider.get("secrets-backends")
if secret_backends_list:
package_name = provider['package-name']
all_secret_backends[package_name] = {
'name': provider['name'],
'secrets_backends': secret_backends_list,
}
return all_secret_backends
def _prepare_auth_backend_data():
package_data = load_package_data()
all_auth_backends = {}
for provider in package_data:
auth_backends_list = provider.get("auth-backends")
if auth_backends_list:
package_name = provider['package-name']
all_auth_backends[package_name] = {
'name': provider['name'],
'auth_backends': auth_backends_list
}
return all_auth_backends
def _prepare_logging_data():
package_data = load_package_data()
all_logging = {}
for provider in package_data:
logging_handlers = provider.get("logging")
if logging_handlers:
package_name = provider['package-name']
all_logging[package_name] = {
'name': provider['name'],
'handlers': logging_handlers
}
return all_logging
def _generate_provider_intersphinx_mapping():
airflow_mapping = {}
for_production = os.environ.get('AIRFLOW_FOR_PRODUCTION',
'false') == 'true'
current_version = 'stable' if for_production else 'latest'
for provider in load_package_data():
package_name = provider['package-name']
if os.environ.get('AIRFLOW_PACKAGE_NAME') == package_name:
continue
provider_base_url = f'/docs/{package_name}/{current_version}/'
doc_inventory = f'{DOCS_DIR}/_build/docs/{package_name}/{current_version}/objects.inv'
cache_inventory = f'{DOCS_DIR}/_inventory_cache/{package_name}/objects.inv'
# Skip adding the mapping if the path does not exist
if not os.path.exists(doc_inventory) and not os.path.exists(
cache_inventory):
continue
airflow_mapping[package_name] = (
# base URI
provider_base_url,
(doc_inventory
if os.path.exists(doc_inventory) else cache_inventory, ),
)
for pkg_name in ["apache-airflow", 'helm-chart']:
if os.environ.get('AIRFLOW_PACKAGE_NAME') == pkg_name:
continue
doc_inventory = f'{DOCS_DIR}/_build/docs/{pkg_name}/{current_version}/objects.inv'
cache_inventory = f'{DOCS_DIR}/_inventory_cache/{pkg_name}/objects.inv'
airflow_mapping[pkg_name] = (
# base URI
f'/docs/{pkg_name}/latest/',
(doc_inventory
if os.path.exists(doc_inventory) else cache_inventory, ),
)
for pkg_name in ['apache-airflow-providers', 'docker-stack']:
if os.environ.get('AIRFLOW_PACKAGE_NAME') == pkg_name:
continue
doc_inventory = f'{DOCS_DIR}/_build/docs/{pkg_name}/objects.inv'
cache_inventory = f'{DOCS_DIR}/_inventory_cache/{pkg_name}/objects.inv'
airflow_mapping[pkg_name] = (
# base URI
f'/docs/{pkg_name}/',
(doc_inventory
if os.path.exists(doc_inventory) else cache_inventory, ),
)
return airflow_mapping
def _generate_provider_intersphinx_mapping():
airflow_mapping = {}
for provider in load_package_data():
package_name = provider['package-name']
if os.environ.get('AIRFLOW_PACKAGE_NAME') == package_name:
continue
# For local build and S3, use relative URLS.
# For RTD, use absolute URLs
if IS_RTD:
provider_base_url = f"{S3_DOC_URL}/docs/{package_name}/latest/"
else:
provider_base_url = f'/docs/{package_name}/latest/'
airflow_mapping[package_name] = (
# base URI
provider_base_url,
# Index locations list
# If passed None, this will try to fetch the index from `[base_url]/objects.inv`
# If we pass a path containing `://` then we will try to index from the given address.
# Otherwise, it will try to read the local file
#
# In this case, the local index will be read. If unsuccessful, the remote index
# will be fetched.
(
f'{DOCS_DIR}/_build/docs/{package_name}/latest/objects.inv',
f'{S3_DOC_URL}/docs/{package_name}/latest/objects.inv',
),
)
if os.environ.get('AIRFLOW_PACKAGE_NAME') != 'apache-airflow':
airflow_mapping['apache-airflow'] = (
# base URI
'/docs/apache-airflow/latest/',
# Index locations list
# If passed None, this will try to fetch the index from `[base_url]/objects.inv`
# If we pass a path containing `://` then we will try to index from the given address.
# Otherwise, it will try to read the local file
#
# In this case, the local index will be read. If unsuccessful, the remote index
# will be fetched.
(
f'{DOCS_DIR}/_build/docs/apache-airflow/latest/objects.inv',
'https://airflow.readthedocs.io/en/latest/objects.inv',
),
)
return airflow_mapping
def _generate_provider_intersphinx_mapping():
airflow_mapping = {}
for_production = os.environ.get('AIRFLOW_FOR_PRODUCTION',
'false') == 'true'
current_version = 'stable' if for_production else 'latest'
for provider in load_package_data():
package_name = provider['package-name']
if os.environ.get('AIRFLOW_PACKAGE_NAME') == package_name:
continue
provider_base_url = f'/docs/{package_name}/{current_version}/'
doc_inventory = f'{DOCS_DIR}/_build/docs/{package_name}/{current_version}/objects.inv'
cache_inventory = f'{DOCS_DIR}/_inventory_cache/{package_name}/objects.inv'
airflow_mapping[package_name] = (
# base URI
provider_base_url,
(doc_inventory
if os.path.exists(doc_inventory) else cache_inventory, ),
)
if os.environ.get('AIRFLOW_PACKAGE_NAME') != 'apache-airflow':
doc_inventory = f'{DOCS_DIR}/_build/docs/apache-airflow/{current_version}/objects.inv'
cache_inventory = f'{DOCS_DIR}/_inventory_cache/apache-airflow/objects.inv'
airflow_mapping['apache-airflow'] = (
# base URI
f'/docs/apache-airflow/{current_version}/',
(doc_inventory
if os.path.exists(doc_inventory) else cache_inventory, ),
)
if os.environ.get('AIRFLOW_PACKAGE_NAME') != 'apache-airflow-providers':
doc_inventory = f'{DOCS_DIR}/_build/docs/apache-airflow-providers/objects.inv'
cache_inventory = f'{DOCS_DIR}/_inventory_cache/apache-airflow-providers/objects.inv'
airflow_mapping['apache-airflow-providers'] = (
# base URI
'/docs/apache-airflow-providers/',
(doc_inventory
if os.path.exists(doc_inventory) else cache_inventory, ),
)
return airflow_mapping
CONF_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__)))
INVENTORY_CACHE_DIR = os.path.join(CONF_DIR, '_inventory_cache')
ROOT_DIR = os.path.abspath(os.path.join(CONF_DIR, os.pardir))
FOR_PRODUCTION = os.environ.get('AIRFLOW_FOR_PRODUCTION', 'false') == 'true'
# By default (e.g. on RTD), build docs for `airflow` package
PACKAGE_NAME = os.environ.get('AIRFLOW_PACKAGE_NAME', 'apache-airflow')
PACKAGE_DIR: Optional[str]
if PACKAGE_NAME == 'apache-airflow':
PACKAGE_DIR = os.path.join(ROOT_DIR, 'airflow')
PACKAGE_VERSION = airflow.__version__
elif PACKAGE_NAME.startswith('apache-airflow-providers-'):
from provider_yaml_utils import load_package_data # pylint: disable=no-name-in-module
ALL_PROVIDER_YAMLS = load_package_data()
try:
CURRENT_PROVIDER = next(
provider_yaml for provider_yaml in ALL_PROVIDER_YAMLS
if provider_yaml['package-name'] == PACKAGE_NAME)
except StopIteration:
raise Exception(
f"Could not find provider.yaml file for package: {PACKAGE_NAME}")
PACKAGE_DIR = CURRENT_PROVIDER['package-dir']
PACKAGE_VERSION = 'master'
else:
PACKAGE_DIR = None
PACKAGE_VERSION = 'master'
# Adds to environment variables for easy access from other plugins like airflow_intersphinx.
os.environ['AIRFLOW_PACKAGE_NAME'] = PACKAGE_NAME
if PACKAGE_DIR:
def _generate_provider_intersphinx_mapping():
airflow_mapping = {}
for_production = os.environ.get('AIRFLOW_FOR_PRODUCTION',
'false') == 'true'
current_version = 'stable' if for_production else 'latest'
for provider in load_package_data():
package_name = provider['package-name']
if os.environ.get('AIRFLOW_PACKAGE_NAME') == package_name:
continue
provider_base_url = f'/docs/{package_name}/{current_version}/'
airflow_mapping[package_name] = (
# base URI
provider_base_url,
# Index locations list
# If passed None, this will try to fetch the index from `[base_url]/objects.inv`
# If we pass a path containing `://` then we will try to index from the given address.
# Otherwise, it will try to read the local file
#
# In this case, the local index will be read. If unsuccessful, the remote index
# will be fetched.
(
f'{DOCS_DIR}/_build/docs/{package_name}/{current_version}/objects.inv',
f'{S3_DOC_URL}/docs/{package_name}/latest/objects.inv',
),
)
if os.environ.get('AIRFLOW_PACKAGE_NAME') != 'apache-airflow':
airflow_mapping['apache-airflow'] = (
# base URI
f'/docs/apache-airflow/{current_version}/',
# Index locations list
# If passed None, this will try to fetch the index from `[base_url]/objects.inv`
# If we pass a path containing `://` then we will try to index from the given address.
# Otherwise, it will try to read the local file
#
# In this case, the local index will be read. If unsuccessful, the remote index
# will be fetched.
(
f'{DOCS_DIR}/_build/docs/apache-airflow/{current_version}/objects.inv',
f'{S3_DOC_URL}/docs/apache-airflow/latest/objects.inv',
),
)
if os.environ.get('AIRFLOW_PACKAGE_NAME') != 'apache-airflow-providers':
airflow_mapping['apache-airflow-providers'] = (
# base URI
'/docs/apache-airflow-providers/',
# Index locations list
# If passed None, this will try to fetch the index from `[base_url]/objects.inv`
# If we pass a path containing `://` then we will try to index from the given address.
# Otherwise, it will try to read the local file
#
# In this case, the local index will be read. If unsuccessful, the remote index
# will be fetched.
(
f'{DOCS_DIR}/_build/docs/apache-airflow-providers/objects.inv',
f'{S3_DOC_URL}/docs/apache-airflow-providers/objects.inv',
),
)
return airflow_mapping