def _add_provider_info_from_local_source_file(self, path,
package_name) -> None:
"""
Parses found provider.yaml file and adds found provider to the dictionary.
:param path: full file path of the provider.yaml file
:param package_name: name of the package
"""
try:
log.debug("Loading %s from %s", package_name, path)
with open(path) as provider_yaml_file:
provider_info = yaml.safe_load(provider_yaml_file)
self._provider_schema_validator.validate(provider_info)
version = provider_info['versions'][0]
if package_name not in self._provider_dict:
self._provider_dict[package_name] = ProviderInfo(
version, provider_info, 'source')
else:
log.warning(
"The providers for package '%s' could not be registered because providers for that "
"package name have already been registered",
package_name,
)
except Exception as e:
log.warning("Error when loading '%s'", path, exc_info=e)
def _parse_yaml_file(
file_path: str) -> Tuple[Dict[str, List[str]], List[FileSyntaxError]]:
"""
Parse a file in the YAML format.
:param file_path: The location of the file that will be processed.
:type file_path: str
:return: Tuple with mapping of key and list of values and list of syntax errors
"""
with open(file_path) as f:
content = f.read()
if not content:
return {}, [FileSyntaxError(line_no=1, message="The file is empty.")]
try:
secrets = yaml.safe_load(content)
except yaml.MarkedYAMLError as e:
return {}, [
FileSyntaxError(line_no=e.problem_mark.line, message=str(e))
]
if not isinstance(secrets, dict):
return {}, [
FileSyntaxError(line_no=1,
message="The file should contain the object.")
]
return secrets, []
def deserialize_model_file(path: str) -> k8s.V1Pod:
"""
:param path: Path to the file
:return: a kubernetes.client.models.V1Pod
Unfortunately we need access to the private method
``_ApiClient__deserialize_model`` from the kubernetes client.
This issue is tracked here; https://github.com/kubernetes-client/python/issues/977.
"""
if os.path.exists(path):
with open(path) as stream:
pod = yaml.safe_load(stream)
else:
pod = yaml.safe_load(path)
return PodGenerator.deserialize_model_dict(pod)
def _load_body_to_dict(body):
try:
body_dict = yaml.safe_load(body)
except yaml.YAMLError as e:
raise AirflowException(
f"Exception when loading resource definition: {e}\n")
return body_dict
def default_config_yaml() -> List[dict]:
"""
Read Airflow configs from YAML file
:return: Python dictionary containing configs & their info
"""
with open(_default_config_file_path('config.yml')) as config_file:
return yaml.safe_load(config_file)
def prepare_template(self) -> None:
# if no file is specified, skip
if not isinstance(self.build_raw, str):
return
with open(self.build_raw) as file:
if any(self.build_raw.endswith(ext) for ext in ['.yaml', '.yml']):
self.build = yaml.safe_load(file.read())
if self.build_raw.endswith('.json'):
self.build = json.loads(file.read())
def default_config_yaml() -> dict:
"""
Read Airflow configs from YAML file
:return: Python dictionary containing configs & their info
"""
import airflow.utils.yaml as yaml
with open(_default_config_file_path('config.yml')) as config_file:
return yaml.safe_load(config_file)
def _get_kube_config_loader_for_yaml_file(filename, **kwargs) -> Optional[RefreshKubeConfigLoader]:
"""
Adapted from the upstream _get_kube_config_loader_for_yaml_file function, changed
KubeConfigLoader to RefreshKubeConfigLoader
"""
with open(filename) as f:
return RefreshKubeConfigLoader(
config_dict=yaml.safe_load(f),
config_base_path=os.path.abspath(os.path.dirname(filename)),
**kwargs,
)
