def _format_connections(conns: List[Connection], fmt: str) -> str:
if fmt == '.env':
connections_env = ""
for conn in conns:
connections_env += f"{conn.conn_id}={conn.get_uri()}\n"
return connections_env
connections_dict = {}
for conn in conns:
connections_dict[conn.conn_id] = {
'conn_type': conn.conn_type,
'description': conn.description,
'host': conn.host,
'login': conn.login,
'password': conn.password,
'schema': conn.schema,
'port': conn.port,
'extra': conn.extra,
}
if fmt == '.yaml':
return yaml.dump(connections_dict)
if fmt == '.json':
return json.dumps(connections_dict, indent=2)
return json.dumps(connections_dict)
def _format_connections(conns: List[Connection], file_format: str,
serialization_format: str) -> str:
if serialization_format == 'json':
serializer_func = lambda x: json.dumps(_connection_to_dict(x))
elif serialization_format == 'uri':
serializer_func = Connection.get_uri
else:
raise SystemExit(
f"Received unexpected value for `--serialization-format`: {serialization_format!r}"
)
if file_format == '.env':
connections_env = ""
for conn in conns:
connections_env += f"{conn.conn_id}={serializer_func(conn)}\n"
return connections_env
connections_dict = {}
for conn in conns:
connections_dict[conn.conn_id] = _connection_to_dict(conn)
if file_format == '.yaml':
return yaml.dump(connections_dict)
if file_format == '.json':
return json.dumps(connections_dict, indent=2)
return json.dumps(connections_dict)
def generate_pod_yaml(args):
"""Generates yaml files for each task in the DAG. Used for testing output of KubernetesExecutor"""
execution_date = args.execution_date
dag = get_dag(subdir=args.subdir, dag_id=args.dag_id)
yaml_output_path = args.output_path
dr = DagRun(dag.dag_id, execution_date=execution_date)
kube_config = KubeConfig()
for task in dag.tasks:
ti = TaskInstance(task, None)
ti.dag_run = dr
pod = PodGenerator.construct_pod(
dag_id=args.dag_id,
task_id=ti.task_id,
pod_id=create_pod_id(args.dag_id, ti.task_id),
try_number=ti.try_number,
kube_image=kube_config.kube_image,
date=ti.execution_date,
args=ti.command_as_list(),
pod_override_object=PodGenerator.from_obj(ti.executor_config),
scheduler_job_id="worker-config",
namespace=kube_config.executor_namespace,
base_worker_pod=PodGenerator.deserialize_model_file(kube_config.pod_template_file),
)
pod_mutation_hook(pod)
api_client = ApiClient()
date_string = pod_generator.datetime_to_label_safe_datestring(execution_date)
yaml_file_name = f"{args.dag_id}_{ti.task_id}_{date_string}.yml"
os.makedirs(os.path.dirname(yaml_output_path + "/airflow_yaml_output/"), exist_ok=True)
with open(yaml_output_path + "/airflow_yaml_output/" + yaml_file_name, "w") as output:
sanitized_pod = api_client.sanitize_for_serialization(pod)
output.write(yaml.dump(sanitized_pod))
print(f"YAML output can be found at {yaml_output_path}/airflow_yaml_output/")
def dry_run(self) -> None:
"""
Prints out the pod definition that would be created by this operator.
Does not include labels specific to the task instance (since there isn't
one in a dry_run) and excludes all empty elements.
"""
pod = self.build_pod_request_obj()
print(yaml.dump(_prune_dict(pod.to_dict(), mode='strict')))
def print_as_yaml(self, data: Dict):
"""Renders dict as yaml text representation"""
yaml_content = yaml.dump(data)
self.print(Syntax(yaml_content, "yaml", theme="ansi_dark"),
soft_wrap=True)
def generate_config_file(
self,
eks_cluster_name: str,
pod_namespace: Optional[str],
pod_username: Optional[str] = None,
pod_context: Optional[str] = None,
) -> Generator[str, None, None]:
"""
Writes the kubeconfig file given an EKS Cluster.
:param eks_cluster_name: The name of the cluster to generate kubeconfig file for.
:param pod_namespace: The namespace to run within kubernetes.
"""
if pod_username:
warnings.warn(
"This pod_username parameter is deprecated, because changing the value does not make any "
"visible changes to the user.",
DeprecationWarning,
stacklevel=2,
)
if pod_context:
warnings.warn(
"This pod_context parameter is deprecated, because changing the value does not make any "
"visible changes to the user.",
DeprecationWarning,
stacklevel=2,
)
# Set up the client
eks_client = self.conn
# Get cluster details
cluster = eks_client.describe_cluster(name=eks_cluster_name)
cluster_cert = cluster["cluster"]["certificateAuthority"]["data"]
cluster_ep = cluster["cluster"]["endpoint"]
cluster_config = {
"apiVersion":
"v1",
"kind":
"Config",
"clusters": [{
"cluster": {
"server": cluster_ep,
"certificate-authority-data": cluster_cert
},
"name": eks_cluster_name,
}],
"contexts": [{
"context": {
"cluster": eks_cluster_name,
"namespace": pod_namespace,
"user": _POD_USERNAME,
},
"name": _CONTEXT_NAME,
}],
"current-context":
_CONTEXT_NAME,
"preferences": {},
"users": [{
"name": _POD_USERNAME,
"user": {
"exec": {
"apiVersion":
AUTHENTICATION_API_VERSION,
"command":
sys.executable,
"args": [
"-m",
"airflow.providers.amazon.aws.utils.eks_get_token",
*(["--region-name", self.region_name]
if self.region_name is not None else []),
*(["--aws-conn-id", self.aws_conn_id]
if self.aws_conn_id is not None else []),
"--cluster-name",
eks_cluster_name,
],
"env": [{
"name": "AIRFLOW__LOGGING__LOGGING_LEVEL",
"value": "FATAL",
}],
"interactiveMode":
"Never",
}
},
}],
}
config_text = yaml.dump(cluster_config, default_flow_style=False)
with tempfile.NamedTemporaryFile(mode='w') as config_file:
config_file.write(config_text)
config_file.flush()
yield config_file.name
