def _get_util(self, util): env = {**self.env, **util.get('env', {}), **{"ES_SERVER": var_loader.get_elastic_url()}, **{"KUBEADMIN_PASSWORD": environ.get("KUBEADMIN_PASSWORD", "")}} return BashOperator( task_id=f"{util['name']}", depends_on_past=False, bash_command=f"{constants.root_dag_dir}/scripts/utils/run_scale_ci_diagnosis.sh -w {util['workload']} -c {util['command']} ", retries=3, dag=self.dag, env=env, executor_config=self.exec_config )
def _get_task(self, operation="install", trigger_rule="all_success"): bash_script = "" # Merge all variables, prioritizing Airflow Secrets over git based vars config = { **self.vars, **self.install_secrets, **{ "es_server": var_loader.get_elastic_url() } } config['pullsecret'] = json.dumps( config['openshift_install_pull_secret']) config['version'] = config['openshift_release'] config['build'] = config['openshift_build'] # Required Environment Variables for Install script env = { "SSHKEY_TOKEN": config['sshkey_token'], "OPENSHIFT_WORKER_COUNT": config['openshift_worker_count'], "BAREMETAL_NETWORK_CIDR": config['baremetal_network_cidr'], "BAREMETAL_NETWORK_VLAN": config['baremetal_network_vlan'], "OPENSHIFT_BASE_DOMAIN": config['openshift_base_domain'], "KUBECONFIG_PATH": config['kubeconfig_path'], **self._insert_kube_env() } if operation == "install": config['worker_count'] = config['openshift_worker_count'] bash_script = f"{constants.root_dag_dir}/scripts/install/baremetal_install.sh" else: config['worker_count'] = config['openshift_worker_scaleup_target'] bash_script = f"{constants.root_dag_dir}/scripts/install/baremetal_scaleup.sh" # Dump all vars to json file for Ansible to pick up with open( f"/tmp/{self.version}-{self.platform}-{self.profile}-{operation}-task.json", 'w') as json_file: json.dump(config, json_file, sort_keys=True, indent=4) return BashOperator( task_id=f"{operation}-cluster", depends_on_past=False, bash_command= f"{bash_script} -p {self.platform} -v {self.version} -j /tmp/{self.version}-{self.platform}-{self.profile}-{operation}-task.json -o {operation} ", retries=3, dag=self.dag, trigger_rule=trigger_rule, executor_config=self.exec_config, env=env)
def __init__(self, dag, version, release_stream, latest_release, platform, profile): self.exec_config = var_loader.get_default_executor_config() # General DAG Configuration self.dag = dag self.platform = platform # e.g. aws self.version = version # e.g. 4.6/4.7, major.minor only # true release stream to follow. Nightlies, CI, etc. self.release_stream = release_stream self.latest_release = latest_release # latest relase from the release stream self.profile = profile # e.g. default/ovn # Specific Task Configuration self.vars = var_loader.build_task_vars(task="install", version=version, platform=platform, profile=profile) # Airflow Variables self.ansible_orchestrator = Variable.get("ansible_orchestrator", deserialize_json=True) self.install_secrets = Variable.get(f"openshift_install_config", deserialize_json=True) self.aws_creds = Variable.get("aws_creds", deserialize_json=True) self.gcp_creds = Variable.get("gcp_creds", deserialize_json=True) self.azure_creds = Variable.get("azure_creds", deserialize_json=True) self.ocp_pull_secret = Variable.get("osp_ocp_pull_creds") self.openstack_creds = Variable.get("openstack_creds", deserialize_json=True) # Merge all variables, prioritizing Airflow Secrets over git based vars self.config = { **self.vars, **self.ansible_orchestrator, **self.install_secrets, **self.aws_creds, **self.gcp_creds, **self.azure_creds, **self.openstack_creds, **self.latest_release, **{ "es_server": var_loader.get_elastic_url() } } super().__init__()
def get_index_task(self): env = { **self.env, **{"ES_SERVER": var_loader.get_elastic_url()}, **environ } return BashOperator( task_id=f"index_{self.task}", depends_on_past=False, bash_command=f"{constants.root_dag_dir}/scripts/index.sh ", retries=3, dag=self.dag, trigger_rule="all_done", executor_config=self.exec_config, env=env )
def _get_benchmark(self, benchmark): env = { **self.env, **benchmark.get('env', {}), **{ "ES_SERVER": var_loader.get_elastic_url() } } return BashOperator( task_id=f"{benchmark['name']}", depends_on_past=False, bash_command= f"{constants.root_dag_dir}/scripts/run_benchmark.sh -w {benchmark['workload']} -c {benchmark['command']} ", retries=0, dag=self.dag, env=env, executor_config=self.exec_config)
def _get_task(self, trigger_rule="all_success"): bash_script = f"{constants.root_dag_dir}/scripts/install/baremetal_deploy_webfuse.sh" # Merge all variables, prioritizing Airflow Secrets over git based vars config = { **self.vars, **self.install_secrets, **{ "es_server": var_loader.get_elastic_url() } } config['version'] = config['openshift_release'] config['build'] = config['openshift_build'] # Required Environment Variables for Install script env = { "SSHKEY_TOKEN": config['sshkey_token'], "ORCHESTRATION_HOST": config['provisioner_hostname'], "ORCHESTRATION_USER": config['provisioner_user'], "WEBFUSE_SKIPTAGS": config['webfuse_skiptags'], "WEBFUSE_PLAYBOOK": config['webfuse_playbook'], **self._insert_kube_env() } # Dump all vars to json file for Ansible to pick up with open( f"/tmp/{self.version}-{self.platform}-{self.profile}-task.json", 'w') as json_file: json.dump(config, json_file, sort_keys=True, indent=4) return BashOperator( task_id="deploy-webfuse", depends_on_past=False, bash_command= f"{bash_script} -p {self.platform} -v {self.version} -j /tmp/{self.version}-{self.platform}-{self.profile}-task.json -o deploy_app ", retries=3, dag=self.dag, trigger_rule=trigger_rule, executor_config=self.exec_config, env=env)
def _get_task(self, operation="install", trigger_rule="all_success"): playbook_operations = {} if operation == "install": playbook_operations = { "openshift_cleanup": True, "openshift_debug_config": False, "openshift_install": True, "openshift_post_config": True, "openshift_post_install": True } else: playbook_operations = { "openshift_cleanup": True, "openshift_debug_config": False, "openshift_install": False, "openshift_post_config": False, "openshift_post_install": False } # Merge all variables, prioritizing Airflow Secrets over git based vars config = { **self.vars, **self.ansible_orchestrator, **self.install_secrets, **self.aws_creds, **self.gcp_creds, **self.azure_creds, **playbook_operations, **var_loader.get_latest_release_from_stream( self.release_stream_base_url, self.release_stream), **{ "es_server": var_loader.get_elastic_url() } } git_user = var_loader.get_git_user() if git_user == 'cloud-bulldozer': config[ 'openshift_cluster_name'] = f"ci-{self.version}-{self.platform}-{self.profile}" else: config[ 'openshift_cluster_name'] = f"{git_user}-{self.version}-{self.platform}-{self.profile}" config['dynamic_deploy_path'] = f"{config['openshift_cluster_name']}" config[ 'kubeconfig_path'] = f"/root/{config['dynamic_deploy_path']}/auth/kubeconfig" # Required Environment Variables for Install script env = { "SSHKEY_TOKEN": config['sshkey_token'], "ORCHESTRATION_HOST": config['orchestration_host'], "ORCHESTRATION_USER": config['orchestration_user'], "OPENSHIFT_CLUSTER_NAME": config['openshift_cluster_name'], "DEPLOY_PATH": config['dynamic_deploy_path'], "KUBECONFIG_NAME": f"{self.version}-{self.platform}-{self.profile}-kubeconfig", **self._insert_kube_env() } # Dump all vars to json file for Ansible to pick up with open( f"/tmp/{self.version}-{self.platform}-{self.profile}-{operation}-task.json", 'w') as json_file: json.dump(config, json_file, sort_keys=True, indent=4) return BashOperator( task_id=f"{operation}", depends_on_past=False, bash_command= f"{constants.root_dag_dir}/scripts/install_cluster.sh -p {self.platform} -v {self.version} -j /tmp/{self.version}-{self.platform}-{self.profile}-{operation}-task.json -o {operation}", retries=3, dag=self.dag, trigger_rule=trigger_rule, executor_config=self.exec_config, env=env)