def __init__(self, dag, config: DagConfig, release: OpenshiftRelease): # General DAG Configuration self.dag = dag self.release = release self.config = config self.exec_config = executor.get_executor_config_with_cluster_access( self.config, self.release) # Specific Task Configuration self.env = { "REL_PLATFORM": self.release.platform, "THANOS_RECEIVER_URL": var_loader.get_secret("thanos_receiver_url"), "LOKI_RECEIVER_URL": var_loader.get_secret("loki_receiver_url") } if self.release.platform == "baremetal": self.install_vars = var_loader.build_task_vars(release, task="install") self.baremetal_install_secrets = var_loader.get_secret( f"baremetal_openshift_install_config", deserialize_json=True) self.config = { **self.install_vars, **self.baremetal_install_secrets } self.env = { **self.env, "SSHKEY_TOKEN": self.config['sshkey_token'], "ORCHESTRATION_USER": self.config['provisioner_user'], "ORCHESTRATION_HOST": self.config['provisioner_hostname'] }
def __init__(self, dag, config: DagConfig, release: OpenshiftRelease): # General DAG Configuration self.dag = dag self.release = release self.dag_config = config self.release_name = release.get_release_name(delimiter="-") self.cluster_name = release._generate_cluster_name() # Airflow Variables self.ansible_orchestrator = var_loader.get_secret( "ansible_orchestrator", deserialize_json=True) self.exec_config = executor.get_default_executor_config( self.dag_config) # Merge all variables, prioritizing Airflow Secrets over git based vars self.config = { **self.ansible_orchestrator, **{ "es_server": var_loader.get_secret('elasticsearch'), "thanos_receiver_url": var_loader.get_secret('thanos_receiver_url'), "loki_receiver_url": var_loader.get_secret('loki_receiver_url') } } self.env = { "OPENSHIFT_CLUSTER_NAME": self.cluster_name, "KUBECONFIG_NAME": f"{self.release_name}-kubeconfig", "KUBEADMIN_NAME": f"{self.release_name}-kubeadmin", **self._insert_kube_env() }
def __init__(self, dag, config: DagConfig, release: OpenshiftRelease, task_group="benchmarks"): # General DAG Configuration self.dag = dag self.release = release self.task_group = task_group self.dag_config = config self.exec_config = executor.get_executor_config_with_cluster_access( self.dag_config, self.release) self.snappy_creds = var_loader.get_secret("snappy_creds", deserialize_json=True) self.es_gold = var_loader.get_secret("es_gold") self.es_server_baseline = var_loader.get_secret("es_server_baseline") # Specific Task Configuration self.vars = var_loader.build_task_vars(release=self.release, task=self.task_group) self.git_name = self._git_name() self.env = { "SNAPPY_DATA_SERVER_URL": self.snappy_creds['server'], "SNAPPY_DATA_SERVER_USERNAME": self.snappy_creds['username'], "SNAPPY_DATA_SERVER_PASSWORD": self.snappy_creds['password'], "SNAPPY_USER_FOLDER": self.git_name, "PLATFORM": self.release.platform, "TASK_GROUP": self.task_group, "ES_GOLD": self.es_gold, "ES_SERVER_BASELINE": self.es_server_baseline } self.env.update(self.dag_config.dependencies) if self.release.platform == "baremetal": self.install_vars = var_loader.build_task_vars(release, task="install") self.baremetal_install_secrets = var_loader.get_secret( f"baremetal_openshift_install_config", deserialize_json=True) self.config = { **self.install_vars, **self.baremetal_install_secrets } self.env = { **self.env, "SSHKEY_TOKEN": self.config['sshkey_token'], "ORCHESTRATION_USER": self.config['provisioner_user'], "ORCHESTRATION_HOST": self.config['provisioner_hostname'] }
def __init__(self, dag, config: DagConfig, release: OpenshiftRelease): # General DAG Configuration self.dag = dag self.release = release self.config = config self.release_name = release.get_release_name(delimiter="-") self.aws_creds = var_loader.get_secret("aws_creds", deserialize_json=True) # Specific Task Configuration self.vars = var_loader.build_task_vars( release=self.release, task="install") self.all_vars = { **self.vars, **self.aws_creds, } # Dump all vars to json file for Ansible to pick up with open(f"/tmp/{self.release_name}-postinstall-task.json", 'w') as json_file: json.dump(self.all_vars, json_file, sort_keys=True, indent=4) super().__init__() self.exec_config = executor.get_executor_config_with_cluster_access(self.config, self.release, executor_image="airflow-managed-services")
def _get_benchmark(self, benchmark): env = { **self.env, **benchmark.get('env', {}), **{ "ES_SERVER": var_loader.get_secret('elasticsearch'), "KUBEADMIN_PASSWORD": environ.get("KUBEADMIN_PASSWORD", "") } } task_prefix = f"{self.task_group}-" task = BashOperator( task_id= f"{task_prefix if self.task_group != 'benchmarks' else ''}{benchmark['name']}", depends_on_past=False, bash_command= f"{constants.root_dag_dir}/scripts/run_benchmark.sh -w {benchmark['workload']} -c {benchmark['command']} ", retries=0, trigger_rule=benchmark.get("trigger_rule", "all_success"), dag=self.dag, env=env, do_xcom_push=True, execution_timeout=timedelta(seconds=21600), executor_config=self.exec_config) self._add_indexer(task) return task
def __init__(self, dag, config: DagConfig, release: OpenshiftRelease): super().__init__(dag, config, release) self.exec_config = executor.get_executor_config_with_cluster_access( self.dag_config, self.release, executor_image="airflow-managed-services") self.hypershift_pull_secret = var_loader.get_secret( "hypershift_pull_secret")
def __init__(self, root_dag_dir): with open(f"{root_dag_dir}/manifest.yaml") as manifest_file: try: self.yaml = yaml.safe_load(manifest_file) except yaml.YAMLError as exc: print(exc) self.releases = [] self.release_stream_base_url = var_loader.get_secret("release_stream_base_url") self.get_latest_releases()
def _get_task(self, operation="install", trigger_rule="all_success"): bash_script = "" # Merge all variables, prioritizing Airflow Secrets over git based vars config = { **self.vars, **self.baremetal_install_secrets, **{ "es_server": var_loader.get_secret('elasticsearch') } } config['pullsecret'] = json.dumps( config['openshift_install_pull_secret']) config['version'] = self.release.release_stream config['build'] = self.release.build # Required Environment Variables for Install script env = { "SSHKEY_TOKEN": config['sshkey_token'], "OPENSHIFT_WORKER_COUNT": config['openshift_worker_count'], "BAREMETAL_NETWORK_CIDR": config['baremetal_network_cidr'], "BAREMETAL_NETWORK_VLAN": config['baremetal_network_vlan'], "OPENSHIFT_BASE_DOMAIN": config['openshift_base_domain'], "JETSKI_SKIPTAGS": config['jetski_skiptags'], "KUBECONFIG_PATH": config['kubeconfig_path'], "KUBECONFIG_NAME": f"{self.release_name}-kubeconfig", "KUBEADMIN_NAME": f"{self.release_name}-kubeadmin", "CURRENT_WORKER_COUNT": config['openshift_worker_count'], "TARGET_WORKER_COUNT": config['openshift_worker_scaleup_target'], "SCALE_STEP": config['openshift_worker_scale_step'], **self._insert_kube_env() } if operation == "install": config['worker_count'] = config['openshift_worker_count'] bash_script = f"{constants.root_dag_dir}/scripts/install/baremetal_install.sh" else: config['worker_count'] = config['openshift_worker_scaleup_target'] bash_script = f"{constants.root_dag_dir}/scripts/install/baremetal_scaleup.sh" # Dump all vars to json file for Ansible to pick up with open(f"/tmp/{self.release_name}-{operation}-task.json", 'w') as json_file: json.dump(config, json_file, sort_keys=True, indent=4) return BashOperator( task_id=f"{operation}-cluster", depends_on_past=False, bash_command= f"{bash_script} -p {self.release.platform} -v {self.release.version} -j /tmp/{self.release_name}-{operation}-task.json -o {operation} ", retries=3, dag=self.dag, trigger_rule=trigger_rule, executor_config=self.exec_config, env=env)
def __init__(self, dag, config: DagConfig, release: OpenshiftRelease, task_group="benchmarks"): # General DAG Configuration self.dag = dag self.release = release self.task_group = task_group self.dag_config = config self.exec_config = executor.get_executor_config_with_cluster_access(self.dag_config, self.release) self.snappy_creds = var_loader.get_secret("snappy_creds", deserialize_json=True) self.es_gold = var_loader.get_secret("es_gold") self.es_server_baseline = var_loader.get_secret("es_server_baseline") # Specific Task Configuration self.vars = var_loader.build_task_vars( release=self.release, task=self.task_group) self.git_name=self._git_name() self.env = { "SNAPPY_DATA_SERVER_URL": self.snappy_creds['server'], "SNAPPY_DATA_SERVER_USERNAME": self.snappy_creds['username'], "SNAPPY_DATA_SERVER_PASSWORD": self.snappy_creds['password'], "SNAPPY_USER_FOLDER": self.git_name, "PLATFORM": self.release.platform, "TASK_GROUP": self.task_group, "ES_GOLD": self.es_gold, "ES_SERVER_BASELINE": self.es_server_baseline } self.env.update(self.dag_config.dependencies) if self.release.platform == "baremetal": self.install_vars = var_loader.build_task_vars( release, task="install") self.baremetal_install_secrets = var_loader.get_secret( f"baremetal_openshift_install_config", deserialize_json=True) self.config = { **self.install_vars, **self.baremetal_install_secrets } self.env = { **self.env, "SSHKEY_TOKEN": self.config['sshkey_token'], "ORCHESTRATION_USER": self.config['provisioner_user'], "ORCHESTRATION_HOST": self.config['provisioner_hostname'] } if self.release.platform == "rosa": self.rosa_creds = var_loader.get_secret("rosa_creds", deserialize_json=True) self.aws_creds = var_loader.get_secret("aws_creds", deserialize_json=True) self.environment = self.vars["environment"] if "environment" in self.vars else "staging" self.env = { **self.env, "ROSA_CLUSTER_NAME": release._generate_cluster_name(), "ROSA_ENVIRONMENT": self.environment, "ROSA_TOKEN": self.rosa_creds['rosa_token_'+self.environment], "AWS_ACCESS_KEY_ID": self.aws_creds['aws_access_key_id'], "AWS_SECRET_ACCESS_KEY": self.aws_creds['aws_secret_access_key'], "AWS_DEFAULT_REGION": self.aws_creds['aws_region_for_openshift'] }
def _get_util(self, util): env = { **self.env, **util.get('env', {}), **{ "ES_SERVER": var_loader.get_secret('elasticsearch') }, **{ "KUBEADMIN_PASSWORD": environ.get("KUBEADMIN_PASSWORD", "") } } return BashOperator( task_id=f"{util['name']}", depends_on_past=False, bash_command= f"{constants.root_dag_dir}/scripts/utils/run_scale_ci_diagnosis.sh -w {util['workload']} -c {util['command']} ", retries=3, dag=self.dag, env=env, executor_config=self.exec_config)
def get_index_task(self): env = { **self.env, **{"ES_SERVER": var_loader.get_secret('elasticsearch')}, **environ } if self.task != "install": command = f'UUID={{{{ ti.xcom_pull("{self.task}") }}}} {constants.root_dag_dir}/scripts/index.sh ' else: command = f'{constants.root_dag_dir}/scripts/index.sh ' return BashOperator( task_id=f"index-{self.task}", depends_on_past=False, bash_command=command, retries=3, dag=self.dag, trigger_rule="all_done", executor_config=self.exec_config, env=env )
def _get_task(self, trigger_rule="all_success"): bash_script = f"{constants.root_dag_dir}/scripts/install/baremetal_deploy_webfuse.sh" # Merge all variables, prioritizing Airflow Secrets over git based vars config = { **self.vars, **self.baremetal_install_secrets, **{ "es_server": var_loader.get_secret('elasticsearch') } } config['version'] = self.release.release_stream config['build'] = self.release.build # Required Environment Variables for Install script env = { "SSHKEY_TOKEN": config['sshkey_token'], "ORCHESTRATION_HOST": config['provisioner_hostname'], "ORCHESTRATION_USER": config['provisioner_user'], "WEBFUSE_SKIPTAGS": config['webfuse_skiptags'], "WEBFUSE_PLAYBOOK": config['webfuse_playbook'], **self._insert_kube_env() } # Dump all vars to json file for Ansible to pick up with open(f"/tmp/{self.release_name}-task.json", 'w') as json_file: json.dump(config, json_file, sort_keys=True, indent=4) return BashOperator( task_id="deploy-webfuse", depends_on_past=False, bash_command= f"{bash_script} -p {self.release.platform} -v {self.release.version} -j /tmp/{self.release_name}-task.json -o deploy_app ", retries=3, dag=self.dag, trigger_rule=trigger_rule, executor_config=self.exec_config, env=env)
def __init__(self, dag, config: DagConfig, release: OpenshiftRelease): # General DAG Configuration self.dag = dag self.release = release self.config = config self.exec_config = executor.get_executor_config_with_cluster_access( self.config, self.release) self.snappy_creds = var_loader.get_secret("snappy_creds", deserialize_json=True) # Specific Task Configuration self.vars = var_loader.build_task_vars(release=self.release, task="utils") self.git_name = self._git_name() self.env = { "SNAPPY_DATA_SERVER_URL": self.snappy_creds['server'], "SNAPPY_DATA_SERVER_USERNAME": self.snappy_creds['username'], "SNAPPY_DATA_SERVER_PASSWORD": self.snappy_creds['password'], "SNAPPY_USER_FOLDER": self.git_name } self.env.update(self.config.dependencies)
def __init__(self, dag, config: DagConfig, release: OpenshiftRelease): # General DAG Configuration self.dag = dag self.release = release self.dag_config = config self.release_name = release.get_release_name(delimiter="-") self.cluster_name = release._generate_cluster_name() # Specific Task Configuration self.vars = var_loader.build_task_vars(release, task="install") # Airflow Variables self.ansible_orchestrator = var_loader.get_secret( "ansible_orchestrator", deserialize_json=True) self.install_secrets = var_loader.get_secret( f"openshift_install_config", deserialize_json=True) self.aws_creds = var_loader.get_secret("aws_creds", deserialize_json=True) self.gcp_creds = var_loader.get_secret("gcp_creds", deserialize_json=True) self.azure_creds = var_loader.get_secret("azure_creds", deserialize_json=True) self.ocp_pull_secret = var_loader.get_secret("osp_ocp_pull_creds") self.openstack_creds = var_loader.get_secret("openstack_creds", deserialize_json=True) self.rosa_creds = var_loader.get_secret("rosa_creds", deserialize_json=True) self.rhacs_creds = var_loader.get_secret("rhacs_creds", deserialize_json=True) self.rogcp_creds = var_loader.get_secret("rogcp_creds") self.release_stream_base_url = var_loader.get_secret( "release_stream_base_url") self.exec_config = executor.get_default_executor_config( self.dag_config) # Merge all variables, prioritizing Airflow Secrets over git based vars self.config = { **self.vars, **self.ansible_orchestrator, **self.install_secrets, **self.aws_creds, **self.gcp_creds, **self.azure_creds, **self.openstack_creds, **self.rosa_creds, **self.rhacs_creds, **self.release.get_latest_release(self.release_stream_base_url), **{ "es_server": var_loader.get_secret('elasticsearch'), "thanos_receiver_url": var_loader.get_secret('thanos_receiver_url'), "loki_receiver_url": var_loader.get_secret('loki_receiver_url') } } super().__init__()
def test_get_secret(self, mocker): mocker.patch('openshift_nightlies.util.var_loader.get_overrides', return_value={}) mocker.patch('airflow.models.Variable.get') var_loader.get_secret("foo") Variable.get.assert_called_once_with("foo", deserialize_json=False)
def __init__(self, dag, config: DagConfig, release: BaremetalRelease): self.baremetal_install_secrets = var_loader.get_secret( f"baremetal_openshift_install_config", deserialize_json=True) super().__init__(dag, config, release) self.exec_config = executor.get_default_executor_config( self.dag_config, executor_image="airflow-jetski")
def test_get_overridden_secret(self, mocker): mocker.patch('openshift_nightlies.util.var_loader.get_overrides', return_value={"foo": "bar"}) assert var_loader.get_secret("foo") == "bar"