def __init__(self, dag, config: DagConfig, release: OpenshiftRelease, task_group="benchmarks"):
# General DAG Configuration
self.dag = dag
self.release = release
self.task_group = task_group
self.dag_config = config
self.exec_config = executor.get_executor_config_with_cluster_access(self.dag_config, self.release)
self.snappy_creds = var_loader.get_secret("snappy_creds", deserialize_json=True)
self.es_gold = var_loader.get_secret("es_gold")
self.es_server_baseline = var_loader.get_secret("es_server_baseline")
# Specific Task Configuration
self.vars = var_loader.build_task_vars(
release=self.release, task=self.task_group)
self.git_name=self._git_name()
self.env = {
"SNAPPY_DATA_SERVER_URL": self.snappy_creds['server'],
"SNAPPY_DATA_SERVER_USERNAME": self.snappy_creds['username'],
"SNAPPY_DATA_SERVER_PASSWORD": self.snappy_creds['password'],
"SNAPPY_USER_FOLDER": self.git_name,
"PLATFORM": self.release.platform,
"TASK_GROUP": self.task_group,
"ES_GOLD": self.es_gold,
"ES_SERVER_BASELINE": self.es_server_baseline
}
self.env.update(self.dag_config.dependencies)
if self.release.platform == "baremetal":
self.install_vars = var_loader.build_task_vars(
release, task="install")
self.baremetal_install_secrets = var_loader.get_secret(
f"baremetal_openshift_install_config", deserialize_json=True)
self.config = {
**self.install_vars,
**self.baremetal_install_secrets
}
self.env = {
**self.env,
"SSHKEY_TOKEN": self.config['sshkey_token'],
"ORCHESTRATION_USER": self.config['provisioner_user'],
"ORCHESTRATION_HOST": self.config['provisioner_hostname']
}
if self.release.platform == "rosa":
self.rosa_creds = var_loader.get_secret("rosa_creds", deserialize_json=True)
self.aws_creds = var_loader.get_secret("aws_creds", deserialize_json=True)
self.environment = self.vars["environment"] if "environment" in self.vars else "staging"
self.env = {
**self.env,
"ROSA_CLUSTER_NAME": release._generate_cluster_name(),
"ROSA_ENVIRONMENT": self.environment,
"ROSA_TOKEN": self.rosa_creds['rosa_token_'+self.environment],
"AWS_ACCESS_KEY_ID": self.aws_creds['aws_access_key_id'],
"AWS_SECRET_ACCESS_KEY": self.aws_creds['aws_secret_access_key'],
"AWS_DEFAULT_REGION": self.aws_creds['aws_region_for_openshift']
}
def __init__(self,
dag,
config: DagConfig,
release: OpenshiftRelease,
task_group="benchmarks"):
# General DAG Configuration
self.dag = dag
self.release = release
self.task_group = task_group
self.dag_config = config
self.exec_config = executor.get_executor_config_with_cluster_access(
self.dag_config, self.release)
self.snappy_creds = var_loader.get_secret("snappy_creds",
deserialize_json=True)
self.es_gold = var_loader.get_secret("es_gold")
self.es_server_baseline = var_loader.get_secret("es_server_baseline")
# Specific Task Configuration
self.vars = var_loader.build_task_vars(release=self.release,
task=self.task_group)
self.git_name = self._git_name()
self.env = {
"SNAPPY_DATA_SERVER_URL": self.snappy_creds['server'],
"SNAPPY_DATA_SERVER_USERNAME": self.snappy_creds['username'],
"SNAPPY_DATA_SERVER_PASSWORD": self.snappy_creds['password'],
"SNAPPY_USER_FOLDER": self.git_name,
"PLATFORM": self.release.platform,
"TASK_GROUP": self.task_group,
"ES_GOLD": self.es_gold,
"ES_SERVER_BASELINE": self.es_server_baseline
}
self.env.update(self.dag_config.dependencies)
if self.release.platform == "baremetal":
self.install_vars = var_loader.build_task_vars(release,
task="install")
self.baremetal_install_secrets = var_loader.get_secret(
f"baremetal_openshift_install_config", deserialize_json=True)
self.config = {
**self.install_vars,
**self.baremetal_install_secrets
}
self.env = {
**self.env, "SSHKEY_TOKEN": self.config['sshkey_token'],
"ORCHESTRATION_USER": self.config['provisioner_user'],
"ORCHESTRATION_HOST": self.config['provisioner_hostname']
}
def __init__(self, dag, config: DagConfig, release: OpenshiftRelease):
# General DAG Configuration
self.dag = dag
self.release = release
self.config = config
self.exec_config = executor.get_executor_config_with_cluster_access(
self.config, self.release)
# Specific Task Configuration
self.env = {
"REL_PLATFORM": self.release.platform,
"THANOS_RECEIVER_URL":
var_loader.get_secret("thanos_receiver_url"),
"LOKI_RECEIVER_URL": var_loader.get_secret("loki_receiver_url")
}
if self.release.platform == "baremetal":
self.install_vars = var_loader.build_task_vars(release,
task="install")
self.baremetal_install_secrets = var_loader.get_secret(
f"baremetal_openshift_install_config", deserialize_json=True)
self.config = {
**self.install_vars,
**self.baremetal_install_secrets
}
self.env = {
**self.env, "SSHKEY_TOKEN": self.config['sshkey_token'],
"ORCHESTRATION_USER": self.config['provisioner_user'],
"ORCHESTRATION_HOST": self.config['provisioner_hostname']
}
def __init__(self, dag, config: DagConfig, release: OpenshiftRelease):
# General DAG Configuration
self.dag = dag
self.release = release
self.config = config
self.release_name = release.get_release_name(delimiter="-")
self.aws_creds = var_loader.get_secret("aws_creds", deserialize_json=True)
# Specific Task Configuration
self.vars = var_loader.build_task_vars(
release=self.release, task="install")
self.all_vars = {
**self.vars,
**self.aws_creds,
}
# Dump all vars to json file for Ansible to pick up
with open(f"/tmp/{self.release_name}-postinstall-task.json", 'w') as json_file:
json.dump(self.all_vars, json_file, sort_keys=True, indent=4)
super().__init__()
self.exec_config = executor.get_executor_config_with_cluster_access(self.config, self.release, executor_image="airflow-managed-services")
def __init__(self, dag, config: DagConfig, release: OpenshiftRelease):
# General DAG Configuration
self.dag = dag
self.release = release
self.dag_config = config
self.release_name = release.get_release_name(delimiter="-")
self.cluster_name = release._generate_cluster_name()
# Specific Task Configuration
self.vars = var_loader.build_task_vars(release, task="install")
# Airflow Variables
self.ansible_orchestrator = var_loader.get_secret(
"ansible_orchestrator", deserialize_json=True)
self.install_secrets = var_loader.get_secret(
f"openshift_install_config", deserialize_json=True)
self.aws_creds = var_loader.get_secret("aws_creds",
deserialize_json=True)
self.gcp_creds = var_loader.get_secret("gcp_creds",
deserialize_json=True)
self.azure_creds = var_loader.get_secret("azure_creds",
deserialize_json=True)
self.ocp_pull_secret = var_loader.get_secret("osp_ocp_pull_creds")
self.openstack_creds = var_loader.get_secret("openstack_creds",
deserialize_json=True)
self.rosa_creds = var_loader.get_secret("rosa_creds",
deserialize_json=True)
self.rhacs_creds = var_loader.get_secret("rhacs_creds",
deserialize_json=True)
self.rogcp_creds = var_loader.get_secret("rogcp_creds")
self.exec_config = executor.get_default_executor_config(
self.dag_config)
# Merge all variables, prioritizing Airflow Secrets over git based vars
self.config = {
**self.vars,
**self.ansible_orchestrator,
**self.install_secrets,
**self.aws_creds,
**self.gcp_creds,
**self.azure_creds,
**self.openstack_creds,
**self.rosa_creds,
**self.rhacs_creds,
**{
"es_server": var_loader.get_secret('elasticsearch'),
"thanos_receiver_url": var_loader.get_secret('thanos_receiver_url'),
"loki_receiver_url": var_loader.get_secret('loki_receiver_url')
}
}
if self.config.get('openshift_install_binary_url',
"") == "" or self.config(
'openshift_client_location', "") == "":
self.config = {**self.config, **self.release.get_latest_release()}
super().__init__()
def test_build_task_vars(self, valid_openshift_release, test_config_dir,
test_tasks_dir):
assert var_loader.build_task_vars(valid_openshift_release,
task="test",
config_dir=str(test_config_dir),
task_dir=str(test_tasks_dir)) == {
"task": "test",
"default": "override",
"new_field": "merge",
"platform": None
}
def __init__(self, dag, config: DagConfig, release: OpenshiftRelease):
# General DAG Configuration
self.dag = dag
self.release = release
self.config = config
self.exec_config = executor.get_executor_config_with_cluster_access(
self.config, self.release)
self.snappy_creds = var_loader.get_secret("snappy_creds",
deserialize_json=True)
# Specific Task Configuration
self.vars = var_loader.build_task_vars(release=self.release,
task="utils")
self.git_name = self._git_name()
self.env = {
"SNAPPY_DATA_SERVER_URL": self.snappy_creds['server'],
"SNAPPY_DATA_SERVER_USERNAME": self.snappy_creds['username'],
"SNAPPY_DATA_SERVER_PASSWORD": self.snappy_creds['password'],
"SNAPPY_USER_FOLDER": self.git_name
}
self.env.update(self.config.dependencies)
def __init__(self, dag, config: DagConfig, release: OpenshiftRelease, task):
# General DAG Configuration
self.dag = dag
self.release = release
self.config = config
self.exec_config = executor.get_executor_config_with_cluster_access(self.config, self.release)
# Specific Task Configuration
self.vars = var_loader.build_task_vars(release, task="index")
# Upstream task this is to index
self.task = task
self.env = {
"RELEASE_STREAM": self.release.release_stream,
"TASK": self.task
}
self.git_user = var_loader.get_git_user()
if self.git_user == 'cloud-bulldozer':
self.env["ES_INDEX"] = "perf_scale_ci"
else:
self.env["ES_INDEX"] = f"{self.git_user}_playground"