def construct_gabi_oc_resource(name: str, users: Iterable[str]) -> OpenshiftResource:
body = {
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {"name": name, "annotations": {"qontract.recycle": "true"}},
"data": {"authorized-users.yaml": "\n".join(users)},
}
return OpenshiftResource(
body, QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION, error_details=name
)
def build_probe(provider: EndpointMonitoringProvider,
endpoints: list[Endpoint]) -> Optional[OpenshiftResource]:
blackbox_exporter = provider.blackboxExporter
if blackbox_exporter:
prober_url = parse_prober_url(blackbox_exporter.exporterUrl)
body: dict[str, Any] = {
"apiVersion": "monitoring.coreos.com/v1",
"kind": "Probe",
"metadata": {
"name": provider.name,
"namespace": blackbox_exporter.namespace.get("name"),
"labels": {
"prometheus": "app-sre"
}
},
"spec": {
"jobName": provider.name,
"interval": provider.checkInterval or "10s",
"module": blackbox_exporter.module,
"prober": prober_url,
"targets": {
"staticConfig": {
"relabelingConfigs": [
{
"action": "labeldrop",
"regex": "namespace"
}
],
"labels": provider.metric_labels,
"static": [
ep.url for ep in endpoints
]
}
}
}
}
if provider.timeout:
body["spec"]["scrapeTimeout"] = provider.timeout
return OpenshiftResource(
body,
QONTRACT_INTEGRATION,
QONTRACT_INTEGRATION_VERSION
)
else:
return None
def run(dry_run, enable_deletion=False):
settings = queries.get_app_interface_settings()
accounts = queries.get_state_aws_accounts()
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
queries_list = collect_queries(settings=settings)
remove_candidates = []
for query in queries_list:
query_name = query["name"]
# Checking the sql-query state:
# - No state: up for execution.
# - State is a timestamp: executed and up for removal
# after the JOB_TTL
# - State is 'DONE': executed and removed.
try:
query_state = state[query_name]
is_cronjob = query.get("schedule")
if query_state != "DONE" and not is_cronjob:
remove_candidates.append({
"name": query_name,
"timestamp": query_state,
"output": query["output"],
})
continue
except KeyError:
pass
image_repository = "quay.io/app-sre"
use_pull_secret = False
sql_query_settings = settings.get("sqlQuery")
if sql_query_settings:
use_pull_secret = True
image_repository = sql_query_settings["imageRepository"]
pull_secret = sql_query_settings["pullSecret"]
secret_resource = orb.fetch_provider_vault_secret(
path=pull_secret["path"],
version=pull_secret["version"],
name=query_name,
labels=pull_secret["labels"] or {},
annotations=pull_secret["annotations"] or {},
type=pull_secret["type"],
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
)
job_yaml = process_template(query,
image_repository=image_repository,
use_pull_secret=use_pull_secret)
job = yaml.safe_load(job_yaml)
job_resource = OpenshiftResource(job, QONTRACT_INTEGRATION,
QONTRACT_INTEGRATION_VERSION)
oc_map = OC_Map(
namespaces=[query["namespace"]],
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=None,
)
if use_pull_secret:
openshift_apply(dry_run, oc_map, query, secret_resource)
if query["output"] == "encrypted":
render_kwargs = {
"GPG_KEY": query["name"],
"PUBLIC_GPG_KEY": query["public_gpg_key"],
}
template = jinja2.Template(CONFIGMAP_TEMPLATE)
configmap_yaml = template.render(**render_kwargs)
configmap = yaml.safe_load(configmap_yaml)
configmap_resource = OpenshiftResource(
configmap, QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION)
openshift_apply(dry_run, oc_map, query, configmap_resource)
openshift_apply(dry_run, oc_map, query, job_resource)
if not dry_run:
state[query_name] = time.time()
for candidate in remove_candidates:
if time.time() < candidate["timestamp"] + JOB_TTL:
continue
try:
query = collect_queries(query_name=candidate["name"],
settings=settings)[0]
except IndexError:
raise RuntimeError(f'sql-query {candidate["name"]} not present'
f"in the app-interface while its Job is still "
f"not removed from the cluster. Manual clean "
f"up is needed.")
oc_map = OC_Map(
namespaces=[query["namespace"]],
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=None,
)
resource_types = ["Job", "Secret"]
if candidate["output"] == "encrypted":
resource_types.append("ConfigMap")
for resource_type in resource_types:
openshift_delete(dry_run, oc_map, query, resource_type,
enable_deletion)
if not dry_run:
state[candidate["name"]] = "DONE"
def run(dry_run, enable_deletion=False):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
queries_list = collect_queries(settings=settings)
remove_candidates = []
for query in queries_list:
query_name = query['name']
# Checking the sql-query state:
# - No state: up for execution.
# - State is a timestamp: executed and up for removal
# after the JOB_TTL
# - State is 'DONE': executed and removed.
try:
query_state = state[query_name]
is_cronjob = query.get('schedule')
if query_state != 'DONE' and not is_cronjob:
remove_candidates.append({
'name': query_name,
'timestamp': query_state,
'output': query['output']
})
continue
except KeyError:
pass
image_repository = 'quay.io/app-sre'
use_pull_secret = False
sql_query_settings = settings.get('sqlQuery')
if sql_query_settings:
use_pull_secret = True
image_repository = sql_query_settings['imageRepository']
pull_secret = sql_query_settings['pullSecret']
secret_resource = orb.fetch_provider_vault_secret(
path=pull_secret['path'],
version=pull_secret['version'],
name=query_name,
labels=pull_secret['labels'] or {},
annotations=pull_secret['annotations'] or {},
type=pull_secret['type'],
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION)
job_yaml = process_template(query,
image_repository=image_repository,
use_pull_secret=use_pull_secret)
job = yaml.safe_load(job_yaml)
job_resource = OpenshiftResource(job, QONTRACT_INTEGRATION,
QONTRACT_INTEGRATION_VERSION)
oc_map = OC_Map(namespaces=[query['namespace']],
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=None)
if use_pull_secret:
openshift_apply(dry_run, oc_map, query, secret_resource)
if query['output'] == 'encrypted':
render_kwargs = {
'GPG_KEY': query['name'],
'PUBLIC_GPG_KEY': query['public_gpg_key']
}
template = jinja2.Template(CONFIGMAP_TEMPLATE)
configmap_yaml = template.render(**render_kwargs)
configmap = yaml.safe_load(configmap_yaml)
configmap_resource = OpenshiftResource(
configmap, QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION)
openshift_apply(dry_run, oc_map, query, configmap_resource)
openshift_apply(dry_run, oc_map, query, job_resource)
if not dry_run:
state[query_name] = time.time()
for candidate in remove_candidates:
if time.time() < candidate['timestamp'] + JOB_TTL:
continue
try:
query = collect_queries(query_name=candidate['name'],
settings=settings)[0]
except IndexError:
raise RuntimeError(f'sql-query {candidate["name"]} not present'
f'in the app-interface while its Job is still '
f'not removed from the cluster. Manual clean '
f'up is needed.')
oc_map = OC_Map(namespaces=[query['namespace']],
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=None)
resource_types = ['Job', 'Secret']
if candidate['output'] == 'encrypted':
resource_types.append('ConfigMap')
for resource_type in resource_types:
openshift_delete(dry_run, oc_map, query, resource_type,
enable_deletion)
if not dry_run:
state[candidate['name']] = 'DONE'
