def run(dry_run, enable_deletion=False):
settings = queries.get_app_interface_settings()
accounts = queries.get_state_aws_accounts()
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
queries_list = collect_queries(settings=settings)
remove_candidates = []
for query in queries_list:
query_name = query["name"]
# Checking the sql-query state:
# - No state: up for execution.
# - State is a timestamp: executed and up for removal
# after the JOB_TTL
# - State is 'DONE': executed and removed.
try:
query_state = state[query_name]
is_cronjob = query.get("schedule")
if query_state != "DONE" and not is_cronjob:
remove_candidates.append({
"name": query_name,
"timestamp": query_state,
"output": query["output"],
})
continue
except KeyError:
pass
image_repository = "quay.io/app-sre"
use_pull_secret = False
sql_query_settings = settings.get("sqlQuery")
if sql_query_settings:
use_pull_secret = True
image_repository = sql_query_settings["imageRepository"]
pull_secret = sql_query_settings["pullSecret"]
secret_resource = orb.fetch_provider_vault_secret(
path=pull_secret["path"],
version=pull_secret["version"],
name=query_name,
labels=pull_secret["labels"] or {},
annotations=pull_secret["annotations"] or {},
type=pull_secret["type"],
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
)
job_yaml = process_template(query,
image_repository=image_repository,
use_pull_secret=use_pull_secret)
job = yaml.safe_load(job_yaml)
job_resource = OpenshiftResource(job, QONTRACT_INTEGRATION,
QONTRACT_INTEGRATION_VERSION)
oc_map = OC_Map(
namespaces=[query["namespace"]],
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=None,
)
if use_pull_secret:
openshift_apply(dry_run, oc_map, query, secret_resource)
if query["output"] == "encrypted":
render_kwargs = {
"GPG_KEY": query["name"],
"PUBLIC_GPG_KEY": query["public_gpg_key"],
}
template = jinja2.Template(CONFIGMAP_TEMPLATE)
configmap_yaml = template.render(**render_kwargs)
configmap = yaml.safe_load(configmap_yaml)
configmap_resource = OpenshiftResource(
configmap, QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION)
openshift_apply(dry_run, oc_map, query, configmap_resource)
openshift_apply(dry_run, oc_map, query, job_resource)
if not dry_run:
state[query_name] = time.time()
for candidate in remove_candidates:
if time.time() < candidate["timestamp"] + JOB_TTL:
continue
try:
query = collect_queries(query_name=candidate["name"],
settings=settings)[0]
except IndexError:
raise RuntimeError(f'sql-query {candidate["name"]} not present'
f"in the app-interface while its Job is still "
f"not removed from the cluster. Manual clean "
f"up is needed.")
oc_map = OC_Map(
namespaces=[query["namespace"]],
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=None,
)
resource_types = ["Job", "Secret"]
if candidate["output"] == "encrypted":
resource_types.append("ConfigMap")
for resource_type in resource_types:
openshift_delete(dry_run, oc_map, query, resource_type,
enable_deletion)
if not dry_run:
state[candidate["name"]] = "DONE"
def construct_resources(namespaces):
for namespace in namespaces:
namespace_name = namespace["name"]
acme = namespace.get("openshiftAcme", {})
# Get the linked acme schema settings
acme_config = acme.get("config", {})
image = acme_config.get("image")
acme_overrides = acme_config.get("overrides", {})
default_name = 'openshift-acme'
default_rbac_api_version = 'authorization.openshift.io/v1'
deployment_name = \
acme_overrides.get('deploymentName') or default_name
serviceaccount_name = \
acme_overrides.get('serviceaccountName') or default_name
role_name = \
acme_overrides.get('roleName') or default_name
rolebinding_name = \
acme_overrides.get('roleName') or default_name
rbac_api_version = \
acme_overrides.get('rbacApiVersion') or default_rbac_api_version
# Create the resources and append them to the namespace
namespace["resources"] = []
namespace["resources"].append(
process_template(
ACME_DEPLOYMENT, {
'deployment_name': deployment_name,
'image': image,
'serviceaccount_name': serviceaccount_name
}))
namespace["resources"].append(
process_template(ACME_SERVICEACCOUNT,
{'serviceaccount_name': serviceaccount_name}))
namespace["resources"].append(
process_template(ACME_ROLE, {
'role_name': role_name,
'role_api_version': rbac_api_version
}))
namespace["resources"].append(
process_template(
ACME_ROLEBINDING, {
'role_name': role_name,
'rolebinding_name': rolebinding_name,
'rolebinding_api_version': rbac_api_version,
'serviceaccount_name': serviceaccount_name,
'namespace_name': namespace_name
}))
# If acme-account Secret is defined, add it to the namespace
acme_account_secret = acme.get("accountSecret", {})
if acme_account_secret:
namespace["resources"].append(
orb.fetch_provider_vault_secret(
acme_account_secret['path'],
acme_account_secret['version'],
'acme-account',
labels={'kubernetes.io/acme.type': 'account'},
annotations={},
type='Opaque',
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION))
return namespaces
def run(dry_run, enable_deletion=False):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
queries_list = collect_queries(settings=settings)
remove_candidates = []
for query in queries_list:
query_name = query['name']
# Checking the sql-query state:
# - No state: up for execution.
# - State is a timestamp: executed and up for removal
# after the JOB_TTL
# - State is 'DONE': executed and removed.
try:
query_state = state[query_name]
is_cronjob = query.get('schedule')
if query_state != 'DONE' and not is_cronjob:
remove_candidates.append({
'name': query_name,
'timestamp': query_state,
'output': query['output']
})
continue
except KeyError:
pass
image_repository = 'quay.io/app-sre'
use_pull_secret = False
sql_query_settings = settings.get('sqlQuery')
if sql_query_settings:
use_pull_secret = True
image_repository = sql_query_settings['imageRepository']
pull_secret = sql_query_settings['pullSecret']
secret_resource = orb.fetch_provider_vault_secret(
path=pull_secret['path'],
version=pull_secret['version'],
name=query_name,
labels=pull_secret['labels'] or {},
annotations=pull_secret['annotations'] or {},
type=pull_secret['type'],
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION)
job_yaml = process_template(query,
image_repository=image_repository,
use_pull_secret=use_pull_secret)
job = yaml.safe_load(job_yaml)
job_resource = OpenshiftResource(job, QONTRACT_INTEGRATION,
QONTRACT_INTEGRATION_VERSION)
oc_map = OC_Map(namespaces=[query['namespace']],
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=None)
if use_pull_secret:
openshift_apply(dry_run, oc_map, query, secret_resource)
if query['output'] == 'encrypted':
render_kwargs = {
'GPG_KEY': query['name'],
'PUBLIC_GPG_KEY': query['public_gpg_key']
}
template = jinja2.Template(CONFIGMAP_TEMPLATE)
configmap_yaml = template.render(**render_kwargs)
configmap = yaml.safe_load(configmap_yaml)
configmap_resource = OpenshiftResource(
configmap, QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION)
openshift_apply(dry_run, oc_map, query, configmap_resource)
openshift_apply(dry_run, oc_map, query, job_resource)
if not dry_run:
state[query_name] = time.time()
for candidate in remove_candidates:
if time.time() < candidate['timestamp'] + JOB_TTL:
continue
try:
query = collect_queries(query_name=candidate['name'],
settings=settings)[0]
except IndexError:
raise RuntimeError(f'sql-query {candidate["name"]} not present'
f'in the app-interface while its Job is still '
f'not removed from the cluster. Manual clean '
f'up is needed.')
oc_map = OC_Map(namespaces=[query['namespace']],
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=None)
resource_types = ['Job', 'Secret']
if candidate['output'] == 'encrypted':
resource_types.append('ConfigMap')
for resource_type in resource_types:
openshift_delete(dry_run, oc_map, query, resource_type,
enable_deletion)
if not dry_run:
state[candidate['name']] = 'DONE'
def run(dry_run, enable_deletion=False):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
queries_list = collect_queries()
remove_candidates = []
for query in queries_list:
query_name = query['name']
# Checking the sql-query state:
# - No state: up for execution.
# - State is a timestamp: executed and up for removal
# after the JOB_TTL
# - State is 'DONE': executed and removed.
try:
query_state = state[query_name]
is_cronjob = query.get('schedule')
if query_state != 'DONE' and not is_cronjob:
remove_candidates.append({
'name': query_name,
'timestamp': query_state
})
continue
except KeyError:
pass
image_repository = 'quay.io/app-sre'
use_pull_secret = False
sql_query_settings = settings.get('sqlQuery')
if sql_query_settings:
use_pull_secret = True
image_repository = sql_query_settings['imageRepository']
pull_secret = sql_query_settings['pullSecret']
secret_resource = orb.fetch_provider_vault_secret(
path=pull_secret['path'],
version=pull_secret['version'],
name=query_name,
labels=pull_secret['labels'] or {},
annotations=pull_secret['annotations'] or {},
type=pull_secret['type'],
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION)
job_yaml = process_template(query,
image_repository=image_repository,
use_pull_secret=use_pull_secret)
job = yaml.safe_load(job_yaml)
job_resource = OpenshiftResource(job, QONTRACT_INTEGRATION,
QONTRACT_INTEGRATION_VERSION)
oc_map = OC_Map(namespaces=[query['namespace']],
integration=QONTRACT_INTEGRATION,
settings=queries.get_app_interface_settings(),
internal=None)
if use_pull_secret:
openshift_base.apply(dry_run=dry_run,
oc_map=oc_map,
cluster=query['cluster'],
namespace=query['namespace']['name'],
resource_type=secret_resource.kind,
resource=secret_resource,
wait_for_namespace=False)
openshift_base.apply(dry_run=dry_run,
oc_map=oc_map,
cluster=query['cluster'],
namespace=query['namespace']['name'],
resource_type=job_resource.kind,
resource=job_resource,
wait_for_namespace=False)
if not dry_run:
state[query_name] = time.time()
for candidate in remove_candidates:
if time.time() < candidate['timestamp'] + JOB_TTL:
continue
try:
query = collect_queries(query_name=candidate['name'])[0]
except IndexError:
raise RuntimeError(f'sql-query {candidate["name"]} not present'
f'in the app-interface while its Job is still '
f'not removed from the cluster. Manual clean '
f'up is needed.')
oc_map = OC_Map(namespaces=[query['namespace']],
integration=QONTRACT_INTEGRATION,
settings=queries.get_app_interface_settings(),
internal=None)
try:
openshift_base.delete(dry_run=dry_run,
oc_map=oc_map,
cluster=query['cluster'],
namespace=query['namespace']['name'],
resource_type='job',
name=query['name'],
enable_deletion=enable_deletion)
except StatusCodeError:
LOG.exception("Error removing ['%s' '%s' 'job' '%s']",
query['cluster'], query['namespace']['name'],
query['name'])
try:
openshift_base.delete(dry_run=dry_run,
oc_map=oc_map,
cluster=query['cluster'],
namespace=query['namespace']['name'],
resource_type='Secret',
name=query['name'],
enable_deletion=enable_deletion)
except StatusCodeError:
LOG.exception("Error removing ['%s' '%s' 'Secret' '%s']",
query['cluster'], query['namespace']['name'],
query['name'])
if not dry_run:
state[candidate['name']] = 'DONE'