def openshift_delete(dry_run, oc_map, query, resource_type, enable_deletion):
try:
openshift_base.delete(dry_run=dry_run,
oc_map=oc_map,
cluster=query['cluster'],
namespace=query['namespace']['name'],
resource_type=resource_type,
name=query['name'],
enable_deletion=enable_deletion)
except StatusCodeError:
LOG.exception("Error removing ['%s' '%s' '%s' '%s']", query['cluster'],
query['namespace']['name'], resource_type, query['name'])
def run(dry_run=False, enable_deletion=False):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
queries_list = collect_queries()
remove_candidates = []
for query in queries_list:
query_name = query['name']
# Checking the sql-query state:
# - No state: up for execution.
# - State is a timestamp: executed and up for removal
# after the JOB_TTL
# - State is 'DONE': executed and removed.
try:
query_state = state[query_name]
if query_state != 'DONE':
remove_candidates.append({
'name': query_name,
'timestamp': query_state
})
continue
except KeyError:
pass
job_yaml = process_template(query)
job = yaml.safe_load(job_yaml)
job_resource = OpenshiftResource(job, QONTRACT_INTEGRATION,
QONTRACT_INTEGRATION_VERSION)
oc_map = OC_Map(namespaces=[query['namespace']],
integration=QONTRACT_INTEGRATION,
settings=queries.get_app_interface_settings(),
internal=None)
openshift_base.apply(dry_run=dry_run,
oc_map=oc_map,
cluster=query['cluster'],
namespace=query['namespace']['name'],
resource_type='job',
resource=job_resource)
if not dry_run:
state[query_name] = time.time()
for candidate in remove_candidates:
if time.time() < candidate['timestamp'] + JOB_TTL:
continue
try:
query = collect_queries(query_name=candidate['name'])[0]
except IndexError:
raise RuntimeError(f'sql-query {candidate["name"]} not present'
f'in the app-interface while its Job is still '
f'not removed from the cluster. Manual clean '
f'up is needed.')
oc_map = OC_Map(namespaces=[query['namespace']],
integration=QONTRACT_INTEGRATION,
settings=queries.get_app_interface_settings(),
internal=None)
openshift_base.delete(dry_run=dry_run,
oc_map=oc_map,
cluster=query['cluster'],
namespace=query['namespace']['name'],
resource_type='job',
name=query['name'],
enable_deletion=enable_deletion)
if not dry_run:
state[candidate['name']] = 'DONE'
def run(dry_run, enable_deletion=False):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
queries_list = collect_queries()
remove_candidates = []
for query in queries_list:
query_name = query['name']
# Checking the sql-query state:
# - No state: up for execution.
# - State is a timestamp: executed and up for removal
# after the JOB_TTL
# - State is 'DONE': executed and removed.
try:
query_state = state[query_name]
is_cronjob = query.get('schedule')
if query_state != 'DONE' and not is_cronjob:
remove_candidates.append({
'name': query_name,
'timestamp': query_state
})
continue
except KeyError:
pass
image_repository = 'quay.io/app-sre'
use_pull_secret = False
sql_query_settings = settings.get('sqlQuery')
if sql_query_settings:
use_pull_secret = True
image_repository = sql_query_settings['imageRepository']
pull_secret = sql_query_settings['pullSecret']
secret_resource = orb.fetch_provider_vault_secret(
path=pull_secret['path'],
version=pull_secret['version'],
name=query_name,
labels=pull_secret['labels'] or {},
annotations=pull_secret['annotations'] or {},
type=pull_secret['type'],
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION)
job_yaml = process_template(query,
image_repository=image_repository,
use_pull_secret=use_pull_secret)
job = yaml.safe_load(job_yaml)
job_resource = OpenshiftResource(job, QONTRACT_INTEGRATION,
QONTRACT_INTEGRATION_VERSION)
oc_map = OC_Map(namespaces=[query['namespace']],
integration=QONTRACT_INTEGRATION,
settings=queries.get_app_interface_settings(),
internal=None)
if use_pull_secret:
openshift_base.apply(dry_run=dry_run,
oc_map=oc_map,
cluster=query['cluster'],
namespace=query['namespace']['name'],
resource_type=secret_resource.kind,
resource=secret_resource,
wait_for_namespace=False)
openshift_base.apply(dry_run=dry_run,
oc_map=oc_map,
cluster=query['cluster'],
namespace=query['namespace']['name'],
resource_type=job_resource.kind,
resource=job_resource,
wait_for_namespace=False)
if not dry_run:
state[query_name] = time.time()
for candidate in remove_candidates:
if time.time() < candidate['timestamp'] + JOB_TTL:
continue
try:
query = collect_queries(query_name=candidate['name'])[0]
except IndexError:
raise RuntimeError(f'sql-query {candidate["name"]} not present'
f'in the app-interface while its Job is still '
f'not removed from the cluster. Manual clean '
f'up is needed.')
oc_map = OC_Map(namespaces=[query['namespace']],
integration=QONTRACT_INTEGRATION,
settings=queries.get_app_interface_settings(),
internal=None)
try:
openshift_base.delete(dry_run=dry_run,
oc_map=oc_map,
cluster=query['cluster'],
namespace=query['namespace']['name'],
resource_type='job',
name=query['name'],
enable_deletion=enable_deletion)
except StatusCodeError:
LOG.exception("Error removing ['%s' '%s' 'job' '%s']",
query['cluster'], query['namespace']['name'],
query['name'])
try:
openshift_base.delete(dry_run=dry_run,
oc_map=oc_map,
cluster=query['cluster'],
namespace=query['namespace']['name'],
resource_type='Secret',
name=query['name'],
enable_deletion=enable_deletion)
except StatusCodeError:
LOG.exception("Error removing ['%s' '%s' 'Secret' '%s']",
query['cluster'], query['namespace']['name'],
query['name'])
if not dry_run:
state[candidate['name']] = 'DONE'