def collect_state():
state = []
saas_files = queries.get_saas_files()
for saas_file in saas_files:
saas_file_path = saas_file['path']
saas_file_name = saas_file['name']
saas_file_parameters = json.loads(saas_file.get('parameters') or '{}')
resource_templates = saas_file['resourceTemplates']
for resource_template in resource_templates:
resource_template_name = resource_template['name']
resource_template_parameters = \
json.loads(resource_template.get('parameters') or '{}')
for target in resource_template['targets']:
namespace = target['namespace']['name']
cluster = target['namespace']['cluster']['name']
target_ref = target['ref']
target_parameters = \
json.loads(target.get('parameters') or '{}')
parameters = {}
parameters.update(saas_file_parameters)
parameters.update(resource_template_parameters)
parameters.update(target_parameters)
state.append({
'saas_file_path': saas_file_path,
'saas_file_name': saas_file_name,
'resource_template_name': resource_template_name,
'cluster': cluster,
'namespace': namespace,
'ref': target_ref,
'parameters': parameters
})
return state
def collect_owners():
owners = {}
saas_files = queries.get_saas_files(v1=True, v2=True)
for saas_file in saas_files:
saas_file_name = saas_file['name']
owners[saas_file_name] = set()
owner_roles = saas_file.get('roles')
if not owner_roles:
continue
for owner_role in owner_roles:
owner_users = owner_role.get('users')
if not owner_users:
continue
for owner_user in owner_users:
owner_username = owner_user['org_username']
if owner_user.get('tag_on_merge_requests'):
owner_username = f'@{owner_username}'
owners[saas_file_name].add(owner_username)
# make owners suitable for json dump
ans = {}
for k, v in owners.items():
ans[k] = list(v)
return ans
def collect_state():
state = []
saas_files = queries.get_saas_files(v1=True, v2=True)
for saas_file in saas_files:
saas_file_path = saas_file['path']
saas_file_name = saas_file['name']
saas_file_parameters = json.loads(saas_file.get('parameters') or '{}')
saas_file_definitions = {
'managed_resource_types':
saas_file['managedResourceTypes'],
'image_patterns':
saas_file['imagePatterns'],
'use_channel_in_image_tag':
saas_file.get('use_channel_in_image_tag') or False,
}
resource_templates = saas_file['resourceTemplates']
for resource_template in resource_templates:
resource_template_name = resource_template['name']
resource_template_parameters = \
json.loads(resource_template.get('parameters') or '{}')
resource_template_url = resource_template['url']
for target in resource_template['targets']:
namespace_info = target['namespace']
namespace = namespace_info['name']
cluster = namespace_info['cluster']['name']
environment = namespace_info['environment']['name']
target_ref = target['ref']
target_delete = target.get('delete')
target_parameters = \
json.loads(target.get('parameters') or '{}')
parameters = {}
parameters.update(saas_file_parameters)
parameters.update(resource_template_parameters)
parameters.update(target_parameters)
state.append({
'saas_file_path':
saas_file_path,
'saas_file_name':
saas_file_name,
'resource_template_name':
resource_template_name,
'cluster':
cluster,
'namespace':
namespace,
'environment':
environment,
'url':
resource_template_url,
'ref':
target_ref,
'parameters':
parameters,
'saas_file_definitions':
copy.deepcopy(saas_file_definitions),
'delete':
target_delete,
})
return state
def saas_dev(ctx, app_name=None, saas_file_name=None, env_name=None):
if env_name in [None, '']:
print('env-name must be defined')
return
saas_files = queries.get_saas_files(saas_file_name, env_name, app_name,
v1=True, v2=True)
if not saas_files:
print('no saas files found')
sys.exit(1)
for saas_file in saas_files:
saas_file_parameters = \
json.loads(saas_file.get('parameters') or '{}')
for rt in saas_file['resourceTemplates']:
url = rt['url']
path = rt['path']
rt_parameters = \
json.loads(rt.get('parameters') or '{}')
for target in rt['targets']:
target_parameters = \
json.loads(target.get('parameters') or '{}')
namespace = target['namespace']
namespace_name = namespace['name']
environment = namespace['environment']
if environment['name'] != env_name:
continue
ref = target['ref']
environment_parameters = \
json.loads(environment.get('parameters') or '{}')
parameters = {}
parameters.update(environment_parameters)
parameters.update(saas_file_parameters)
parameters.update(rt_parameters)
parameters.update(target_parameters)
for replace_key, replace_value in parameters.items():
if not isinstance(replace_value, str):
continue
replace_pattern = '${' + replace_key + '}'
for k, v in parameters.items():
if not isinstance(v, str):
continue
if replace_pattern in v:
parameters[k] = \
v.replace(replace_pattern, replace_value)
parameters_cmd = ''
for k, v in parameters.items():
parameters_cmd += f" -p {k}=\"{v}\""
raw_url = \
url.replace('github.com', 'raw.githubusercontent.com')
if 'gitlab' in raw_url:
raw_url += '/raw'
raw_url += '/' + ref
raw_url += path
cmd = "oc process --local --ignore-unknown-parameters" + \
f"{parameters_cmd} -f {raw_url}" + \
f" | oc apply -n {namespace_name} -f - --dry-run"
print(cmd)
def run(dry_run, thread_pool_size=10):
saas_files = queries.get_saas_files()
if not saas_files:
logging.error('no saas files found')
sys.exit(1)
# Remove saas-file targets that are disabled
for saas_file in saas_files[:]:
resource_templates = saas_file['resourceTemplates']
for rt in resource_templates[:]:
targets = rt['targets']
for target in targets[:]:
if target['disable']:
targets.remove(target)
instance = queries.get_gitlab_instance()
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
gl = GitLabApi(instance, settings=settings)
jenkins_map = jenkins_base.get_jenkins_map()
saasherder = SaasHerder(saas_files,
thread_pool_size=thread_pool_size,
gitlab=gl,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
settings=settings,
accounts=accounts)
trigger_specs = saasherder.get_moving_commits_diff(dry_run)
already_triggered = []
error = False
for job_spec in trigger_specs:
saas_file_name = job_spec['saas_file_name']
env_name = job_spec['env_name']
instance_name = job_spec['instance_name']
job_name = get_openshift_saas_deploy_job_name(saas_file_name, env_name,
settings)
if job_name not in already_triggered:
logging.info(['trigger_job', instance_name, job_name])
if dry_run:
already_triggered.append(job_name)
if not dry_run:
jenkins = jenkins_map[instance_name]
try:
if job_name not in already_triggered:
jenkins.trigger_job(job_name)
already_triggered.append(job_name)
saasherder.update_moving_commit(job_spec)
except Exception:
error = True
logging.error(
f"could not trigger job {job_name} in {instance_name}.")
if error:
sys.exit(1)
def run(dry_run, thread_pool_size=10):
saas_files = queries.get_saas_files()
if not saas_files:
logging.error('no saas files found')
sys.exit(1)
instance = queries.get_gitlab_instance()
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
gl = GitLabApi(instance, settings=settings)
jenkins_map = jenkins_base.get_jenkins_map()
saasherder = SaasHerder(
saas_files,
thread_pool_size=thread_pool_size,
gitlab=gl,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
settings=settings,
accounts=accounts)
trigger_specs = saasherder.get_configs_diff()
already_triggered = []
error = True # enter loop
while error:
error = False
for job_spec in trigger_specs:
saas_file_name = job_spec['saas_file_name']
env_name = job_spec['env_name']
instance_name = job_spec['instance_name']
job_name = get_openshift_saas_deploy_job_name(
saas_file_name, env_name, settings)
if job_name not in already_triggered:
logging.info(['trigger_job', instance_name, job_name])
if dry_run:
already_triggered.append(job_name)
if not dry_run:
jenkins = jenkins_map[instance_name]
try:
if job_name not in already_triggered:
jenkins.trigger_job(job_name)
already_triggered.append(job_name)
saasherder.update_config(job_spec)
except Exception as e:
error = True
logging.error(
f"could not trigger job {job_name} " +
f"in {instance_name}. details: {str(e)}"
)
if error:
time.sleep(10) # add to contants module once created
def run(dry_run):
saas_files = queries.get_saas_files()
settings = queries.get_app_interface_settings()
saasherder = SaasHerder(saas_files,
thread_pool_size=1,
gitlab=None,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
settings=settings)
if not saasherder.valid:
sys.exit(1)
def run(dry_run,
thread_pool_size=10,
saas_file_name=None,
env_name=None,
defer=None):
saas_files = queries.get_saas_files(saas_file_name, env_name)
if not saas_files:
logging.error('no saas files found')
sys.exit(1)
instance = queries.get_gitlab_instance()
desired_jenkins_instances = [s['instance']['name'] for s in saas_files]
jenkins_map = jenkins_base.get_jenkins_map(
desired_instances=desired_jenkins_instances)
settings = queries.get_app_interface_settings()
try:
gl = GitLabApi(instance, settings=settings)
except Exception:
# allow execution without access to gitlab
# as long as there are no access attempts.
gl = None
saasherder = SaasHerder(saas_files,
thread_pool_size=thread_pool_size,
gitlab=gl,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
settings=settings,
jenkins_map=jenkins_map)
if not saasherder.valid:
sys.exit(1)
if len(saasherder.namespaces) == 0:
logging.warning('no targets found')
sys.exit(0)
ri, oc_map = ob.fetch_current_state(
namespaces=saasherder.namespaces,
thread_pool_size=thread_pool_size,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION)
defer(lambda: oc_map.cleanup())
saasherder.populate_desired_state(ri)
# if saas_file_name is defined, the integration
# is being called from multiple running instances
ob.realize_data(dry_run,
oc_map,
ri,
caller=saas_file_name,
wait_for_namespace=True,
no_dry_run_skip_compare=True,
take_over=saasherder.take_over)
if ri.has_error_registered():
sys.exit(1)
def run(dry_run):
saas_files = queries.get_saas_files(v1=True, v2=True)
settings = queries.get_app_interface_settings()
saasherder = SaasHerder(
saas_files,
thread_pool_size=1,
gitlab=None,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
settings=settings,
validate=True)
app_int_repos = queries.get_repos()
missing_repos = [r for r in saasherder.repo_urls
if r not in app_int_repos]
for r in missing_repos:
logging.error(f'repo is missing from codeComponents: {r}')
if not saasherder.valid or missing_repos:
sys.exit(ExitCodes.ERROR)
def run(dry_run=False,
thread_pool_size=10,
saas_file_name=None,
env_name=None,
defer=None):
saas_files = queries.get_saas_files(saas_file_name, env_name)
if not saas_files:
logging.error('no saas files found')
sys.exit(1)
instance = queries.get_gitlab_instance()
settings = queries.get_app_interface_settings()
try:
gl = GitLabApi(instance, settings=settings)
except Exception:
# allow execution without access to gitlab
# as long as there are no access attempts.
gl = None
saasherder = SaasHerder(saas_files,
thread_pool_size=thread_pool_size,
gitlab=gl,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
settings=settings)
if not saasherder.valid:
sys.exit(1)
ri, oc_map = ob.fetch_current_state(
namespaces=saasherder.namespaces,
thread_pool_size=thread_pool_size,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION)
defer(lambda: oc_map.cleanup())
saasherder.populate_desired_state(ri)
# if saas_file_name is defined, the integration
# is being called from multiple running instances
ob.realize_data(dry_run, oc_map, ri, caller=saas_file_name)
if ri.has_error_registered():
sys.exit(1)
def collect_state():
state = []
saas_files = queries.get_saas_files()
for saas_file in saas_files:
saas_file_path = saas_file['path']
saas_file_name = saas_file['name']
saas_file_parameters = json.loads(saas_file.get('parameters') or '{}')
saas_file_definitions = {
'managed_resource_types': saas_file['managedResourceTypes'],
'image_patterns': saas_file['imagePatterns']
}
resource_templates = saas_file['resourceTemplates']
for resource_template in resource_templates:
resource_template_name = resource_template['name']
resource_template_parameters = \
json.loads(resource_template.get('parameters') or '{}')
for target in resource_template['targets']:
namespace_info = target['namespace']
namespace = namespace_info['name']
cluster = namespace_info['cluster']['name']
environment = namespace_info['environment']['name']
target_ref = target['ref']
target_parameters = \
json.loads(target.get('parameters') or '{}')
parameters = {}
parameters.update(saas_file_parameters)
parameters.update(resource_template_parameters)
parameters.update(target_parameters)
state.append({
'saas_file_path': saas_file_path,
'saas_file_name': saas_file_name,
'resource_template_name': resource_template_name,
'cluster': cluster,
'namespace': namespace,
'environment': environment,
'ref': target_ref,
'parameters': parameters,
'saas_file_definitions': saas_file_definitions
})
return state
def collect_state():
state = []
saas_files = queries.get_saas_files()
for saas_file in saas_files:
saas_file_path = saas_file['path']
saas_file_name = saas_file['name']
resource_templates = saas_file['resourceTemplates']
for resource_template in resource_templates:
resource_template_name = resource_template['name']
for target in resource_template['targets']:
namespace = target['namespace']['name']
cluster = target['namespace']['cluster']['name']
target_hash = target['hash']
state.append({
'saas_file_path': saas_file_path,
'saas_file_name': saas_file_name,
'resource_template_name': resource_template_name,
'cluster': cluster,
'namespace': namespace,
'hash': target_hash
})
return state
def run(dry_run=False, thread_pool_size=10, defer=None):
instance = queries.get_gitlab_instance()
settings = queries.get_app_interface_settings()
aws_accounts = queries.get_aws_accounts()
gl = GitLabApi(instance, settings=settings)
saas_files = queries.get_saas_files()
saasherder = SaasHerder(saas_files,
gitlab=gl,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
settings=settings)
ri, oc_map = ob.fetch_current_state(
namespaces=saasherder.namespaces,
thread_pool_size=thread_pool_size,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION)
defer(lambda: oc_map.cleanup())
saasherder.populate_desired_state(ri)
enable_deletion = False if ri.has_error_registered() else True
ob.realize_data(dry_run, oc_map, ri, enable_deletion=enable_deletion)
saasherder.slack_notify(dry_run, aws_accounts, ri)
def collect_owners():
owners = {}
saas_files = queries.get_saas_files()
for saas_file in saas_files:
saas_file_name = saas_file['name']
owners[saas_file_name] = set()
owner_roles = saas_file.get('roles')
if not owner_roles:
continue
for owner_role in owner_roles:
owner_users = owner_role.get('users')
if not owner_users:
continue
for owner_user in owner_users:
owner_username = owner_user['org_username']
owners[saas_file_name].add(owner_username)
# make owners suitable for json dump
for k in owners:
owners[k] = list(owners[k])
return owners
def collect_saas_file_configs():
# collect a list of jobs per saas file per environment.
# each saas_file_config should have the structure described
# in the above query.
# to make things understandable, each variable used to form
# the structure will be called `jc_<variable>` (jenkins config).
saas_file_configs = []
saas_files = queries.get_saas_files()
settings = queries.get_app_interface_settings()
for saas_file in saas_files:
saas_file_name = saas_file['name']
jc_instance = saas_file['instance']
# currently ignoring the actual Slack workspace
# as that is configured in Jenkins.
# revisit this if we support more then a single Slack workspace.
slack_channel = saas_file['slack']['channel']
for resource_template in saas_file['resourceTemplates']:
for target in resource_template['targets']:
namespace = target['namespace']
env_name = namespace['environment']['name']
upstream = target.get('upstream', '')
job_template_name = settings['saasDeployJobTemplate']
if upstream:
job_template_name += '-with-upstream'
app_name = namespace['app']['name']
jc_name = get_openshift_saas_deploy_job_name(
saas_file_name, env_name, settings)
existing_configs = \
[c for c in saas_file_configs if c['name'] == jc_name]
if existing_configs:
continue
# each config is a list with a single item
# with the following structure:
# project:
# name: 'openshift-saas-deploy-{saas_file_name}-{env_name}'
# saas_file_name: '{saas_file_name}'
# env_name: '{env_name}'
# app_name: '{app_name}'
# slack_channel: '{slack_channel}'
# jobs:
# - 'openshift-saas-deploy':
# display_name: display name of the job
jc_config = json.dumps([{
'project': {
'name': jc_name,
'saas_file_name': saas_file_name,
'env_name': env_name,
'app_name': app_name,
'slack_channel': slack_channel,
'upstream': upstream,
'jobs': [{
job_template_name: {
'display_name': jc_name
}
}]
}
}])
saas_file_configs.append({
'name': jc_name,
'instance': jc_instance,
'type': 'jobs',
'config': jc_config
})
return saas_file_configs, settings
def get_apps_data(date, month_delta=1, thread_pool_size=10):
apps = queries.get_apps()
saas_files = queries.get_saas_files()
jjb, _ = init_jjb()
jenkins_map = jenkins_base.get_jenkins_map()
time_limit = date - relativedelta(months=month_delta)
timestamp_limit = \
int(time_limit.replace(tzinfo=timezone.utc).timestamp())
settings = queries.get_app_interface_settings()
secret_reader = SecretReader(settings=settings)
secret_content = secret_reader.read_all({'path': DASHDOTDB_SECRET})
dashdotdb_url = secret_content['url']
dashdotdb_user = secret_content['username']
dashdotdb_pass = secret_content['password']
auth = (dashdotdb_user, dashdotdb_pass)
vuln_metrics = requests.get(
f'{dashdotdb_url}/api/v1/imagemanifestvuln/metrics', auth=auth).text
validt_metrics = requests.get(
f'{dashdotdb_url}/api/v1/deploymentvalidation/metrics', auth=auth).text
slo_metrics = requests.get(
f'{dashdotdb_url}/api/v1/serviceslometrics/metrics', auth=auth).text
namespaces = queries.get_namespaces()
build_jobs = jjb.get_all_jobs(job_types=['build'])
jobs_to_get = build_jobs.copy()
saas_deploy_jobs = []
for saas_file in saas_files:
saas_file_name = saas_file['name']
for template in saas_file["resourceTemplates"]:
for target in template["targets"]:
job = {}
job['env'] = target["namespace"]["environment"]["name"]
job['app'] = target["namespace"]["app"]["name"]
job['cluster'] = target['namespace']['cluster']['name']
job['namespace'] = target['namespace']['name']
job['name'] = get_openshift_saas_deploy_job_name(
saas_file_name, job['env'], settings)
job['saas_file_name'] = saas_file_name
job['instance'] = saas_file["instance"]["name"]
saas_deploy_jobs.append(job)
if job['instance'] not in jobs_to_get:
jobs_to_get[job['instance']] = [job]
else:
jobs_to_get[job['instance']].append(job)
job_history = get_build_history_pool(jenkins_map, jobs_to_get,
timestamp_limit, thread_pool_size)
for app in apps:
if not app['codeComponents']:
continue
app_name = app['name']
logging.info(f"collecting post-deploy jobs "
f"information for {app_name}")
post_deploy_jobs = {}
for saas_file in saas_files:
if saas_file['app']['name'] != app_name:
continue
resource_types = saas_file['managedResourceTypes']
# Only jobs of these types are expected to have a
# further post-deploy job
if not any([
'Deployment' in resource_types, 'DeploymentConfig'
not in resource_types
]):
continue
for resource_template in saas_file['resourceTemplates']:
for target in resource_template['targets']:
cluster = target['namespace']['cluster']['name']
namespace = target['namespace']['name']
post_deploy_jobs[cluster] = {}
post_deploy_jobs[cluster][namespace] = False
for saas_file in saas_files:
if saas_file['app']['name'] != app_name:
continue
resource_types = saas_file['managedResourceTypes']
if 'Job' not in resource_types:
continue
for resource_template in saas_file['resourceTemplates']:
for target in resource_template['targets']:
cluster = target['namespace']['cluster']['name']
namespace = target['namespace']['name']
# This block skips the check if the cluster/namespace
# has no Deployment/DeploymentConfig job associated.
if cluster not in post_deploy_jobs:
continue
if namespace not in post_deploy_jobs[cluster]:
continue
# Post-deploy job must depend on a openshift-saas-deploy
# job
if target['upstream'] is None:
continue
if target['upstream'].startswith('openshift-saas-deploy-'):
post_deploy_jobs[cluster][namespace] = True
app['post_deploy_jobs'] = post_deploy_jobs
logging.info(f"collecting promotion history for {app_name}")
app["promotions"] = {}
for job in saas_deploy_jobs:
if job['app'] != app_name:
continue
if job['name'] not in job_history:
continue
history = job_history[job["name"]]
saas_file_name = job['saas_file_name']
if saas_file_name not in app["promotions"]:
app["promotions"][saas_file_name] = [{
"env":
job["env"],
"cluster":
job["cluster"],
"namespace":
job["namespace"],
**history
}]
else:
app["promotions"][saas_file_name].append({
"env":
job["env"],
"cluster":
job["cluster"],
"namespace":
job["namespace"],
**history
})
logging.info(f"collecting merge activity for {app_name}")
app['merge_activity'] = {}
code_repos = [
c['url'] for c in app['codeComponents']
if c['resource'] == 'upstream'
]
for jobs in build_jobs.values():
for job in jobs:
try:
repo_url = get_repo_url(job)
except KeyError:
continue
if repo_url not in code_repos:
continue
if job['name'] not in job_history:
continue
history = job_history[job['name']]
if repo_url not in app["merge_activity"]:
app["merge_activity"][repo_url] = [{
"branch": job["branch"],
**history
}]
else:
app["merge_activity"][repo_url].append({
"branch":
job["branch"],
**history
})
logging.info(f"collecting dashdotdb information for {app_name}")
app_namespaces = []
for namespace in namespaces:
if namespace['app']['name'] != app['name']:
continue
app_namespaces.append(namespace)
vuln_mx = {}
validt_mx = {}
slo_mx = {}
for family in text_string_to_metric_families(vuln_metrics):
for sample in family.samples:
if sample.name == 'imagemanifestvuln_total':
for app_namespace in app_namespaces:
cluster = sample.labels['cluster']
if app_namespace['cluster']['name'] != cluster:
continue
namespace = sample.labels['namespace']
if app_namespace['name'] != namespace:
continue
severity = sample.labels['severity']
if cluster not in vuln_mx:
vuln_mx[cluster] = {}
if namespace not in vuln_mx[cluster]:
vuln_mx[cluster][namespace] = {}
if severity not in vuln_mx[cluster][namespace]:
value = int(sample.value)
vuln_mx[cluster][namespace][severity] = value
for family in text_string_to_metric_families(validt_metrics):
for sample in family.samples:
if sample.name == 'deploymentvalidation_total':
for app_namespace in app_namespaces:
cluster = sample.labels['cluster']
if app_namespace['cluster']['name'] != cluster:
continue
namespace = sample.labels['namespace']
if app_namespace['name'] != namespace:
continue
validation = sample.labels['validation']
# dvo: fail == 1, pass == 0, py: true == 1, false == 0
# so: ({false|pass}, {true|fail})
status = ('Passed',
'Failed')[int(sample.labels['status'])]
if cluster not in validt_mx:
validt_mx[cluster] = {}
if namespace not in validt_mx[cluster]:
validt_mx[cluster][namespace] = {}
if validation not in validt_mx[cluster][namespace]:
validt_mx[cluster][namespace][validation] = {}
if status not in validt_mx[cluster][namespace][
validation]: # noqa: E501
validt_mx[cluster][namespace][validation][
status] = {} # noqa: E501
value = int(sample.value)
validt_mx[cluster][namespace][validation][
status] = value # noqa: E501
for family in text_string_to_metric_families(slo_metrics):
for sample in family.samples:
if sample.name == 'serviceslometrics':
for app_namespace in app_namespaces:
cluster = sample.labels['cluster']
if app_namespace['cluster']['name'] != cluster:
continue
namespace = sample.labels['namespace']
if app_namespace['name'] != namespace:
continue
slo_doc_name = sample.labels['slodoc']
slo_name = sample.labels['name']
if cluster not in slo_mx:
slo_mx[cluster] = {}
if namespace not in slo_mx[cluster]:
slo_mx[cluster][namespace] = {}
if slo_doc_name not in slo_mx[cluster][namespace]: # pylint: disable=line-too-long # noqa: E501
slo_mx[cluster][namespace][slo_doc_name] = {}
if slo_name not in slo_mx[cluster][namespace][
slo_doc_name]: # noqa: E501
slo_mx[cluster][namespace][slo_doc_name][
slo_name] = { # noqa: E501
sample.labels['type']: sample.value
}
else:
slo_mx[cluster][namespace][slo_doc_name][
slo_name].update({ # pylint: disable=line-too-long # noqa: E501
sample.labels['type']:
sample.value
})
app['container_vulnerabilities'] = vuln_mx
app['deployment_validations'] = validt_mx
app['service_slo'] = slo_mx
return apps
def run(
dry_run,
thread_pool_size=10,
io_dir="throughput/",
saas_file_name=None,
env_name=None,
gitlab_project_id=None,
defer=None,
):
all_saas_files = queries.get_saas_files(v1=True, v2=True)
saas_files = queries.get_saas_files(saas_file_name, env_name, v1=True, v2=True)
app_interface_settings = queries.get_app_interface_settings()
if not saas_files:
logging.error("no saas files found")
sys.exit(ExitCodes.ERROR)
# notify different outputs (publish results, slack notifications)
# we only do this if:
# - this is not a dry run
# - there is a single saas file deployed
notify = not dry_run and len(saas_files) == 1
if notify:
saas_file = saas_files[0]
slack_info = saas_file.get("slack")
if slack_info:
slack = slackapi_from_slack_workspace(
slack_info,
app_interface_settings,
QONTRACT_INTEGRATION,
init_usergroups=False,
)
# support built-in start and end slack notifications
# only in v2 saas files
if saas_file["apiVersion"] == "v2":
ri = ResourceInventory()
console_url = compose_console_url(saas_file, saas_file_name, env_name)
# deployment result notification
defer(
lambda: slack_notify(
saas_file_name,
env_name,
slack,
ri,
console_url,
in_progress=False,
)
)
# deployment start notification
slack_notifications = slack_info.get("notifications")
if slack_notifications and slack_notifications.get("start"):
slack_notify(
saas_file_name,
env_name,
slack,
ri,
console_url,
in_progress=True,
)
else:
slack = None
instance = queries.get_gitlab_instance()
# instance exists in v1 saas files only
desired_jenkins_instances = [
s["instance"]["name"] for s in saas_files if s.get("instance")
]
jenkins_map = jenkins_base.get_jenkins_map(
desired_instances=desired_jenkins_instances
)
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
try:
gl = GitLabApi(instance, settings=settings)
except Exception:
# allow execution without access to gitlab
# as long as there are no access attempts.
gl = None
saasherder = SaasHerder(
saas_files,
thread_pool_size=thread_pool_size,
gitlab=gl,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
settings=settings,
jenkins_map=jenkins_map,
accounts=accounts,
)
if len(saasherder.namespaces) == 0:
logging.warning("no targets found")
sys.exit(ExitCodes.SUCCESS)
ri, oc_map = ob.fetch_current_state(
namespaces=saasherder.namespaces,
thread_pool_size=thread_pool_size,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
init_api_resources=True,
cluster_admin=saasherder.cluster_admin,
)
defer(oc_map.cleanup)
saasherder.populate_desired_state(ri)
# validate that this deployment is valid
# based on promotion information in targets
if not saasherder.validate_promotions():
logging.error("invalid promotions")
ri.register_error()
sys.exit(ExitCodes.ERROR)
# if saas_file_name is defined, the integration
# is being called from multiple running instances
actions = ob.realize_data(
dry_run,
oc_map,
ri,
thread_pool_size,
caller=saas_file_name,
wait_for_namespace=True,
no_dry_run_skip_compare=(not saasherder.compare),
take_over=saasherder.take_over,
)
if not dry_run:
if saasherder.publish_job_logs:
try:
ob.follow_logs(oc_map, actions, io_dir)
except Exception as e:
logging.error(str(e))
ri.register_error()
try:
ob.validate_data(oc_map, actions)
except Exception as e:
logging.error(str(e))
ri.register_error()
# publish results of this deployment
# based on promotion information in targets
success = not ri.has_error_registered()
# only publish promotions for deployment jobs (a single saas file)
if notify:
# Auto-promote next stages only if there are changes in the
# promoting stage. This prevents trigger promotions on job re-runs
auto_promote = len(actions) > 0
mr_cli = mr_client_gateway.init(gitlab_project_id=gitlab_project_id)
saasherder.publish_promotions(success, all_saas_files, mr_cli, auto_promote)
if not success:
sys.exit(ExitCodes.ERROR)
# send human readable notifications to slack
# we only do this if:
# - this is not a dry run
# - there is a single saas file deployed
# - output is 'events'
# - no errors were registered
if notify and slack and actions and slack_info.get("output") == "events":
for action in actions:
message = (
f"[{action['cluster']}] "
+ f"{action['kind']} {action['name']} {action['action']}"
)
slack.chat_post_message(message)
def run(dry_run,
thread_pool_size=10,
io_dir='throughput/',
saas_file_name=None,
env_name=None,
gitlab_project_id=None,
defer=None):
all_saas_files = queries.get_saas_files()
saas_files = queries.get_saas_files(saas_file_name, env_name)
if not saas_files:
logging.error('no saas files found')
sys.exit(ExitCodes.ERROR)
instance = queries.get_gitlab_instance()
desired_jenkins_instances = [s['instance']['name'] for s in saas_files]
jenkins_map = jenkins_base.get_jenkins_map(
desired_instances=desired_jenkins_instances)
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
try:
gl = GitLabApi(instance, settings=settings)
except Exception:
# allow execution without access to gitlab
# as long as there are no access attempts.
gl = None
saasherder = SaasHerder(saas_files,
thread_pool_size=thread_pool_size,
gitlab=gl,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
settings=settings,
jenkins_map=jenkins_map,
accounts=accounts)
if len(saasherder.namespaces) == 0:
logging.warning('no targets found')
sys.exit(ExitCodes.SUCCESS)
ri, oc_map = ob.fetch_current_state(
namespaces=saasherder.namespaces,
thread_pool_size=thread_pool_size,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
init_api_resources=True)
defer(lambda: oc_map.cleanup())
saasherder.populate_desired_state(ri)
# validate that this deployment is valid
# based on promotion information in targets
if not saasherder.validate_promotions():
logging.error('invalid promotions')
sys.exit(ExitCodes.ERROR)
# if saas_file_name is defined, the integration
# is being called from multiple running instances
actions = ob.realize_data(dry_run,
oc_map,
ri,
caller=saas_file_name,
wait_for_namespace=True,
no_dry_run_skip_compare=(not saasherder.compare),
take_over=saasherder.take_over)
if not dry_run:
if saasherder.publish_job_logs:
try:
ob.follow_logs(oc_map, actions, io_dir)
except Exception as e:
logging.error(str(e))
ri.register_error()
try:
ob.validate_data(oc_map, actions)
except Exception as e:
logging.error(str(e))
ri.register_error()
# publish results of this deployment
# based on promotion information in targets
success = not ri.has_error_registered()
# only publish promotions for deployment jobs (a single saas file)
if not dry_run and len(saasherder.saas_files) == 1:
mr_cli = mr_client_gateway.init(gitlab_project_id=gitlab_project_id)
saasherder.publish_promotions(success, all_saas_files, mr_cli)
if not success:
sys.exit(ExitCodes.ERROR)
# send human readable notifications to slack
# we only do this if:
# - this is not a dry run
# - there is a single saas file deployed
# - output is 'events'
# - no errors were registered
if not dry_run and len(saasherder.saas_files) == 1:
saas_file = saasherder.saas_files[0]
slack_info = saas_file.get('slack')
if slack_info and actions and slack_info.get('output') == 'events':
slack = init_slack(slack_info,
QONTRACT_INTEGRATION,
init_usergroups=False)
for action in actions:
message = \
f"[{action['cluster']}] " + \
f"{action['kind']} {action['name']} {action['action']}"
slack.chat_post_message(message)
def setup(thread_pool_size, internal, use_jump_host, integration,
integration_version, v1, v2):
"""Setup required resources for triggering integrations
Args:
thread_pool_size (int): Thread pool size to use
internal (bool): Should run for internal/extrenal/all clusters
use_jump_host (bool): Should use jump host to reach clusters
integration (string): Name of calling integration
integration_version (string): Version of calling integration
v1 (bool): Should trigger for v1 SaaS files
v2 (bool): Should trigger for v2 SaaS files
Returns:
saasherder (SaasHerder): a SaasHerder instance
jenkins_map (dict): Instance names with JenkinsApi instances
oc_map (OC_Map): a dictionary of OC clients per cluster
settings (dict): App-interface settings
error (bool): True if one happened, False otherwise
"""
saas_files = queries.get_saas_files(v1=v1, v2=v2)
if not saas_files:
logging.error("no saas files found")
return None, None, None, None, True
saas_files = [sf for sf in saas_files if is_in_shard(sf["name"])]
# Remove saas-file targets that are disabled
for saas_file in saas_files[:]:
resource_templates = saas_file["resourceTemplates"]
for rt in resource_templates[:]:
targets = rt["targets"]
for target in targets[:]:
if target["disable"]:
targets.remove(target)
instance = queries.get_gitlab_instance()
settings = queries.get_app_interface_settings()
accounts = queries.get_state_aws_accounts()
gl = GitLabApi(instance, settings=settings)
jenkins_map = jenkins_base.get_jenkins_map()
pipelines_providers = queries.get_pipelines_providers()
tkn_provider_namespaces = [
pp["namespace"] for pp in pipelines_providers
if pp["provider"] == "tekton"
]
oc_map = OC_Map(
namespaces=tkn_provider_namespaces,
integration=integration,
settings=settings,
internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size,
)
saasherder = SaasHerder(
saas_files,
thread_pool_size=thread_pool_size,
gitlab=gl,
integration=integration,
integration_version=integration_version,
settings=settings,
jenkins_map=jenkins_map,
accounts=accounts,
)
return saasherder, jenkins_map, oc_map, settings, False
def get_apps_data(date, month_delta=1):
apps = queries.get_apps()
saas_files = queries.get_saas_files()
jjb, _ = init_jjb()
saas_jobs = jjb.get_all_jobs(job_types=['saas-deploy', 'promote-to-prod'])
build_master_jobs = jjb.get_all_jobs(job_types=['build-master'])
jenkins_map = jenkins_base.get_jenkins_map()
time_limit = date - relativedelta(months=month_delta)
timestamp_limit = \
int(time_limit.replace(tzinfo=timezone.utc).timestamp())
saas_build_history = \
get_build_history(jenkins_map, saas_jobs, timestamp_limit)
build_master_build_history = \
get_build_history(jenkins_map, build_master_jobs, timestamp_limit)
settings = queries.get_app_interface_settings()
secret_reader = SecretReader(settings=settings)
secret_content = secret_reader.read_all({'path': DASHDOTDB_SECRET})
dashdotdb_url = secret_content['url']
dashdotdb_user = secret_content['username']
dashdotdb_pass = secret_content['password']
metrics = requests.get(f'{dashdotdb_url}/api/v1/metrics',
auth=(dashdotdb_user, dashdotdb_pass)).text
namespaces = queries.get_namespaces()
for app in apps:
if not app['codeComponents']:
continue
app_name = app['name']
logging.info(f"collecting post-deploy jobs "
f"information for {app_name}")
post_deploy_jobs = {}
for saas_file in saas_files:
if saas_file['app']['name'] != app_name:
continue
resource_types = saas_file['managedResourceTypes']
# Only jobs of these types are expected to have a
# further post-deploy job
if not any([
'Deployment' in resource_types, 'DeploymentConfig'
not in resource_types
]):
continue
for resource_template in saas_file['resourceTemplates']:
for target in resource_template['targets']:
cluster = target['namespace']['cluster']['name']
namespace = target['namespace']['name']
post_deploy_jobs[cluster] = {}
post_deploy_jobs[cluster][namespace] = False
for saas_file in saas_files:
if saas_file['app']['name'] != app_name:
continue
resource_types = saas_file['managedResourceTypes']
if 'Job' not in resource_types:
continue
for resource_template in saas_file['resourceTemplates']:
for target in resource_template['targets']:
cluster = target['namespace']['cluster']['name']
namespace = target['namespace']['name']
# This block skips the check if the cluster/namespace
# has no Deployment/DeploymentConfig job associated.
if cluster not in post_deploy_jobs:
continue
if namespace not in post_deploy_jobs[cluster]:
continue
# Post-deploy job must depend on a openshift-saas-deploy
# job
if target['upstream'] is None:
continue
if target['upstream'].startswith('openshift-saas-deploy-'):
post_deploy_jobs[cluster][namespace] = True
app['post_deploy_jobs'] = post_deploy_jobs
logging.info(f"collecting promotions for {app_name}")
app['promotions'] = {}
saas_repos = [
c['url'] for c in app['codeComponents']
if c['resource'] == 'saasrepo'
]
for sr in saas_repos:
sr_history = saas_build_history.get(sr)
if not sr_history:
continue
successes = [h for h in sr_history if h == 'SUCCESS']
app['promotions'][sr] = (len(sr_history), len(successes))
logging.info(f"collecting merge activity for {app_name}")
app['merge_activity'] = {}
code_repos = [
c['url'] for c in app['codeComponents']
if c['resource'] == 'upstream'
]
for cr in code_repos:
cr_history = build_master_build_history.get(cr)
if not cr_history:
continue
successes = [h for h in cr_history if h == 'SUCCESS']
app['merge_activity'][cr] = (len(cr_history), len(successes))
logging.info(f"collecting vulnerabilities information for {app_name}")
app_namespaces = []
for namespace in namespaces:
if namespace['app']['name'] != app['name']:
continue
app_namespaces.append(namespace)
app_metrics = {}
for family in text_string_to_metric_families(metrics):
for sample in family.samples:
if sample.name != 'imagemanifestvuln_total':
continue
for app_namespace in app_namespaces:
cluster = sample.labels['cluster']
if app_namespace['cluster']['name'] != cluster:
continue
namespace = sample.labels['namespace']
if app_namespace['name'] != namespace:
continue
severity = sample.labels['severity']
if cluster not in app_metrics:
app_metrics[cluster] = {}
if namespace not in app_metrics[cluster]:
app_metrics[cluster][namespace] = {}
if severity not in app_metrics[cluster][namespace]:
value = int(sample.value)
app_metrics[cluster][namespace][severity] = value
app['container_vulnerabilities'] = app_metrics
return apps
def run(dry_run,
thread_pool_size=10,
saas_file_name=None,
env_name=None,
defer=None):
saas_files = queries.get_saas_files(saas_file_name, env_name)
if not saas_files:
logging.error('no saas files found')
sys.exit(1)
instance = queries.get_gitlab_instance()
desired_jenkins_instances = [s['instance']['name'] for s in saas_files]
jenkins_map = jenkins_base.get_jenkins_map(
desired_instances=desired_jenkins_instances)
settings = queries.get_app_interface_settings()
try:
gl = GitLabApi(instance, settings=settings)
except Exception:
# allow execution without access to gitlab
# as long as there are no access attempts.
gl = None
saasherder = SaasHerder(saas_files,
thread_pool_size=thread_pool_size,
gitlab=gl,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
settings=settings,
jenkins_map=jenkins_map)
if not saasherder.valid:
sys.exit(1)
if len(saasherder.namespaces) == 0:
logging.warning('no targets found')
sys.exit(0)
ri, oc_map = ob.fetch_current_state(
namespaces=saasherder.namespaces,
thread_pool_size=thread_pool_size,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
init_api_resources=True)
defer(lambda: oc_map.cleanup())
saasherder.populate_desired_state(ri)
# if saas_file_name is defined, the integration
# is being called from multiple running instances
actions = ob.realize_data(dry_run,
oc_map,
ri,
caller=saas_file_name,
wait_for_namespace=True,
no_dry_run_skip_compare=(not saasherder.compare),
take_over=saasherder.take_over)
if ri.has_error_registered():
sys.exit(1)
# send human readable notifications to slack
# we only do this if:
# - this is not a dry run
# - there is a single saas file deployed
# - output is 'events'
# - no errors were registered
if not dry_run and len(saasherder.saas_files) == 1:
saas_file = saasherder.saas_files[0]
slack_info = saas_file.get('slack')
if slack_info and actions and slack_info.get('output') == 'events':
slack = init_slack(slack_info, QONTRACT_INTEGRATION)
for action in actions:
message = \
f"[{action['cluster']}] " + \
f"{action['kind']} {action['name']} {action['action']}"
slack.chat_post_message(message)
def collect_saas_file_configs(settings, instance_name=None):
# collect a list of jobs per saas file per environment.
# each saas_file_config should have the structure described
# in the above query.
# to make things understandable, each variable used to form
# the structure will be called `jc_<variable>` (jenkins config).
saas_file_configs = []
repo_urls = set()
saas_files = queries.get_saas_files()
job_template_name = settings['saasDeployJobTemplate']
for saas_file in saas_files:
saas_file_name = saas_file['name']
jc_instance = saas_file['instance']
if instance_name is not None and jc_instance['name'] != instance_name:
continue
app_name = saas_file['app']['name']
# currently ignoring the actual Slack workspace
# as that is configured in Jenkins.
# revisit this if we support more then a single Slack workspace.
output = saas_file['slack'].get('output') or 'publish'
# if the output type is 'publish', we send notifications
# to the selected slack_channel
slack_channel = \
saas_file['slack']['channel'] \
if output == 'publish' \
else 'dev-null'
slack_notify_start = False
slack_notifications = saas_file['slack'].get('notifications')
if slack_notifications:
start = slack_notifications.get('start')
if start:
slack_notify_start = True
timeout = saas_file.get('timeout', None)
for resource_template in saas_file['resourceTemplates']:
url = resource_template['url']
repo_urls.add(url)
for target in resource_template['targets']:
env_name = target['namespace']['environment']['name']
upstream = target.get('upstream') or ''
final_job_template_name = \
f'{job_template_name}-with-upstream' if upstream \
else job_template_name
jc_name = get_openshift_saas_deploy_job_name(
saas_file_name, env_name, settings)
existing_configs = \
[c for c in saas_file_configs if c['name'] == jc_name]
if existing_configs:
# if upstream is defined - append it to existing upstreams
if upstream:
# should be exactly one
jc_data = existing_configs[0]['data']
project = jc_data['project']
# append upstream to existing upstreams
project['upstream'] += f',{upstream}'
# update job template name if needed
job_definition = project['jobs'][0]
if job_template_name in job_definition:
job_definition[final_job_template_name] = \
job_definition.pop(job_template_name)
continue
# each config is a list with a single item
# with the following structure:
# project:
# name: 'openshift-saas-deploy-{saas_file_name}-{env_name}'
# saas_file_name: '{saas_file_name}'
# env_name: '{env_name}'
# app_name: '{app_name}'
# slack_channel: '{slack_channel}'
# slack_notify_start: '{slack_notify_start}
# jobs:
# - 'openshift-saas-deploy':
# display_name: display name of the job
jc_data = {
'project': {
'name':
jc_name,
'saas_file_name':
saas_file_name,
'env_name':
env_name,
'app_name':
app_name,
'slack_channel':
slack_channel,
'slack_notify_start':
slack_notify_start,
'upstream':
upstream,
'jobs': [{
final_job_template_name: {
'display_name': jc_name
}
}]
}
}
if timeout:
jc_data['project']['timeout'] = timeout
saas_file_configs.append({
'name': jc_name,
'instance': jc_instance,
'type': 'jobs',
'data': jc_data
})
for saas_file_config in saas_file_configs:
jc_data = saas_file_config.pop('data')
saas_file_config['config'] = json.dumps([jc_data])
return saas_file_configs, repo_urls
def collect_saas_file_configs(settings, instance_name=None):
# collect a list of jobs per saas file per environment.
# each saas_file_config should have the structure described
# in the above query.
# to make things understandable, each variable used to form
# the structure will be called `jc_<variable>` (jenkins config).
saas_file_configs = []
repo_urls = set()
saas_files = queries.get_saas_files()
job_template_name = settings["saasDeployJobTemplate"]
for saas_file in saas_files:
saas_file_name = saas_file["name"]
jc_instance = saas_file["instance"]
if instance_name is not None and jc_instance["name"] != instance_name:
continue
app_name = saas_file["app"]["name"]
# currently ignoring the actual Slack workspace
# as that is configured in Jenkins.
# revisit this if we support more then a single Slack workspace.
output = saas_file["slack"].get("output") or "publish"
# if the output type is 'publish', we send notifications
# to the selected slack_channel
slack_channel = (saas_file["slack"]["channel"]
if output == "publish" else "dev-null")
slack_notify_start = False
slack_notifications = saas_file["slack"].get("notifications")
if slack_notifications:
start = slack_notifications.get("start")
if start:
slack_notify_start = True
timeout = saas_file.get("timeout", None)
for resource_template in saas_file["resourceTemplates"]:
url = resource_template["url"]
repo_urls.add(url)
for target in resource_template["targets"]:
env_name = target["namespace"]["environment"]["name"]
upstream = target.get("upstream") or ""
final_job_template_name = (f"{job_template_name}-with-upstream"
if upstream else job_template_name)
jc_name = get_openshift_saas_deploy_job_name(
saas_file_name, env_name, settings)
existing_configs = [
c for c in saas_file_configs if c["name"] == jc_name
]
if existing_configs:
# if upstream is defined - append it to existing upstreams
if upstream:
# should be exactly one
jc_data = existing_configs[0]["data"]
project = jc_data["project"]
# append upstream to existing upstreams
project["upstream"] += f",{upstream}"
# update job template name if needed
job_definition = project["jobs"][0]
if job_template_name in job_definition:
job_definition[
final_job_template_name] = job_definition.pop(
job_template_name)
continue
# each config is a list with a single item
# with the following structure:
# project:
# name: 'openshift-saas-deploy-{saas_file_name}-{env_name}'
# saas_file_name: '{saas_file_name}'
# env_name: '{env_name}'
# app_name: '{app_name}'
# slack_channel: '{slack_channel}'
# slack_notify_start: '{slack_notify_start}
# jobs:
# - 'openshift-saas-deploy':
# display_name: display name of the job
jc_data = {
"project": {
"name":
jc_name,
"saas_file_name":
saas_file_name,
"env_name":
env_name,
"app_name":
app_name,
"slack_channel":
slack_channel,
"slack_notify_start":
slack_notify_start,
"upstream":
upstream,
"jobs": [{
final_job_template_name: {
"display_name": jc_name
}
}],
}
}
if timeout:
jc_data["project"]["timeout"] = timeout
saas_file_configs.append({
"name": jc_name,
"instance": jc_instance,
"type": "jobs",
"data": jc_data,
})
for saas_file_config in saas_file_configs:
jc_data = saas_file_config.pop("data")
saas_file_config["config"] = json.dumps([jc_data])
return saas_file_configs, repo_urls