def run(thread_pool_size=10, defer=None):
oc_map = tb.get_oc_map(QONTRACT_E2E_TEST)
defer(lambda: oc_map.cleanup())
pattern = tb.get_namespaces_pattern()
threaded.run(test_cluster, oc_map.clusters(), thread_pool_size,
oc_map=oc_map,
pattern=pattern)
def __init__(self,
clusters=None,
namespaces=None,
integration='',
e2e_test='',
settings=None,
internal=None,
use_jump_host=True,
thread_pool_size=1,
init_projects=False,
init_api_resources=False):
self.oc_map = {}
self.calling_integration = integration
self.calling_e2e_test = e2e_test
self.settings = settings
self.internal = internal
self.use_jump_host = use_jump_host
self.thread_pool_size = thread_pool_size
self.init_projects = init_projects
self.init_api_resources = init_api_resources
self._lock = Lock()
if clusters and namespaces:
raise KeyError('expected only one of clusters or namespaces.')
elif clusters:
threaded.run(self.init_oc_client, clusters, self.thread_pool_size)
elif namespaces:
clusters = [ns_info['cluster'] for ns_info in namespaces]
threaded.run(self.init_oc_client, clusters, self.thread_pool_size)
else:
raise KeyError('expected one of clusters or namespaces.')
def fetch_current_state(namespaces=None,
clusters=None,
thread_pool_size=None,
integration=None,
integration_version=None,
override_managed_types=None,
internal=None,
use_jump_host=True,
init_api_resources=False):
ri = ResourceInventory()
settings = queries.get_app_interface_settings()
oc_map = OC_Map(namespaces=namespaces,
clusters=clusters,
integration=integration,
settings=settings,
internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size,
init_api_resources=init_api_resources)
state_specs = \
init_specs_to_fetch(
ri,
oc_map,
namespaces=namespaces,
clusters=clusters,
override_managed_types=override_managed_types
)
threaded.run(populate_current_state, state_specs, thread_pool_size,
ri=ri,
integration=integration,
integration_version=integration_version)
return ri, oc_map
def run(thread_pool_size=10, defer=None):
oc_map = tb.get_oc_map(QONTRACT_E2E_TEST)
defer(lambda: oc_map.cleanup())
ns_under_test = tb.get_test_namespace_name()
threaded.run(test_cluster,
oc_map.clusters(),
thread_pool_size,
oc_map=oc_map,
ns_under_test=ns_under_test)
def populate_desired_state(self, ri):
results = threaded.run(self.init_populate_desired_state_specs,
self.saas_files, self.thread_pool_size)
desired_state_specs = \
[item for sublist in results for item in sublist]
promotions = threaded.run(self.populate_desired_state_saas_file,
desired_state_specs,
self.thread_pool_size,
ri=ri)
self.promotions = promotions
def run(self):
slo_documents = queries.get_slo_documents()
service_slos = threaded.run(func=self._get_service_slo,
iterable=slo_documents,
thread_pool_size=self.thread_pool_size)
threaded.run(func=self._post,
iterable=service_slos,
thread_pool_size=self.thread_pool_size)
def run(dry_run, thread_pool_size=10):
instance = queries.get_gitlab_instance()
settings = queries.get_app_interface_settings()
repos = queries.get_repos_gitlab_owner(server=instance['url'])
threaded.run(act,
repos,
thread_pool_size,
dry_run=dry_run,
instance=instance,
settings=settings)
def run(self, cname=None):
validation_list = ('operator_replica', 'operator_request_limit')
for validation in validation_list:
LOG.debug('%s Processing validation: %s', self.logmarker,
validation)
validations = threaded.run(func=self._get_deploymentvalidation,
iterable=self._get_clusters(cname),
thread_pool_size=self.thread_pool_size,
validation=validation)
threaded.run(func=self._post,
iterable=validations,
thread_pool_size=self.thread_pool_size)
def run(dry_run, gitlab_project_id=None, thread_pool_size=10):
accounts = queries.get_aws_accounts()
settings = queries.get_app_interface_settings()
aws = AWSApi(thread_pool_size, accounts, settings=settings)
existing_keys = aws.get_users_keys()
existing_keys_list = [
key for user_key in existing_keys.values()
for keys in user_key.values() for key in keys
]
logging.info('found {} existing keys'.format(len(existing_keys_list)))
app_int_github_repos = queries.get_repos(server='https://github.com')
all_repos = get_all_repos_to_scan(app_int_github_repos)
logging.info('about to scan {} repos'.format(len(all_repos)))
results = threaded.run(git_secrets.scan_history,
all_repos,
thread_pool_size,
existing_keys=existing_keys_list)
all_leaked_keys = [key for keys in results for key in keys]
deleted_keys = aws_sos.get_deleted_keys(accounts)
keys_to_delete = \
[{'account': account, 'key': key}
for key in all_leaked_keys
for account, user_keys in existing_keys.items()
if key in [uk for uks in user_keys.values() for uk in uks]
and key not in deleted_keys[account]]
aws_sos.act(dry_run, gitlab_project_id, accounts, keys_to_delete)
def init_ecr_auth_tokens(self, accounts):
accounts_with_ecr = [a for a in accounts if a.get('ecrs')]
if not accounts_with_ecr:
return
auth_tokens = {}
results = threaded.run(self.get_tf_secrets, accounts_with_ecr,
self.thread_pool_size)
account_secrets = {account: secret for account, secret in results}
for account in accounts_with_ecr:
account_name = account['name']
account_secret = account_secrets[account_name]
access_key = account_secret['aws_access_key_id']
secret_key = account_secret['aws_secret_access_key']
ecrs = account['ecrs']
for ecr in ecrs:
region_name = ecr['region']
session = boto3.Session(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region_name,
)
client = session.client('ecr')
token = client.get_authorization_token()
auth_tokens[f"{account_name}/{region_name}"] = token
self.auth_tokens = auth_tokens
def get_build_history_pool(jenkins_map, jobs,
timestamp_limit, thread_pool_size):
history_to_get = []
for instance, jobs in jobs.items():
jenkins = jenkins_map[instance]
for job in jobs:
job['jenkins'] = jenkins
job['timestamp_limit'] = timestamp_limit
history_to_get.append(job)
result = run(func=get_build_history,
iterable=history_to_get,
thread_pool_size=thread_pool_size)
history = {}
for job in result:
build_history = job.get('build_history')
if not build_history:
continue
successes = [_ for _ in build_history if _ == 'SUCCESS']
history[job['name']] = {
"total": len(build_history),
"success": len(successes)
}
return history
def run(dry_run,
gitlab_project_id=None,
thread_pool_size=10,
enable_deletion=False,
send_mails=False):
settings = queries.get_app_interface_settings()
users = queries.get_users()
g = init_github()
results = threaded.run(get_user_company, users, thread_pool_size, github=g)
users_to_delete = get_users_to_delete(results)
if not dry_run and enable_deletion:
mr_cli = mr_client_gateway.init(gitlab_project_id=gitlab_project_id)
for user in users_to_delete:
username = user['username']
paths = user['paths']
logging.info(['delete_user', username])
if not dry_run:
if send_mails:
send_email_notification(user, settings)
elif enable_deletion:
mr = CreateDeleteUser(username, paths)
mr.submit(cli=mr_cli)
else:
msg = ('\'delete\' action is not enabled. '
'Please run the integration manually '
'with the \'--enable-deletion\' flag.')
logging.warning(msg)
def run(self):
clusters = queries.get_clusters()
oc_map = OC_Map(clusters=clusters,
integration=QONTRACT_INTEGRATION,
settings=self.settings,
use_jump_host=True,
thread_pool_size=self.thread_pool_size)
manifests = threaded.run(func=self._get_imagemanifestvuln,
iterable=oc_map.clusters(),
thread_pool_size=self.thread_pool_size,
oc_map=oc_map)
threaded.run(func=self._post,
iterable=manifests,
thread_pool_size=self.thread_pool_size)
def init_specs(self):
wd_specs = \
[{'name': name, 'wd': wd}
for name, wd in self.working_dirs.items()]
results = threaded.run(self.terraform_init, wd_specs,
self.thread_pool_size)
self.specs = \
[{'name': name, 'tf': tf} for name, tf in results]
def fetch_data(namespaces,
thread_pool_size,
internal,
use_jump_host,
init_api_resources=False):
ri = ResourceInventory()
settings = queries.get_app_interface_settings()
oc_map = OC_Map(namespaces=namespaces,
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size,
init_api_resources=init_api_resources)
state_specs = ob.init_specs_to_fetch(ri, oc_map, namespaces=namespaces)
threaded.run(fetch_states, state_specs, thread_pool_size, ri=ri)
return oc_map, ri
def apply(self):
errors = False
results = threaded.run(self.terraform_apply, self.specs,
self.thread_pool_size)
for error in results:
if error:
errors = True
return errors
def fetch_current_state(thread_pool_size, internal, use_jump_host):
clusters = queries.get_clusters(minimal=True)
settings = queries.get_app_interface_settings()
oc_map = OC_Map(clusters=clusters, integration=QONTRACT_INTEGRATION,
settings=settings, internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size)
results = threaded.run(get_cluster_users, oc_map.clusters(),
thread_pool_size, oc_map=oc_map)
current_state = [item for sublist in results for item in sublist]
return oc_map, current_state
def fetch_current_state(dry_run, namespaces, thread_pool_size,
internal, use_jump_host, account_name):
ri = ResourceInventory()
if dry_run:
return ri, None
settings = queries.get_app_interface_settings()
oc_map = OC_Map(namespaces=namespaces, integration=QONTRACT_INTEGRATION,
settings=settings, internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size)
state_specs = \
ob.init_specs_to_fetch(
ri,
oc_map,
namespaces=namespaces,
override_managed_types=['Secret']
)
threaded.run(populate_oc_resources, state_specs, thread_pool_size, ri=ri,
account_name=account_name)
return ri, oc_map
def fetch_current_state(thread_pool_size):
clusters = queries.get_clusters()
clusters = [c for c in clusters if c.get('ocm') is not None]
current_state = []
settings = queries.get_app_interface_settings()
ocm_map = OCMMap(clusters=clusters, integration=QONTRACT_INTEGRATION,
settings=settings)
groups_list = openshift_groups.create_groups_list(clusters, oc_map=ocm_map)
results = threaded.run(get_cluster_state, groups_list, thread_pool_size,
ocm_map=ocm_map)
current_state = [item for sublist in results for item in sublist]
return ocm_map, current_state
def _check_images(self, options):
saas_file_name = options['saas_file_name']
resource_template_name = options['resource_template_name']
html_url = options['html_url']
resources = options['resources']
image_auth = options['image_auth']
image_patterns = options['image_patterns']
error_prefix = \
f"[{saas_file_name}/{resource_template_name}] {html_url}:"
images_list = threaded.run(self._collect_images, resources,
self.available_thread_pool_size)
images = {item for sublist in images_list for item in sublist}
if not images:
return False # no errors
errors = threaded.run(self._check_image, images,
self.available_thread_pool_size,
image_patterns=image_patterns,
image_auth=image_auth,
error_prefix=error_prefix)
error = True in errors
return error
def run(dry_run, thread_pool_size=10):
namespaces = queries.get_namespaces()
tfrs_to_mirror = []
for namespace in namespaces:
if namespace['terraformResources'] is None:
continue
for tfr in namespace['terraformResources']:
if tfr['provider'] != 'ecr':
continue
if tfr['mirror'] is None:
continue
tfrs_to_mirror.append(tfr)
work_list = threaded.run(EcrMirror,
tfrs_to_mirror,
thread_pool_size=thread_pool_size,
dry_run=dry_run)
threaded.run(worker, work_list, thread_pool_size=thread_pool_size)
def run(dry_run, gitlab_project_id=None, thread_pool_size=10):
users = init_users()
user_specs = threaded.run(init_user_spec, users, thread_pool_size)
users_to_delete = [(username, paths) for username, delete, paths
in user_specs if delete]
if not dry_run:
mr_cli = mr_client_gateway.init(gitlab_project_id=gitlab_project_id,
sqs_or_gitlab='gitlab')
for username, paths in users_to_delete:
logging.info(['delete_user', username])
if not dry_run:
mr = CreateDeleteUser(username, paths)
mr.submit(cli=mr_cli)
def plan(self, enable_deletion):
errors = False
disabled_deletions_detected = False
results = threaded.run(self.terraform_plan,
self.specs,
self.thread_pool_size,
enable_deletion=enable_deletion)
self.deleted_users = []
for disabled_deletion_detected, deleted_users, error in results:
if error:
errors = True
if disabled_deletion_detected:
disabled_deletions_detected = True
self.deleted_users.extend(deleted_users)
return disabled_deletions_detected, errors
def fetch_current_state(thread_pool_size, internal, use_jump_host):
clusters = queries.get_clusters()
ocm_clusters = [c['name'] for c in clusters if c.get('ocm') is not None]
current_state = []
settings = queries.get_app_interface_settings()
oc_map = OC_Map(clusters=clusters, integration=QONTRACT_INTEGRATION,
settings=settings, internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size)
groups_list = create_groups_list(clusters, oc_map)
results = threaded.run(get_cluster_state, groups_list, thread_pool_size,
oc_map=oc_map)
current_state = [item for sublist in results for item in sublist]
return oc_map, current_state, ocm_clusters
def init_sessions_and_resources(self, accounts):
results = threaded.run(self.get_tf_secrets, accounts,
self.thread_pool_size)
self.sessions = {}
self.resources = {}
for account, secret in results:
access_key = secret['aws_access_key_id']
secret_key = secret['aws_secret_access_key']
region_name = secret['region']
session = boto3.Session(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region_name,
)
self.sessions[account] = session
self.resources[account] = {}
def run(dry_run, thread_pool_size=10):
instance = queries.get_gitlab_instance()
settings = queries.get_app_interface_settings()
gl = GitLabApi(instance, settings=settings)
repos = queries.get_repos(server=gl.server)
app_sre = gl.get_app_sre_group_users()
results = threaded.run(get_members_to_add,
repos,
thread_pool_size,
gl=gl,
app_sre=app_sre)
members_to_add = [item for sublist in results for item in sublist]
for m in members_to_add:
logging.info(['add_maintainer', m["repo"], m["user"].username])
if not dry_run:
gl.add_project_member(m["repo"], m["user"])
def run(dry_run,
thread_pool_size=10,
internal=None,
use_jump_host=True,
defer=None):
oc_map, desired_state = get_desired_state(internal, use_jump_host,
thread_pool_size)
defer(lambda: oc_map.cleanup())
results = threaded.run(check_ns_exists,
desired_state,
thread_pool_size,
oc_map=oc_map)
specs_to_create = [spec for spec, create in results if create]
for spec in specs_to_create:
logging.info(['create', spec['cluster'], spec['namespace']])
if not dry_run:
create_new_project(spec, oc_map)
def run(dry_run, thread_pool_size=10, io_dir='throughput/'):
saas_file_owners_diffs = read_saas_file_owners_diffs(io_dir)
if len(saas_file_owners_diffs) == 0:
return
available_thread_pool_size = \
threaded.estimate_available_thread_pool_size(
thread_pool_size,
len(saas_file_owners_diffs))
exit_codes = threaded.run(
osd_run_wrapper,
saas_file_owners_diffs,
thread_pool_size,
dry_run=dry_run,
available_thread_pool_size=available_thread_pool_size)
if [ec for ec in exit_codes if ec]:
sys.exit(1)
def check_prometheus_rules(rules, thread_pool_size):
'''Returns a list of dicts with failed rule checks'''
# flatten the list of prometheus rules to have a list of dicts
rules_to_check = []
for path, cluster_data in rules.items():
for cluster, namespace_data in cluster_data.items():
for namespace, rule_data in namespace_data.items():
rules_to_check.append({
'path': path,
'cluster': cluster,
'namespace': namespace,
'spec': rule_data['spec']
})
result = threaded.run(func=check_rule,
iterable=rules_to_check,
thread_pool_size=thread_pool_size)
# return invalid rules
return [rule for rule in result if not rule['check_result']]
def run(dry_run,
print_only=False,
enable_deletion=False,
io_dir='throughput/',
thread_pool_size=10,
internal=None,
use_jump_host=True,
light=False,
vault_output_path='',
extra_labels=None):
account_names = [
name for index, name in enumerate(sorted(get_accounts_names()))
if is_in_shard_round_robin(name, index)
]
if not account_names:
logging.warning("No accounts in shards")
return
exit_codes = threaded.run(
tfr_run_wrapper,
account_names,
thread_pool_size,
dry_run=dry_run,
print_only=print_only,
enable_deletion=enable_deletion,
io_dir=io_dir,
internal_thread_pool_size=thread_pool_size,
internal=internal,
use_jump_host=use_jump_host,
light=light,
vault_output_path=vault_output_path,
extra_labels=extra_labels,
)
if any(exit_codes):
sys.exit(ExitCodes.ERROR)
