def run(self, cname=None):
validation_list = threaded.run(func=self._get_validation_names,
iterable=self._get_clusters(cname),
thread_pool_size=self.thread_pool_size,
filter='deployment_validation_operator')
validation_names = {}
if validation_list:
validation_names = {
v['cluster']: v['data']
for v in validation_list if v
}
clusters = self._get_clusters(cname)
self._get_token()
for cluster in clusters:
cluster_name = cluster['name']
if cluster_name not in validation_names:
LOG.debug('%s Skipping cluster: %s', self.logmarker,
cluster_name)
continue
LOG.debug('%s Processing cluster: %s', self.logmarker,
cluster_name)
validations = threaded.run(func=self._get_deploymentvalidation,
iterable=validation_names[cluster_name],
thread_pool_size=self.thread_pool_size,
clusterinfo=cluster)
threaded.run(func=self._post,
iterable=validations,
thread_pool_size=self.thread_pool_size)
self._close_token()
def fetch_current_state(dry_run, namespaces, thread_pool_size, internal,
use_jump_host, account_name):
ri = ResourceInventory()
if dry_run:
return ri, None
settings = queries.get_app_interface_settings()
oc_map = OC_Map(namespaces=namespaces,
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size)
state_specs = \
ob.init_specs_to_fetch(
ri,
oc_map,
namespaces=namespaces,
override_managed_types=['Secret']
)
threaded.run(populate_oc_resources,
state_specs,
thread_pool_size,
ri=ri,
account_name=account_name)
return ri, oc_map
def fetch_data(
namespaces,
thread_pool_size,
internal,
use_jump_host,
init_api_resources=False,
overrides=None,
):
ri = ResourceInventory()
settings = queries.get_app_interface_settings()
logging.debug(f"Overriding keys {overrides}")
oc_map = OC_Map(
namespaces=namespaces,
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size,
init_api_resources=init_api_resources,
)
state_specs = ob.init_specs_to_fetch(ri,
oc_map,
namespaces=namespaces,
override_managed_types=overrides)
threaded.run(fetch_states, state_specs, thread_pool_size, ri=ri)
return oc_map, ri
def realize(
inventory: LabelInventory,
state: State,
oc_map: OC_Map,
dry_run: bool,
thread_pool_size: int,
) -> None:
"""
Apply the changes in the state store and on the namespaces
"""
for cluster, namespace, types in inventory:
if inventory.errors(cluster, namespace):
continue
upd_managed = types.get(UPDATED_MANAGED, [])
if upd_managed:
key = state_key(cluster, namespace)
_LOG.debug(f"Updating state store: {key}: {upd_managed}")
if not dry_run:
state.add(key, upd_managed, force=True)
# Potential exceptions will get raised up
threaded.run(
label,
inventory,
thread_pool_size,
oc_map=oc_map,
dry_run=dry_run,
inventory=inventory,
)
def run(dry_run, thread_pool_size=10):
instance = queries.get_gitlab_instance()
settings = queries.get_app_interface_settings()
repos = queries.get_repos_gitlab_owner(server=instance['url'])
threaded.run(act, repos, thread_pool_size,
dry_run=dry_run,
instance=instance,
settings=settings)
def run(thread_pool_size=10, defer=None):
oc_map = tb.get_oc_map(QONTRACT_E2E_TEST)
defer(oc_map.cleanup)
ns_under_test = tb.get_test_namespace_name()
threaded.run(test_cluster,
oc_map.clusters(),
thread_pool_size,
oc_map=oc_map,
ns_under_test=ns_under_test)
def run(thread_pool_size=10, defer=None):
oc_map = tb.get_oc_map(QONTRACT_E2E_TEST)
defer(oc_map.cleanup)
pattern = tb.get_namespaces_pattern()
threaded.run(test_cluster,
oc_map.clusters(),
thread_pool_size,
oc_map=oc_map,
pattern=pattern)
def run(self):
slo_documents = queries.get_slo_documents()
service_slos = threaded.run(func=self._get_service_slo,
iterable=slo_documents,
thread_pool_size=self.thread_pool_size)
self._get_token()
threaded.run(func=self._post,
iterable=service_slos,
thread_pool_size=self.thread_pool_size)
self._close_token()
def get_build_history_pool(jenkins_map, jobs, timestamp_limit,
thread_pool_size):
history_to_get = []
for instance, jobs in jobs.items():
jenkins = jenkins_map[instance]
for job in jobs:
job['jenkins'] = jenkins
job['timestamp_limit'] = timestamp_limit
history_to_get.append(job)
result = run(func=get_build_history,
iterable=history_to_get,
thread_pool_size=thread_pool_size)
history = {}
for job in result:
build_history = job.get('build_history')
if not build_history:
continue
successes = [_ for _ in build_history if _ == 'SUCCESS']
history[job['name']] = {
"total": len(build_history),
"success": len(successes)
}
return history
def run(dry_run: bool,
thread_pool_size=10,
internal: Optional[bool] = None,
use_jump_host=True,
defer=None):
all_namespaces = queries.get_namespaces(minimal=True)
shard_namespaces, duplicates = get_shard_namespaces(all_namespaces)
desired_state = get_desired_state(shard_namespaces)
settings = queries.get_app_interface_settings()
oc_map = OC_Map(namespaces=shard_namespaces,
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size,
init_projects=True)
defer(oc_map.cleanup)
results = threaded.run(manage_namespaces,
desired_state,
thread_pool_size,
return_exceptions=True,
dry_run=dry_run,
oc_map=oc_map)
err = check_results(desired_state, results)
if err or duplicates:
sys.exit(ExitCodes.ERROR)
def run(dry_run, print_to_file=None,
enable_deletion=False, io_dir='throughput/',
thread_pool_size=10, internal=None, use_jump_host=True,
light=False, vault_output_path='', extra_labels=None):
account_names = [
name for index, name in enumerate(sorted(get_accounts_names()))
if is_in_shard_round_robin(name, index)
]
if not account_names:
logging.warning("No accounts in shards")
return
exit_codes = threaded.run(
tfr_run_wrapper, account_names, thread_pool_size,
dry_run=dry_run,
print_to_file=print_to_file,
enable_deletion=enable_deletion,
io_dir=io_dir,
internal_thread_pool_size=thread_pool_size,
internal=internal,
use_jump_host=use_jump_host,
light=light,
vault_output_path=vault_output_path,
extra_labels=extra_labels,
)
if any(exit_codes):
sys.exit(ExitCodes.ERROR)
def init_ecr_auth_tokens(self, accounts):
accounts_with_ecr = [a for a in accounts if a.get('ecrs')]
if not accounts_with_ecr:
return
auth_tokens = {}
results = threaded.run(self.get_tf_secrets, accounts_with_ecr,
self.thread_pool_size)
account_secrets = dict(results)
for account in accounts_with_ecr:
account_name = account['name']
account_secret = account_secrets[account_name]
access_key = account_secret['aws_access_key_id']
secret_key = account_secret['aws_secret_access_key']
ecrs = account['ecrs']
for ecr in ecrs:
region_name = ecr['region']
session = Session(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region_name,
)
client = session.client('ecr')
token = client.get_authorization_token()
auth_tokens[f"{account_name}/{region_name}"] = token
self.auth_tokens = auth_tokens
def run(dry_run, gitlab_project_id=None, thread_pool_size=10):
accounts = queries.get_aws_accounts()
settings = queries.get_app_interface_settings()
aws = AWSApi(thread_pool_size, accounts, settings=settings)
existing_keys = aws.get_users_keys()
existing_keys_list = [
key for user_key in existing_keys.values()
for keys in user_key.values() for key in keys
]
logging.info("found {} existing keys".format(len(existing_keys_list)))
app_int_github_repos = queries.get_repos(server="https://github.com")
all_repos = get_all_repos_to_scan(app_int_github_repos)
logging.info("about to scan {} repos".format(len(all_repos)))
results = threaded.run(
git_secrets.scan_history,
all_repos,
thread_pool_size,
existing_keys=existing_keys_list,
)
all_leaked_keys = [key for keys in results for key in keys]
deleted_keys = aws_sos.get_deleted_keys(accounts)
keys_to_delete = [
{
"account": account,
"key": key
} for key in all_leaked_keys
for account, user_keys in existing_keys.items()
if key in [uk for uks in user_keys.values()
for uk in uks] and key not in deleted_keys[account]
]
aws_sos.act(dry_run, gitlab_project_id, accounts, keys_to_delete)
def realize_data(dry_run,
oc_map: OC_Map,
ri: ResourceInventory,
thread_pool_size,
take_over=False,
caller=None,
wait_for_namespace=False,
no_dry_run_skip_compare=False,
override_enable_deletion=None,
recycle_pods=True):
"""
Realize the current state to the desired state.
:param dry_run: run in dry-run mode
:param oc_map: a dictionary containing oc client per cluster
:param ri: a ResourceInventory containing current and desired states
:param thread_pool_size: Thread pool size to use for parallelism
:param take_over: manage resource types in a namespace exclusively
:param caller: name of the calling entity.
enables multiple running instances of the same integration
to deploy to the same namespace
:param wait_for_namespace: wait for namespace to exist before applying
:param no_dry_run_skip_compare: when running without dry-run, skip compare
:param override_enable_deletion: override calculated enable_deletion value
:param recycle_pods: should pods be recycled if a dependency changed
"""
args = locals()
del args['thread_pool_size']
results = threaded.run(_realize_resource_data, ri, thread_pool_size,
**args)
return list(itertools.chain.from_iterable(results))
def run(
dry_run,
gitlab_project_id=None,
thread_pool_size=10,
enable_deletion=False,
send_mails=False,
):
settings = queries.get_app_interface_settings()
users = queries.get_users()
g = init_github()
results = threaded.run(get_user_company, users, thread_pool_size, github=g)
users_to_delete = get_users_to_delete(results)
if not dry_run and enable_deletion:
mr_cli = mr_client_gateway.init(gitlab_project_id=gitlab_project_id)
for user in users_to_delete:
username = user["username"]
paths = user["paths"]
logging.info(["delete_user", username])
if not dry_run:
if send_mails:
send_email_notification(user, settings)
elif enable_deletion:
mr = CreateDeleteUser(username, paths)
mr.submit(cli=mr_cli)
else:
msg = ("'delete' action is not enabled. "
"Please run the integration manually "
"with the '--enable-deletion' flag.")
logging.warning(msg)
def __init__(self,
clusters=None,
namespaces=None,
integration='',
e2e_test='',
settings=None,
internal=None,
use_jump_host=True,
thread_pool_size=1,
init_projects=False,
init_api_resources=False,
cluster_admin=False):
self.oc_map = {}
self.calling_integration = integration
self.calling_e2e_test = e2e_test
self.settings = settings
self.internal = internal
self.use_jump_host = use_jump_host
self.thread_pool_size = thread_pool_size
self.init_projects = init_projects
self.init_api_resources = init_api_resources
self._lock = Lock()
self.jh_ports = {}
if clusters and namespaces:
raise KeyError('expected only one of clusters or namespaces.')
elif clusters:
threaded.run(self.init_oc_client,
clusters,
self.thread_pool_size,
cluster_admin=cluster_admin)
elif namespaces:
clusters = []
cluster_names = []
for ns_info in namespaces:
cluster = ns_info['cluster']
name = cluster['name']
if name not in cluster_names:
cluster_names.append(name)
clusters.append(cluster)
threaded.run(self.init_oc_client,
clusters,
self.thread_pool_size,
cluster_admin=cluster_admin)
else:
raise KeyError('expected one of clusters or namespaces.')
def init_specs(self):
wd_specs = \
[{'name': name, 'wd': wd}
for name, wd in self.working_dirs.items()]
results = threaded.run(self.terraform_init, wd_specs,
self.thread_pool_size)
self.specs = \
[{'name': name, 'tf': tf} for name, tf in results]
def fetch_data(namespaces,
thread_pool_size,
internal,
use_jump_host,
init_api_resources=False):
ri = ResourceInventory()
settings = queries.get_app_interface_settings()
oc_map = OC_Map(namespaces=namespaces,
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size,
init_api_resources=init_api_resources)
state_specs = ob.init_specs_to_fetch(ri, oc_map, namespaces=namespaces)
threaded.run(fetch_states, state_specs, thread_pool_size, ri=ri)
return oc_map, ri
def run(self):
clusters = queries.get_clusters()
oc_map = OC_Map(clusters=clusters,
integration=QONTRACT_INTEGRATION,
settings=self.settings,
use_jump_host=True,
thread_pool_size=self.thread_pool_size)
manifests = threaded.run(func=self._get_imagemanifestvuln,
iterable=oc_map.clusters(),
thread_pool_size=self.thread_pool_size,
oc_map=oc_map)
self._get_token()
threaded.run(func=self._post,
iterable=manifests,
thread_pool_size=self.thread_pool_size)
self._close_token()
def apply(self):
errors = False
results = threaded.run(self.terraform_apply, self.specs,
self.thread_pool_size)
for error in results:
if error:
errors = True
return errors
def fetch_current_state(
namespaces=None,
clusters=None,
thread_pool_size=None,
integration=None,
integration_version=None,
override_managed_types=None,
internal=None,
use_jump_host=True,
init_api_resources=False,
cluster_admin=False,
):
ri = ResourceInventory()
settings = queries.get_app_interface_settings()
oc_map = OC_Map(
namespaces=namespaces,
clusters=clusters,
integration=integration,
settings=settings,
internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size,
init_api_resources=init_api_resources,
cluster_admin=cluster_admin,
)
state_specs = init_specs_to_fetch(
ri,
oc_map,
namespaces=namespaces,
clusters=clusters,
override_managed_types=override_managed_types,
)
threaded.run(
populate_current_state,
state_specs,
thread_pool_size,
ri=ri,
integration=integration,
integration_version=integration_version,
)
return ri, oc_map
def validate_users_github(users, thread_pool_size):
ok = True
g = init_github()
results = threaded.run(get_github_user, users, thread_pool_size, github=g)
for org_username, gb_username, gh_login in results:
if gb_username != gh_login:
logging.error(
"Github username is case sensitive in OSD. "
f"User {org_username} github_username should be: {gh_login}.")
ok = False
return ok
def validate_users_github(users, thread_pool_size):
ok = True
g = init_github()
results = threaded.run(get_github_user, users, thread_pool_size, github=g)
for org_username, gb_username, gh_login in results:
if gb_username != gh_login:
logging.error("Github username is case sensitive in OSD. "
f"User {org_username} is expecting to have "
f"the github username of {gh_login}, "
f"but the username specified in "
f"app-interface is {gb_username}")
ok = False
return ok
def run(dry_run, thread_pool_size=10):
namespaces = queries.get_namespaces()
tfrs_to_mirror = []
for namespace in namespaces:
if namespace["terraformResources"] is None:
continue
for tfr in namespace["terraformResources"]:
if tfr["provider"] != "ecr":
continue
if tfr["mirror"] is None:
continue
tfrs_to_mirror.append(tfr)
work_list = threaded.run(EcrMirror,
tfrs_to_mirror,
thread_pool_size=thread_pool_size,
dry_run=dry_run)
threaded.run(worker, work_list, thread_pool_size=thread_pool_size)
def plan(self, enable_deletion):
errors = False
disabled_deletions_detected = False
results = threaded.run(self.terraform_plan, self.specs,
self.thread_pool_size,
enable_deletion=enable_deletion)
self.deleted_users = []
for disabled_deletion_detected, deleted_users, error in results:
if error:
errors = True
if disabled_deletion_detected:
disabled_deletions_detected = True
self.deleted_users.extend(deleted_users)
return disabled_deletions_detected, errors
def fetch_current_state(thread_pool_size, internal, use_jump_host):
clusters = queries.get_clusters(minimal=True)
settings = queries.get_app_interface_settings()
oc_map = OC_Map(clusters=clusters,
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size)
results = threaded.run(get_cluster_users,
oc_map.clusters(include_errors=True),
thread_pool_size,
oc_map=oc_map)
current_state = list(itertools.chain.from_iterable(results))
return oc_map, current_state
def fetch_current_state(thread_pool_size, internal, use_jump_host):
clusters = [c for c in queries.get_clusters() if is_in_shard(c['name'])]
ocm_clusters = [c['name'] for c in clusters if c.get('ocm') is not None]
current_state = []
settings = queries.get_app_interface_settings()
oc_map = OC_Map(clusters=clusters, integration=QONTRACT_INTEGRATION,
settings=settings, internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size)
groups_list = create_groups_list(clusters, oc_map)
results = threaded.run(get_cluster_state, groups_list, thread_pool_size,
oc_map=oc_map)
current_state = list(itertools.chain.from_iterable(results))
return oc_map, current_state, ocm_clusters
def init_sessions_and_resources(self, accounts: Iterable[Account]):
results = threaded.run(self.get_tf_secrets, accounts,
self.thread_pool_size)
self.sessions: Dict[str, Session] = {}
self.resources: Dict[str, Any] = {}
for account, secret in results:
access_key = secret['aws_access_key_id']
secret_key = secret['aws_secret_access_key']
region_name = secret['region']
session = Session(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region_name,
)
self.sessions[account] = session
self.resources[account] = {}
def fetch_current_state(thread_pool_size):
clusters = queries.get_clusters()
clusters = [c for c in clusters if c.get("ocm") is not None]
current_state = []
settings = queries.get_app_interface_settings()
ocm_map = OCMMap(clusters=clusters,
integration=QONTRACT_INTEGRATION,
settings=settings)
groups_list = openshift_groups.create_groups_list(clusters, oc_map=ocm_map)
results = threaded.run(get_cluster_state,
groups_list,
thread_pool_size,
ocm_map=ocm_map)
current_state = list(itertools.chain.from_iterable(results))
return ocm_map, current_state
def run(dry_run, thread_pool_size=10):
instance = queries.get_gitlab_instance()
settings = queries.get_app_interface_settings()
gl = GitLabApi(instance, settings=settings)
repos = queries.get_repos(server=gl.server)
app_sre = gl.get_app_sre_group_users()
results = threaded.run(get_members_to_add,
repos,
thread_pool_size,
gl=gl,
app_sre=app_sre)
members_to_add = list(itertools.chain.from_iterable(results))
for m in members_to_add:
logging.info(['add_maintainer', m["repo"], m["user"].username])
if not dry_run:
gl.add_project_member(m["repo"], m["user"])