def version_history(ctx):
settings = queries.get_app_interface_settings()
clusters = queries.get_clusters()
clusters = [c for c in clusters if c.get('upgradePolicy') is not None]
ocm_map = OCMMap(clusters=clusters, settings=settings)
history = ous.get_version_history(dry_run=True,
upgrade_policies=[],
ocm_map=ocm_map)
results = []
for ocm_name, history_data in history.items():
for version, version_data in history_data['versions'].items():
if not version:
continue
for workload, workload_data in version_data['workloads'].items():
item = {
'ocm': ocm_name,
'version': parse_semver(version),
'workload': workload,
'soak_days': round(workload_data['soak_days'], 2),
'clusters': ', '.join(workload_data['reporting']),
}
results.append(item)
columns = ['ocm', 'version', 'workload', 'soak_days', 'clusters']
ctx.obj['options']['to_string'] = True
print_output(ctx.obj['options'], results, columns)
def clusters_egress_ips(ctx):
settings = queries.get_app_interface_settings()
clusters = queries.get_clusters()
clusters = [c for c in clusters
if c.get('ocm') is not None
and c.get('awsInfrastructureManagementAccounts') is not None]
ocm_map = OCMMap(clusters=clusters, settings=settings)
results = []
for cluster in clusters:
cluster_name = cluster['name']
management_account = tfvpc._get_default_management_account(cluster)
account = tfvpc._build_infrastructure_assume_role(
management_account,
cluster,
ocm_map.get(cluster_name)
)
account['resourcesDefaultRegion'] = \
management_account['resourcesDefaultRegion']
aws_api = AWSApi(1, [account], settings=settings)
egress_ips = \
aws_api.get_cluster_nat_gateways_egress_ips(account)
item = {
'cluster': cluster_name,
'egress_ips': ', '.join(sorted(egress_ips))
}
results.append(item)
columns = ['cluster', 'egress_ips']
print_output(ctx.obj['options'], results, columns)
def run(dry_run=False, thread_pool_size=10):
settings = queries.get_app_interface_settings()
clusters = queries.get_clusters()
clusters = [c for c in clusters if c.get('ocm') is not None]
ocm_map = OCMMap(clusters=clusters,
integration=QONTRACT_INTEGRATION,
settings=settings)
current_state = ocm_map.cluster_specs()
desired_state = {
c['name']: {
'spec': c['spec'],
'network': c['network']
}
for c in clusters
}
error = False
for k, desired_spec in desired_state.items():
current_spec = current_state[k]
if current_spec != desired_spec:
logging.error(
'[%s] desired spec %s is different from current spec %s', k,
desired_spec, current_spec)
error = True
if error:
sys.exit(1)
def run(dry_run,
thread_pool_size=10,
internal=None,
use_jump_host=True,
defer=None):
settings = queries.get_app_interface_settings()
accounts = queries.get_state_aws_accounts()
clusters = [c for c in queries.get_clusters(minimal=True) if c.get("ocm")]
oc_map = OC_Map(
clusters=clusters,
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size,
)
defer(oc_map.cleanup)
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
if not dry_run:
slack = slackapi_from_queries(QONTRACT_INTEGRATION)
now = datetime.utcnow()
for cluster in oc_map.clusters(include_errors=True):
oc = oc_map.get(cluster)
if not oc:
logging.log(level=oc.log_level, msg=oc.message)
continue
upgrade_config = oc.get(
namespace="openshift-managed-upgrade-operator",
kind="UpgradeConfig",
allow_not_found=True,
)["items"]
if not upgrade_config:
logging.debug(f"[{cluster}] UpgradeConfig not found.")
continue
[upgrade_config] = upgrade_config
upgrade_spec = upgrade_config["spec"]
upgrade_at = upgrade_spec["upgradeAt"]
version = upgrade_spec["desired"]["version"]
upgrade_at_obj = datetime.strptime(upgrade_at, "%Y-%m-%dT%H:%M:%SZ")
state_key = f"{cluster}-{upgrade_at}"
# if this is the first iteration in which 'now' had passed
# the upgrade at date time, we send a notification
if upgrade_at_obj < now:
if state.exists(state_key):
# already notified
continue
logging.info(["cluster_upgrade", cluster])
if not dry_run:
state.add(state_key)
usergroup = f"{cluster}-cluster"
usergroup_id = slack.get_usergroup_id(usergroup)
slack.chat_post_message(
f"Heads up <!subteam^{usergroup_id}>! " +
f"cluster `{cluster}` is currently " +
f"being upgraded to version `{version}`")
def fetch_current_state():
current_state = []
current_failed = []
current_deleting = []
settings = queries.get_app_interface_settings()
clusters = [c for c in queries.get_clusters()
if c.get('ocm') is not None]
ocm_map = OCMMap(clusters=clusters, integration=QONTRACT_INTEGRATION,
settings=settings)
for cluster_info in clusters:
cluster = cluster_info['name']
ocm = ocm_map.get(cluster)
role_grants = ocm.get_aws_infrastructure_access_role_grants(cluster)
for user_arn, access_level, state, _ in role_grants:
item = {
'cluster': cluster,
'user_arn': user_arn,
'access_level': access_level
}
if state == STATUS_FAILED:
current_failed.append(item)
elif state == STATUS_DELETING:
current_deleting.append(item)
else:
current_state.append(item)
return ocm_map, current_state, current_failed, current_deleting
def clusters_egress_ips(ctx):
settings = queries.get_app_interface_settings()
clusters = queries.get_clusters()
clusters = [
c for c in clusters if c.get('ocm') is not None
and c.get('awsInfrastructureAccess') is not None
]
ocm_map = OCMMap(clusters=clusters, settings=settings)
results = []
for cluster in clusters:
cluster_name = cluster['name']
account = tfvpc.aws_account_from_infrastructure_access(
cluster, 'network-mgmt', ocm_map)
aws_api = AWSApi(1, [account], settings=settings)
egress_ips = \
aws_api.get_cluster_nat_gateways_egress_ips(account)
item = {
'cluster': cluster_name,
'egress_ips': ', '.join(sorted(egress_ips))
}
results.append(item)
columns = ['cluster', 'egress_ips']
print_output(ctx.obj['options'], results, columns)
def clusters_network(ctx, name):
clusters = queries.get_clusters()
if name:
clusters = [c for c in clusters if c['name'] == name]
columns = ['name', 'network.vpc', 'network.service', 'network.pod']
print_output(ctx.obj['output'], clusters, columns)
def get_desired_state(slack):
desired_state = []
all_users = queries.get_roles()
all_clusters = queries.get_clusters(minimal=True)
clusters = [c for c in all_clusters
if c.get('auth') and c['auth'].get('team')
and c.get('ocm')]
openshift_users_desired_state = \
openshift_users.fetch_desired_state(oc_map=None)
for cluster in clusters:
cluster_name = cluster['name']
cluster_users = [u['user'] for u in openshift_users_desired_state
if u['cluster'] == cluster_name]
usergroup = cluster['auth']['team']
try:
ugid = slack.get_usergroup_id(usergroup)
except UsergroupNotFoundException:
logging.warning(f'Usergroup {usergroup} not found')
continue
user_names = [slack_usergroups.get_slack_username(u)
for u in all_users
if include_user(u, cluster_name, cluster_users)]
users = slack.get_users_by_names(user_names)
channels = slack.get_channels_by_names([slack.chat_kwargs['channel']])
desired_state.append({
"workspace": slack.workspace_name,
"usergroup": usergroup,
"usergroup_id": ugid,
"users": users,
"channels": channels,
"description": f'Users with access to the {cluster_name} cluster',
})
return desired_state
def clusters(ctx, name):
clusters = queries.get_clusters()
if name:
clusters = [c for c in clusters if c['name'] == name]
columns = ['name', 'consoleUrl', 'kibanaUrl', 'prometheusUrl']
print_output(ctx.obj['output'], clusters, columns)
def root_owner(ctx, cluster, namespace, kind, name):
settings = queries.get_app_interface_settings()
clusters = [
c for c in queries.get_clusters(minimal=True) if c['name'] == cluster
]
oc_map = OC_Map(clusters=clusters,
integration='qontract-cli',
thread_pool_size=1,
settings=settings,
init_api_resources=True)
oc = oc_map.get(cluster)
obj = oc.get(namespace, kind, name)
root_owner = oc.get_obj_root_owner(namespace,
obj,
allow_not_found=True,
allow_not_controller=True)
# TODO(mafriedm): fix this
# do not sort
ctx.obj['options']['sort'] = False
# a bit hacky, but ¯\_(ツ)_/¯
if ctx.obj['options']['output'] != 'json':
ctx.obj['options']['output'] = 'yaml'
print_output(ctx.obj['options'], root_owner)
def run(dry_run, thread_pool_size=10):
settings = queries.get_app_interface_settings()
clusters = queries.get_clusters()
clusters = [c for c in clusters if c.get('ocm') is not None]
ocm_map = OCMMap(clusters=clusters, integration=QONTRACT_INTEGRATION,
settings=settings)
current_state, pending_state = ocm_map.cluster_specs()
desired_state = {c['name']: {'spec': c['spec'], 'network': c['network']}
for c in clusters}
error = False
for cluster_name, desired_spec in desired_state.items():
current_spec = current_state.get(cluster_name)
if current_spec and current_spec != desired_spec:
logging.error(
'[%s] desired spec %s is different from current spec %s',
cluster_name, desired_spec, current_spec)
error = True
if error:
sys.exit(1)
for cluster_name, desired_spec in desired_state.items():
if cluster_name in current_state or cluster_name in pending_state:
continue
logging.info(['create_cluster', cluster_name])
if not dry_run:
ocm = ocm_map.get(cluster_name)
ocm.create_cluster(cluster_name, desired_spec)
def setup(dry_run, print_only, thread_pool_size, internal,
use_jump_host, account_name, extra_labels):
gqlapi = gql.get_api()
accounts = queries.get_aws_accounts()
if account_name:
accounts = [n for n in accounts
if n['name'] == account_name]
if not accounts:
raise ValueError(f"aws account {account_name} is not found")
extra_labels['shard_key'] = account_name
settings = queries.get_app_interface_settings()
namespaces = gqlapi.query(TF_NAMESPACES_QUERY)['namespaces']
tf_namespaces = filter_tf_namespaces(namespaces, account_name)
ri, oc_map = fetch_current_state(dry_run, tf_namespaces, thread_pool_size,
internal, use_jump_host, account_name)
ts, working_dirs = init_working_dirs(accounts, thread_pool_size,
oc_map=oc_map,
settings=settings)
tf = Terraform(QONTRACT_INTEGRATION,
QONTRACT_INTEGRATION_VERSION,
QONTRACT_TF_PREFIX,
accounts,
working_dirs,
thread_pool_size)
existing_secrets = tf.get_terraform_output_secrets()
clusters = [c for c in queries.get_clusters()
if c.get('ocm') is not None]
ocm_map = OCMMap(clusters=clusters, integration=QONTRACT_INTEGRATION,
settings=settings)
ts.populate_resources(tf_namespaces, existing_secrets, account_name,
ocm_map=ocm_map)
ts.dump(print_only, existing_dirs=working_dirs)
return ri, oc_map, tf, tf_namespaces
def run(dry_run,
thread_pool_size=10,
internal=None,
use_jump_host=True,
defer=None):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
clusters = [c for c in queries.get_clusters(minimal=True) if c.get('ocm')]
oc_map = OC_Map(clusters=clusters,
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size)
defer(oc_map.cleanup)
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
if not dry_run:
slack = init_slack_workspace(QONTRACT_INTEGRATION)
now = datetime.utcnow()
for cluster in oc_map.clusters(include_errors=True):
oc = oc_map.get(cluster)
if not oc:
logging.log(level=oc.log_level, msg=oc.message)
continue
upgrade_config = oc.get(namespace='openshift-managed-upgrade-operator',
kind='UpgradeConfig',
allow_not_found=True)['items']
if not upgrade_config:
logging.debug(f'[{cluster}] UpgradeConfig not found.')
continue
[upgrade_config] = upgrade_config
upgrade_spec = upgrade_config['spec']
upgrade_at = upgrade_spec['upgradeAt']
version = upgrade_spec['desired']['version']
upgrade_at_obj = datetime.strptime(upgrade_at, '%Y-%m-%dT%H:%M:%SZ')
state_key = f'{cluster}-{upgrade_at}'
# if this is the first iteration in which 'now' had passed
# the upgrade at date time, we send a notification
if upgrade_at_obj < now:
if state.exists(state_key):
# already notified
continue
logging.info(['cluster_upgrade', cluster])
if not dry_run:
state.add(state_key)
usergroup = f'{cluster}-cluster'
usergroup_id = slack.get_usergroup_id(usergroup)
slack.chat_post_message(
f'Heads up <!subteam^{usergroup_id}>! ' +
f'cluster `{cluster}` is currently ' +
f'being upgraded to version `{version}`')
def clusters_network(ctx, name):
clusters = queries.get_clusters()
if name:
clusters = [c for c in clusters if c['name'] == name]
columns = ['name', 'network.vpc', 'network.service', 'network.pod']
# TODO(mafriedm): fix this
# do not sort
ctx.obj['options']['sort'] = False
print_output(ctx.obj['options'], clusters, columns)
def fetch_current_state(thread_pool_size, internal, use_jump_host):
clusters = queries.get_clusters(minimal=True)
settings = queries.get_app_interface_settings()
oc_map = OC_Map(clusters=clusters, integration=QONTRACT_INTEGRATION,
settings=settings, internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size)
results = threaded.run(get_cluster_users, oc_map.clusters(),
thread_pool_size, oc_map=oc_map)
current_state = [item for sublist in results for item in sublist]
return oc_map, current_state
def bot_login(ctx, cluster_name):
clusters = queries.get_clusters()
clusters = [c for c in clusters if c['name'] == cluster_name]
if len(clusters) == 0:
print(f"{cluster_name} not found.")
sys.exit(1)
cluster = clusters[0]
settings = queries.get_app_interface_settings()
server = cluster['serverUrl']
token = secret_reader.read(cluster['automationToken'], settings=settings)
print(f"oc login --server {server} --token {token}")
def run(dry_run):
clusters = queries.get_clusters()
clusters = [c for c in clusters if c.get("additionalRouters") is not None]
if not clusters:
logging.debug(
"No additionalRouters definitions found in app-interface")
sys.exit(ExitCodes.SUCCESS)
ocm_map, current_state = fetch_current_state(clusters)
desired_state = fetch_desired_state(clusters)
diffs = calculate_diff(current_state, desired_state)
act(dry_run, diffs, ocm_map)
def fetch_current_state(thread_pool_size):
clusters = queries.get_clusters()
clusters = [c for c in clusters if c.get('ocm') is not None]
current_state = []
settings = queries.get_app_interface_settings()
ocm_map = OCMMap(clusters=clusters, integration=QONTRACT_INTEGRATION,
settings=settings)
groups_list = openshift_groups.create_groups_list(clusters, oc_map=ocm_map)
results = threaded.run(get_cluster_state, groups_list, thread_pool_size,
ocm_map=ocm_map)
current_state = [item for sublist in results for item in sublist]
return ocm_map, current_state
def run(dry_run, gitlab_project_id=None, thread_pool_size=10):
clusters = queries.get_clusters()
clusters = [c for c in clusters if c.get('upgradePolicy') is not None]
if not clusters:
logging.debug("No upgradePolicy definitions found in app-interface")
sys.exit(0)
ocm_map, current_state = fetch_current_state(clusters)
desired_state = fetch_desired_state(clusters)
version_history = get_version_history(dry_run, desired_state, ocm_map)
diffs = calculate_diff(
current_state, desired_state, ocm_map, version_history)
act(dry_run, diffs, ocm_map)
def run(dry_run, vault_input_path=''):
if not vault_input_path:
logging.error('must supply vault input path')
sys.exit(1)
settings = queries.get_app_interface_settings()
clusters = [c for c in queries.get_clusters()
if c.get('ocm') is not None
and c.get('auth') is not None]
ocm_map, current_state = fetch_current_state(clusters, settings)
desired_state, error = \
fetch_desired_state(clusters, vault_input_path, settings)
if error:
sys.exit(1)
act(dry_run, ocm_map, current_state, desired_state)
def run(dry_run, gitlab_project_id=None, thread_pool_size=10):
clusters = queries.get_clusters()
clusters = [c for c in clusters if c.get("addons") is not None]
if not clusters:
logging.debug("No Addon definitions found in app-interface")
sys.exit(ExitCodes.SUCCESS)
ocm_map, current_state = fetch_current_state(clusters)
desired_state = fetch_desired_state(clusters)
diffs = calculate_diff(current_state, desired_state)
err = act(dry_run, diffs, ocm_map)
if err:
sys.exit(ExitCodes.ERROR)
def run(dry_run, gitlab_project_id=None, thread_pool_size=10):
clusters = queries.get_clusters()
clusters = [c for c in clusters if c.get('machinePools') is not None]
if not clusters:
logging.debug("No machinePools definitions found in app-interface")
sys.exit(0)
ocm_map, current_state = fetch_current_state(clusters)
desired_state = fetch_desired_state(clusters)
diffs, err = calculate_diff(current_state, desired_state)
act(dry_run, diffs, ocm_map)
if err:
sys.exit(1)
def fetch_desired_state(settings):
desired_state = []
clusters = [
c for c in queries.get_clusters() if c.get('peering') is not None
]
ocm_map = OCMMap(clusters=clusters,
integration=QONTRACT_INTEGRATION,
settings=settings)
for cluster_info in clusters:
cluster = cluster_info['name']
ocm = ocm_map.get(cluster)
peering_info = cluster_info['peering']
# requester is the cluster's AWS account
requester = {
'vpc_id': peering_info['vpc_id'],
'cidr_block': cluster_info['network']['vpc'],
'region': cluster_info['spec']['region']
}
peer_connections = peering_info['connections']
for peer_connection in peer_connections:
connection_name = peer_connection['name']
peer_vpc = peer_connection['vpc']
# accepter is the peered AWS account
accepter = {
'vpc_id': peer_vpc['vpc_id'],
'cidr_block': peer_vpc['cidr_block'],
'region': peer_vpc['region']
}
account = peer_vpc['account']
# assume_role is the role to assume to provision the
# peering connection request, through the accepter AWS account.
# this may change in the future -
# in case we add support for peerings between clusters.
account['assume_role'] = \
ocm.get_aws_infrastructure_access_terraform_assume_role(
cluster,
peer_vpc['account']['uid'],
peer_vpc['account']['terraformUsername']
)
# assume_region is the region in which the requester resides
account['assume_region'] = requester['region']
item = {
'connection_name': connection_name,
'requester': requester,
'accepter': accepter,
'account': account
}
desired_state.append(item)
return desired_state
def cluster_upgrades(ctx, name):
settings = queries.get_app_interface_settings()
clusters = queries.get_clusters()
clusters_ocm = [
c for c in clusters
if c.get('ocm') is not None and c.get('auth') is not None
]
ocm_map = OCMMap(clusters=clusters_ocm, settings=settings)
clusters_data = []
for c in clusters:
if name and c['name'] != name:
continue
if not c.get('spec'):
continue
data = {
'name': c['name'],
'upgrade': c['spec']['upgrade'],
'id': c['spec']['id'],
'external_id': c['spec'].get('external_id'),
}
upgrade_policy = c['upgradePolicy']
if upgrade_policy:
data['upgradePolicy'] = upgrade_policy.get('schedule_type')
if data.get('upgradePolicy') == 'automatic':
data['schedule'] = c['upgradePolicy']['schedule']
ocm = ocm_map.get(c['name'])
if ocm:
upgrade_policy = ocm.get_upgrade_policies(c['name'])
next_run = upgrade_policy[0].get('next_run')
if next_run:
data['next_run'] = next_run
else:
data['upgradePolicy'] = 'manual'
clusters_data.append(data)
clusters_data = sorted(clusters_data, key=lambda k: k['upgrade'])
columns = ['name', 'upgrade', 'upgradePolicy', 'schedule', 'next_run']
print_output(ctx.obj['output'], clusters_data, columns)
def fetch_current_state(thread_pool_size, internal, use_jump_host):
clusters = queries.get_clusters(minimal=True)
settings = queries.get_app_interface_settings()
oc_map = OC_Map(clusters=clusters,
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size)
results = threaded.run(get_cluster_users,
oc_map.clusters(include_errors=True),
thread_pool_size,
oc_map=oc_map)
current_state = list(itertools.chain.from_iterable(results))
return oc_map, current_state
def get_desired_state(slack):
"""
Get the desired state of the Slack cluster usergroups.
:param slack: client for calling Slack API
:type slack: reconcile.utils.slack_api.SlackApi
:return: desired state data, keys are workspace -> usergroup
(ex. state['coreos']['app-sre-ic']
:rtype: dict
"""
desired_state = {}
all_users = queries.get_roles()
all_clusters = queries.get_clusters(minimal=True)
clusters = [
c for c in all_clusters
if c.get("auth") and c["auth"].get("team") and c.get("ocm")
]
openshift_users_desired_state = openshift_users.fetch_desired_state(
oc_map=None)
for cluster in clusters:
cluster_name = cluster["name"]
cluster_users = [
u["user"] for u in openshift_users_desired_state
if u["cluster"] == cluster_name
]
usergroup = cluster["auth"]["team"]
try:
ugid = slack.get_usergroup_id(usergroup)
except UsergroupNotFoundException:
logging.warning(f"Usergroup {usergroup} not found")
continue
user_names = [
slack_usergroups.get_slack_username(u) for u in all_users
if include_user(u, cluster_name, cluster_users)
]
users = slack.get_users_by_names(user_names)
channels = slack.get_channels_by_names([slack.channel])
desired_state.setdefault(slack.workspace_name, {})[usergroup] = {
"workspace": slack.workspace_name,
"usergroup": usergroup,
"usergroup_id": ugid,
"users": users,
"channels": channels,
"description": f"Users with access to the {cluster_name} cluster",
}
return desired_state
def fetch_current_state(thread_pool_size, internal, use_jump_host):
clusters = queries.get_clusters()
ocm_clusters = [c['name'] for c in clusters if c.get('ocm') is not None]
current_state = []
settings = queries.get_app_interface_settings()
oc_map = OC_Map(clusters=clusters, integration=QONTRACT_INTEGRATION,
settings=settings, internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size)
groups_list = create_groups_list(clusters, oc_map)
results = threaded.run(get_cluster_state, groups_list, thread_pool_size,
oc_map=oc_map)
current_state = [item for sublist in results for item in sublist]
return oc_map, current_state, ocm_clusters
def fetch_current_state(thread_pool_size):
clusters = queries.get_clusters()
clusters = [c for c in clusters if c.get("ocm") is not None]
current_state = []
settings = queries.get_app_interface_settings()
ocm_map = OCMMap(clusters=clusters,
integration=QONTRACT_INTEGRATION,
settings=settings)
groups_list = openshift_groups.create_groups_list(clusters, oc_map=ocm_map)
results = threaded.run(get_cluster_state,
groups_list,
thread_pool_size,
ocm_map=ocm_map)
current_state = list(itertools.chain.from_iterable(results))
return ocm_map, current_state
def run(self):
clusters = queries.get_clusters()
oc_map = OC_Map(clusters=clusters,
integration=QONTRACT_INTEGRATION,
settings=self.settings, use_jump_host=True,
thread_pool_size=self.thread_pool_size)
manifests = threaded.run(func=self._get_imagemanifestvuln,
iterable=oc_map.clusters(),
thread_pool_size=self.thread_pool_size,
oc_map=oc_map)
threaded.run(func=self._post,
iterable=manifests,
thread_pool_size=self.thread_pool_size)
def fetch_current_state(thread_pool_size, internal, use_jump_host):
clusters = [c for c in queries.get_clusters() if is_in_shard(c['name'])]
ocm_clusters = [c['name'] for c in clusters if c.get('ocm') is not None]
current_state = []
settings = queries.get_app_interface_settings()
oc_map = OC_Map(clusters=clusters, integration=QONTRACT_INTEGRATION,
settings=settings, internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size)
groups_list = create_groups_list(clusters, oc_map)
results = threaded.run(get_cluster_state, groups_list, thread_pool_size,
oc_map=oc_map)
current_state = list(itertools.chain.from_iterable(results))
return oc_map, current_state, ocm_clusters