def fetch_desired_state():
gqlapi = gql.get_api()
result = gqlapi.query(QUAY_ORG_QUERY)
state = AggregatedList()
for role in result['roles']:
permissions = [
process_permission(p) for p in role['permissions']
if p.get('service') == 'quay-membership'
]
if permissions:
members = []
def append_quay_username_members(member):
quay_username = member.get('quay_username')
if quay_username:
members.append(quay_username)
for user in role['users']:
append_quay_username_members(user)
for bot in role['bots']:
append_quay_username_members(bot)
for p in permissions:
state.add(p, members)
return state
def fetch_current_state(quay_api_store):
state = AggregatedList()
for org_key, org_data in quay_api_store.items():
quay_api = org_data['api']
teams = org_data['teams']
if not teams:
continue
for team in teams:
try:
members = quay_api.list_team_members(team)
except QuayTeamNotFoundException:
logging.warning(
"Attempted to list members for team %s in "
"org %s/%s, but it doesn't exist", team, org_key.instance,
org_key.org_name)
else:
# Teams are only added to the state if they exist so that
# there is a proper diff between the desired and current state.
state.add(
{
'service': 'quay-membership',
'org': org_key,
'team': team
}, members)
return state
def fetch_desired_state():
gqlapi = gql.get_api()
result = gqlapi.query(QUAY_ORG_QUERY)
state = AggregatedList()
for role in result['roles']:
permissions = list(filter(
lambda p: p.get('service') == 'quay-membership',
role['permissions']
))
if permissions:
members = []
def append_quay_username_members(member):
quay_username = member.get('quay_username')
if quay_username:
members.append(quay_username)
for user in role['users']:
append_quay_username_members(user)
for bot in role['bots']:
append_quay_username_members(bot)
list(map(lambda p: state.add(p, members), permissions))
return state
def fetch_current_state(quay_api_store):
state = AggregatedList()
for name, data in quay_api_store.items():
quay_api = data['api']
for repo in quay_api.list_images():
params = {
'org': name,
'repo': repo['name']
}
public = repo['is_public']
description = repo['description']
if description is None:
description = ''
item = {
'public': public,
'description': description.strip()
}
state.add(params, item)
return state
def fetch_desired_state():
gqlapi = gql.get_api()
roles = expiration.filter(gqlapi.query(QUAY_ORG_QUERY)['roles'])
state = AggregatedList()
for role in roles:
permissions = [
process_permission(p)
for p in role['permissions']
if p.get('service') == 'quay-membership'
]
if permissions:
members = []
for user in role['users'] + role['bots']:
quay_username = user.get('quay_username')
if quay_username:
members.append(quay_username)
for p in permissions:
state.add(p, members)
return state
def get_items_by_params(state, params):
h = AggregatedList.hash_params(params)
for group in state:
this_h = AggregatedList.hash_params(group['params'])
if h == this_h:
return sorted(group['items'])
return False
def test_unknown_diff_on(self):
left = AggregatedList()
right = AggregatedList()
runner = AggregatedDiffRunner(left.diff(right))
with pytest.raises(Exception):
runner.register("qwerty", lambda p, i: True, lambda p: True)
def fetch_current_state(quay_api_store):
state = AggregatedList()
for name, org_data in quay_api_store.items():
quay_api = org_data['api']
teams = org_data['teams']
if not teams:
continue
for team in teams:
members = quay_api.list_team_members(team)
state.add({
'service': 'quay-membership',
'org': name,
'team': team
}, members)
return state
def test_diff_update_delete():
left = AggregatedList()
right = AggregatedList()
left.add({"a": 1}, ["qwerty1", "qwerty2"])
right.add({"a": 1}, ["qwerty1"])
diff = left.diff(right)
assert diff["insert"] == []
assert diff["delete"] == []
assert not diff["update-insert"]
assert diff["update-delete"] == [{"items": ["qwerty2"], "params": {"a": 1}}]
def fetch_desired_state():
gqlapi = gql.get_api()
result = gqlapi.query(QUAY_REPOS_QUERY)
state = AggregatedList()
for app in result['apps']:
quay_repos = app.get('quayRepos')
if quay_repos is None:
continue
for quay_repo in quay_repos:
name = quay_repo['org']['name']
for repo in quay_repo['items']:
params = {'org': name, 'repo': repo['name']}
# Avoiding duplicates
try:
state.get(params)
logging.error(['Repository %s/%s defined more than once'],
params['org'], params['repo'])
sys.exit(ExitCodes.ERROR)
except KeyError:
pass
item = {
'public': repo['public'],
'description': repo['description'].strip()
}
state.add(params, item)
return state
def test_add_item(self):
alist = AggregatedList()
params = {'a': 1, 'b': 2}
items = ["qwerty"]
alist.add(params, items)
assert len(alist.dump()) == 1
assert alist.dump()[0]['items'] == items
assert alist.dump()[0]['params'] == params
def test_add_item():
alist = AggregatedList()
params = {"a": 1, "b": 2}
items = ["qwerty"]
alist.add(params, items)
assert len(alist.dump()) == 1
assert alist.dump()[0]["items"] == items
assert alist.dump()[0]["params"] == params
def test_diff_update_delete(self):
left = AggregatedList()
right = AggregatedList()
left.add({'a': 1}, ["qwerty1", "qwerty2"])
right.add({'a': 1}, ["qwerty1"])
diff = left.diff(right)
assert diff['insert'] == []
assert diff['delete'] == []
assert diff['update-insert'] == []
assert diff['update-delete'] == [{
'items': ['qwerty2'],
'params': {
'a': 1
}
}]
def test_add_repeated_item():
alist = AggregatedList()
params = {'a': 1, 'b': 2}
item = "qwerty"
items = [item, item]
alist.add(params, items)
assert len(alist.dump()) == 1
assert alist.dump()[0]['items'] == [item]
assert alist.dump()[0]['params'] == params
def test_diff_insert():
left = AggregatedList()
right = AggregatedList()
right.add({"a": 1}, ["qwerty"])
diff = left.diff(right)
assert not diff["delete"]
assert not diff["update-insert"]
assert not diff["update-delete"]
assert diff["insert"] == [{"params": {"a": 1}, "items": ["qwerty"]}]
def test_diff_insert(self):
left = AggregatedList()
right = AggregatedList()
right.add({'a': 1}, ["qwerty"])
diff = left.diff(right)
assert diff['delete'] == []
assert diff['update-insert'] == []
assert diff['update-delete'] == []
assert diff['insert'] == [{'params': {'a': 1}, 'items': ['qwerty']}]
def test_run_cond_false():
left = AggregatedList()
right = AggregatedList()
right.add({"on": "insert"}, ["qwerty"])
runner = AggregatedDiffRunner(left.diff(right))
recorder = []
runner.register("insert", lambda p, i: recorder.append("True"), lambda p: False)
runner.run()
assert not recorder
def test_run_cond_false(self):
left = AggregatedList()
right = AggregatedList()
right.add({'on': 'insert'}, ["qwerty"])
runner = AggregatedDiffRunner(left.diff(right))
recorder = []
runner.register("insert", lambda p, i: recorder.append('True'),
lambda p: False)
runner.run()
assert recorder == []
def test_add_different_params(self):
alist = AggregatedList()
params1 = {'b': 1, 'a': 2}
items1 = ["qwerty1"]
params2 = {'a': 1, 'b': 3}
items2 = ["qwerty2"]
alist.add(params1, items1)
alist.add(params2, items2)
assert len(alist.dump()) == 2
hp1 = AggregatedList.hash_params(params1)
hp2 = AggregatedList.hash_params(params2)
assert alist.get_by_params_hash(hp1)['items'] == items1
assert alist.get_by_params_hash(hp2)['items'] == items2
def test_add_different_params():
alist = AggregatedList()
params1 = {"b": 1, "a": 2}
items1 = ["qwerty1"]
params2 = {"a": 1, "b": 3}
items2 = ["qwerty2"]
alist.add(params1, items1)
alist.add(params2, items2)
assert len(alist.dump()) == 2
hp1 = AggregatedList.hash_params(params1)
hp2 = AggregatedList.hash_params(params2)
assert alist.get_by_params_hash(hp1)["items"] == items1
assert alist.get_by_params_hash(hp2)["items"] == items2
def fetch_current_state(gh_api_store):
state = AggregatedList()
for org_name in gh_api_store.orgs():
g = gh_api_store.github(org_name)
raw_gh_api = gh_api_store.raw_github_api(org_name)
managed_teams = gh_api_store.managed_teams(org_name)
# if 'managedTeams' is not specified
# we manage all teams
is_managed = managed_teams is None or len(managed_teams) == 0
org, teams = get_org_and_teams(g, org_name)
org_members = None
if is_managed:
org_members = get_members(org)
org_members.extend(raw_gh_api.org_invitations(org_name))
org_members = [m.lower() for m in org_members]
all_team_members = []
for team in teams:
if not is_managed and team.name not in managed_teams:
continue
members = get_members(team)
members.extend(raw_gh_api.team_invitations(org.id, team.id))
members = [m.lower() for m in members]
all_team_members.extend(members)
state.add(
{
"service": "github-org-team",
"org": org_name,
"team": team.name
},
members,
)
all_team_members = list(set(all_team_members))
members = org_members or all_team_members
state.add(
{
"service": "github-org",
"org": org_name,
},
members,
)
return state
def fetch_current_state(gh_api_store):
state = AggregatedList()
for org_name in gh_api_store.orgs():
g = gh_api_store.github(org_name)
raw_gh_api = gh_api_store.raw_github_api(org_name)
managed_teams = gh_api_store.managed_teams(org_name)
# if 'managedTeams' is not specified
# we manage all teams
is_managed = managed_teams is None or len(managed_teams) == 0
org = g.get_organization(org_name)
org_members = None
if is_managed:
org_members = [member.login for member in org.get_members()]
org_members.extend(raw_gh_api.org_invitations(org_name))
org_members = [m.lower() for m in org_members]
all_team_members = []
for team in org.get_teams():
if not is_managed and team.name not in managed_teams:
continue
members = [member.login for member in team.get_members()]
members.extend(raw_gh_api.team_invitations(team.id))
members = [m.lower() for m in members]
all_team_members.extend(members)
state.add(
{
'service': 'github-org-team',
'org': org_name,
'team': team.name
},
members
)
all_team_members = list(set(all_team_members))
members = org_members or all_team_members
state.add(
{
'service': 'github-org',
'org': org_name,
},
members
)
return state
def fetch_desired_state(infer_clusters=True):
gqlapi = gql.get_api()
state = AggregatedList()
roles = expiration.filter(gqlapi.query(ROLES_QUERY)["roles"])
for role in roles:
permissions = list(
filter(
lambda p: p.get("service") in
["github-org", "github-org-team"],
role["permissions"],
))
if not permissions:
continue
members = []
for user in role["users"]:
members.append(user["github_username"])
for bot in role["bots"]:
if "github_username" in bot:
members.append(bot["github_username"])
members = [m.lower() for m in members]
for permission in permissions:
if permission["service"] == "github-org":
state.add(permission, members)
elif permission["service"] == "github-org-team":
state.add(permission, members)
state.add(
{
"service": "github-org",
"org": permission["org"],
},
members,
)
if not infer_clusters:
return state
clusters = gqlapi.query(CLUSTERS_QUERY)["clusters"]
openshift_users_desired_state = openshift_users.fetch_desired_state(
oc_map=None)
for cluster in clusters:
if not cluster["auth"]:
continue
cluster_name = cluster["name"]
members = [
ou["user"].lower() for ou in openshift_users_desired_state
if ou["cluster"] == cluster_name
]
state.add(
{
"service": "github-org",
"org": cluster["auth"]["org"],
},
members,
)
if cluster["auth"]["service"] == "github-org-team":
state.add(
{
"service": "github-org-team",
"org": cluster["auth"]["org"],
"team": cluster["auth"]["team"],
},
members,
)
return state
def test_run():
left = AggregatedList()
right = AggregatedList()
# test insert
right.add({"on": "insert"}, ["i"])
# test delete
left.add({"on": "delete"}, ["d"])
# test update-insert
left.add({"on": "update-insert"}, ["ui1"])
right.add({"on": "update-insert"}, ["ui1", "ui2"])
# test update-delete
left.add({"on": "update-delete"}, ["ud1", "ud2"])
right.add({"on": "update-delete"}, ["ud1"])
on_insert = []
on_delete = []
on_update_insert = []
on_update_delete = []
def recorder(ls):
return lambda p, i: ls.append([p, i])
runner = AggregatedDiffRunner(left.diff(right))
runner.register("insert", recorder(on_insert))
runner.register("delete", recorder(on_delete))
runner.register("update-insert", recorder(on_update_insert))
runner.register("update-delete", recorder(on_update_delete))
runner.run()
assert on_insert == [[{"on": "insert"}, ["i"]]]
assert on_delete == [[{"on": "delete"}, ["d"]]]
assert on_update_insert == [[{"on": "update-insert"}, ["ui2"]]]
assert on_update_delete == [[{"on": "update-delete"}, ["ud2"]]]
def test_get_py_params_hash():
alist = AggregatedList()
params1 = {"a": 1, "b": 2, "c": 3}
params2 = {"b": 2, "c": 3, "a": 1}
params3 = {"c": 3, "a": 1, "b": 2}
params4 = {"a": 1, "c": 3, "b": 2}
params5 = {"a": 1}
items1 = ["qwerty1"]
items2 = ["qwerty2"]
alist.add(params1, items1)
alist.add(params2, items1)
alist.add(params3, items1)
alist.add(params4, items1)
alist.add(params5, items2)
hp1 = AggregatedList.hash_params(params1)
hp2 = AggregatedList.hash_params(params2)
hp3 = AggregatedList.hash_params(params3)
hp4 = AggregatedList.hash_params(params4)
hp5 = AggregatedList.hash_params(params5)
assert hp1 == hp2
assert hp1 == hp2
assert hp1 == hp3
assert hp1 == hp4
assert hp1 != hp5
assert alist.get_by_params_hash(hp1)["items"] == items1
assert alist.get_by_params_hash(hp5)["items"] == items2
def fetch_desired_state(infer_clusters=True):
gqlapi = gql.get_api()
state = AggregatedList()
roles = gqlapi.query(ROLES_QUERY)['roles']
for role in roles:
permissions = list(filter(
lambda p: p.get('service') in ['github-org', 'github-org-team'],
role['permissions']
))
if not permissions:
continue
members = []
for user in role['users']:
members.append(user['github_username'])
for bot in role['bots']:
if 'github_username' in bot:
members.append(bot['github_username'])
members = [m.lower() for m in members]
for permission in permissions:
if permission['service'] == 'github-org':
state.add(permission, members)
elif permission['service'] == 'github-org-team':
state.add(permission, members)
state.add({
'service': 'github-org',
'org': permission['org'],
}, members)
if not infer_clusters:
return state
clusters = gqlapi.query(CLUSTERS_QUERY)['clusters']
openshift_users_desired_state = \
openshift_users.fetch_desired_state(oc_map=None)
for cluster in clusters:
if not cluster['auth']:
continue
cluster_name = cluster['name']
members = [ou['user'].lower()
for ou in openshift_users_desired_state
if ou['cluster'] == cluster_name]
state.add({
'service': 'github-org',
'org': cluster['auth']['org'],
}, members)
if cluster['auth']['service'] == 'github-org-team':
state.add({
'service': 'github-org-team',
'org': cluster['auth']['org'],
'team': cluster['auth']['team'],
}, members)
return state
def test_get_py_params_hash(self):
alist = AggregatedList()
params1 = {'a': 1, 'b': 2, 'c': 3}
params2 = {'b': 2, 'c': 3, 'a': 1}
params3 = {'c': 3, 'a': 1, 'b': 2}
params4 = {'a': 1, 'c': 3, 'b': 2}
params5 = {'a': 1}
items1 = ["qwerty1"]
items2 = ["qwerty2"]
alist.add(params1, items1)
alist.add(params2, items1)
alist.add(params3, items1)
alist.add(params4, items1)
alist.add(params5, items2)
hp1 = AggregatedList.hash_params(params1)
hp2 = AggregatedList.hash_params(params2)
hp3 = AggregatedList.hash_params(params3)
hp4 = AggregatedList.hash_params(params4)
hp5 = AggregatedList.hash_params(params5)
assert hp1 == hp2
assert hp1 == hp2
assert hp1 == hp3
assert hp1 == hp4
assert hp1 != hp5
assert alist.get_by_params_hash(hp1)['items'] == items1
assert alist.get_by_params_hash(hp5)['items'] == items2
def test_run(self):
left = AggregatedList()
right = AggregatedList()
# test insert
right.add({'on': 'insert'}, ["i"])
# test delete
left.add({'on': 'delete'}, ["d"])
# test update-insert
left.add({'on': 'update-insert'}, ["ui1"])
right.add({'on': 'update-insert'}, ["ui1", "ui2"])
# test update-delete
left.add({'on': 'update-delete'}, ["ud1", "ud2"])
right.add({'on': 'update-delete'}, ["ud1"])
on_insert = []
on_delete = []
on_update_insert = []
on_update_delete = []
def recorder(l):
return lambda p, i: l.append([p, i])
runner = AggregatedDiffRunner(left.diff(right))
runner.register("insert", recorder(on_insert))
runner.register("delete", recorder(on_delete))
runner.register("update-insert", recorder(on_update_insert))
runner.register("update-delete", recorder(on_update_delete))
runner.run()
assert on_insert == [[{'on': 'insert'}, ['i']]]
assert on_delete == [[{'on': 'delete'}, ['d']]]
assert on_update_insert == [[{'on': 'update-insert'}, ['ui2']]]
assert on_update_delete == [[{'on': 'update-delete'}, ['ud2']]]