def run(dry_run=False):
instance = queries.get_gitlab_instance()
settings = queries.get_app_interface_settings()
app_int_repos = queries.get_repos()
gl = GitLabApi(instance, settings=settings)
project_requests = instance['projectRequests'] or []
error = False
for pr in project_requests:
group = pr['group']
group_id, existing_projects = gl.get_group_id_and_projects(group)
requested_projects = pr['projects']
projects_to_create = [
p for p in requested_projects if p not in existing_projects
]
for p in projects_to_create:
project_url = gl.get_project_url(group, p)
if project_url not in app_int_repos:
logging.error(f'{project_url} missing from all codeComponents')
error = True
continue
logging.info(['create_project', group, p])
if not dry_run:
gl.create_project(group_id, p)
sys.exit(error)
def run(dry_run, gitlab_project_id=None, thread_pool_size=10):
accounts = queries.get_aws_accounts()
settings = queries.get_app_interface_settings()
aws = AWSApi(thread_pool_size, accounts, settings=settings)
existing_keys = aws.get_users_keys()
existing_keys_list = [
key for user_key in existing_keys.values()
for keys in user_key.values() for key in keys
]
logging.info("found {} existing keys".format(len(existing_keys_list)))
app_int_github_repos = queries.get_repos(server="https://github.com")
all_repos = get_all_repos_to_scan(app_int_github_repos)
logging.info("about to scan {} repos".format(len(all_repos)))
results = threaded.run(
git_secrets.scan_history,
all_repos,
thread_pool_size,
existing_keys=existing_keys_list,
)
all_leaked_keys = [key for keys in results for key in keys]
deleted_keys = aws_sos.get_deleted_keys(accounts)
keys_to_delete = [
{
"account": account,
"key": key
} for key in all_leaked_keys
for account, user_keys in existing_keys.items()
if key in [uk for uks in user_keys.values()
for uk in uks] and key not in deleted_keys[account]
]
aws_sos.act(dry_run, gitlab_project_id, accounts, keys_to_delete)
def run(dry_run=False, thread_pool_size=10):
instance = queries.get_gitlab_instance()
settings = queries.get_app_interface_settings()
gl = GitLabApi(instance, settings=settings)
repos = queries.get_repos(server=gl.server)
app_sre = gl.get_app_sre_group_users()
results = threaded.run(get_members_to_add, repos, thread_pool_size,
gl=gl, app_sre=app_sre)
members_to_add = [item for sublist in results for item in sublist]
for m in members_to_add:
logging.info(['add_maintainer', m["repo"], m["user"].username])
if not dry_run:
gl.add_project_member(m["repo"], m["user"])
def run(dry_run):
instance = queries.get_gitlab_instance()
settings = queries.get_app_interface_settings()
gl = GitLabApi(instance, settings=settings)
previous_urls = queries.get_jenkins_instances_previous_urls()
repos = queries.get_repos(server=gl.server)
for repo in repos:
hooks = gl.get_project_hooks(repo)
for hook in hooks:
hook_url = hook.url
for previous_url in previous_urls:
if hook_url.startswith(previous_url):
logging.info(['delete_hook', repo, hook_url])
if not dry_run:
hook.delete()
def validate_repos_and_admins(jjb):
jjb_repos = jjb.get_repos()
app_int_repos = queries.get_repos()
missing_repos = [r for r in jjb_repos if r not in app_int_repos]
if missing_repos:
msg = 'repos are missing from codeComponents: ' + missing_repos
raise Exception(msg)
jjb_admins = jjb.get_admins()
app_int_users = queries.get_users()
app_int_bots = queries.get_bots()
github_usernames = \
[u.get('github_username') for u in app_int_users] + \
[b.get('github_username') for b in app_int_bots]
unknown_admins = [a for a in jjb_admins if a not in github_usernames]
if unknown_admins:
logging.warning('user file not found for: {}'.format(unknown_admins))
def validate_repos_and_admins(jjb):
jjb_repos = jjb.get_repos()
app_int_repos = queries.get_repos()
missing_repos = [r for r in jjb_repos if r not in app_int_repos]
for r in missing_repos:
logging.error('repo is missing from codeComponents: {}'.format(r))
jjb_admins = jjb.get_admins()
app_int_users = queries.get_users()
app_int_bots = queries.get_bots()
github_usernames = \
[u.get('github_username') for u in app_int_users] + \
[b.get('github_username') for b in app_int_bots]
unknown_admins = [a for a in jjb_admins if a not in github_usernames]
for a in unknown_admins:
logging.warning('admin is missing from users: {}'.format(a))
if missing_repos:
sys.exit(1)
def run(dry_run):
saas_files = queries.get_saas_files(v1=True, v2=True)
settings = queries.get_app_interface_settings()
saasherder = SaasHerder(
saas_files,
thread_pool_size=1,
gitlab=None,
integration=QONTRACT_INTEGRATION,
integration_version=QONTRACT_INTEGRATION_VERSION,
settings=settings,
validate=True)
app_int_repos = queries.get_repos()
missing_repos = [r for r in saasherder.repo_urls
if r not in app_int_repos]
for r in missing_repos:
logging.error(f'repo is missing from codeComponents: {r}')
if not saasherder.valid or missing_repos:
sys.exit(ExitCodes.ERROR)
def validate_repos_and_admins(jjb, additional_repo_urls):
jjb_repos = jjb.get_repos()
jjb_repos.update(additional_repo_urls)
app_int_repos = queries.get_repos()
missing_repos = [r for r in jjb_repos if r not in app_int_repos]
for r in missing_repos:
logging.error(f"repo is missing from codeComponents: {r}")
jjb_admins = jjb.get_admins()
app_int_users = queries.get_users()
app_int_bots = queries.get_bots()
external_users = queries.get_external_users()
github_usernames = ([u.get("github_username") for u in app_int_users] +
[b.get("github_username") for b in app_int_bots] +
[u.get("github_username") for u in external_users])
unknown_admins = [a for a in jjb_admins if a not in github_usernames]
for a in unknown_admins:
logging.warning("admin is missing from users: {}".format(a))
if missing_repos:
sys.exit(1)
def repos(ctx):
repos = queries.get_repos()
repos = [{'url': r} for r in repos]
columns = ['url']
print_output(ctx.obj['output'], repos, columns)
