def run(dry_run=False): instance = queries.get_gitlab_instance() settings = queries.get_app_interface_settings() app_int_repos = queries.get_repos() gl = GitLabApi(instance, settings=settings) project_requests = instance['projectRequests'] or [] error = False for pr in project_requests: group = pr['group'] group_id, existing_projects = gl.get_group_id_and_projects(group) requested_projects = pr['projects'] projects_to_create = [ p for p in requested_projects if p not in existing_projects ] for p in projects_to_create: project_url = gl.get_project_url(group, p) if project_url not in app_int_repos: logging.error(f'{project_url} missing from all codeComponents') error = True continue logging.info(['create_project', group, p]) if not dry_run: gl.create_project(group_id, p) sys.exit(error)
def run(dry_run, gitlab_project_id=None, thread_pool_size=10): accounts = queries.get_aws_accounts() settings = queries.get_app_interface_settings() aws = AWSApi(thread_pool_size, accounts, settings=settings) existing_keys = aws.get_users_keys() existing_keys_list = [ key for user_key in existing_keys.values() for keys in user_key.values() for key in keys ] logging.info("found {} existing keys".format(len(existing_keys_list))) app_int_github_repos = queries.get_repos(server="https://github.com") all_repos = get_all_repos_to_scan(app_int_github_repos) logging.info("about to scan {} repos".format(len(all_repos))) results = threaded.run( git_secrets.scan_history, all_repos, thread_pool_size, existing_keys=existing_keys_list, ) all_leaked_keys = [key for keys in results for key in keys] deleted_keys = aws_sos.get_deleted_keys(accounts) keys_to_delete = [ { "account": account, "key": key } for key in all_leaked_keys for account, user_keys in existing_keys.items() if key in [uk for uks in user_keys.values() for uk in uks] and key not in deleted_keys[account] ] aws_sos.act(dry_run, gitlab_project_id, accounts, keys_to_delete)
def run(dry_run=False, thread_pool_size=10): instance = queries.get_gitlab_instance() settings = queries.get_app_interface_settings() gl = GitLabApi(instance, settings=settings) repos = queries.get_repos(server=gl.server) app_sre = gl.get_app_sre_group_users() results = threaded.run(get_members_to_add, repos, thread_pool_size, gl=gl, app_sre=app_sre) members_to_add = [item for sublist in results for item in sublist] for m in members_to_add: logging.info(['add_maintainer', m["repo"], m["user"].username]) if not dry_run: gl.add_project_member(m["repo"], m["user"])
def run(dry_run): instance = queries.get_gitlab_instance() settings = queries.get_app_interface_settings() gl = GitLabApi(instance, settings=settings) previous_urls = queries.get_jenkins_instances_previous_urls() repos = queries.get_repos(server=gl.server) for repo in repos: hooks = gl.get_project_hooks(repo) for hook in hooks: hook_url = hook.url for previous_url in previous_urls: if hook_url.startswith(previous_url): logging.info(['delete_hook', repo, hook_url]) if not dry_run: hook.delete()
def validate_repos_and_admins(jjb): jjb_repos = jjb.get_repos() app_int_repos = queries.get_repos() missing_repos = [r for r in jjb_repos if r not in app_int_repos] if missing_repos: msg = 'repos are missing from codeComponents: ' + missing_repos raise Exception(msg) jjb_admins = jjb.get_admins() app_int_users = queries.get_users() app_int_bots = queries.get_bots() github_usernames = \ [u.get('github_username') for u in app_int_users] + \ [b.get('github_username') for b in app_int_bots] unknown_admins = [a for a in jjb_admins if a not in github_usernames] if unknown_admins: logging.warning('user file not found for: {}'.format(unknown_admins))
def validate_repos_and_admins(jjb): jjb_repos = jjb.get_repos() app_int_repos = queries.get_repos() missing_repos = [r for r in jjb_repos if r not in app_int_repos] for r in missing_repos: logging.error('repo is missing from codeComponents: {}'.format(r)) jjb_admins = jjb.get_admins() app_int_users = queries.get_users() app_int_bots = queries.get_bots() github_usernames = \ [u.get('github_username') for u in app_int_users] + \ [b.get('github_username') for b in app_int_bots] unknown_admins = [a for a in jjb_admins if a not in github_usernames] for a in unknown_admins: logging.warning('admin is missing from users: {}'.format(a)) if missing_repos: sys.exit(1)
def run(dry_run): saas_files = queries.get_saas_files(v1=True, v2=True) settings = queries.get_app_interface_settings() saasherder = SaasHerder( saas_files, thread_pool_size=1, gitlab=None, integration=QONTRACT_INTEGRATION, integration_version=QONTRACT_INTEGRATION_VERSION, settings=settings, validate=True) app_int_repos = queries.get_repos() missing_repos = [r for r in saasherder.repo_urls if r not in app_int_repos] for r in missing_repos: logging.error(f'repo is missing from codeComponents: {r}') if not saasherder.valid or missing_repos: sys.exit(ExitCodes.ERROR)
def validate_repos_and_admins(jjb, additional_repo_urls): jjb_repos = jjb.get_repos() jjb_repos.update(additional_repo_urls) app_int_repos = queries.get_repos() missing_repos = [r for r in jjb_repos if r not in app_int_repos] for r in missing_repos: logging.error(f"repo is missing from codeComponents: {r}") jjb_admins = jjb.get_admins() app_int_users = queries.get_users() app_int_bots = queries.get_bots() external_users = queries.get_external_users() github_usernames = ([u.get("github_username") for u in app_int_users] + [b.get("github_username") for b in app_int_bots] + [u.get("github_username") for u in external_users]) unknown_admins = [a for a in jjb_admins if a not in github_usernames] for a in unknown_admins: logging.warning("admin is missing from users: {}".format(a)) if missing_repos: sys.exit(1)
def repos(ctx): repos = queries.get_repos() repos = [{'url': r} for r in repos] columns = ['url'] print_output(ctx.obj['output'], repos, columns)