def setup_gerrit_groups(gerritperms_path, admin_username, admin_email): """Generate groups file""" cwd = os.getcwd() os.chdir(gerritperms_path) try: query = 'SELECT name, group_uuid FROM account_groups' cmd = ['java', '-jar', WAR_PATH, 'gsql', '-d', SITE_PATH, '-c', query] result = subprocess.check_output(cmd) if result: # parse file and generate groups output = result.splitlines() with open('groups', 'w') as f: for item in output[2:]: # split between name and id data = item.split('|') if len(data) == 2: group_cfg = ('%s\t%s\n' % (data[1].strip(), data[0].strip())) f.write(group_cfg) cmds = [['git', 'config', '--global', 'user.name', admin_username], ['git', 'config', '--global', 'user.email', admin_email], ['git', 'commit', '-a', '-m', '"%s"' % (INITIAL_PERMISSIONS_COMMIT_MSG)], ['git', 'push', 'repo', 'meta/config:meta/config']] for cmd in cmds: common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=gerritperms_path) else: msg = 'Failed to query gerrit db for groups' raise GerritConfigurationException(msg) finally: os.chdir(cwd)
def is_permissions_initialised(repo_name, repo_path): """ The All-Projects.git repository is created by the Gerrit charm and configured by this charm. In order for it to be deemed initialised we need: 1. expected branches 2. initial permissions commit message """ if repo_is_initialised("%s/%s" % (GIT_PATH, repo_name)): cmd = ['git', 'log', '--grep', INITIAL_PERMISSIONS_COMMIT_MSG] stdout = common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=repo_path) if stdout and INITIAL_PERMISSIONS_COMMIT_MSG in stdout: return True # NOTE(dosaboy): the ci-configurator used to set the same commit # message as the gerrit charm i.e. "Initial permissions" so if we don't # find the new-style message we then check for > 1 of the old-style # message. old_style_msg = "Initial permissions" cmd = ['git', 'log', '--grep', old_style_msg] stdout = common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=repo_path) if stdout: count = 0 for line in stdout.split('\n'): if old_style_msg in line: count += 1 if count > 1: return True return False
def setup_gerrit_groups(gerritperms_path, admin_username, admin_email): """Generate groups file""" cwd = os.getcwd() os.chdir(gerritperms_path) try: query = 'SELECT name, group_uuid FROM account_groups' cmd = ['java', '-jar', WAR_PATH, 'gsql', '-d', SITE_PATH, '-c', query] result = subprocess.check_output(cmd) if result: # parse file and generate groups output = result.splitlines() with open('groups', 'w') as f: for item in output[2:]: # split between name and id data = item.split('|') if len(data) == 2: group_cfg = ('%s\t%s\n' % (data[1].strip(), data[0].strip())) f.write(group_cfg) cmds = [['git', 'config', '--global', 'user.name', admin_username], ['git', 'config', '--global', 'user.email', admin_email], [ 'git', 'commit', '-a', '-m', '"%s"' % (INITIAL_PERMISSIONS_COMMIT_MSG) ], ['git', 'push', 'repo', 'meta/config:meta/config']] for cmd in cmds: common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=gerritperms_path) else: msg = 'Failed to query gerrit db for groups' raise GerritConfigurationException(msg) finally: os.chdir(cwd)
def config_changed(): # setup identity to reach private LP resources common.ensure_user() common.install_ssh_keys() lp_user = config('lp-login') if lp_user: cmd = ['bzr', 'launchpad-login', lp_user] common.run_as_user(cmd=cmd, user=common.CI_USER) # NOTE: this will overwrite existing configs so relation hooks will have to # re-run in order for settings to be re-applied. bundled_repo = os.path.join(charm_dir(), common.LOCAL_CONFIG_REPO) conf_repo = config('config-repo') conf_repo_rcs = config('config-repo-rcs') if os.path.exists(bundled_repo) and os.path.isdir(bundled_repo): common.update_configs_from_charm(bundled_repo) run_relation_hooks() elif is_valid_config_repo(conf_repo_rcs, conf_repo): common.update_configs_from_repo(conf_repo_rcs, conf_repo, config('config-repo-revision')) run_relation_hooks() if config('schedule-updates'): schedule = config('update-frequency') cron.schedule_repo_updates( schedule, common.CI_USER, common.CI_CONFIG_DIR, conf_repo_rcs, jjb.JOBS_CONFIG_DIR)
def _update_jenkins_jobs(): if not write_jjb_config(): log('Could not write jenkins-job-builder config, skipping ' 'jobs update.') return if not os.path.isdir(JOBS_CONFIG_DIR): log( 'Could not find jobs-config directory at expected location, ' 'skipping jenkins-jobs update (%s)' % JOBS_CONFIG_DIR, ERROR) return save_context() # inform hook where to find the context json dump os.environ['JJB_CHARM_CONTEXT'] = CHARM_CONTEXT_DUMP os.environ['JJB_JOBS_CONFIG_DIR'] = JOBS_CONFIG_DIR hook = os.path.join(JOBS_CONFIG_DIR, 'update') if not os.path.isfile(hook): log( 'Could not find jobs-config update hook at expected location: ' + "%s, continuing anyways." % hook, ERROR) else: log('Calling jenkins-job-builder repo update hook: %s.' % hook) subprocess.check_call(hook) # call jenkins-jobs to actually update jenkins # TODO: Call 'jenkins-job test' to validate configs before updating? log('Updating jobs in jenkins.') # call jenkins-jobs update, wait for jenkins to be available if needed # because it comes after a restart, so needs time for attempt in range(MAX_RETRIES): try: cmd = [_get_jjb_cmd(), 'update', JOBS_CONFIG_DIR] # Run as the CI_USER so the cache will be primed with the correct # permissions (rather than root:root). common.run_as_user(cmd=cmd, user=common.CI_USER) except HTTPError as err: if err.code == 503: # sleep for a while, retry time.sleep(SLEEP_TIME) log('Jenkins is still not available, retrying') continue else: log('Error updating jobs, check jjb settings and retry', ERROR) except Exception as e: log( 'Error updating jobs, check jjb settings and retry: %s' % str(e), ERROR) raise break
def _update_jenkins_jobs(): if not write_jjb_config(): log('Could not write jenkins-job-builder config, skipping ' 'jobs update.') return if not os.path.isdir(JOBS_CONFIG_DIR): log('Could not find jobs-config directory at expected location, ' 'skipping jenkins-jobs update (%s)' % JOBS_CONFIG_DIR, ERROR) return hook = os.path.join(JOBS_CONFIG_DIR, 'update') if not os.path.isfile(hook): log('Could not find jobs-config update hook at expected location: %s' % hook, ERROR) return save_context() # inform hook where to find the context json dump os.environ['JJB_CHARM_CONTEXT'] = CHARM_CONTEXT_DUMP os.environ['JJB_JOBS_CONFIG_DIR'] = JOBS_CONFIG_DIR log('Calling jenkins-job-builder repo update hook: %s.' % hook) subprocess.check_call(hook) # call jenkins-jobs to actually update jenkins # TODO: Call 'jenkins-job test' to validate configs before updating? log('Updating jobs in jenkins.') # call jenkins-jobs update, wait for jenkins to be available if needed # because it comes after a restart, so needs time for attempt in range(MAX_RETRIES): try: cmd = ['jenkins-jobs', 'update', JOBS_CONFIG_DIR] # Run as the CI_USER so the cache will be primed with the correct # permissions (rather than root:root). common.run_as_user(cmd=cmd, user=common.CI_USER) except urllib2.HTTPError, err: if err.code == 503: # sleep for a while, retry time.sleep(SLEEP_TIME) log('Jenkins is still not available, retrying') continue else: log('Error updating jobs, check jjb settings and retry', ERROR) except Exception as e: log('Error updating jobs, check jjb settings and retry: %s' % str(e), ERROR)
def create_projects(admin_username, admin_email, admin_privkey, base_url, projects, branches, git_host, tmpdir): """Globally create all projects and repositories, clone and push""" cmd = ["chown", "%s:%s" % (GERRIT_USER, GERRIT_USER), tmpdir] subprocess.check_call(cmd) os.chmod(tmpdir, 0774) gerrit_client = GerritClient(host='localhost', user=admin_username, port=SSH_PORT, key_file=admin_privkey) try: for project in projects: name, repo = project.itervalues() if not gerrit_client.create_project(name): log("failed to create project in gerrit - skipping setup " "for '%s'" % (name)) continue git_srv_path = os.path.join(GIT_PATH, name) repo_path = os.path.join(tmpdir, name.replace('/', '')) repo_url = 'https://%s/%s' % (base_url, repo) gerrit_remote_url = "%s/%s.git" % (GIT_PATH, repo) # Only proceed if the repo has NOT been successfully initialised. if repo_is_initialised(gerrit_remote_url, branches): log("Repository '%s' already initialised - skipping" % (git_srv_path), level=INFO) continue # Git config may not have been set yet so just in case. cmds = [['git', 'config', '--global', 'user.name', admin_username], ['git', 'config', '--global', 'user.email', admin_email]] log("Cloning git repository '%s'" % (repo_url)) cmds.append(['git', 'clone', repo_url, repo_path]) for cmd in cmds: common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=tmpdir) # Setup the .gitreview file to point to this repo by default (as # opposed to upstream openstack). host = get_gerrit_hostname(git_host) cmds = setup_gitreview(repo_path, name, host) cmds.append(['git', 'remote', 'add', 'gerrit', gerrit_remote_url]) cmds.append(['git', 'fetch', '--all']) for cmd in cmds: common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=repo_path) # Push to each branch if needed for branch in branches: branch = branch.strip() try: cmd = ['git', 'show-branch', 'gerrit/%s' % (branch)] common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=repo_path) except Exception: # branch does not exist, create it ref = 'HEAD:refs/heads/%s' % branch cmds = [['git', 'checkout', branch], ['git', 'pull'], ['git', 'push', '--force', 'gerrit', ref]] for cmd in cmds: common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=repo_path) gerrit_client.flush_cache() except Exception as exc: msg = ('project setup failed (%s)' % str(exc)) log(msg, ERROR) raise exc
def update_permissions(admin_username, admin_email, admin_privkey): if not os.path.isdir(PERMISSIONS_DIR): log('Gerrit permissions directory not found @ %s, skipping ' 'permissions refresh.' % PERMISSIONS_DIR, level=WARNING) return False # create launchpad directory and setup permissions if not os.path.isdir(LAUNCHPAD_DIR): os.mkdir(LAUNCHPAD_DIR) cmd = ['chown', "%s:%s" % (GERRIT_USER, GERRIT_USER), LAUNCHPAD_DIR] subprocess.check_call(cmd) os.chmod(LAUNCHPAD_DIR, 0774) # check if we have creds, push to dir if config('lp-credentials-file'): creds = b64decode(config('lp-credentials-file')) with open(os.path.join(LAUNCHPAD_DIR, 'creds'), 'w') as f: f.write(creds) # if we have teams and schedule, update cronjob if config('lp-schedule'): command = ('%s %s %s > %s 2>&1' % (os.path.join(os.environ['CHARM_DIR'], 'scripts', 'query_lp_members.py'), admin_username, admin_privkey, LOGS_PATH+'/launchpad_sync.log')) cron.schedule_generic_job( config('lp-schedule'), 'root', 'launchpad_sync', command) repo_name = 'All-Projects.git' repo_url = ('ssh://%s@localhost:%s/%s' % (admin_username, SSH_PORT, repo_name)) # parse groups file and create groups gerrit_client = GerritClient(host='localhost', user=admin_username, port=SSH_PORT, key_file=admin_privkey) with open(GROUPS_CONFIG_FILE, 'r') as f: groups_config = yaml.load(f) # Create group(s) for group, _ in groups_config.items(): gerrit_client.create_group(group) # Update git repo with permissions log('Installing gerrit permissions from %s.' % PERMISSIONS_DIR) try: tmppath = tempfile.mkdtemp('', 'gerritperms') if tmppath: cmd = ["chown", "%s:%s" % (GERRIT_USER, GERRIT_USER), tmppath] subprocess.check_call(cmd) os.chmod(tmppath, 0774) config_ref = 'refs/meta/config:refs/remotes/origin/meta/config' for cmd in [['git', 'init'], ['git', 'remote', 'add', 'repo', repo_url], ['git', 'fetch', 'repo', config_ref], ['git', 'checkout', 'meta/config']]: common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=tmppath) common.sync_dir(os.path.join(PERMISSIONS_DIR, 'All-Projects'), tmppath) # Only proceed if the repo has NOT been successfully initialised. if is_permissions_initialised(repo_name, tmppath): log("%s is already initialised - skipping update permissions" % (repo_name), level=INFO) return False try: setup_gerrit_groups(tmppath, admin_username, admin_email) except GerritConfigurationException as exc: log(str(exc), level=ERROR) return False else: log('Failed to create permissions temporary directory', level=ERROR) return False except Exception as e: log('Failed to create permissions: %s' % str(e), level=ERROR) return True
def update_permissions(admin_username, admin_email, admin_privkey): if not os.path.isdir(PERMISSIONS_DIR): log('Gerrit permissions directory not found @ %s, skipping ' 'permissions refresh.' % PERMISSIONS_DIR, level=WARNING) return False # create launchpad directory and setup permissions if not os.path.isdir(LAUNCHPAD_DIR): os.mkdir(LAUNCHPAD_DIR) cmd = ['chown', "%s:%s" % (GERRIT_USER, GERRIT_USER), LAUNCHPAD_DIR] subprocess.check_call(cmd) os.chmod(LAUNCHPAD_DIR, 0774) # check if we have creds, push to dir if config('lp-credentials-file'): creds = b64decode(config('lp-credentials-file')) with open(os.path.join(LAUNCHPAD_DIR, 'creds'), 'w') as f: f.write(creds) # if we have teams and schedule, update cronjob if config('lp-schedule'): command = ('%s %s %s > %s 2>&1' % (os.path.join(os.environ['CHARM_DIR'], 'scripts', 'query_lp_members.py'), admin_username, admin_privkey, LOGS_PATH + '/launchpad_sync.log')) cron.schedule_generic_job(config('lp-schedule'), 'root', 'launchpad_sync', command) repo_name = 'All-Projects.git' repo_url = ('ssh://%s@localhost:%s/%s' % (admin_username, SSH_PORT, repo_name)) # parse groups file and create groups gerrit_client = GerritClient(host='localhost', user=admin_username, port=SSH_PORT, key_file=admin_privkey) with open(GROUPS_CONFIG_FILE, 'r') as f: groups_config = yaml.load(f) # Create group(s) for group, _ in groups_config.items(): gerrit_client.create_group(group) # Update git repo with permissions log('Installing gerrit permissions from %s.' % PERMISSIONS_DIR) try: tmppath = tempfile.mkdtemp('', 'gerritperms') if tmppath: cmd = ["chown", "%s:%s" % (GERRIT_USER, GERRIT_USER), tmppath] subprocess.check_call(cmd) os.chmod(tmppath, 0774) config_ref = 'refs/meta/config:refs/remotes/origin/meta/config' for cmd in [['git', 'init'], ['git', 'remote', 'add', 'repo', repo_url], ['git', 'fetch', 'repo', config_ref], ['git', 'checkout', 'meta/config']]: common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=tmppath) common.sync_dir(os.path.join(PERMISSIONS_DIR, 'All-Projects'), tmppath) # Only proceed if the repo has NOT been successfully initialised. if is_permissions_initialised(repo_name, tmppath): log("%s is already initialised - skipping update permissions" % (repo_name), level=INFO) return False try: setup_gerrit_groups(tmppath, admin_username, admin_email) except GerritConfigurationException as exc: log(str(exc), level=ERROR) return False else: log('Failed to create permissions temporary directory', level=ERROR) return False except Exception as e: log('Failed to create permissions: %s' % str(e), level=ERROR) return True