def setup_gerrit_groups(gerritperms_path, admin_username, admin_email):
"""Generate groups file"""
cwd = os.getcwd()
os.chdir(gerritperms_path)
try:
query = 'SELECT name, group_uuid FROM account_groups'
cmd = ['java', '-jar', WAR_PATH, 'gsql', '-d', SITE_PATH, '-c', query]
result = subprocess.check_output(cmd)
if result:
# parse file and generate groups
output = result.splitlines()
with open('groups', 'w') as f:
for item in output[2:]:
# split between name and id
data = item.split('|')
if len(data) == 2:
group_cfg = ('%s\t%s\n' %
(data[1].strip(), data[0].strip()))
f.write(group_cfg)
cmds = [['git', 'config', '--global', 'user.name',
admin_username],
['git', 'config', '--global', 'user.email',
admin_email],
['git', 'commit', '-a', '-m', '"%s"' %
(INITIAL_PERMISSIONS_COMMIT_MSG)],
['git', 'push', 'repo', 'meta/config:meta/config']]
for cmd in cmds:
common.run_as_user(user=GERRIT_USER, cmd=cmd,
cwd=gerritperms_path)
else:
msg = 'Failed to query gerrit db for groups'
raise GerritConfigurationException(msg)
finally:
os.chdir(cwd)
def is_permissions_initialised(repo_name, repo_path):
""" The All-Projects.git repository is created by the Gerrit charm and
configured by this charm. In order for it to be deemed initialised we need:
1. expected branches
2. initial permissions commit message
"""
if repo_is_initialised("%s/%s" % (GIT_PATH, repo_name)):
cmd = ['git', 'log', '--grep', INITIAL_PERMISSIONS_COMMIT_MSG]
stdout = common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=repo_path)
if stdout and INITIAL_PERMISSIONS_COMMIT_MSG in stdout:
return True
# NOTE(dosaboy): the ci-configurator used to set the same commit
# message as the gerrit charm i.e. "Initial permissions" so if we don't
# find the new-style message we then check for > 1 of the old-style
# message.
old_style_msg = "Initial permissions"
cmd = ['git', 'log', '--grep', old_style_msg]
stdout = common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=repo_path)
if stdout:
count = 0
for line in stdout.split('\n'):
if old_style_msg in line:
count += 1
if count > 1:
return True
return False
def is_permissions_initialised(repo_name, repo_path):
""" The All-Projects.git repository is created by the Gerrit charm and
configured by this charm. In order for it to be deemed initialised we need:
1. expected branches
2. initial permissions commit message
"""
if repo_is_initialised("%s/%s" % (GIT_PATH, repo_name)):
cmd = ['git', 'log', '--grep', INITIAL_PERMISSIONS_COMMIT_MSG]
stdout = common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=repo_path)
if stdout and INITIAL_PERMISSIONS_COMMIT_MSG in stdout:
return True
# NOTE(dosaboy): the ci-configurator used to set the same commit
# message as the gerrit charm i.e. "Initial permissions" so if we don't
# find the new-style message we then check for > 1 of the old-style
# message.
old_style_msg = "Initial permissions"
cmd = ['git', 'log', '--grep', old_style_msg]
stdout = common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=repo_path)
if stdout:
count = 0
for line in stdout.split('\n'):
if old_style_msg in line:
count += 1
if count > 1:
return True
return False
def setup_gerrit_groups(gerritperms_path, admin_username, admin_email):
"""Generate groups file"""
cwd = os.getcwd()
os.chdir(gerritperms_path)
try:
query = 'SELECT name, group_uuid FROM account_groups'
cmd = ['java', '-jar', WAR_PATH, 'gsql', '-d', SITE_PATH, '-c', query]
result = subprocess.check_output(cmd)
if result:
# parse file and generate groups
output = result.splitlines()
with open('groups', 'w') as f:
for item in output[2:]:
# split between name and id
data = item.split('|')
if len(data) == 2:
group_cfg = ('%s\t%s\n' %
(data[1].strip(), data[0].strip()))
f.write(group_cfg)
cmds = [['git', 'config', '--global', 'user.name', admin_username],
['git', 'config', '--global', 'user.email', admin_email],
[
'git', 'commit', '-a', '-m',
'"%s"' % (INITIAL_PERMISSIONS_COMMIT_MSG)
], ['git', 'push', 'repo', 'meta/config:meta/config']]
for cmd in cmds:
common.run_as_user(user=GERRIT_USER,
cmd=cmd,
cwd=gerritperms_path)
else:
msg = 'Failed to query gerrit db for groups'
raise GerritConfigurationException(msg)
finally:
os.chdir(cwd)
def config_changed():
# setup identity to reach private LP resources
common.ensure_user()
common.install_ssh_keys()
lp_user = config('lp-login')
if lp_user:
cmd = ['bzr', 'launchpad-login', lp_user]
common.run_as_user(cmd=cmd, user=common.CI_USER)
# NOTE: this will overwrite existing configs so relation hooks will have to
# re-run in order for settings to be re-applied.
bundled_repo = os.path.join(charm_dir(), common.LOCAL_CONFIG_REPO)
conf_repo = config('config-repo')
conf_repo_rcs = config('config-repo-rcs')
if os.path.exists(bundled_repo) and os.path.isdir(bundled_repo):
common.update_configs_from_charm(bundled_repo)
run_relation_hooks()
elif is_valid_config_repo(conf_repo_rcs, conf_repo):
common.update_configs_from_repo(conf_repo_rcs,
conf_repo,
config('config-repo-revision'))
run_relation_hooks()
if config('schedule-updates'):
schedule = config('update-frequency')
cron.schedule_repo_updates(
schedule, common.CI_USER, common.CI_CONFIG_DIR, conf_repo_rcs,
jjb.JOBS_CONFIG_DIR)
def _update_jenkins_jobs():
if not write_jjb_config():
log('Could not write jenkins-job-builder config, skipping '
'jobs update.')
return
if not os.path.isdir(JOBS_CONFIG_DIR):
log(
'Could not find jobs-config directory at expected location, '
'skipping jenkins-jobs update (%s)' % JOBS_CONFIG_DIR, ERROR)
return
save_context()
# inform hook where to find the context json dump
os.environ['JJB_CHARM_CONTEXT'] = CHARM_CONTEXT_DUMP
os.environ['JJB_JOBS_CONFIG_DIR'] = JOBS_CONFIG_DIR
hook = os.path.join(JOBS_CONFIG_DIR, 'update')
if not os.path.isfile(hook):
log(
'Could not find jobs-config update hook at expected location: ' +
"%s, continuing anyways." % hook, ERROR)
else:
log('Calling jenkins-job-builder repo update hook: %s.' % hook)
subprocess.check_call(hook)
# call jenkins-jobs to actually update jenkins
# TODO: Call 'jenkins-job test' to validate configs before updating?
log('Updating jobs in jenkins.')
# call jenkins-jobs update, wait for jenkins to be available if needed
# because it comes after a restart, so needs time
for attempt in range(MAX_RETRIES):
try:
cmd = [_get_jjb_cmd(), 'update', JOBS_CONFIG_DIR]
# Run as the CI_USER so the cache will be primed with the correct
# permissions (rather than root:root).
common.run_as_user(cmd=cmd, user=common.CI_USER)
except HTTPError as err:
if err.code == 503:
# sleep for a while, retry
time.sleep(SLEEP_TIME)
log('Jenkins is still not available, retrying')
continue
else:
log('Error updating jobs, check jjb settings and retry', ERROR)
except Exception as e:
log(
'Error updating jobs, check jjb settings and retry: %s' %
str(e), ERROR)
raise
break
def _update_jenkins_jobs():
if not write_jjb_config():
log('Could not write jenkins-job-builder config, skipping '
'jobs update.')
return
if not os.path.isdir(JOBS_CONFIG_DIR):
log('Could not find jobs-config directory at expected location, '
'skipping jenkins-jobs update (%s)' % JOBS_CONFIG_DIR, ERROR)
return
hook = os.path.join(JOBS_CONFIG_DIR, 'update')
if not os.path.isfile(hook):
log('Could not find jobs-config update hook at expected location: %s' %
hook, ERROR)
return
save_context()
# inform hook where to find the context json dump
os.environ['JJB_CHARM_CONTEXT'] = CHARM_CONTEXT_DUMP
os.environ['JJB_JOBS_CONFIG_DIR'] = JOBS_CONFIG_DIR
log('Calling jenkins-job-builder repo update hook: %s.' % hook)
subprocess.check_call(hook)
# call jenkins-jobs to actually update jenkins
# TODO: Call 'jenkins-job test' to validate configs before updating?
log('Updating jobs in jenkins.')
# call jenkins-jobs update, wait for jenkins to be available if needed
# because it comes after a restart, so needs time
for attempt in range(MAX_RETRIES):
try:
cmd = ['jenkins-jobs', 'update', JOBS_CONFIG_DIR]
# Run as the CI_USER so the cache will be primed with the correct
# permissions (rather than root:root).
common.run_as_user(cmd=cmd, user=common.CI_USER)
except urllib2.HTTPError, err:
if err.code == 503:
# sleep for a while, retry
time.sleep(SLEEP_TIME)
log('Jenkins is still not available, retrying')
continue
else:
log('Error updating jobs, check jjb settings and retry', ERROR)
except Exception as e:
log('Error updating jobs, check jjb settings and retry: %s' %
str(e), ERROR)
def create_projects(admin_username, admin_email, admin_privkey, base_url,
projects, branches, git_host, tmpdir):
"""Globally create all projects and repositories, clone and push"""
cmd = ["chown", "%s:%s" % (GERRIT_USER, GERRIT_USER), tmpdir]
subprocess.check_call(cmd)
os.chmod(tmpdir, 0774)
gerrit_client = GerritClient(host='localhost', user=admin_username,
port=SSH_PORT, key_file=admin_privkey)
try:
for project in projects:
name, repo = project.itervalues()
if not gerrit_client.create_project(name):
log("failed to create project in gerrit - skipping setup "
"for '%s'" % (name))
continue
git_srv_path = os.path.join(GIT_PATH, name)
repo_path = os.path.join(tmpdir, name.replace('/', ''))
repo_url = 'https://%s/%s' % (base_url, repo)
gerrit_remote_url = "%s/%s.git" % (GIT_PATH, repo)
# Only proceed if the repo has NOT been successfully initialised.
if repo_is_initialised(gerrit_remote_url, branches):
log("Repository '%s' already initialised - skipping" %
(git_srv_path), level=INFO)
continue
# Git config may not have been set yet so just in case.
cmds = [['git', 'config', '--global', 'user.name', admin_username],
['git', 'config', '--global', 'user.email', admin_email]]
log("Cloning git repository '%s'" % (repo_url))
cmds.append(['git', 'clone', repo_url, repo_path])
for cmd in cmds:
common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=tmpdir)
# Setup the .gitreview file to point to this repo by default (as
# opposed to upstream openstack).
host = get_gerrit_hostname(git_host)
cmds = setup_gitreview(repo_path, name, host)
cmds.append(['git', 'remote', 'add', 'gerrit', gerrit_remote_url])
cmds.append(['git', 'fetch', '--all'])
for cmd in cmds:
common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=repo_path)
# Push to each branch if needed
for branch in branches:
branch = branch.strip()
try:
cmd = ['git', 'show-branch', 'gerrit/%s' % (branch)]
common.run_as_user(user=GERRIT_USER, cmd=cmd,
cwd=repo_path)
except Exception:
# branch does not exist, create it
ref = 'HEAD:refs/heads/%s' % branch
cmds = [['git', 'checkout', branch],
['git', 'pull'],
['git', 'push', '--force', 'gerrit', ref]]
for cmd in cmds:
common.run_as_user(user=GERRIT_USER, cmd=cmd,
cwd=repo_path)
gerrit_client.flush_cache()
except Exception as exc:
msg = ('project setup failed (%s)' % str(exc))
log(msg, ERROR)
raise exc
def update_permissions(admin_username, admin_email, admin_privkey):
if not os.path.isdir(PERMISSIONS_DIR):
log('Gerrit permissions directory not found @ %s, skipping '
'permissions refresh.' % PERMISSIONS_DIR, level=WARNING)
return False
# create launchpad directory and setup permissions
if not os.path.isdir(LAUNCHPAD_DIR):
os.mkdir(LAUNCHPAD_DIR)
cmd = ['chown', "%s:%s" % (GERRIT_USER, GERRIT_USER), LAUNCHPAD_DIR]
subprocess.check_call(cmd)
os.chmod(LAUNCHPAD_DIR, 0774)
# check if we have creds, push to dir
if config('lp-credentials-file'):
creds = b64decode(config('lp-credentials-file'))
with open(os.path.join(LAUNCHPAD_DIR, 'creds'), 'w') as f:
f.write(creds)
# if we have teams and schedule, update cronjob
if config('lp-schedule'):
command = ('%s %s %s > %s 2>&1' %
(os.path.join(os.environ['CHARM_DIR'], 'scripts',
'query_lp_members.py'), admin_username, admin_privkey,
LOGS_PATH+'/launchpad_sync.log'))
cron.schedule_generic_job(
config('lp-schedule'), 'root', 'launchpad_sync', command)
repo_name = 'All-Projects.git'
repo_url = ('ssh://%s@localhost:%s/%s' % (admin_username, SSH_PORT,
repo_name))
# parse groups file and create groups
gerrit_client = GerritClient(host='localhost', user=admin_username,
port=SSH_PORT, key_file=admin_privkey)
with open(GROUPS_CONFIG_FILE, 'r') as f:
groups_config = yaml.load(f)
# Create group(s)
for group, _ in groups_config.items():
gerrit_client.create_group(group)
# Update git repo with permissions
log('Installing gerrit permissions from %s.' % PERMISSIONS_DIR)
try:
tmppath = tempfile.mkdtemp('', 'gerritperms')
if tmppath:
cmd = ["chown", "%s:%s" % (GERRIT_USER, GERRIT_USER), tmppath]
subprocess.check_call(cmd)
os.chmod(tmppath, 0774)
config_ref = 'refs/meta/config:refs/remotes/origin/meta/config'
for cmd in [['git', 'init'],
['git', 'remote', 'add', 'repo', repo_url],
['git', 'fetch', 'repo', config_ref],
['git', 'checkout', 'meta/config']]:
common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=tmppath)
common.sync_dir(os.path.join(PERMISSIONS_DIR, 'All-Projects'),
tmppath)
# Only proceed if the repo has NOT been successfully initialised.
if is_permissions_initialised(repo_name, tmppath):
log("%s is already initialised - skipping update permissions" %
(repo_name), level=INFO)
return False
try:
setup_gerrit_groups(tmppath, admin_username, admin_email)
except GerritConfigurationException as exc:
log(str(exc), level=ERROR)
return False
else:
log('Failed to create permissions temporary directory',
level=ERROR)
return False
except Exception as e:
log('Failed to create permissions: %s' % str(e), level=ERROR)
return True
def create_projects(admin_username, admin_email, admin_privkey, base_url,
projects, branches, git_host, tmpdir):
"""Globally create all projects and repositories, clone and push"""
cmd = ["chown", "%s:%s" % (GERRIT_USER, GERRIT_USER), tmpdir]
subprocess.check_call(cmd)
os.chmod(tmpdir, 0774)
gerrit_client = GerritClient(host='localhost',
user=admin_username,
port=SSH_PORT,
key_file=admin_privkey)
try:
for project in projects:
name, repo = project.itervalues()
if not gerrit_client.create_project(name):
log("failed to create project in gerrit - skipping setup "
"for '%s'" % (name))
continue
git_srv_path = os.path.join(GIT_PATH, name)
repo_path = os.path.join(tmpdir, name.replace('/', ''))
repo_url = 'https://%s/%s' % (base_url, repo)
gerrit_remote_url = "%s/%s.git" % (GIT_PATH, repo)
# Only proceed if the repo has NOT been successfully initialised.
if repo_is_initialised(gerrit_remote_url, branches):
log("Repository '%s' already initialised - skipping" %
(git_srv_path),
level=INFO)
continue
# Git config may not have been set yet so just in case.
cmds = [['git', 'config', '--global', 'user.name', admin_username],
['git', 'config', '--global', 'user.email', admin_email]]
log("Cloning git repository '%s'" % (repo_url))
cmds.append(['git', 'clone', repo_url, repo_path])
for cmd in cmds:
common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=tmpdir)
# Setup the .gitreview file to point to this repo by default (as
# opposed to upstream openstack).
host = get_gerrit_hostname(git_host)
cmds = setup_gitreview(repo_path, name, host)
cmds.append(['git', 'remote', 'add', 'gerrit', gerrit_remote_url])
cmds.append(['git', 'fetch', '--all'])
for cmd in cmds:
common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=repo_path)
# Push to each branch if needed
for branch in branches:
branch = branch.strip()
try:
cmd = ['git', 'show-branch', 'gerrit/%s' % (branch)]
common.run_as_user(user=GERRIT_USER,
cmd=cmd,
cwd=repo_path)
except Exception:
# branch does not exist, create it
ref = 'HEAD:refs/heads/%s' % branch
cmds = [['git', 'checkout', branch], ['git', 'pull'],
['git', 'push', '--force', 'gerrit', ref]]
for cmd in cmds:
common.run_as_user(user=GERRIT_USER,
cmd=cmd,
cwd=repo_path)
gerrit_client.flush_cache()
except Exception as exc:
msg = ('project setup failed (%s)' % str(exc))
log(msg, ERROR)
raise exc
def update_permissions(admin_username, admin_email, admin_privkey):
if not os.path.isdir(PERMISSIONS_DIR):
log('Gerrit permissions directory not found @ %s, skipping '
'permissions refresh.' % PERMISSIONS_DIR,
level=WARNING)
return False
# create launchpad directory and setup permissions
if not os.path.isdir(LAUNCHPAD_DIR):
os.mkdir(LAUNCHPAD_DIR)
cmd = ['chown', "%s:%s" % (GERRIT_USER, GERRIT_USER), LAUNCHPAD_DIR]
subprocess.check_call(cmd)
os.chmod(LAUNCHPAD_DIR, 0774)
# check if we have creds, push to dir
if config('lp-credentials-file'):
creds = b64decode(config('lp-credentials-file'))
with open(os.path.join(LAUNCHPAD_DIR, 'creds'), 'w') as f:
f.write(creds)
# if we have teams and schedule, update cronjob
if config('lp-schedule'):
command = ('%s %s %s > %s 2>&1' %
(os.path.join(os.environ['CHARM_DIR'], 'scripts',
'query_lp_members.py'), admin_username,
admin_privkey, LOGS_PATH + '/launchpad_sync.log'))
cron.schedule_generic_job(config('lp-schedule'), 'root',
'launchpad_sync', command)
repo_name = 'All-Projects.git'
repo_url = ('ssh://%s@localhost:%s/%s' %
(admin_username, SSH_PORT, repo_name))
# parse groups file and create groups
gerrit_client = GerritClient(host='localhost',
user=admin_username,
port=SSH_PORT,
key_file=admin_privkey)
with open(GROUPS_CONFIG_FILE, 'r') as f:
groups_config = yaml.load(f)
# Create group(s)
for group, _ in groups_config.items():
gerrit_client.create_group(group)
# Update git repo with permissions
log('Installing gerrit permissions from %s.' % PERMISSIONS_DIR)
try:
tmppath = tempfile.mkdtemp('', 'gerritperms')
if tmppath:
cmd = ["chown", "%s:%s" % (GERRIT_USER, GERRIT_USER), tmppath]
subprocess.check_call(cmd)
os.chmod(tmppath, 0774)
config_ref = 'refs/meta/config:refs/remotes/origin/meta/config'
for cmd in [['git', 'init'],
['git', 'remote', 'add', 'repo', repo_url],
['git', 'fetch', 'repo', config_ref],
['git', 'checkout', 'meta/config']]:
common.run_as_user(user=GERRIT_USER, cmd=cmd, cwd=tmppath)
common.sync_dir(os.path.join(PERMISSIONS_DIR, 'All-Projects'),
tmppath)
# Only proceed if the repo has NOT been successfully initialised.
if is_permissions_initialised(repo_name, tmppath):
log("%s is already initialised - skipping update permissions" %
(repo_name),
level=INFO)
return False
try:
setup_gerrit_groups(tmppath, admin_username, admin_email)
except GerritConfigurationException as exc:
log(str(exc), level=ERROR)
return False
else:
log('Failed to create permissions temporary directory',
level=ERROR)
return False
except Exception as e:
log('Failed to create permissions: %s' % str(e), level=ERROR)
return True
