def main():
parser = argparse.ArgumentParser()
parser.add_argument('hook')
# common
parser.add_argument('--change', default=None)
parser.add_argument('--change-url', default=None)
parser.add_argument('--project', default=None)
parser.add_argument('--branch', default=None)
parser.add_argument('--commit', default=None)
parser.add_argument('--topic', default=None)
parser.add_argument('--change-owner', default=None)
# patchset-abandoned
parser.add_argument('--abandoner', default=None)
parser.add_argument('--reason', default=None)
# change-merged
parser.add_argument('--submitter', default=None)
parser.add_argument('--newrev', default=None)
# patchset-created
parser.add_argument('--uploader', default=None)
parser.add_argument('--patchset', default=None)
parser.add_argument('--is-draft', default=None)
parser.add_argument('--kind', default=None)
# for Welcome Message
parser.add_argument('--ssh-user',
dest='ssh_user',
help='The gerrit welcome message user')
parser.add_argument('--ssh-key',
dest='ssh_key',
help='The gerrit welcome message SSH key file')
parser.add_argument('--message-file',
dest='message_file',
default=None,
help='The gerrit welcome message')
# Don't actually post the message
parser.add_argument('--dryrun', dest='dryrun', action='store_true')
parser.add_argument('--no-dryrun', dest='dryrun', action='store_false')
parser.set_defaults(dryrun=False)
l.setup_logging_arguments(parser)
args = parser.parse_args()
l.configure_logging(args)
# they're a first-timer, post the message on 1st patchset
if is_newbie(args.uploader) and args.patchset == '1' and not args.dryrun:
post_message(args.commit, args.ssh_user, args.ssh_key,
args.message_file)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('user', help='The gerrit admin user')
parser.add_argument('ssh_key', help='The gerrit admin SSH key file')
parser.add_argument('--age',
dest='age',
default='1w',
help='The minimum age of a review to expire')
l.setup_logging_arguments(parser)
options = parser.parse_args()
l.configure_logging(options)
GERRIT_USER = options.user
GERRIT_SSH_KEY = options.ssh_key
EXPIRY_AGE = options.age
logger.info('Starting expire reviews')
logger.info('Connecting to Gerrit')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('localhost',
username=GERRIT_USER,
key_filename=GERRIT_SSH_KEY,
port=29418)
# Query all reviewed with no activity for 1 week
logger.info('Searching no activity on negative review for 1 week')
stdin, stdout, stderr = ssh.exec_command(
'gerrit query --current-patch-set --all-approvals'
' --format JSON status:reviewed age:' + EXPIRY_AGE)
for line in stdout:
row = json.loads(line)
if 'rowCount' not in row and 'open' in row and row['open']:
# Search for negative approvals
for approval in row['currentPatchSet']['approvals']:
if approval['value'] in ('-1', '-2'):
expire_patch_set(ssh, row['currentPatchSet']['revision'],
row['subject'])
break
logger.info('End expire review')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('hook')
# common
parser.add_argument('--change', default=None)
parser.add_argument('--change-url', default=None)
parser.add_argument('--project', default=None)
parser.add_argument('--branch', default=None)
parser.add_argument('--commit', default=None)
parser.add_argument('--topic', default=None)
parser.add_argument('--change-owner', default=None)
# patchset-abandoned
parser.add_argument('--abandoner', default=None)
parser.add_argument('--reason', default=None)
# change-merged
parser.add_argument('--submitter', default=None)
parser.add_argument('--newrev', default=None)
# patchset-created
parser.add_argument('--uploader', default=None)
parser.add_argument('--patchset', default=None)
parser.add_argument('--is-draft', default=None)
parser.add_argument('--kind', default=None)
# for Welcome Message
parser.add_argument('--ssh-user', dest='ssh_user',
help='The gerrit welcome message user')
parser.add_argument('--ssh-key', dest='ssh_key',
help='The gerrit welcome message SSH key file')
parser.add_argument('--message-file', dest='message_file', default=None,
help='The gerrit welcome message')
# Don't actually post the message
parser.add_argument('--dryrun', dest='dryrun', action='store_true')
parser.add_argument('--no-dryrun', dest='dryrun', action='store_false')
parser.set_defaults(dryrun=False)
l.setup_logging_arguments(parser)
args = parser.parse_args()
l.configure_logging(args)
# they're a first-timer, post the message on 1st patchset
if is_newbie(args.uploader) and args.patchset == '1' and not args.dryrun:
post_message(args.commit, args.ssh_user, args.ssh_key,
args.message_file)
def main():
parser = argparse.ArgumentParser(description='Register projects in Zanata')
l.setup_logging_arguments(parser)
args = parser.parse_args()
l.configure_logging(args)
registry = u.ProjectsRegistry(PROJECTS_YAML)
rest_service = t.ZanataRestService(ZANATA_URL, ZANATA_USER, ZANATA_KEY)
log.info("Registering projects in Zanata")
for entry in registry.configs_list:
project = entry['project']
if not p.has_translations(project):
continue
log.info("Processing project %s" % project)
(org, name) = project.split('/')
try:
translation_proect = t.TranslationProject(rest_service, name)
translation_proect.register()
except ValueError as e:
log.error(e)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('user', help='The gerrit admin user')
parser.add_argument('ssh_key', help='The gerrit admin SSH key file')
parser.add_argument('--age', dest='age', default='1w',
help='The minimum age of a review to expire')
l.setup_logging_arguments(parser)
options = parser.parse_args()
l.configure_logging(options)
GERRIT_USER = options.user
GERRIT_SSH_KEY = options.ssh_key
EXPIRY_AGE = options.age
logger.info('Starting expire reviews')
logger.info('Connecting to Gerrit')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('localhost', username=GERRIT_USER,
key_filename=GERRIT_SSH_KEY, port=29418)
# Query all reviewed with no activity for 1 week
logger.info('Searching no activity on negative review for 1 week')
stdin, stdout, stderr = ssh.exec_command(
'gerrit query --current-patch-set --all-approvals'
' --format JSON status:reviewed age:' + EXPIRY_AGE)
for line in stdout:
row = json.loads(line)
if 'rowCount' not in row:
# Search for negative approvals
for approval in row['currentPatchSet']['approvals']:
if approval['value'] in ('-1', '-2'):
expire_patch_set(ssh,
row['currentPatchSet']['revision'],
row['subject'])
break
logger.info('End expire review')
def main():
parser = argparse.ArgumentParser(description='Manage projects')
l.setup_logging_arguments(parser)
parser.add_argument('--nocleanup', action='store_true',
help='do not remove temp directories')
parser.add_argument('projects', metavar='project', nargs='*',
help='name of project(s) to process')
args = parser.parse_args()
l.configure_logging(args)
default_has_github = registry.get_defaults('has-github', True)
LOCAL_GIT_DIR = registry.get_defaults('local-git-dir', '/var/lib/git')
JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
'/var/lib/jeepyb')
ACL_DIR = registry.get_defaults('acl-dir')
GERRIT_HOST = registry.get_defaults('gerrit-host')
GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
GERRIT_USER = registry.get_defaults('gerrit-user')
GERRIT_KEY = registry.get_defaults('gerrit-key')
GERRIT_GITID = registry.get_defaults('gerrit-committer')
GERRIT_REPLICATE = registry.get_defaults('gerrit-replicate', True)
GERRIT_OS_SYSTEM_USER = registry.get_defaults('gerrit-system-user',
'gerrit2')
GERRIT_OS_SYSTEM_GROUP = registry.get_defaults('gerrit-system-group',
'gerrit2')
DEFAULT_HOMEPAGE = registry.get_defaults('homepage')
DEFAULT_HAS_ISSUES = registry.get_defaults('has-issues', False)
DEFAULT_HAS_DOWNLOADS = registry.get_defaults('has-downloads', False)
DEFAULT_HAS_WIKI = registry.get_defaults('has-wiki', False)
GITHUB_SECURE_CONFIG = registry.get_defaults(
'github-config',
'/etc/github/github-projects.secure.config')
gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST,
GERRIT_USER,
GERRIT_PORT,
GERRIT_KEY)
project_list = gerrit.listProjects()
ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
for section in registry.configs_list:
project = section['project']
if args.projects and project not in args.projects:
continue
try:
log.info("Processing project: %s" % project)
# Figure out all of the options
options = section.get('options', dict())
description = section.get('description', None)
homepage = section.get('homepage', DEFAULT_HOMEPAGE)
upstream = section.get('upstream', None)
upstream_prefix = section.get('upstream-prefix', None)
track_upstream = 'track-upstream' in options
repo_path = os.path.join(JEEPYB_CACHE_DIR, project)
# If this project doesn't want to use gerrit, exit cleanly.
if 'no-gerrit' in options:
continue
project_git = "%s.git" % project
remote_url = "ssh://%s:%s/%s" % (
GERRIT_HOST,
GERRIT_PORT,
project)
git_opts = dict(upstream=upstream,
repo_path=repo_path,
remote_url=remote_url)
acl_config = section.get(
'acl-config',
'%s.config' % os.path.join(ACL_DIR, project))
# Create the project in Gerrit first, since it will fail
# spectacularly if its project directory or local replica
# already exist on disk
project_created = create_gerrit_project(
project, project_list, gerrit)
# Create the repo for the local git mirror
create_local_mirror(
LOCAL_GIT_DIR, project_git,
GERRIT_OS_SYSTEM_USER, GERRIT_OS_SYSTEM_GROUP)
if not os.path.exists(repo_path) or project_created:
# We don't have a local copy already, get one
# Make Local repo
push_string = make_local_copy(
repo_path, project, project_list,
git_opts, ssh_env, upstream, GERRIT_HOST, GERRIT_PORT,
project_git, GERRIT_GITID)
else:
# We do have a local copy of it already, make sure it's
# in shape to have work done.
update_local_copy(
repo_path, track_upstream, git_opts, ssh_env)
description = (
find_description_override(repo_path) or description)
if project_created:
push_to_gerrit(
repo_path, project, push_string, remote_url, ssh_env)
if GERRIT_REPLICATE:
gerrit.replicate(project)
# If we're configured to track upstream, make sure we have
# upstream's refs, and then push them to the appropriate
# branches in gerrit
if track_upstream:
sync_upstream(repo_path, project, ssh_env, upstream_prefix)
if acl_config:
process_acls(
acl_config, project, ACL_DIR, section,
remote_url, repo_path, ssh_env, gerrit, GERRIT_GITID)
if 'has-github' in options or default_has_github:
created = create_github_project(
DEFAULT_HAS_ISSUES, DEFAULT_HAS_DOWNLOADS,
DEFAULT_HAS_WIKI, GITHUB_SECURE_CONFIG,
options, project, description, homepage)
if created and GERRIT_REPLICATE:
gerrit.replicate(project)
except Exception:
log.exception(
"Problems creating %s, moving on." % project)
continue
finally:
os.unlink(ssh_env['GIT_SSH'])
def main():
parser = argparse.ArgumentParser(description='Manage projects')
l.setup_logging_arguments(parser)
parser.add_argument('--nocleanup',
action='store_true',
help='do not remove temp directories')
parser.add_argument('projects',
metavar='project',
nargs='*',
help='name of project(s) to process')
args = parser.parse_args()
l.configure_logging(args)
default_has_github = registry.get_defaults('has-github', True)
LOCAL_GIT_DIR = registry.get_defaults('local-git-dir', '/var/lib/git')
JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
'/var/lib/jeepyb')
ACL_DIR = registry.get_defaults('acl-dir')
GERRIT_HOST = registry.get_defaults('gerrit-host')
GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
GERRIT_USER = registry.get_defaults('gerrit-user')
GERRIT_KEY = registry.get_defaults('gerrit-key')
GERRIT_GITID = registry.get_defaults('gerrit-committer')
GERRIT_REPLICATE = registry.get_defaults('gerrit-replicate', True)
GERRIT_OS_SYSTEM_USER = registry.get_defaults('gerrit-system-user',
'gerrit2')
GERRIT_OS_SYSTEM_GROUP = registry.get_defaults('gerrit-system-group',
'gerrit2')
DEFAULT_HOMEPAGE = registry.get_defaults('homepage')
DEFAULT_HAS_ISSUES = registry.get_defaults('has-issues', False)
DEFAULT_HAS_DOWNLOADS = registry.get_defaults('has-downloads', False)
DEFAULT_HAS_WIKI = registry.get_defaults('has-wiki', False)
GITHUB_SECURE_CONFIG = registry.get_defaults(
'github-config', '/etc/github/github-projects.secure.config')
gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST, GERRIT_USER, GERRIT_PORT,
GERRIT_KEY)
project_list = gerrit.listProjects()
ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
for section in registry.configs_list:
project = section['project']
if args.projects and project not in args.projects:
continue
try:
log.info("Processing project: %s" % project)
# Figure out all of the options
options = section.get('options', dict())
description = section.get('description', None)
homepage = section.get('homepage', DEFAULT_HOMEPAGE)
upstream = section.get('upstream', None)
upstream_prefix = section.get('upstream-prefix', None)
track_upstream = 'track-upstream' in options
repo_path = os.path.join(JEEPYB_CACHE_DIR, project)
# If this project doesn't want to use gerrit, exit cleanly.
if 'no-gerrit' in options:
continue
project_git = "%s.git" % project
remote_url = "ssh://%s:%s/%s" % (GERRIT_HOST, GERRIT_PORT,
project)
git_opts = dict(upstream=upstream,
repo_path=repo_path,
remote_url=remote_url)
acl_config = section.get(
'acl-config', '%s.config' % os.path.join(ACL_DIR, project))
# Create the project in Gerrit first, since it will fail
# spectacularly if its project directory or local replica
# already exist on disk
project_created = create_gerrit_project(
project, project_list, gerrit)
# Create the repo for the local git mirror
create_local_mirror(LOCAL_GIT_DIR, project_git,
GERRIT_OS_SYSTEM_USER,
GERRIT_OS_SYSTEM_GROUP)
if not os.path.exists(repo_path) or project_created:
# We don't have a local copy already, get one
# Make Local repo
push_string = make_local_copy(repo_path, project,
project_list, git_opts,
ssh_env, upstream,
GERRIT_HOST, GERRIT_PORT,
project_git, GERRIT_GITID)
else:
# We do have a local copy of it already, make sure it's
# in shape to have work done.
update_local_copy(repo_path, track_upstream, git_opts,
ssh_env)
description = (find_description_override(repo_path)
or description)
if project_created:
push_to_gerrit(repo_path, project, push_string, remote_url,
ssh_env)
if GERRIT_REPLICATE:
gerrit.replicate(project)
# If we're configured to track upstream, make sure we have
# upstream's refs, and then push them to the appropriate
# branches in gerrit
if track_upstream:
sync_upstream(repo_path, project, ssh_env, upstream_prefix)
if acl_config:
process_acls(acl_config, project, ACL_DIR, section,
remote_url, repo_path, ssh_env, gerrit,
GERRIT_GITID)
if 'has-github' in options or default_has_github:
created = create_github_project(DEFAULT_HAS_ISSUES,
DEFAULT_HAS_DOWNLOADS,
DEFAULT_HAS_WIKI,
GITHUB_SECURE_CONFIG,
options, project,
description, homepage)
if created and GERRIT_REPLICATE:
gerrit.replicate(project)
except Exception:
log.exception("Problems creating %s, moving on." % project)
continue
finally:
os.unlink(ssh_env['GIT_SSH'])
def main():
parser = argparse.ArgumentParser(description='Manage projects')
l.setup_logging_arguments(parser)
parser.add_argument('--nocleanup', action='store_true',
help='do not remove temp directories')
parser.add_argument('projects', metavar='project', nargs='*',
help='name of project(s) to process')
args = parser.parse_args()
l.configure_logging(args)
JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
'/var/lib/jeepyb')
IMPORT_DIR = os.path.join(JEEPYB_CACHE_DIR, 'import')
GERRIT_HOST = registry.get_defaults('gerrit-host')
GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
GERRIT_USER = registry.get_defaults('gerrit-user')
GERRIT_KEY = registry.get_defaults('gerrit-key')
GERRIT_GITID = registry.get_defaults('gerrit-committer')
PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache')
project_cache = {}
if os.path.exists(PROJECT_CACHE_FILE):
project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read())
gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST,
GERRIT_USER,
GERRIT_PORT,
GERRIT_KEY)
project_list = gerrit.listProjects()
ssh_env = u.make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
for section in registry.configs_list:
project = section['project']
if args.projects and project not in args.projects:
continue
try:
log.info("Processing project: %s" % project)
# Figure out all of the options
options = section.get('options', dict())
track_upstream = 'track-upstream' in options
if not track_upstream:
continue
# If this project doesn't want to use gerrit, exit cleanly.
if 'no-gerrit' in options:
continue
upstream = section.get('upstream', None)
upstream_prefix = section.get('upstream-prefix', None)
repo_path = os.path.join(IMPORT_DIR, project)
project_git = "%s.git" % project
remote_url = "ssh://%s:%s/%s" % (
GERRIT_HOST,
GERRIT_PORT,
project)
git_opts = dict(upstream=upstream,
repo_path=repo_path,
remote_url=remote_url)
project_cache.setdefault(project, {})
if not project_cache[project]['pushed-to-gerrit']:
continue
# Make Local repo
if not os.path.exists(repo_path):
u.make_local_copy(
repo_path, project, project_list,
git_opts, ssh_env, upstream, GERRIT_HOST,
GERRIT_PORT, project_git, GERRIT_GITID)
else:
update_local_copy(
repo_path, track_upstream, git_opts, ssh_env)
fsck_repo(repo_path)
sync_upstream(repo_path, project, ssh_env, upstream_prefix)
except Exception:
log.exception(
"Problems creating %s, moving on." % project)
continue
finally:
os.unlink(ssh_env['GIT_SSH'])
def main():
parser = argparse.ArgumentParser()
l.setup_logging_arguments(parser)
parser.add_argument('--message-file',
dest='message_file',
default=None,
help='The close pull request message')
args = parser.parse_args()
l.configure_logging(args)
if args.message_file:
try:
with open(args.message_file, 'r') as _file:
pull_request_text = _file.read()
except (OSError, IOError):
log.exception("Could not open close pull request message file")
raise
else:
pull_request_text = MESSAGE
GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG',
'/etc/github/github.secure.config')
secure_config = ConfigParser.ConfigParser()
secure_config.read(GITHUB_SECURE_CONFIG)
registry = u.ProjectsRegistry()
if secure_config.has_option("github", "oauth_token"):
ghub = github.Github(secure_config.get("github", "oauth_token"))
else:
ghub = github.Github(secure_config.get("github", "username"),
secure_config.get("github", "password"))
orgs = ghub.get_user().get_orgs()
orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
for section in registry.configs_list:
project = section['project']
# Make sure we're using GitHub for this project:
if not p.has_github(project):
continue
# Make sure we're supposed to close pull requests for this project:
if 'options' in section and 'has-pull-requests' in section['options']:
continue
# Find the project's repo
project_split = project.split('/', 1)
# Handle errors in case the repo or the organization doesn't exists
try:
if len(project_split) > 1:
org = orgs_dict[project_split[0].lower()]
repo = org.get_repo(project_split[1])
else:
repo = ghub.get_user().get_repo(project)
except (KeyError, github.GithubException):
log.exception("Could not find project %s on GitHub." % project)
continue
# Close each pull request
pull_requests = repo.get_pulls("open")
for req in pull_requests:
vars = dict(project=project)
issue_data = {"url": repo.url + "/issues/" + str(req.number)}
issue = github.Issue.Issue(requester=req._requester,
headers={},
attributes=issue_data,
completed=True)
issue.create_comment(pull_request_text % vars)
req.edit(state="closed")
def main():
parser = argparse.ArgumentParser()
l.setup_logging_arguments(parser)
parser.add_argument('--message-file', dest='message_file', default=None,
help='The close pull request message')
args = parser.parse_args()
l.configure_logging(args)
if args.message_file:
try:
with open(args.message_file, 'r') as _file:
pull_request_text = _file.read()
except (OSError, IOError):
log.exception("Could not open close pull request message file")
raise
else:
pull_request_text = MESSAGE
GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG',
'/etc/github/github.secure.config')
secure_config = ConfigParser.ConfigParser()
secure_config.read(GITHUB_SECURE_CONFIG)
registry = u.ProjectsRegistry()
if secure_config.has_option("github", "oauth_token"):
ghub = github.Github(secure_config.get("github", "oauth_token"))
else:
ghub = github.Github(secure_config.get("github", "username"),
secure_config.get("github", "password"))
orgs = ghub.get_user().get_orgs()
orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
for section in registry.configs_list:
project = section['project']
# Make sure we're using GitHub for this project:
if not p.has_github(project):
continue
# Make sure we're supposed to close pull requests for this project:
if 'options' in section and 'has-pull-requests' in section['options']:
continue
# Find the project's repo
project_split = project.split('/', 1)
# Handle errors in case the repo or the organization doesn't exists
try:
if len(project_split) > 1:
org = orgs_dict[project_split[0].lower()]
repo = org.get_repo(project_split[1])
else:
repo = ghub.get_user().get_repo(project)
except (KeyError, github.GithubException):
log.exception("Could not find project %s on GitHub." % project)
continue
# Close each pull request
pull_requests = repo.get_pulls("open")
for req in pull_requests:
vars = dict(project=project)
issue_data = {"url": repo.url + "/issues/" + str(req.number)}
issue = github.Issue.Issue(requester=req._requester,
headers={},
attributes=issue_data,
completed=True)
issue.create_comment(pull_request_text % vars)
req.edit(state="closed")
def main():
parser = argparse.ArgumentParser(description='Manage projects')
l.setup_logging_arguments(parser)
parser.add_argument('--nocleanup', action='store_true',
help='do not remove temp directories')
parser.add_argument('projects', metavar='project', nargs='*',
help='name of project(s) to process')
args = parser.parse_args()
l.configure_logging(args)
JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
'/var/lib/jeepyb')
IMPORT_DIR = os.path.join(JEEPYB_CACHE_DIR, 'import')
GERRIT_HOST = registry.get_defaults('gerrit-host')
GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
GERRIT_USER = registry.get_defaults('gerrit-user')
GERRIT_KEY = registry.get_defaults('gerrit-key')
GERRIT_GITID = registry.get_defaults('gerrit-committer')
PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache')
project_cache = {}
if os.path.exists(PROJECT_CACHE_FILE):
project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read())
gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST,
GERRIT_USER,
GERRIT_PORT,
GERRIT_KEY)
project_list = gerrit.listProjects()
ssh_env = u.make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
for section in registry.configs_list:
project = section['project']
if args.projects and project not in args.projects:
continue
try:
log.info("Processing project: %s" % project)
# Figure out all of the options
options = section.get('options', dict())
track_upstream = 'track-upstream' in options
if not track_upstream:
continue
# If this project doesn't want to use gerrit, exit cleanly.
if 'no-gerrit' in options:
continue
upstream = section.get('upstream', None)
upstream_prefix = section.get('upstream-prefix', None)
repo_path = os.path.join(IMPORT_DIR, project)
project_git = "%s.git" % project
remote_url = "ssh://%s:%s/%s" % (
GERRIT_HOST,
GERRIT_PORT,
project)
git_opts = dict(upstream=upstream,
repo_path=repo_path,
remote_url=remote_url)
project_cache.setdefault(project, {})
if not project_cache[project]['pushed-to-gerrit']:
continue
# Make Local repo
if not os.path.exists(repo_path):
u.make_local_copy(
repo_path, project, project_list,
git_opts, ssh_env, upstream, GERRIT_HOST,
GERRIT_PORT, project_git, GERRIT_GITID)
else:
update_local_copy(
repo_path, track_upstream, git_opts, ssh_env)
u.fsck_repo(repo_path)
sync_upstream(repo_path, project, ssh_env, upstream_prefix)
except Exception:
log.exception(
"Problems creating %s, moving on." % project)
continue
finally:
os.unlink(ssh_env['GIT_SSH'])
def main():
parser = argparse.ArgumentParser(description='Manage projects')
l.setup_logging_arguments(parser)
parser.add_argument('--nocleanup',
action='store_true',
help='do not remove temp directories')
parser.add_argument('projects',
metavar='project',
nargs='*',
help='name of project(s) to process')
args = parser.parse_args()
l.configure_logging(args)
default_has_github = registry.get_defaults('has-github', True)
LOCAL_GIT_DIR = registry.get_defaults('local-git-dir', '/var/lib/git')
JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
'/var/lib/jeepyb')
ACL_DIR = registry.get_defaults('acl-dir')
GERRIT_HOST = registry.get_defaults('gerrit-host')
GITREVIEW_GERRIT_HOST = registry.get_defaults('gitreview-gerrit-host',
GERRIT_HOST)
GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
GITREVIEW_GERRIT_PORT = int(
registry.get_defaults('gitreview-gerrit-port', GERRIT_PORT))
GERRIT_USER = registry.get_defaults('gerrit-user')
GERRIT_KEY = registry.get_defaults('gerrit-key')
GERRIT_GITID = registry.get_defaults('gerrit-committer')
GERRIT_REPLICATE = registry.get_defaults('gerrit-replicate', True)
GERRIT_OS_SYSTEM_USER = registry.get_defaults('gerrit-system-user',
'gerrit2')
GERRIT_OS_SYSTEM_GROUP = registry.get_defaults('gerrit-system-group',
'gerrit2')
DEFAULT_HOMEPAGE = registry.get_defaults('homepage')
DEFAULT_HAS_ISSUES = registry.get_defaults('has-issues', False)
DEFAULT_HAS_DOWNLOADS = registry.get_defaults('has-downloads', False)
DEFAULT_HAS_WIKI = registry.get_defaults('has-wiki', False)
GITHUB_SECURE_CONFIG = registry.get_defaults(
'github-config', '/etc/github/github-projects.secure.config')
PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache')
project_cache = {}
if os.path.exists(PROJECT_CACHE_FILE):
project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read())
acl_cache = {}
for acl_file in glob.glob(os.path.join(ACL_DIR, '*/*.config')):
sha256 = hashlib.sha256()
sha256.update(open(acl_file, 'r').read())
acl_cache[acl_file] = sha256.hexdigest()
gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST, GERRIT_USER, GERRIT_PORT,
GERRIT_KEY)
project_list = gerrit.listProjects()
ssh_env = u.make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
for section in registry.configs_list:
project = section['project']
if args.projects and project not in args.projects:
continue
try:
log.info("Processing project: %s" % project)
# Figure out all of the options
options = section.get('options', dict())
description = section.get('description', None)
homepage = section.get('homepage', DEFAULT_HOMEPAGE)
upstream = section.get('upstream', None)
repo_path = os.path.join(JEEPYB_CACHE_DIR, project)
# If this project doesn't want to use gerrit, exit cleanly.
if 'no-gerrit' in options:
continue
project_git = "%s.git" % project
remote_url = "ssh://%s:%s/%s" % (GERRIT_HOST, GERRIT_PORT,
project)
git_opts = dict(upstream=upstream,
repo_path=repo_path,
remote_url=remote_url)
acl_config = section.get(
'acl-config', '%s.config' % os.path.join(ACL_DIR, project))
project_cache.setdefault(project, {})
# Create the project in Gerrit first, since it will fail
# spectacularly if its project directory or local replica
# already exist on disk
project_created = project_cache[project].get(
'project-created', False)
if not project_created:
try:
project_created = create_gerrit_project(
project, project_list, gerrit)
project_cache[project]['project-created'] = True
except Exception:
project_cache[project]['project-created'] = False
continue
pushed_to_gerrit = project_cache[project].get(
'pushed-to-gerrit', False)
if not pushed_to_gerrit:
# We haven't pushed to gerrit, so grab the repo again
if os.path.exists(repo_path):
shutil.rmtree(repo_path)
# Make Local repo
push_string = u.make_local_copy(repo_path, project,
project_list, git_opts,
ssh_env, upstream,
GITREVIEW_GERRIT_HOST,
GITREVIEW_GERRIT_PORT,
project_git, GERRIT_GITID)
description = (find_description_override(repo_path)
or description)
u.fsck_repo(repo_path)
if push_string:
push_to_gerrit(repo_path, project, push_string,
remote_url, ssh_env)
project_cache[project]['pushed-to-gerrit'] = True
if GERRIT_REPLICATE:
gerrit.replicate(project)
# Create the repo for the local git mirror
create_local_mirror(LOCAL_GIT_DIR, project_git,
GERRIT_OS_SYSTEM_USER,
GERRIT_OS_SYSTEM_GROUP)
if acl_config:
acl_sha = acl_cache.get(acl_config)
if project_cache[project].get('acl-sha') != acl_sha:
if not os.path.exists(repo_path):
u.make_local_copy(repo_path, project, project_list,
git_opts, ssh_env, upstream,
GERRIT_HOST, GERRIT_PORT,
project_git, GERRIT_GITID)
process_acls(acl_config, project, ACL_DIR, section,
remote_url, repo_path, ssh_env, gerrit,
GERRIT_GITID)
project_cache[project]['acl-sha'] = acl_sha
else:
log.info("%s has matching sha, skipping ACLs", project)
if 'has-github' in options or default_has_github:
created = create_update_github_project(
DEFAULT_HAS_ISSUES, DEFAULT_HAS_DOWNLOADS,
DEFAULT_HAS_WIKI, GITHUB_SECURE_CONFIG, options,
project, description, homepage, project_cache[project])
if created and GERRIT_REPLICATE:
gerrit.replicate(project)
project_cache[project]['created-in-github'] = created
except Exception:
log.exception("Problems creating %s, moving on." % project)
continue
finally:
# Clean up after ourselves - this repo has no use
if os.path.exists(repo_path):
shutil.rmtree(repo_path)
finally:
with open(PROJECT_CACHE_FILE, 'w') as cache_out:
log.info("Writing cache file %s", PROJECT_CACHE_FILE)
cache_out.write(json.dumps(project_cache, sort_keys=True,
indent=2))
os.unlink(ssh_env['GIT_SSH'])
def main():
parser = argparse.ArgumentParser(description='Manage projects')
l.setup_logging_arguments(parser)
parser.add_argument('--nocleanup', action='store_true',
help='do not remove temp directories')
parser.add_argument('projects', metavar='project', nargs='*',
help='name of project(s) to process')
args = parser.parse_args()
l.configure_logging(args)
default_has_github = registry.get_defaults('has-github', True)
LOCAL_GIT_DIR = registry.get_defaults('local-git-dir', '/var/lib/git')
JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
'/var/lib/jeepyb')
ACL_DIR = registry.get_defaults('acl-dir')
GERRIT_HOST = registry.get_defaults('gerrit-host')
GITREVIEW_GERRIT_HOST = registry.get_defaults(
'gitreview-gerrit-host', GERRIT_HOST)
GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
GITREVIEW_GERRIT_PORT = int(registry.get_defaults(
'gitreview-gerrit-port', GERRIT_PORT))
GERRIT_USER = registry.get_defaults('gerrit-user')
GERRIT_KEY = registry.get_defaults('gerrit-key')
GERRIT_GITID = registry.get_defaults('gerrit-committer')
GERRIT_REPLICATE = registry.get_defaults('gerrit-replicate', True)
GERRIT_OS_SYSTEM_USER = registry.get_defaults('gerrit-system-user',
'gerrit2')
GERRIT_OS_SYSTEM_GROUP = registry.get_defaults('gerrit-system-group',
'gerrit2')
DEFAULT_HOMEPAGE = registry.get_defaults('homepage')
DEFAULT_HAS_ISSUES = registry.get_defaults('has-issues', False)
DEFAULT_HAS_DOWNLOADS = registry.get_defaults('has-downloads', False)
DEFAULT_HAS_WIKI = registry.get_defaults('has-wiki', False)
GITHUB_SECURE_CONFIG = registry.get_defaults(
'github-config',
'/etc/github/github-projects.secure.config')
PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache')
project_cache = {}
if os.path.exists(PROJECT_CACHE_FILE):
project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read())
acl_cache = {}
for acl_file in glob.glob(os.path.join(ACL_DIR, '*/*.config')):
sha256 = hashlib.sha256()
sha256.update(open(acl_file, 'r').read())
acl_cache[acl_file] = sha256.hexdigest()
gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST,
GERRIT_USER,
GERRIT_PORT,
GERRIT_KEY)
project_list = gerrit.listProjects()
ssh_env = u.make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
# Collect processed errors,if any
process_errors = []
for section in registry.configs_list:
project = section['project']
if args.projects and project not in args.projects:
continue
try:
log.info("Processing project: %s" % project)
# Figure out all of the options
options = section.get('options', dict())
description = section.get('description', None)
homepage = section.get('homepage', DEFAULT_HOMEPAGE)
upstream = section.get('upstream', None)
repo_path = os.path.join(JEEPYB_CACHE_DIR, project)
# If this project doesn't want to use gerrit, exit cleanly.
if 'no-gerrit' in options:
continue
project_git = "%s.git" % project
remote_url = "ssh://%s:%s/%s" % (
GERRIT_HOST,
GERRIT_PORT,
project)
git_opts = dict(upstream=upstream,
repo_path=repo_path,
remote_url=remote_url)
acl_config = section.get(
'acl-config',
'%s.config' % os.path.join(ACL_DIR, project))
project_cache.setdefault(project, {})
# Create the project in Gerrit first, since it will fail
# spectacularly if its project directory or local replica
# already exist on disk
project_created = project_cache[project].get(
'project-created', False)
if not project_created:
try:
project_created = create_gerrit_project(
project, project_list, gerrit)
project_cache[project]['project-created'] = True
except Exception:
project_cache[project]['project-created'] = False
continue
pushed_to_gerrit = project_cache[project].get(
'pushed-to-gerrit', False)
if not pushed_to_gerrit:
# We haven't pushed to gerrit, so grab the repo again
if os.path.exists(repo_path):
shutil.rmtree(repo_path)
# Make Local repo
push_string = u.make_local_copy(
repo_path, project, project_list,
git_opts, ssh_env, upstream, GITREVIEW_GERRIT_HOST,
GITREVIEW_GERRIT_PORT, project_git, GERRIT_GITID)
description = (
find_description_override(repo_path)
or description)
u.fsck_repo(repo_path)
if push_string:
push_to_gerrit(
repo_path, project, push_string,
remote_url, ssh_env)
project_cache[project]['pushed-to-gerrit'] = True
if GERRIT_REPLICATE:
gerrit.replicate(project)
# Create the repo for the local git mirror
create_local_mirror(
LOCAL_GIT_DIR, project_git,
GERRIT_OS_SYSTEM_USER, GERRIT_OS_SYSTEM_GROUP)
if acl_config:
acl_sha = acl_cache.get(acl_config)
if project_cache[project].get('acl-sha') != acl_sha:
if not os.path.exists(repo_path):
u.make_local_copy(
repo_path, project, project_list,
git_opts, ssh_env, upstream, GERRIT_HOST,
GERRIT_PORT, project_git, GERRIT_GITID)
process_acls(
acl_config, project, ACL_DIR, section,
remote_url, repo_path, ssh_env, gerrit,
GERRIT_GITID)
project_cache[project]['acl-sha'] = acl_sha
else:
log.info("%s has matching sha, skipping ACLs",
project)
if 'has-github' in options or default_has_github:
created = create_update_github_project(
DEFAULT_HAS_ISSUES, DEFAULT_HAS_DOWNLOADS,
DEFAULT_HAS_WIKI, GITHUB_SECURE_CONFIG,
options, project, description, homepage,
project_cache[project])
if created and GERRIT_REPLICATE:
gerrit.replicate(project)
project_cache[project]['created-in-github'] = created
except Exception:
msg = "Problems creating %s, moving on." % project
log.exception(msg)
process_errors.append(msg)
continue
finally:
# Clean up after ourselves - this repo has no use
if os.path.exists(repo_path):
shutil.rmtree(repo_path)
finally:
with open(PROJECT_CACHE_FILE, 'w') as cache_out:
log.info("Writing cache file %s", PROJECT_CACHE_FILE)
cache_out.write(json.dumps(
project_cache, sort_keys=True, indent=2))
os.unlink(ssh_env['GIT_SSH'])
if len(process_errors) > 0:
log.error("%d problems has been caught during run:\n %s" % (
len(process_errors), process_errors))
sys.exit(1)
