def main():
parser = argparse.ArgumentParser(description='Manage projects')
l.setup_logging_arguments(parser)
parser.add_argument('--nocleanup', action='store_true',
help='do not remove temp directories')
parser.add_argument('projects', metavar='project', nargs='*',
help='name of project(s) to process')
args = parser.parse_args()
l.configure_logging(args)
JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
'/var/lib/jeepyb')
IMPORT_DIR = os.path.join(JEEPYB_CACHE_DIR, 'import')
GERRIT_HOST = registry.get_defaults('gerrit-host')
GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
GERRIT_USER = registry.get_defaults('gerrit-user')
GERRIT_KEY = registry.get_defaults('gerrit-key')
GERRIT_GITID = registry.get_defaults('gerrit-committer')
PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache')
project_cache = {}
if os.path.exists(PROJECT_CACHE_FILE):
project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read())
gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST,
GERRIT_USER,
GERRIT_PORT,
GERRIT_KEY)
project_list = gerrit.listProjects()
ssh_env = u.make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
for section in registry.configs_list:
project = section['project']
if args.projects and project not in args.projects:
continue
try:
log.info("Processing project: %s" % project)
# Figure out all of the options
options = section.get('options', dict())
track_upstream = 'track-upstream' in options
if not track_upstream:
continue
# If this project doesn't want to use gerrit, exit cleanly.
if 'no-gerrit' in options:
continue
upstream = section.get('upstream', None)
upstream_prefix = section.get('upstream-prefix', None)
repo_path = os.path.join(IMPORT_DIR, project)
project_git = "%s.git" % project
remote_url = "ssh://%s:%s/%s" % (
GERRIT_HOST,
GERRIT_PORT,
project)
git_opts = dict(upstream=upstream,
repo_path=repo_path,
remote_url=remote_url)
project_cache.setdefault(project, {})
if not project_cache[project]['pushed-to-gerrit']:
continue
# Make Local repo
if not os.path.exists(repo_path):
u.make_local_copy(
repo_path, project, project_list,
git_opts, ssh_env, upstream, GERRIT_HOST,
GERRIT_PORT, project_git, GERRIT_GITID)
else:
update_local_copy(
repo_path, track_upstream, git_opts, ssh_env)
fsck_repo(repo_path)
sync_upstream(repo_path, project, ssh_env, upstream_prefix)
except Exception:
log.exception(
"Problems creating %s, moving on." % project)
continue
finally:
os.unlink(ssh_env['GIT_SSH'])
def main():
parser = argparse.ArgumentParser(description='Manage projects')
l.setup_logging_arguments(parser)
parser.add_argument('--nocleanup',
action='store_true',
help='do not remove temp directories')
parser.add_argument('projects',
metavar='project',
nargs='*',
help='name of project(s) to process')
args = parser.parse_args()
l.configure_logging(args)
default_has_github = registry.get_defaults('has-github', True)
LOCAL_GIT_DIR = registry.get_defaults('local-git-dir', '/var/lib/git')
JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
'/var/lib/jeepyb')
ACL_DIR = registry.get_defaults('acl-dir')
GERRIT_HOST = registry.get_defaults('gerrit-host')
GITREVIEW_GERRIT_HOST = registry.get_defaults('gitreview-gerrit-host',
GERRIT_HOST)
GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
GITREVIEW_GERRIT_PORT = int(
registry.get_defaults('gitreview-gerrit-port', GERRIT_PORT))
GERRIT_USER = registry.get_defaults('gerrit-user')
GERRIT_KEY = registry.get_defaults('gerrit-key')
GERRIT_GITID = registry.get_defaults('gerrit-committer')
GERRIT_REPLICATE = registry.get_defaults('gerrit-replicate', True)
GERRIT_OS_SYSTEM_USER = registry.get_defaults('gerrit-system-user',
'gerrit2')
GERRIT_OS_SYSTEM_GROUP = registry.get_defaults('gerrit-system-group',
'gerrit2')
DEFAULT_HOMEPAGE = registry.get_defaults('homepage')
DEFAULT_HAS_ISSUES = registry.get_defaults('has-issues', False)
DEFAULT_HAS_DOWNLOADS = registry.get_defaults('has-downloads', False)
DEFAULT_HAS_WIKI = registry.get_defaults('has-wiki', False)
GITHUB_SECURE_CONFIG = registry.get_defaults(
'github-config', '/etc/github/github-projects.secure.config')
PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache')
project_cache = {}
if os.path.exists(PROJECT_CACHE_FILE):
project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read())
acl_cache = {}
for acl_file in glob.glob(os.path.join(ACL_DIR, '*/*.config')):
sha256 = hashlib.sha256()
sha256.update(open(acl_file, 'r').read())
acl_cache[acl_file] = sha256.hexdigest()
gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST, GERRIT_USER, GERRIT_PORT,
GERRIT_KEY)
project_list = gerrit.listProjects()
ssh_env = u.make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
for section in registry.configs_list:
project = section['project']
if args.projects and project not in args.projects:
continue
try:
log.info("Processing project: %s" % project)
# Figure out all of the options
options = section.get('options', dict())
description = section.get('description', None)
homepage = section.get('homepage', DEFAULT_HOMEPAGE)
upstream = section.get('upstream', None)
repo_path = os.path.join(JEEPYB_CACHE_DIR, project)
# If this project doesn't want to use gerrit, exit cleanly.
if 'no-gerrit' in options:
continue
project_git = "%s.git" % project
remote_url = "ssh://%s:%s/%s" % (GERRIT_HOST, GERRIT_PORT,
project)
git_opts = dict(upstream=upstream,
repo_path=repo_path,
remote_url=remote_url)
acl_config = section.get(
'acl-config', '%s.config' % os.path.join(ACL_DIR, project))
project_cache.setdefault(project, {})
# Create the project in Gerrit first, since it will fail
# spectacularly if its project directory or local replica
# already exist on disk
project_created = project_cache[project].get(
'project-created', False)
if not project_created:
try:
project_created = create_gerrit_project(
project, project_list, gerrit)
project_cache[project]['project-created'] = True
except Exception:
project_cache[project]['project-created'] = False
continue
pushed_to_gerrit = project_cache[project].get(
'pushed-to-gerrit', False)
if not pushed_to_gerrit:
# We haven't pushed to gerrit, so grab the repo again
if os.path.exists(repo_path):
shutil.rmtree(repo_path)
# Make Local repo
push_string = u.make_local_copy(repo_path, project,
project_list, git_opts,
ssh_env, upstream,
GITREVIEW_GERRIT_HOST,
GITREVIEW_GERRIT_PORT,
project_git, GERRIT_GITID)
description = (find_description_override(repo_path)
or description)
u.fsck_repo(repo_path)
if push_string:
push_to_gerrit(repo_path, project, push_string,
remote_url, ssh_env)
project_cache[project]['pushed-to-gerrit'] = True
if GERRIT_REPLICATE:
gerrit.replicate(project)
# Create the repo for the local git mirror
create_local_mirror(LOCAL_GIT_DIR, project_git,
GERRIT_OS_SYSTEM_USER,
GERRIT_OS_SYSTEM_GROUP)
if acl_config:
acl_sha = acl_cache.get(acl_config)
if project_cache[project].get('acl-sha') != acl_sha:
if not os.path.exists(repo_path):
u.make_local_copy(repo_path, project, project_list,
git_opts, ssh_env, upstream,
GERRIT_HOST, GERRIT_PORT,
project_git, GERRIT_GITID)
process_acls(acl_config, project, ACL_DIR, section,
remote_url, repo_path, ssh_env, gerrit,
GERRIT_GITID)
project_cache[project]['acl-sha'] = acl_sha
else:
log.info("%s has matching sha, skipping ACLs", project)
if 'has-github' in options or default_has_github:
created = create_update_github_project(
DEFAULT_HAS_ISSUES, DEFAULT_HAS_DOWNLOADS,
DEFAULT_HAS_WIKI, GITHUB_SECURE_CONFIG, options,
project, description, homepage, project_cache[project])
if created and GERRIT_REPLICATE:
gerrit.replicate(project)
project_cache[project]['created-in-github'] = created
except Exception:
log.exception("Problems creating %s, moving on." % project)
continue
finally:
# Clean up after ourselves - this repo has no use
if os.path.exists(repo_path):
shutil.rmtree(repo_path)
finally:
with open(PROJECT_CACHE_FILE, 'w') as cache_out:
log.info("Writing cache file %s", PROJECT_CACHE_FILE)
cache_out.write(json.dumps(project_cache, sort_keys=True,
indent=2))
os.unlink(ssh_env['GIT_SSH'])
def main():
parser = argparse.ArgumentParser(description='Manage projects')
l.setup_logging_arguments(parser)
parser.add_argument('--nocleanup', action='store_true',
help='do not remove temp directories')
parser.add_argument('projects', metavar='project', nargs='*',
help='name of project(s) to process')
args = parser.parse_args()
l.configure_logging(args)
JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
'/var/lib/jeepyb')
IMPORT_DIR = os.path.join(JEEPYB_CACHE_DIR, 'import')
GERRIT_HOST = registry.get_defaults('gerrit-host')
GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
GERRIT_USER = registry.get_defaults('gerrit-user')
GERRIT_KEY = registry.get_defaults('gerrit-key')
GERRIT_GITID = registry.get_defaults('gerrit-committer')
PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache')
project_cache = {}
if os.path.exists(PROJECT_CACHE_FILE):
project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read())
gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST,
GERRIT_USER,
GERRIT_PORT,
GERRIT_KEY)
project_list = gerrit.listProjects()
ssh_env = u.make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
for section in registry.configs_list:
project = section['project']
if args.projects and project not in args.projects:
continue
try:
log.info("Processing project: %s" % project)
# Figure out all of the options
options = section.get('options', dict())
track_upstream = 'track-upstream' in options
if not track_upstream:
continue
# If this project doesn't want to use gerrit, exit cleanly.
if 'no-gerrit' in options:
continue
upstream = section.get('upstream', None)
upstream_prefix = section.get('upstream-prefix', None)
repo_path = os.path.join(IMPORT_DIR, project)
project_git = "%s.git" % project
remote_url = "ssh://%s:%s/%s" % (
GERRIT_HOST,
GERRIT_PORT,
project)
git_opts = dict(upstream=upstream,
repo_path=repo_path,
remote_url=remote_url)
project_cache.setdefault(project, {})
if not project_cache[project]['pushed-to-gerrit']:
continue
# Make Local repo
if not os.path.exists(repo_path):
u.make_local_copy(
repo_path, project, project_list,
git_opts, ssh_env, upstream, GERRIT_HOST,
GERRIT_PORT, project_git, GERRIT_GITID)
else:
update_local_copy(
repo_path, track_upstream, git_opts, ssh_env)
u.fsck_repo(repo_path)
sync_upstream(repo_path, project, ssh_env, upstream_prefix)
except Exception:
log.exception(
"Problems creating %s, moving on." % project)
continue
finally:
os.unlink(ssh_env['GIT_SSH'])
def main():
parser = argparse.ArgumentParser(description='Manage projects')
l.setup_logging_arguments(parser)
parser.add_argument('--nocleanup', action='store_true',
help='do not remove temp directories')
parser.add_argument('projects', metavar='project', nargs='*',
help='name of project(s) to process')
args = parser.parse_args()
l.configure_logging(args)
default_has_github = registry.get_defaults('has-github', True)
LOCAL_GIT_DIR = registry.get_defaults('local-git-dir', '/var/lib/git')
JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
'/var/lib/jeepyb')
ACL_DIR = registry.get_defaults('acl-dir')
GERRIT_HOST = registry.get_defaults('gerrit-host')
GITREVIEW_GERRIT_HOST = registry.get_defaults(
'gitreview-gerrit-host', GERRIT_HOST)
GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
GITREVIEW_GERRIT_PORT = int(registry.get_defaults(
'gitreview-gerrit-port', GERRIT_PORT))
GERRIT_USER = registry.get_defaults('gerrit-user')
GERRIT_KEY = registry.get_defaults('gerrit-key')
GERRIT_GITID = registry.get_defaults('gerrit-committer')
GERRIT_REPLICATE = registry.get_defaults('gerrit-replicate', True)
GERRIT_OS_SYSTEM_USER = registry.get_defaults('gerrit-system-user',
'gerrit2')
GERRIT_OS_SYSTEM_GROUP = registry.get_defaults('gerrit-system-group',
'gerrit2')
DEFAULT_HOMEPAGE = registry.get_defaults('homepage')
DEFAULT_HAS_ISSUES = registry.get_defaults('has-issues', False)
DEFAULT_HAS_DOWNLOADS = registry.get_defaults('has-downloads', False)
DEFAULT_HAS_WIKI = registry.get_defaults('has-wiki', False)
GITHUB_SECURE_CONFIG = registry.get_defaults(
'github-config',
'/etc/github/github-projects.secure.config')
PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache')
project_cache = {}
if os.path.exists(PROJECT_CACHE_FILE):
project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read())
acl_cache = {}
for acl_file in glob.glob(os.path.join(ACL_DIR, '*/*.config')):
sha256 = hashlib.sha256()
sha256.update(open(acl_file, 'r').read())
acl_cache[acl_file] = sha256.hexdigest()
gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST,
GERRIT_USER,
GERRIT_PORT,
GERRIT_KEY)
project_list = gerrit.listProjects()
ssh_env = u.make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
# Collect processed errors,if any
process_errors = []
for section in registry.configs_list:
project = section['project']
if args.projects and project not in args.projects:
continue
try:
log.info("Processing project: %s" % project)
# Figure out all of the options
options = section.get('options', dict())
description = section.get('description', None)
homepage = section.get('homepage', DEFAULT_HOMEPAGE)
upstream = section.get('upstream', None)
repo_path = os.path.join(JEEPYB_CACHE_DIR, project)
# If this project doesn't want to use gerrit, exit cleanly.
if 'no-gerrit' in options:
continue
project_git = "%s.git" % project
remote_url = "ssh://%s:%s/%s" % (
GERRIT_HOST,
GERRIT_PORT,
project)
git_opts = dict(upstream=upstream,
repo_path=repo_path,
remote_url=remote_url)
acl_config = section.get(
'acl-config',
'%s.config' % os.path.join(ACL_DIR, project))
project_cache.setdefault(project, {})
# Create the project in Gerrit first, since it will fail
# spectacularly if its project directory or local replica
# already exist on disk
project_created = project_cache[project].get(
'project-created', False)
if not project_created:
try:
project_created = create_gerrit_project(
project, project_list, gerrit)
project_cache[project]['project-created'] = True
except Exception:
project_cache[project]['project-created'] = False
continue
pushed_to_gerrit = project_cache[project].get(
'pushed-to-gerrit', False)
if not pushed_to_gerrit:
# We haven't pushed to gerrit, so grab the repo again
if os.path.exists(repo_path):
shutil.rmtree(repo_path)
# Make Local repo
push_string = u.make_local_copy(
repo_path, project, project_list,
git_opts, ssh_env, upstream, GITREVIEW_GERRIT_HOST,
GITREVIEW_GERRIT_PORT, project_git, GERRIT_GITID)
description = (
find_description_override(repo_path)
or description)
u.fsck_repo(repo_path)
if push_string:
push_to_gerrit(
repo_path, project, push_string,
remote_url, ssh_env)
project_cache[project]['pushed-to-gerrit'] = True
if GERRIT_REPLICATE:
gerrit.replicate(project)
# Create the repo for the local git mirror
create_local_mirror(
LOCAL_GIT_DIR, project_git,
GERRIT_OS_SYSTEM_USER, GERRIT_OS_SYSTEM_GROUP)
if acl_config:
acl_sha = acl_cache.get(acl_config)
if project_cache[project].get('acl-sha') != acl_sha:
if not os.path.exists(repo_path):
u.make_local_copy(
repo_path, project, project_list,
git_opts, ssh_env, upstream, GERRIT_HOST,
GERRIT_PORT, project_git, GERRIT_GITID)
process_acls(
acl_config, project, ACL_DIR, section,
remote_url, repo_path, ssh_env, gerrit,
GERRIT_GITID)
project_cache[project]['acl-sha'] = acl_sha
else:
log.info("%s has matching sha, skipping ACLs",
project)
if 'has-github' in options or default_has_github:
created = create_update_github_project(
DEFAULT_HAS_ISSUES, DEFAULT_HAS_DOWNLOADS,
DEFAULT_HAS_WIKI, GITHUB_SECURE_CONFIG,
options, project, description, homepage,
project_cache[project])
if created and GERRIT_REPLICATE:
gerrit.replicate(project)
project_cache[project]['created-in-github'] = created
except Exception:
msg = "Problems creating %s, moving on." % project
log.exception(msg)
process_errors.append(msg)
continue
finally:
# Clean up after ourselves - this repo has no use
if os.path.exists(repo_path):
shutil.rmtree(repo_path)
finally:
with open(PROJECT_CACHE_FILE, 'w') as cache_out:
log.info("Writing cache file %s", PROJECT_CACHE_FILE)
cache_out.write(json.dumps(
project_cache, sort_keys=True, indent=2))
os.unlink(ssh_env['GIT_SSH'])
if len(process_errors) > 0:
log.error("%d problems has been caught during run:\n %s" % (
len(process_errors), process_errors))
sys.exit(1)