示例#1
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('hook')
    # common
    parser.add_argument('--change', default=None)
    parser.add_argument('--change-url', default=None)
    parser.add_argument('--project', default=None)
    parser.add_argument('--branch', default=None)
    parser.add_argument('--commit', default=None)
    parser.add_argument('--topic', default=None)
    parser.add_argument('--change-owner', default=None)
    # patchset-abandoned
    parser.add_argument('--abandoner', default=None)
    parser.add_argument('--reason', default=None)
    # change-merged
    parser.add_argument('--submitter', default=None)
    parser.add_argument('--newrev', default=None)
    # patchset-created
    parser.add_argument('--uploader', default=None)
    parser.add_argument('--patchset', default=None)
    parser.add_argument('--is-draft', default=None)
    parser.add_argument('--kind', default=None)
    # for Welcome Message
    parser.add_argument('--ssh-user',
                        dest='ssh_user',
                        help='The gerrit welcome message user')
    parser.add_argument('--ssh-key',
                        dest='ssh_key',
                        help='The gerrit welcome message SSH key file')
    parser.add_argument('--message-file',
                        dest='message_file',
                        default=None,
                        help='The gerrit welcome message')
    # Don't actually post the message
    parser.add_argument('--dryrun', dest='dryrun', action='store_true')
    parser.add_argument('--no-dryrun', dest='dryrun', action='store_false')
    parser.set_defaults(dryrun=False)
    l.setup_logging_arguments(parser)

    args = parser.parse_args()

    l.configure_logging(args)

    # they're a first-timer, post the message on 1st patchset
    if is_newbie(args.uploader) and args.patchset == '1' and not args.dryrun:
        post_message(args.commit, args.ssh_user, args.ssh_key,
                     args.message_file)
示例#2
0
def main():

    parser = argparse.ArgumentParser()
    parser.add_argument('user', help='The gerrit admin user')
    parser.add_argument('ssh_key', help='The gerrit admin SSH key file')
    parser.add_argument('--age',
                        dest='age',
                        default='1w',
                        help='The minimum age of a review to expire')
    l.setup_logging_arguments(parser)
    options = parser.parse_args()
    l.configure_logging(options)

    GERRIT_USER = options.user
    GERRIT_SSH_KEY = options.ssh_key
    EXPIRY_AGE = options.age

    logger.info('Starting expire reviews')
    logger.info('Connecting to Gerrit')

    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.connect('localhost',
                username=GERRIT_USER,
                key_filename=GERRIT_SSH_KEY,
                port=29418)

    # Query all reviewed with no activity for 1 week
    logger.info('Searching no activity on negative review for 1 week')
    stdin, stdout, stderr = ssh.exec_command(
        'gerrit query --current-patch-set --all-approvals'
        ' --format JSON status:reviewed age:' + EXPIRY_AGE)

    for line in stdout:
        row = json.loads(line)
        if 'rowCount' not in row and 'open' in row and row['open']:
            # Search for negative approvals
            for approval in row['currentPatchSet']['approvals']:
                if approval['value'] in ('-1', '-2'):
                    expire_patch_set(ssh, row['currentPatchSet']['revision'],
                                     row['subject'])
                    break

    logger.info('End expire review')
示例#3
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('hook')
    # common
    parser.add_argument('--change', default=None)
    parser.add_argument('--change-url', default=None)
    parser.add_argument('--project', default=None)
    parser.add_argument('--branch', default=None)
    parser.add_argument('--commit', default=None)
    parser.add_argument('--topic', default=None)
    parser.add_argument('--change-owner', default=None)
    # patchset-abandoned
    parser.add_argument('--abandoner', default=None)
    parser.add_argument('--reason', default=None)
    # change-merged
    parser.add_argument('--submitter', default=None)
    parser.add_argument('--newrev', default=None)
    # patchset-created
    parser.add_argument('--uploader', default=None)
    parser.add_argument('--patchset', default=None)
    parser.add_argument('--is-draft', default=None)
    parser.add_argument('--kind', default=None)
    # for Welcome Message
    parser.add_argument('--ssh-user', dest='ssh_user',
                        help='The gerrit welcome message user')
    parser.add_argument('--ssh-key', dest='ssh_key',
                        help='The gerrit welcome message SSH key file')
    parser.add_argument('--message-file', dest='message_file', default=None,
                        help='The gerrit welcome message')
    # Don't actually post the message
    parser.add_argument('--dryrun', dest='dryrun', action='store_true')
    parser.add_argument('--no-dryrun', dest='dryrun', action='store_false')
    parser.set_defaults(dryrun=False)
    l.setup_logging_arguments(parser)

    args = parser.parse_args()

    l.configure_logging(args)

    # they're a first-timer, post the message on 1st patchset
    if is_newbie(args.uploader) and args.patchset == '1' and not args.dryrun:
        post_message(args.commit, args.ssh_user, args.ssh_key,
                     args.message_file)
示例#4
0
def main():
    parser = argparse.ArgumentParser(description='Register projects in Zanata')
    l.setup_logging_arguments(parser)
    args = parser.parse_args()
    l.configure_logging(args)

    registry = u.ProjectsRegistry(PROJECTS_YAML)
    rest_service = t.ZanataRestService(ZANATA_URL, ZANATA_USER, ZANATA_KEY)
    log.info("Registering projects in Zanata")
    for entry in registry.configs_list:
        project = entry['project']
        if not p.has_translations(project):
            continue
        log.info("Processing project %s" % project)
        (org, name) = project.split('/')
        try:
            translation_proect = t.TranslationProject(rest_service, name)
            translation_proect.register()
        except ValueError as e:
            log.error(e)
def main():

    parser = argparse.ArgumentParser()
    parser.add_argument('user', help='The gerrit admin user')
    parser.add_argument('ssh_key', help='The gerrit admin SSH key file')
    parser.add_argument('--age', dest='age', default='1w',
                        help='The minimum age of a review to expire')
    l.setup_logging_arguments(parser)
    options = parser.parse_args()
    l.configure_logging(options)

    GERRIT_USER = options.user
    GERRIT_SSH_KEY = options.ssh_key
    EXPIRY_AGE = options.age

    logger.info('Starting expire reviews')
    logger.info('Connecting to Gerrit')

    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.connect('localhost', username=GERRIT_USER,
                key_filename=GERRIT_SSH_KEY, port=29418)

    # Query all reviewed with no activity for 1 week
    logger.info('Searching no activity on negative review for 1 week')
    stdin, stdout, stderr = ssh.exec_command(
        'gerrit query --current-patch-set --all-approvals'
        ' --format JSON status:reviewed age:' + EXPIRY_AGE)

    for line in stdout:
        row = json.loads(line)
        if 'rowCount' not in row:
            # Search for negative approvals
            for approval in row['currentPatchSet']['approvals']:
                if approval['value'] in ('-1', '-2'):
                    expire_patch_set(ssh,
                                     row['currentPatchSet']['revision'],
                                     row['subject'])
                    break

    logger.info('End expire review')
示例#6
0
def main():
    parser = argparse.ArgumentParser(description='Manage projects')
    l.setup_logging_arguments(parser)
    parser.add_argument('--nocleanup', action='store_true',
                        help='do not remove temp directories')
    parser.add_argument('projects', metavar='project', nargs='*',
                        help='name of project(s) to process')
    args = parser.parse_args()
    l.configure_logging(args)

    default_has_github = registry.get_defaults('has-github', True)

    LOCAL_GIT_DIR = registry.get_defaults('local-git-dir', '/var/lib/git')
    JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
                                             '/var/lib/jeepyb')
    ACL_DIR = registry.get_defaults('acl-dir')
    GERRIT_HOST = registry.get_defaults('gerrit-host')
    GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
    GERRIT_USER = registry.get_defaults('gerrit-user')
    GERRIT_KEY = registry.get_defaults('gerrit-key')
    GERRIT_GITID = registry.get_defaults('gerrit-committer')
    GERRIT_REPLICATE = registry.get_defaults('gerrit-replicate', True)
    GERRIT_OS_SYSTEM_USER = registry.get_defaults('gerrit-system-user',
                                                  'gerrit2')
    GERRIT_OS_SYSTEM_GROUP = registry.get_defaults('gerrit-system-group',
                                                   'gerrit2')
    DEFAULT_HOMEPAGE = registry.get_defaults('homepage')
    DEFAULT_HAS_ISSUES = registry.get_defaults('has-issues', False)
    DEFAULT_HAS_DOWNLOADS = registry.get_defaults('has-downloads', False)
    DEFAULT_HAS_WIKI = registry.get_defaults('has-wiki', False)
    GITHUB_SECURE_CONFIG = registry.get_defaults(
        'github-config',
        '/etc/github/github-projects.secure.config')

    gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST,
                                     GERRIT_USER,
                                     GERRIT_PORT,
                                     GERRIT_KEY)
    project_list = gerrit.listProjects()
    ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
    try:

        for section in registry.configs_list:
            project = section['project']
            if args.projects and project not in args.projects:
                continue

            try:
                log.info("Processing project: %s" % project)

                # Figure out all of the options
                options = section.get('options', dict())
                description = section.get('description', None)
                homepage = section.get('homepage', DEFAULT_HOMEPAGE)
                upstream = section.get('upstream', None)
                upstream_prefix = section.get('upstream-prefix', None)
                track_upstream = 'track-upstream' in options
                repo_path = os.path.join(JEEPYB_CACHE_DIR, project)

                # If this project doesn't want to use gerrit, exit cleanly.
                if 'no-gerrit' in options:
                    continue

                project_git = "%s.git" % project
                remote_url = "ssh://%s:%s/%s" % (
                    GERRIT_HOST,
                    GERRIT_PORT,
                    project)
                git_opts = dict(upstream=upstream,
                                repo_path=repo_path,
                                remote_url=remote_url)
                acl_config = section.get(
                    'acl-config',
                    '%s.config' % os.path.join(ACL_DIR, project))

                # Create the project in Gerrit first, since it will fail
                # spectacularly if its project directory or local replica
                # already exist on disk
                project_created = create_gerrit_project(
                    project, project_list, gerrit)

                # Create the repo for the local git mirror
                create_local_mirror(
                    LOCAL_GIT_DIR, project_git,
                    GERRIT_OS_SYSTEM_USER, GERRIT_OS_SYSTEM_GROUP)

                if not os.path.exists(repo_path) or project_created:
                    # We don't have a local copy already, get one

                    # Make Local repo
                    push_string = make_local_copy(
                        repo_path, project, project_list,
                        git_opts, ssh_env, upstream, GERRIT_HOST, GERRIT_PORT,
                        project_git, GERRIT_GITID)
                else:
                    # We do have a local copy of it already, make sure it's
                    # in shape to have work done.
                    update_local_copy(
                        repo_path, track_upstream, git_opts, ssh_env)

                description = (
                    find_description_override(repo_path) or description)

                if project_created:
                    push_to_gerrit(
                        repo_path, project, push_string, remote_url, ssh_env)
                    if GERRIT_REPLICATE:
                        gerrit.replicate(project)

                # If we're configured to track upstream, make sure we have
                # upstream's refs, and then push them to the appropriate
                # branches in gerrit
                if track_upstream:
                    sync_upstream(repo_path, project, ssh_env, upstream_prefix)

                if acl_config:
                    process_acls(
                        acl_config, project, ACL_DIR, section,
                        remote_url, repo_path, ssh_env, gerrit, GERRIT_GITID)

                if 'has-github' in options or default_has_github:
                    created = create_github_project(
                        DEFAULT_HAS_ISSUES, DEFAULT_HAS_DOWNLOADS,
                        DEFAULT_HAS_WIKI, GITHUB_SECURE_CONFIG,
                        options, project, description, homepage)
                    if created and GERRIT_REPLICATE:
                        gerrit.replicate(project)

            except Exception:
                log.exception(
                    "Problems creating %s, moving on." % project)
                continue
    finally:
        os.unlink(ssh_env['GIT_SSH'])
示例#7
0
def main():
    parser = argparse.ArgumentParser(description='Manage projects')
    l.setup_logging_arguments(parser)
    parser.add_argument('--nocleanup',
                        action='store_true',
                        help='do not remove temp directories')
    parser.add_argument('projects',
                        metavar='project',
                        nargs='*',
                        help='name of project(s) to process')
    args = parser.parse_args()
    l.configure_logging(args)

    default_has_github = registry.get_defaults('has-github', True)

    LOCAL_GIT_DIR = registry.get_defaults('local-git-dir', '/var/lib/git')
    JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
                                             '/var/lib/jeepyb')
    ACL_DIR = registry.get_defaults('acl-dir')
    GERRIT_HOST = registry.get_defaults('gerrit-host')
    GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
    GERRIT_USER = registry.get_defaults('gerrit-user')
    GERRIT_KEY = registry.get_defaults('gerrit-key')
    GERRIT_GITID = registry.get_defaults('gerrit-committer')
    GERRIT_REPLICATE = registry.get_defaults('gerrit-replicate', True)
    GERRIT_OS_SYSTEM_USER = registry.get_defaults('gerrit-system-user',
                                                  'gerrit2')
    GERRIT_OS_SYSTEM_GROUP = registry.get_defaults('gerrit-system-group',
                                                   'gerrit2')
    DEFAULT_HOMEPAGE = registry.get_defaults('homepage')
    DEFAULT_HAS_ISSUES = registry.get_defaults('has-issues', False)
    DEFAULT_HAS_DOWNLOADS = registry.get_defaults('has-downloads', False)
    DEFAULT_HAS_WIKI = registry.get_defaults('has-wiki', False)
    GITHUB_SECURE_CONFIG = registry.get_defaults(
        'github-config', '/etc/github/github-projects.secure.config')

    gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST, GERRIT_USER, GERRIT_PORT,
                                     GERRIT_KEY)
    project_list = gerrit.listProjects()
    ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
    try:

        for section in registry.configs_list:
            project = section['project']
            if args.projects and project not in args.projects:
                continue

            try:
                log.info("Processing project: %s" % project)

                # Figure out all of the options
                options = section.get('options', dict())
                description = section.get('description', None)
                homepage = section.get('homepage', DEFAULT_HOMEPAGE)
                upstream = section.get('upstream', None)
                upstream_prefix = section.get('upstream-prefix', None)
                track_upstream = 'track-upstream' in options
                repo_path = os.path.join(JEEPYB_CACHE_DIR, project)

                # If this project doesn't want to use gerrit, exit cleanly.
                if 'no-gerrit' in options:
                    continue

                project_git = "%s.git" % project
                remote_url = "ssh://%s:%s/%s" % (GERRIT_HOST, GERRIT_PORT,
                                                 project)
                git_opts = dict(upstream=upstream,
                                repo_path=repo_path,
                                remote_url=remote_url)
                acl_config = section.get(
                    'acl-config', '%s.config' % os.path.join(ACL_DIR, project))

                # Create the project in Gerrit first, since it will fail
                # spectacularly if its project directory or local replica
                # already exist on disk
                project_created = create_gerrit_project(
                    project, project_list, gerrit)

                # Create the repo for the local git mirror
                create_local_mirror(LOCAL_GIT_DIR, project_git,
                                    GERRIT_OS_SYSTEM_USER,
                                    GERRIT_OS_SYSTEM_GROUP)

                if not os.path.exists(repo_path) or project_created:
                    # We don't have a local copy already, get one

                    # Make Local repo
                    push_string = make_local_copy(repo_path, project,
                                                  project_list, git_opts,
                                                  ssh_env, upstream,
                                                  GERRIT_HOST, GERRIT_PORT,
                                                  project_git, GERRIT_GITID)
                else:
                    # We do have a local copy of it already, make sure it's
                    # in shape to have work done.
                    update_local_copy(repo_path, track_upstream, git_opts,
                                      ssh_env)

                description = (find_description_override(repo_path)
                               or description)

                if project_created:
                    push_to_gerrit(repo_path, project, push_string, remote_url,
                                   ssh_env)
                    if GERRIT_REPLICATE:
                        gerrit.replicate(project)

                # If we're configured to track upstream, make sure we have
                # upstream's refs, and then push them to the appropriate
                # branches in gerrit
                if track_upstream:
                    sync_upstream(repo_path, project, ssh_env, upstream_prefix)

                if acl_config:
                    process_acls(acl_config, project, ACL_DIR, section,
                                 remote_url, repo_path, ssh_env, gerrit,
                                 GERRIT_GITID)

                if 'has-github' in options or default_has_github:
                    created = create_github_project(DEFAULT_HAS_ISSUES,
                                                    DEFAULT_HAS_DOWNLOADS,
                                                    DEFAULT_HAS_WIKI,
                                                    GITHUB_SECURE_CONFIG,
                                                    options, project,
                                                    description, homepage)
                    if created and GERRIT_REPLICATE:
                        gerrit.replicate(project)

            except Exception:
                log.exception("Problems creating %s, moving on." % project)
                continue
    finally:
        os.unlink(ssh_env['GIT_SSH'])
示例#8
0
def main():
    parser = argparse.ArgumentParser(description='Manage projects')
    l.setup_logging_arguments(parser)
    parser.add_argument('--nocleanup', action='store_true',
                        help='do not remove temp directories')
    parser.add_argument('projects', metavar='project', nargs='*',
                        help='name of project(s) to process')
    args = parser.parse_args()
    l.configure_logging(args)

    JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
                                             '/var/lib/jeepyb')
    IMPORT_DIR = os.path.join(JEEPYB_CACHE_DIR, 'import')
    GERRIT_HOST = registry.get_defaults('gerrit-host')
    GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
    GERRIT_USER = registry.get_defaults('gerrit-user')
    GERRIT_KEY = registry.get_defaults('gerrit-key')
    GERRIT_GITID = registry.get_defaults('gerrit-committer')

    PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache')
    project_cache = {}
    if os.path.exists(PROJECT_CACHE_FILE):
        project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read())

    gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST,
                                     GERRIT_USER,
                                     GERRIT_PORT,
                                     GERRIT_KEY)
    project_list = gerrit.listProjects()
    ssh_env = u.make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
    try:

        for section in registry.configs_list:
            project = section['project']
            if args.projects and project not in args.projects:
                continue

            try:
                log.info("Processing project: %s" % project)

                # Figure out all of the options
                options = section.get('options', dict())
                track_upstream = 'track-upstream' in options
                if not track_upstream:
                    continue

                # If this project doesn't want to use gerrit, exit cleanly.
                if 'no-gerrit' in options:
                    continue

                upstream = section.get('upstream', None)
                upstream_prefix = section.get('upstream-prefix', None)
                repo_path = os.path.join(IMPORT_DIR, project)

                project_git = "%s.git" % project
                remote_url = "ssh://%s:%s/%s" % (
                    GERRIT_HOST,
                    GERRIT_PORT,
                    project)
                git_opts = dict(upstream=upstream,
                                repo_path=repo_path,
                                remote_url=remote_url)
                project_cache.setdefault(project, {})
                if not project_cache[project]['pushed-to-gerrit']:
                    continue

                # Make Local repo
                if not os.path.exists(repo_path):
                    u.make_local_copy(
                        repo_path, project, project_list,
                        git_opts, ssh_env, upstream, GERRIT_HOST,
                        GERRIT_PORT, project_git, GERRIT_GITID)
                else:
                    update_local_copy(
                        repo_path, track_upstream, git_opts, ssh_env)

                fsck_repo(repo_path)
                sync_upstream(repo_path, project, ssh_env, upstream_prefix)

            except Exception:
                log.exception(
                    "Problems creating %s, moving on." % project)
                continue
    finally:
        os.unlink(ssh_env['GIT_SSH'])
def main():

    parser = argparse.ArgumentParser()
    l.setup_logging_arguments(parser)
    parser.add_argument('--message-file',
                        dest='message_file',
                        default=None,
                        help='The close pull request message')

    args = parser.parse_args()
    l.configure_logging(args)

    if args.message_file:
        try:
            with open(args.message_file, 'r') as _file:
                pull_request_text = _file.read()
        except (OSError, IOError):
            log.exception("Could not open close pull request message file")
            raise
    else:
        pull_request_text = MESSAGE

    GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG',
                                          '/etc/github/github.secure.config')

    secure_config = ConfigParser.ConfigParser()
    secure_config.read(GITHUB_SECURE_CONFIG)
    registry = u.ProjectsRegistry()

    if secure_config.has_option("github", "oauth_token"):
        ghub = github.Github(secure_config.get("github", "oauth_token"))
    else:
        ghub = github.Github(secure_config.get("github", "username"),
                             secure_config.get("github", "password"))

    orgs = ghub.get_user().get_orgs()
    orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
    for section in registry.configs_list:
        project = section['project']

        # Make sure we're using GitHub for this project:
        if not p.has_github(project):
            continue

        # Make sure we're supposed to close pull requests for this project:
        if 'options' in section and 'has-pull-requests' in section['options']:
            continue

        # Find the project's repo
        project_split = project.split('/', 1)

        # Handle errors in case the repo or the organization doesn't exists
        try:
            if len(project_split) > 1:
                org = orgs_dict[project_split[0].lower()]
                repo = org.get_repo(project_split[1])
            else:
                repo = ghub.get_user().get_repo(project)
        except (KeyError, github.GithubException):
            log.exception("Could not find project %s on GitHub." % project)
            continue

        # Close each pull request
        pull_requests = repo.get_pulls("open")
        for req in pull_requests:
            vars = dict(project=project)
            issue_data = {"url": repo.url + "/issues/" + str(req.number)}
            issue = github.Issue.Issue(requester=req._requester,
                                       headers={},
                                       attributes=issue_data,
                                       completed=True)
            issue.create_comment(pull_request_text % vars)
            req.edit(state="closed")
def main():

    parser = argparse.ArgumentParser()
    l.setup_logging_arguments(parser)
    parser.add_argument('--message-file', dest='message_file', default=None,
                        help='The close pull request message')

    args = parser.parse_args()
    l.configure_logging(args)

    if args.message_file:
        try:
            with open(args.message_file, 'r') as _file:
                pull_request_text = _file.read()
        except (OSError, IOError):
            log.exception("Could not open close pull request message file")
            raise
    else:
        pull_request_text = MESSAGE

    GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG',
                                          '/etc/github/github.secure.config')

    secure_config = ConfigParser.ConfigParser()
    secure_config.read(GITHUB_SECURE_CONFIG)
    registry = u.ProjectsRegistry()

    if secure_config.has_option("github", "oauth_token"):
        ghub = github.Github(secure_config.get("github", "oauth_token"))
    else:
        ghub = github.Github(secure_config.get("github", "username"),
                             secure_config.get("github", "password"))

    orgs = ghub.get_user().get_orgs()
    orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
    for section in registry.configs_list:
        project = section['project']

        # Make sure we're using GitHub for this project:
        if not p.has_github(project):
            continue

        # Make sure we're supposed to close pull requests for this project:
        if 'options' in section and 'has-pull-requests' in section['options']:
            continue

        # Find the project's repo
        project_split = project.split('/', 1)

        # Handle errors in case the repo or the organization doesn't exists
        try:
            if len(project_split) > 1:
                org = orgs_dict[project_split[0].lower()]
                repo = org.get_repo(project_split[1])
            else:
                repo = ghub.get_user().get_repo(project)
        except (KeyError, github.GithubException):
            log.exception("Could not find project %s on GitHub." % project)
            continue

        # Close each pull request
        pull_requests = repo.get_pulls("open")
        for req in pull_requests:
            vars = dict(project=project)
            issue_data = {"url": repo.url + "/issues/" + str(req.number)}
            issue = github.Issue.Issue(requester=req._requester,
                                       headers={},
                                       attributes=issue_data,
                                       completed=True)
            issue.create_comment(pull_request_text % vars)
            req.edit(state="closed")
示例#11
0
def main():
    parser = argparse.ArgumentParser(description='Manage projects')
    l.setup_logging_arguments(parser)
    parser.add_argument('--nocleanup', action='store_true',
                        help='do not remove temp directories')
    parser.add_argument('projects', metavar='project', nargs='*',
                        help='name of project(s) to process')
    args = parser.parse_args()
    l.configure_logging(args)

    JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
                                             '/var/lib/jeepyb')
    IMPORT_DIR = os.path.join(JEEPYB_CACHE_DIR, 'import')
    GERRIT_HOST = registry.get_defaults('gerrit-host')
    GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
    GERRIT_USER = registry.get_defaults('gerrit-user')
    GERRIT_KEY = registry.get_defaults('gerrit-key')
    GERRIT_GITID = registry.get_defaults('gerrit-committer')

    PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache')
    project_cache = {}
    if os.path.exists(PROJECT_CACHE_FILE):
        project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read())

    gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST,
                                     GERRIT_USER,
                                     GERRIT_PORT,
                                     GERRIT_KEY)
    project_list = gerrit.listProjects()
    ssh_env = u.make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
    try:

        for section in registry.configs_list:
            project = section['project']
            if args.projects and project not in args.projects:
                continue

            try:
                log.info("Processing project: %s" % project)

                # Figure out all of the options
                options = section.get('options', dict())
                track_upstream = 'track-upstream' in options
                if not track_upstream:
                    continue

                # If this project doesn't want to use gerrit, exit cleanly.
                if 'no-gerrit' in options:
                    continue

                upstream = section.get('upstream', None)
                upstream_prefix = section.get('upstream-prefix', None)
                repo_path = os.path.join(IMPORT_DIR, project)

                project_git = "%s.git" % project
                remote_url = "ssh://%s:%s/%s" % (
                    GERRIT_HOST,
                    GERRIT_PORT,
                    project)
                git_opts = dict(upstream=upstream,
                                repo_path=repo_path,
                                remote_url=remote_url)
                project_cache.setdefault(project, {})
                if not project_cache[project]['pushed-to-gerrit']:
                    continue

                # Make Local repo
                if not os.path.exists(repo_path):
                    u.make_local_copy(
                        repo_path, project, project_list,
                        git_opts, ssh_env, upstream, GERRIT_HOST,
                        GERRIT_PORT, project_git, GERRIT_GITID)
                else:
                    update_local_copy(
                        repo_path, track_upstream, git_opts, ssh_env)

                u.fsck_repo(repo_path)
                sync_upstream(repo_path, project, ssh_env, upstream_prefix)

            except Exception:
                log.exception(
                    "Problems creating %s, moving on." % project)
                continue
    finally:
        os.unlink(ssh_env['GIT_SSH'])
示例#12
0
def main():
    parser = argparse.ArgumentParser(description='Manage projects')
    l.setup_logging_arguments(parser)
    parser.add_argument('--nocleanup',
                        action='store_true',
                        help='do not remove temp directories')
    parser.add_argument('projects',
                        metavar='project',
                        nargs='*',
                        help='name of project(s) to process')
    args = parser.parse_args()
    l.configure_logging(args)

    default_has_github = registry.get_defaults('has-github', True)

    LOCAL_GIT_DIR = registry.get_defaults('local-git-dir', '/var/lib/git')
    JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
                                             '/var/lib/jeepyb')
    ACL_DIR = registry.get_defaults('acl-dir')
    GERRIT_HOST = registry.get_defaults('gerrit-host')
    GITREVIEW_GERRIT_HOST = registry.get_defaults('gitreview-gerrit-host',
                                                  GERRIT_HOST)
    GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
    GITREVIEW_GERRIT_PORT = int(
        registry.get_defaults('gitreview-gerrit-port', GERRIT_PORT))
    GERRIT_USER = registry.get_defaults('gerrit-user')
    GERRIT_KEY = registry.get_defaults('gerrit-key')
    GERRIT_GITID = registry.get_defaults('gerrit-committer')
    GERRIT_REPLICATE = registry.get_defaults('gerrit-replicate', True)
    GERRIT_OS_SYSTEM_USER = registry.get_defaults('gerrit-system-user',
                                                  'gerrit2')
    GERRIT_OS_SYSTEM_GROUP = registry.get_defaults('gerrit-system-group',
                                                   'gerrit2')
    DEFAULT_HOMEPAGE = registry.get_defaults('homepage')
    DEFAULT_HAS_ISSUES = registry.get_defaults('has-issues', False)
    DEFAULT_HAS_DOWNLOADS = registry.get_defaults('has-downloads', False)
    DEFAULT_HAS_WIKI = registry.get_defaults('has-wiki', False)
    GITHUB_SECURE_CONFIG = registry.get_defaults(
        'github-config', '/etc/github/github-projects.secure.config')
    PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache')
    project_cache = {}
    if os.path.exists(PROJECT_CACHE_FILE):
        project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read())
    acl_cache = {}
    for acl_file in glob.glob(os.path.join(ACL_DIR, '*/*.config')):
        sha256 = hashlib.sha256()
        sha256.update(open(acl_file, 'r').read())
        acl_cache[acl_file] = sha256.hexdigest()

    gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST, GERRIT_USER, GERRIT_PORT,
                                     GERRIT_KEY)
    project_list = gerrit.listProjects()
    ssh_env = u.make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
    try:
        for section in registry.configs_list:
            project = section['project']
            if args.projects and project not in args.projects:
                continue

            try:
                log.info("Processing project: %s" % project)

                # Figure out all of the options
                options = section.get('options', dict())
                description = section.get('description', None)
                homepage = section.get('homepage', DEFAULT_HOMEPAGE)
                upstream = section.get('upstream', None)
                repo_path = os.path.join(JEEPYB_CACHE_DIR, project)

                # If this project doesn't want to use gerrit, exit cleanly.
                if 'no-gerrit' in options:
                    continue

                project_git = "%s.git" % project
                remote_url = "ssh://%s:%s/%s" % (GERRIT_HOST, GERRIT_PORT,
                                                 project)
                git_opts = dict(upstream=upstream,
                                repo_path=repo_path,
                                remote_url=remote_url)
                acl_config = section.get(
                    'acl-config', '%s.config' % os.path.join(ACL_DIR, project))
                project_cache.setdefault(project, {})

                # Create the project in Gerrit first, since it will fail
                # spectacularly if its project directory or local replica
                # already exist on disk
                project_created = project_cache[project].get(
                    'project-created', False)
                if not project_created:
                    try:
                        project_created = create_gerrit_project(
                            project, project_list, gerrit)
                        project_cache[project]['project-created'] = True
                    except Exception:
                        project_cache[project]['project-created'] = False
                        continue

                pushed_to_gerrit = project_cache[project].get(
                    'pushed-to-gerrit', False)
                if not pushed_to_gerrit:
                    # We haven't pushed to gerrit, so grab the repo again
                    if os.path.exists(repo_path):
                        shutil.rmtree(repo_path)

                    # Make Local repo
                    push_string = u.make_local_copy(repo_path, project,
                                                    project_list, git_opts,
                                                    ssh_env, upstream,
                                                    GITREVIEW_GERRIT_HOST,
                                                    GITREVIEW_GERRIT_PORT,
                                                    project_git, GERRIT_GITID)

                    description = (find_description_override(repo_path)
                                   or description)

                    u.fsck_repo(repo_path)

                    if push_string:
                        push_to_gerrit(repo_path, project, push_string,
                                       remote_url, ssh_env)
                    project_cache[project]['pushed-to-gerrit'] = True
                    if GERRIT_REPLICATE:
                        gerrit.replicate(project)

                # Create the repo for the local git mirror
                create_local_mirror(LOCAL_GIT_DIR, project_git,
                                    GERRIT_OS_SYSTEM_USER,
                                    GERRIT_OS_SYSTEM_GROUP)

                if acl_config:
                    acl_sha = acl_cache.get(acl_config)
                    if project_cache[project].get('acl-sha') != acl_sha:

                        if not os.path.exists(repo_path):
                            u.make_local_copy(repo_path, project, project_list,
                                              git_opts, ssh_env, upstream,
                                              GERRIT_HOST, GERRIT_PORT,
                                              project_git, GERRIT_GITID)
                        process_acls(acl_config, project, ACL_DIR, section,
                                     remote_url, repo_path, ssh_env, gerrit,
                                     GERRIT_GITID)
                        project_cache[project]['acl-sha'] = acl_sha
                    else:
                        log.info("%s has matching sha, skipping ACLs", project)

                if 'has-github' in options or default_has_github:
                    created = create_update_github_project(
                        DEFAULT_HAS_ISSUES, DEFAULT_HAS_DOWNLOADS,
                        DEFAULT_HAS_WIKI, GITHUB_SECURE_CONFIG, options,
                        project, description, homepage, project_cache[project])
                    if created and GERRIT_REPLICATE:
                        gerrit.replicate(project)
                    project_cache[project]['created-in-github'] = created

            except Exception:
                log.exception("Problems creating %s, moving on." % project)
                continue
            finally:
                # Clean up after ourselves - this repo has no use
                if os.path.exists(repo_path):
                    shutil.rmtree(repo_path)
    finally:
        with open(PROJECT_CACHE_FILE, 'w') as cache_out:
            log.info("Writing cache file %s", PROJECT_CACHE_FILE)
            cache_out.write(json.dumps(project_cache, sort_keys=True,
                                       indent=2))
        os.unlink(ssh_env['GIT_SSH'])
示例#13
0
def main():
    parser = argparse.ArgumentParser(description='Manage projects')
    l.setup_logging_arguments(parser)
    parser.add_argument('--nocleanup', action='store_true',
                        help='do not remove temp directories')
    parser.add_argument('projects', metavar='project', nargs='*',
                        help='name of project(s) to process')
    args = parser.parse_args()
    l.configure_logging(args)

    default_has_github = registry.get_defaults('has-github', True)

    LOCAL_GIT_DIR = registry.get_defaults('local-git-dir', '/var/lib/git')
    JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir',
                                             '/var/lib/jeepyb')
    ACL_DIR = registry.get_defaults('acl-dir')
    GERRIT_HOST = registry.get_defaults('gerrit-host')
    GITREVIEW_GERRIT_HOST = registry.get_defaults(
        'gitreview-gerrit-host', GERRIT_HOST)
    GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418'))
    GITREVIEW_GERRIT_PORT = int(registry.get_defaults(
        'gitreview-gerrit-port', GERRIT_PORT))
    GERRIT_USER = registry.get_defaults('gerrit-user')
    GERRIT_KEY = registry.get_defaults('gerrit-key')
    GERRIT_GITID = registry.get_defaults('gerrit-committer')
    GERRIT_REPLICATE = registry.get_defaults('gerrit-replicate', True)
    GERRIT_OS_SYSTEM_USER = registry.get_defaults('gerrit-system-user',
                                                  'gerrit2')
    GERRIT_OS_SYSTEM_GROUP = registry.get_defaults('gerrit-system-group',
                                                   'gerrit2')
    DEFAULT_HOMEPAGE = registry.get_defaults('homepage')
    DEFAULT_HAS_ISSUES = registry.get_defaults('has-issues', False)
    DEFAULT_HAS_DOWNLOADS = registry.get_defaults('has-downloads', False)
    DEFAULT_HAS_WIKI = registry.get_defaults('has-wiki', False)
    GITHUB_SECURE_CONFIG = registry.get_defaults(
        'github-config',
        '/etc/github/github-projects.secure.config')
    PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache')
    project_cache = {}
    if os.path.exists(PROJECT_CACHE_FILE):
        project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read())
    acl_cache = {}
    for acl_file in glob.glob(os.path.join(ACL_DIR, '*/*.config')):
        sha256 = hashlib.sha256()
        sha256.update(open(acl_file, 'r').read())
        acl_cache[acl_file] = sha256.hexdigest()

    gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST,
                                     GERRIT_USER,
                                     GERRIT_PORT,
                                     GERRIT_KEY)
    project_list = gerrit.listProjects()
    ssh_env = u.make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
    try:
        # Collect processed errors,if any
        process_errors = []
        for section in registry.configs_list:
            project = section['project']
            if args.projects and project not in args.projects:
                continue

            try:
                log.info("Processing project: %s" % project)

                # Figure out all of the options
                options = section.get('options', dict())
                description = section.get('description', None)
                homepage = section.get('homepage', DEFAULT_HOMEPAGE)
                upstream = section.get('upstream', None)
                repo_path = os.path.join(JEEPYB_CACHE_DIR, project)

                # If this project doesn't want to use gerrit, exit cleanly.
                if 'no-gerrit' in options:
                    continue

                project_git = "%s.git" % project
                remote_url = "ssh://%s:%s/%s" % (
                    GERRIT_HOST,
                    GERRIT_PORT,
                    project)
                git_opts = dict(upstream=upstream,
                                repo_path=repo_path,
                                remote_url=remote_url)
                acl_config = section.get(
                    'acl-config',
                    '%s.config' % os.path.join(ACL_DIR, project))
                project_cache.setdefault(project, {})

                # Create the project in Gerrit first, since it will fail
                # spectacularly if its project directory or local replica
                # already exist on disk
                project_created = project_cache[project].get(
                    'project-created', False)
                if not project_created:
                    try:
                        project_created = create_gerrit_project(
                            project, project_list, gerrit)
                        project_cache[project]['project-created'] = True
                    except Exception:
                        project_cache[project]['project-created'] = False
                        continue

                pushed_to_gerrit = project_cache[project].get(
                    'pushed-to-gerrit', False)
                if not pushed_to_gerrit:
                    # We haven't pushed to gerrit, so grab the repo again
                    if os.path.exists(repo_path):
                        shutil.rmtree(repo_path)

                    # Make Local repo
                    push_string = u.make_local_copy(
                        repo_path, project, project_list,
                        git_opts, ssh_env, upstream, GITREVIEW_GERRIT_HOST,
                        GITREVIEW_GERRIT_PORT, project_git, GERRIT_GITID)

                    description = (
                        find_description_override(repo_path)
                        or description)

                    u.fsck_repo(repo_path)

                    if push_string:
                        push_to_gerrit(
                            repo_path, project, push_string,
                            remote_url, ssh_env)
                    project_cache[project]['pushed-to-gerrit'] = True
                    if GERRIT_REPLICATE:
                        gerrit.replicate(project)

                # Create the repo for the local git mirror
                create_local_mirror(
                    LOCAL_GIT_DIR, project_git,
                    GERRIT_OS_SYSTEM_USER, GERRIT_OS_SYSTEM_GROUP)

                if acl_config:
                    acl_sha = acl_cache.get(acl_config)
                    if project_cache[project].get('acl-sha') != acl_sha:

                        if not os.path.exists(repo_path):
                            u.make_local_copy(
                                repo_path, project, project_list,
                                git_opts, ssh_env, upstream, GERRIT_HOST,
                                GERRIT_PORT, project_git, GERRIT_GITID)
                        process_acls(
                            acl_config, project, ACL_DIR, section,
                            remote_url, repo_path, ssh_env, gerrit,
                            GERRIT_GITID)
                        project_cache[project]['acl-sha'] = acl_sha
                    else:
                        log.info("%s has matching sha, skipping ACLs",
                                 project)

                if 'has-github' in options or default_has_github:
                    created = create_update_github_project(
                        DEFAULT_HAS_ISSUES, DEFAULT_HAS_DOWNLOADS,
                        DEFAULT_HAS_WIKI, GITHUB_SECURE_CONFIG,
                        options, project, description, homepage,
                        project_cache[project])
                    if created and GERRIT_REPLICATE:
                        gerrit.replicate(project)
                    project_cache[project]['created-in-github'] = created

            except Exception:
                msg = "Problems creating %s, moving on." % project
                log.exception(msg)
                process_errors.append(msg)
                continue
            finally:
                # Clean up after ourselves - this repo has no use
                if os.path.exists(repo_path):
                    shutil.rmtree(repo_path)
    finally:
        with open(PROJECT_CACHE_FILE, 'w') as cache_out:
            log.info("Writing cache file %s", PROJECT_CACHE_FILE)
            cache_out.write(json.dumps(
                project_cache, sort_keys=True, indent=2))
        os.unlink(ssh_env['GIT_SSH'])
        if len(process_errors) > 0:
            log.error("%d problems has been caught during run:\n %s" % (
                len(process_errors), process_errors))
            sys.exit(1)