def __init__(self, context):
        from vcttesting.docker import Docker, params_from_env
        from vcttesting.hgmo import HgCluster

        if 'HGMO_STATE_FILE' not in os.environ:
            print('Do not know where to store state.')
            print(
                'Set the HGMO_STATE_FILE environment variable and try again.')
            sys.exit(1)

        if 'DOCKER_STATE_FILE' not in os.environ:
            print('Do not where to store Docker state.')
            print(
                'Set the DOCKER_STATE_FILE environment variable and try again.'
            )
            sys.exit(1)

        docker_url, tls = params_from_env(os.environ)
        docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls)
        if not docker.is_alive():
            print('Docker not available')
            sys.exit(1)
        self.c = HgCluster(
            docker,
            os.environ['HGMO_STATE_FILE'],
            ldap_image=os.environ.get('DOCKER_LDAP_IMAGE'),
            master_image=os.environ.get('DOCKER_HGMASTER_IMAGE'),
            web_image=os.environ.get('DOCKER_HGWEB_IMAGE'),
            pulse_image=os.environ.get('DOCKER_PULSE_IMAGE'))
Exemplo n.º 2
0
    def __init__(self, context):
        from vcttesting.docker import Docker, params_from_env
        from vcttesting.hgmo import HgCluster

        if 'DOCKER_STATE_FILE' not in os.environ:
            print('Do not where to store Docker state.')
            print('Set the DOCKER_STATE_FILE environment variable and try again.')
            sys.exit(1)

        docker_url, tls = params_from_env(os.environ)
        docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls)
        if not docker.is_alive():
            print('Docker not available')
            sys.exit(1)
        self.c = HgCluster(docker)
    def __init__(self, context):
        from vcttesting.docker import Docker, params_from_env
        from vcttesting.hgmo import HgCluster

        if 'HGMO_STATE_FILE' not in os.environ:
            print('Do not know where to store state.')
            print('Set the HGMO_STATE_FILE environment variable and try again.')
            sys.exit(1)

        if 'DOCKER_STATE_FILE' not in os.environ:
            print('Do not where to store Docker state.')
            print('Set the DOCKER_STATE_FILE environment variable and try again.')
            sys.exit(1)

        docker_url, tls = params_from_env(os.environ)
        docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls)
        self.c = HgCluster(docker, os.environ['HGMO_STATE_FILE'],
                           ldap_image=os.environ.get('DOCKER_LDAP_IMAGE'),
                           master_image=os.environ.get('DOCKER_HGMASTER_IMAGE'),
                           web_image=os.environ.get('DOCKER_HGWEB_IMAGE'))
Exemplo n.º 4
0
class HgmoCommands(object):
    def __init__(self, context):
        from vcttesting.docker import Docker, params_from_env
        from vcttesting.hgmo import HgCluster

        if 'HGMO_STATE_FILE' not in os.environ:
            print('Do not know where to store state.')
            print('Set the HGMO_STATE_FILE environment variable and try again.')
            sys.exit(1)

        if 'DOCKER_STATE_FILE' not in os.environ:
            print('Do not where to store Docker state.')
            print('Set the DOCKER_STATE_FILE environment variable and try again.')
            sys.exit(1)

        docker_url, tls = params_from_env(os.environ)
        docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls)
        if not docker.is_alive():
            print('Docker not available')
            sys.exit(1)
        self.c = HgCluster(docker, os.environ['HGMO_STATE_FILE'],
                           ldap_image=os.environ.get('DOCKER_LDAP_IMAGE'),
                           master_image=os.environ.get('DOCKER_HGMASTER_IMAGE'),
                           web_image=os.environ.get('DOCKER_HGWEB_IMAGE'),
                           pulse_image=os.environ.get('DOCKER_PULSE_IMAGE'))

    @Command('start', category='hgmo',
             description='Start a hg.mozilla.org cluster')
    @CommandArgument('--master-ssh-port', type=int,
                     help='Port number on which SSH server should listen')
    def start(self, master_ssh_port=None):
        s = self.c.start(master_ssh_port=master_ssh_port,
                         coverage='CODE_COVERAGE' in os.environ)
        print('SSH Hostname: %s' % s['master_ssh_hostname'])
        print('SSH Port: %s' % s['master_ssh_port'])
        print('LDAP URI: %s' % s['ldap_uri'])
        for url in s['web_urls']:
            print('Web URL: %s' % url)
        print('Pulse: %s:%d' % (s['pulse_hostname'], s['pulse_hostport']))

    @Command('shellinit', category='hgmo',
             description='Print shell commands to export variables')
    def shellinit(self):
        print('export SSH_CID=%s' % self.c.master_id)
        print('export PULSE_HOST=%s' % self.c.pulse_hostname)
        print('export PULSE_PORT=%s' % self.c.pulse_hostport)
        print('export SSH_SERVER=%s' % self.c.master_ssh_hostname)
        print('export SSH_PORT=%d' % self.c.master_ssh_port)
        # Don't export the full value because spaces.
        print('export SSH_HOST_RSA_KEY=%s' % self.c.master_host_rsa_key.split()[1])
        print('export SSH_HOST_ED25519_KEY=%s' % self.c.master_host_ed25519_key.split()[1])
        for i, url in enumerate(self.c.web_urls):
            print('export HGWEB_%d_URL=%s' % (i, url))
        for i, cid in enumerate(self.c.web_ids):
            print('export HGWEB_%d_CID=%s' % (i, cid))
        for i, hostport in enumerate(self.c.kafka_hostports):
            print('export KAFKA_%d_HOSTPORT=%s' % (i, hostport))
        print('export ZOOKEEPER_CONNECT=%s' % self.c.zookeeper_connect)

    @Command('stop', category='hgmo',
             description='Stop a hg.mozilla.org cluster')
    def stop(self):
        self.c.stop()

    @Command('clean', category='hgmo',
             description='Clean up all references to this cluster')
    def clean(self):
        self.c.clean()

    @Command('create-ldap-user', category='hgmo',
             description='Create a new user in LDAP')
    @CommandArgument('email',
                     help='Email address associated with user')
    @CommandArgument('username',
                     help='System account name')
    @CommandArgument('uid', type=int,
                     help='Numeric user ID to associate with user')
    @CommandArgument('fullname',
                     help='Full name of the user')
    @CommandArgument('--key-file',
                     help='Use or create an SSH key')
    @CommandArgument('--scm-level', type=int, choices=(1, 2, 3),
                     help='Add the user to the specified SCM level groups')
    @CommandArgument('--no-hg-access', action='store_true',
                     help='Do not grant Mercurial access to user')
    @CommandArgument('--hg-disabled', action='store_true',
                     help='Set hgAccess to FALSE')
    def create_ldap_user(self, email, username, uid, fullname, key_file=None,
                         scm_level=None, no_hg_access=False, hg_disabled=False):
        self.c.ldap.create_user(email, username, uid, fullname,
                                key_filename=key_file, scm_level=scm_level,
                                hg_access=not no_hg_access,
                                hg_enabled=not hg_disabled)

    @Command('add-ssh-key', category='hgmo',
             description='Add an SSH public key to a user')
    @CommandArgument('email',
                     help='Email address of user to modify')
    @CommandArgument('key',
                     help='SSH public key string')
    def add_ssh_key(self, email, key):
        if key == '-':
            key = sys.stdin.read().strip().encode('utf-8')
        self.c.ldap.add_ssh_key(email, key)

    @Command('add-user-to-group', category='hgmo',
             description='Add a user to an LDAP group')
    @CommandArgument('email',
                     help='Email address of user to modify')
    @CommandArgument('group',
                     help='Name of LDAP group to add user to')
    def add_user_to_group(self, email, group):
        self.c.ldap.add_user_to_group(email, group)

    @Command('create-repo', category='hgmo',
             description='Create a repository in the cluster')
    @CommandArgument('name',
                     help='Name of repository to create')
    @CommandArgument('group', default='scm_level_1',
                     help='LDAP group that owns repo')
    @CommandArgument('--generaldelta', action='store_true',
                     help='Create repository with generaldelta storage')
    def create_repo(self, name, group, generaldelta=True):
        out = self.c.create_repo(name, group=group, generaldelta=generaldelta)
        if out:
            sys.stdout.write(out)

    @Command('aggregate-code-coverage', category='hgmo',
             description='Aggregate code coverage results to a directory')
    @CommandArgument('destdir',
                     help='Directory where to save code coverage files')
    def aggregate_code_coverage(self, destdir):
        self.c.aggregate_code_coverage(destdir)

    @Command('exec', category='hgmo',
             description='Execute a command in a Docker container')
    @CommandArgument('--detach', action='store_true',
                     help='Do not wait for process to finish')
    @CommandArgument('name', help='Name of container to execute inside')
    @CommandArgument('command', help='Command to execute',
                     nargs=argparse.REMAINDER)
    def execute(self, name, command, detach=False):
        if name == 'hgssh':
            cid = self.c.master_id
        elif name == 'pulse':
            cid = self.c.pulse_id
        elif name.startswith('hgweb'):
            i = int(name[5:])
            cid = self.c.web_ids[i]
        else:
            print('invalid name. must be "hgssh" or "hgwebN"')
            return 1

        cmd = ['docker', 'exec']
        if 'TESTTMP' not in os.environ:
            cmd.append('-it')
        if detach:
            cmd.append('-d')

        cmd.append(cid)
        cmd.extend(command)

        return subprocess.call(cmd)

    @Command('download-mirror-ssh-keys', category='hgmo',
             description='Downloads SSH keys used by mirrors')
    @CommandArgument('out_dir', help='Directory in which to write the keys')
    def download_mirror_ssh_keys(self, out_dir):
        priv, pub = self.c.get_mirror_ssh_keys()[0:2]

        with open(os.path.join(out_dir, 'mirror'), 'wb') as fh:
            fh.write(priv)
        os.chmod(os.path.join(out_dir, 'mirror'), stat.S_IRUSR | stat.S_IWUSR)

        with open(os.path.join(out_dir, 'mirror.pub'), 'wb') as fh:
            fh.write(pub)
        print('SSH keys written to %s' % out_dir)
class HgmoCommands(object):
    def __init__(self, context):
        from vcttesting.docker import Docker, params_from_env
        from vcttesting.hgmo import HgCluster

        if 'HGMO_STATE_FILE' not in os.environ:
            print('Do not know where to store state.')
            print('Set the HGMO_STATE_FILE environment variable and try again.')
            sys.exit(1)

        if 'DOCKER_STATE_FILE' not in os.environ:
            print('Do not where to store Docker state.')
            print('Set the DOCKER_STATE_FILE environment variable and try again.')
            sys.exit(1)

        docker_url, tls = params_from_env(os.environ)
        docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls)
        self.c = HgCluster(docker, os.environ['HGMO_STATE_FILE'],
                           ldap_image=os.environ.get('DOCKER_LDAP_IMAGE'),
                           master_image=os.environ.get('DOCKER_HGMASTER_IMAGE'),
                           web_image=os.environ.get('DOCKER_HGWEB_IMAGE'))

    @Command('start', category='hgmo',
             description='Start a hg.mozilla.org cluster')
    @CommandArgument('--master-ssh-port', type=int,
                     help='Port number on which SSH server should listen')
    def start(self, master_ssh_port=None):
        s = self.c.start(master_ssh_port=master_ssh_port,
                         coverage='CODE_COVERAGE' in os.environ)
        print('SSH Hostname: %s' % s['master_ssh_hostname'])
        print('SSH Port: %s' % s['master_ssh_port'])
        print('LDAP URI: %s' % s['ldap_uri'])
        for url in s['web_urls']:
            print('Web URL: %s' % url)

    @Command('shellinit', category='hgmo',
             description='Print shell commands to export variables')
    def shellinit(self):
        print('export SSH_CID=%s' % self.c.master_id)
        print('export SSH_SERVER=%s' % self.c.master_ssh_hostname)
        print('export SSH_PORT=%d' % self.c.master_ssh_port)
        # Don't export the full value because spaces.
        print('export SSH_HOST_KEY=%s' % self.c.master_host_key.split()[1])
        for i, url in enumerate(self.c.web_urls):
            print('export HGWEB_%d_URL=%s' % (i, url))
        for i, cid in enumerate(self.c.web_ids):
            print('export HGWEB_%d_CID=%s' % (i, cid))
        for i, hostport in enumerate(self.c.kafka_hostports):
            print('export KAFKA_%d_HOSTPORT=%s' % (i, hostport))
        print('export ZOOKEEPER_CONNECT=%s' % self.c.zookeeper_connect)

    @Command('stop', category='hgmo',
             description='Stop a hg.mozilla.org cluster')
    def stop(self):
        self.c.stop()

    @Command('clean', category='hgmo',
             description='Clean up all references to this cluster')
    def clean(self):
        self.c.clean()

    @Command('create-ldap-user', category='hgmo',
             description='Create a new user in LDAP')
    @CommandArgument('email',
                     help='Email address associated with user')
    @CommandArgument('username',
                     help='System account name')
    @CommandArgument('uid', type=int,
                     help='Numeric user ID to associate with user')
    @CommandArgument('fullname',
                     help='Full name of the user')
    @CommandArgument('--key-file',
                     help='Use or create an SSH key')
    @CommandArgument('--scm-level', type=int, choices=(1, 2, 3),
                     help='Add the user to the specified SCM level groups')
    @CommandArgument('--no-hg-access', action='store_true',
                     help='Do not grant Mercurial access to user')
    def create_ldap_user(self, email, username, uid, fullname, key_file=None,
                         scm_level=None, no_hg_access=False):
        self.c.ldap.create_user(email, username, uid, fullname,
                                key_filename=key_file, scm_level=scm_level,
                                hg_access=not no_hg_access)

    @Command('add-ssh-key', category='hgmo',
             description='Add an SSH public key to a user')
    @CommandArgument('email',
                     help='Email address of user to modify')
    @CommandArgument('key',
                     help='SSH public key string')
    def add_ssh_key(self, email, key):
        if key == '-':
            key = sys.stdin.read().strip().encode('utf-8')
        self.c.ldap.add_ssh_key(email, key)

    @Command('create-repo', category='hgmo',
             description='Create a repository in the cluster')
    @CommandArgument('name',
                     help='Name of repository to create')
    @CommandArgument('level', type=int, choices=[1, 2, 3], default=1,
                     help='SCM level access for this repository')
    def create_repo(self, name, level):
        out = self.c.create_repo(name, level=level)
        if out:
            sys.stdout.write(out)

    @Command('aggregate-code-coverage', category='hgmo',
             description='Aggregate code coverage results to a directory')
    @CommandArgument('destdir',
                     help='Directory where to save code coverage files')
    def aggregate_code_coverage(self, destdir):
        self.c.aggregate_code_coverage(destdir)

    @Command('exec', category='hgmo',
             description='Execute a command in a Docker container')
    @CommandArgument('--detach', action='store_true',
                     help='Do not wait for process to finish')
    @CommandArgument('name', help='Name of container to execute inside')
    @CommandArgument('command', help='Command to execute',
                     nargs=argparse.REMAINDER)
    def execute(self, name, command, detach=False):
        if name == 'hgssh':
            cid = self.c.master_id
        elif name == 'pulse':
            cid = self.c.pulse_id
        elif name.startswith('hgweb'):
            i = int(name[5:])
            cid = self.c.web_ids[i]
        else:
            print('invalid name. must be "hgssh" or "hgwebN"')
            return 1

        args = '' if 'TESTTMP' in os.environ else '-it'
        if detach:
            args += ' -d '
        args = args.strip()

        return subprocess.call('docker exec %s %s %s' % (
                               args, cid, ' '.join(command)),
                               shell=True)