def __init__(self, context):
        from vcttesting.docker import Docker, params_from_env
        from vcttesting.hgmo import HgCluster

        if 'HGMO_STATE_FILE' not in os.environ:
            print('Do not know where to store state.')
            print(
                'Set the HGMO_STATE_FILE environment variable and try again.')
            sys.exit(1)

        if 'DOCKER_STATE_FILE' not in os.environ:
            print('Do not where to store Docker state.')
            print(
                'Set the DOCKER_STATE_FILE environment variable and try again.'
            )
            sys.exit(1)

        docker_url, tls = params_from_env(os.environ)
        docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls)
        if not docker.is_alive():
            print('Docker not available')
            sys.exit(1)
        self.c = HgCluster(
            docker,
            os.environ['HGMO_STATE_FILE'],
            ldap_image=os.environ.get('DOCKER_LDAP_IMAGE'),
            master_image=os.environ.get('DOCKER_HGMASTER_IMAGE'),
            web_image=os.environ.get('DOCKER_HGWEB_IMAGE'),
            pulse_image=os.environ.get('DOCKER_PULSE_IMAGE'))
def has_docker():
    from vcttesting.docker import Docker, params_from_env

    url, tls = params_from_env(os.environ)

    tf = tempfile.NamedTemporaryFile()
    tf.close()
    d = Docker(tf.name, url, tls=tls)
    return d.is_alive()
def has_docker():
    if 'SKIP_DOCKER_TESTS' in os.environ:
        return False

    from vcttesting.docker import Docker, params_from_env

    url, tls = params_from_env(os.environ)

    tf = tempfile.NamedTemporaryFile()
    tf.close()
    d = Docker(tf.name, url, tls=tls)
    return d.is_alive()
def has_docker():
    if 'SKIP_DOCKER_TESTS' in os.environ:
        return False

    from vcttesting.docker import Docker, params_from_env

    url, tls = params_from_env(os.environ)

    tf = tempfile.NamedTemporaryFile()
    tf.close()
    d = Docker(tf.name, url, tls=tls)
    return d.is_alive()
Exemple #5
0
    def __init__(self, context):
        from vcttesting.docker import Docker, params_from_env
        from vcttesting.hgmo import HgCluster

        if 'DOCKER_STATE_FILE' not in os.environ:
            print('Do not where to store Docker state.')
            print('Set the DOCKER_STATE_FILE environment variable and try again.')
            sys.exit(1)

        docker_url, tls = params_from_env(os.environ)
        docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls)
        if not docker.is_alive():
            print('Docker not available')
            sys.exit(1)
        self.c = HgCluster(docker)
    def __init__(self, context):
        if 'DOCKER_STATE_FILE' in os.environ:
            state_file = os.environ['DOCKER_STATE_FILE']

        # When running from Mercurial tests, use a per-test state file.
        # We can't use HGTMP because it is shared across many tests. We
        # use HGRCPATH as a base, since it is in a test-specific directory.
        elif 'HGRCPATH' in os.environ:
            state_file = os.path.join(os.path.dirname(os.environ['HGRCPATH']),
                                     '.dockerstate')
        else:
            state_file = os.path.join(ROOT, '.dockerstate')

        docker_url, tls = params_from_env(os.environ)
        d = Docker(state_file, docker_url, tls=tls)

        if not d.is_alive():
            print('Docker is not available!')
            sys.exit(1)

        self.d = d
Exemple #7
0
    def __init__(self, context):
        if 'DOCKER_STATE_FILE' in os.environ:
            state_file = os.environ['DOCKER_STATE_FILE']

        # When running from Mercurial tests, use a per-test state file.
        # We can't use HGTMP because it is shared across many tests. We
        # use HGRCPATH as a base, since it is in a test-specific directory.
        elif 'HGRCPATH' in os.environ:
            state_file = os.path.join(os.path.dirname(os.environ['HGRCPATH']),
                                      '.dockerstate')
        else:
            state_file = os.path.join(ROOT, '.dockerstate')

        docker_url, tls = params_from_env(os.environ)
        d = Docker(state_file, docker_url, tls=tls)

        if not d.is_alive():
            print('Docker is not available!')
            sys.exit(1)

        self.d = d
    def __init__(self, context):
        from vcttesting.docker import Docker, params_from_env
        from vcttesting.hgmo import HgCluster

        if 'HGMO_STATE_FILE' not in os.environ:
            print('Do not know where to store state.')
            print('Set the HGMO_STATE_FILE environment variable and try again.')
            sys.exit(1)

        if 'DOCKER_STATE_FILE' not in os.environ:
            print('Do not where to store Docker state.')
            print('Set the DOCKER_STATE_FILE environment variable and try again.')
            sys.exit(1)

        docker_url, tls = params_from_env(os.environ)
        docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls)
        if not docker.is_alive():
            print('Docker not available')
            sys.exit(1)
        self.c = HgCluster(docker, os.environ['HGMO_STATE_FILE'],
                           ldap_image=os.environ.get('DOCKER_LDAP_IMAGE'),
                           master_image=os.environ.get('DOCKER_HGMASTER_IMAGE'),
                           web_image=os.environ.get('DOCKER_HGWEB_IMAGE'))
Exemple #9
0
class MozReview(object):
    """Interface to MozService service.

    This class can be used to create and control MozReview instances.
    """
    def __init__(self,
                 path,
                 web_image=None,
                 hgrb_image=None,
                 ldap_image=None,
                 pulse_image=None,
                 rbweb_image=None,
                 autolanddb_image=None,
                 autoland_image=None,
                 hgweb_image=None,
                 treestatus_image=None):
        if not path:
            raise Exception('You must specify a path to create an instance')
        path = os.path.abspath(path)
        self._path = path

        self.started = False

        self.web_image = web_image
        self.hgrb_image = hgrb_image
        self.ldap_image = ldap_image
        self.pulse_image = pulse_image
        self.rbweb_image = rbweb_image
        self.autolanddb_image = autolanddb_image
        self.autoland_image = autoland_image
        self.hgweb_image = hgweb_image
        self.treestatus_image = treestatus_image

        self._name = os.path.dirname(path)

        if not os.path.exists(path):
            os.mkdir(path)

        keys_path = os.path.join(path, 'keys')
        try:
            os.mkdir(keys_path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        credentials_path = os.path.join(path, 'credentials')
        try:
            os.mkdir(credentials_path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        self._state_path = os.path.join(path, 'state.json')

        docker_state = os.path.join(path, '.dockerstate')

        self._docker_state = docker_state

        self.bugzilla_username = None
        self.bugzilla_password = None
        self.docker_env = {}

        if os.path.exists(self._state_path):
            with open(self._state_path, 'rb') as fh:
                state = json.load(fh)

                for k, v in state.items():
                    setattr(self, k, v)

        # Preserve Docker settings from last time.
        #
        # This was introduced to make watchman happy, as its triggers may not
        # inherit environment variables.
        for k, v in self.docker_env.items():
            os.environ[k] = v

        docker_url, tls = params_from_env(os.environ)
        self._docker = Docker(docker_state, docker_url, tls=tls)

        if not self._docker.is_alive():
            raise DockerNotAvailable('Docker is not available.')

    def get_bugzilla(self, username=None, password=None):
        username = username or self.bugzilla_username or '*****@*****.**'
        password = password or self.bugzilla_password or 'password'

        return Bugzilla(self.bugzilla_url,
                        username=username,
                        password=password)

    def get_reviewboard(self):
        """Obtain a MozReviewBoard instance tied to this MozReview instance."""
        return MozReviewBoard(self._docker,
                              self.rbweb_id,
                              self.reviewboard_url,
                              bugzilla_url=self.bugzilla_url,
                              pulse_host=self.pulse_host,
                              pulse_port=self.pulse_port)

    def get_ldap(self):
        """Obtain an LDAP instance connected to the LDAP server in this instance."""
        return LDAP(self.ldap_uri, 'cn=admin,dc=mozilla', 'password')

    def start(self,
              bugzilla_port=None,
              reviewboard_port=None,
              mercurial_port=None,
              pulse_port=None,
              verbose=False,
              web_image=None,
              hgrb_image=None,
              ldap_image=None,
              ldap_port=None,
              pulse_image=None,
              rbweb_image=None,
              ssh_port=None,
              hgweb_image=None,
              hgweb_port=None,
              autolanddb_image=None,
              autoland_image=None,
              autoland_port=None,
              treestatus_image=None,
              treestatus_port=None,
              max_workers=None):
        """Start a MozReview instance."""
        if self.started:
            raise Exception('MozReview instance has already been started')

        if not bugzilla_port:
            bugzilla_port = get_available_port()
        if not reviewboard_port:
            reviewboard_port = get_available_port()
        if not mercurial_port:
            mercurial_port = get_available_port()
        if not ldap_port:
            ldap_port = get_available_port()
        if not ssh_port:
            ssh_port = get_available_port()
        if not pulse_port:
            pulse_port = get_available_port()
        if not autoland_port:
            autoland_port = get_available_port()
        if not hgweb_port:
            hgweb_port = get_available_port()
        if not treestatus_port:
            treestatus_port = get_available_port()

        web_image = web_image or self.web_image
        hgrb_image = hgrb_image or self.hgrb_image
        ldap_image = ldap_image or self.ldap_image
        pulse_image = pulse_image or self.pulse_image
        rbweb_image = rbweb_image or self.rbweb_image
        autolanddb_image = autolanddb_image or self.autolanddb_image
        autoland_image = autoland_image or self.autoland_image
        hgweb_image = hgweb_image or self.hgweb_image
        treestatus_image = treestatus_image or self.treestatus_image

        if not os.path.exists(os.path.join(ROOT, 'reviewboard-fork')):
            raise Exception('Failed to find reviewboard-fork. '
                            'Please run create-test-environment.')

        self.started = True
        mr_info = self._docker.start_mozreview(
            cluster=self._name,
            http_port=bugzilla_port,
            pulse_port=pulse_port,
            web_image=web_image,
            hgrb_image=hgrb_image,
            ldap_image=ldap_image,
            ldap_port=ldap_port,
            pulse_image=pulse_image,
            rbweb_image=rbweb_image,
            rbweb_port=reviewboard_port,
            ssh_port=ssh_port,
            hg_port=mercurial_port,
            autolanddb_image=autolanddb_image,
            autoland_image=autoland_image,
            autoland_port=autoland_port,
            hgweb_image=hgweb_image,
            hgweb_port=hgweb_port,
            treestatus_image=treestatus_image,
            treestatus_port=treestatus_port,
            max_workers=max_workers,
            verbose=verbose)

        self.bmoweb_id = mr_info['web_id']

        self.bugzilla_url = mr_info['bugzilla_url']
        bugzilla = self.get_bugzilla()

        self.reviewboard_url = mr_info['reviewboard_url']
        self.rbweb_id = mr_info['rbweb_id']

        self.autoland_id = mr_info['autoland_id']
        self.autoland_url = mr_info['autoland_url']
        self.pulse_id = mr_info['pulse_id']
        self.pulse_host = mr_info['pulse_host']
        self.pulse_port = mr_info['pulse_port']

        self.admin_username = bugzilla.username
        self.admin_password = bugzilla.password
        self.hg_rb_username = "******"
        self.hg_rb_email = "*****@*****.**"
        self.hg_rb_password = "******"
        self.ldap_uri = mr_info['ldap_uri']
        self.hgrb_id = mr_info['hgrb_id']
        self.ssh_hostname = mr_info['ssh_hostname']
        self.ssh_port = mr_info['ssh_port']
        self.mercurial_url = mr_info['mercurial_url']
        self.hgweb_id = mr_info['hgweb_id']
        self.hgweb_url = mr_info['hgweb_url']

        self.treestatus_id = mr_info['treestatus_id']
        self.treestatus_url = mr_info['treestatus_url']

        rb = self.get_reviewboard()

        # It is tempting to put the user creation inside the futures
        # block so it runs concurrently. However, there appeared to be
        # race conditions here. The hg_rb_username ("mozreview") user
        # was sometimes not getting created and this led to intermittent
        # test failures in test-auth.t.

        # Ensure admin user is present.
        rb.login_user(bugzilla.username, bugzilla.password)

        # Ensure the mozreview hg user is present and has privileges.
        # This has to occur after the admin user is logged in to avoid
        # race conditions with user IDs.
        rb.create_local_user(self.hg_rb_username, self.hg_rb_email,
                             self.hg_rb_password)

        with limited_threadpoolexecutor(7, max_workers) as e:
            # Ensure admin user had admin privileges.
            e.submit(rb.make_admin, bugzilla.username)

            # Ensure mozreview user has permissions for testing.
            e.submit(rb.grant_permission, self.hg_rb_username,
                     'Can change ldap assocation for all users')

            e.submit(rb.grant_permission, self.hg_rb_username,
                     'Can verify DiffSet legitimacy')

            e.submit(rb.grant_permission, self.hg_rb_username,
                     'Can enable or disable autoland for a repository')

            # Tell hgrb about URLs.
            e.submit(self._docker.execute, self.hgrb_id,
                     ['/set-urls', self.bugzilla_url, self.reviewboard_url])

            # Define site domain and hostname in rbweb. This is necessary so it
            # constructs self-referential URLs properly.
            e.submit(self._docker.execute, self.rbweb_id, [
                '/set-site-url', self.reviewboard_url, self.autoland_url,
                self.bugzilla_url
            ])

            # Tell Bugzilla about Review Board URL.
            e.submit(self._docker.execute, mr_info['web_id'],
                     ['/set-urls', self.reviewboard_url])

        self.create_user_api_key(bugzilla.username, description='mozreview')

        with futures.ThreadPoolExecutor(2) as e:
            f_ssh_ed25519_key = e.submit(
                self._docker.get_file_content, mr_info['hgrb_id'],
                '/etc/mercurial/ssh/ssh_host_ed25519_key.pub')
            f_ssh_rsa_key = e.submit(
                self._docker.get_file_content, mr_info['hgrb_id'],
                '/etc/mercurial/ssh/ssh_host_rsa_key.pub')

        ssh_ed25519_key = f_ssh_ed25519_key.result().split()[0:2]
        ssh_rsa_key = f_ssh_rsa_key.result().split()[0:2]

        hostkeys_path = os.path.join(self._path, 'ssh-known-hosts')
        hoststring = '[%s]:%d' % (mr_info['ssh_hostname'], mr_info['ssh_port'])
        with open(hostkeys_path, 'wb') as fh:
            fh.write('%s %s %s\n' %
                     (hoststring, ssh_ed25519_key[0], ssh_ed25519_key[1]))
            fh.write('%s %s %s\n' %
                     (hoststring, ssh_rsa_key[0], ssh_rsa_key[1]))

        with open(os.path.join(self._path, 'ssh_config'), 'wb') as fh:
            fh.write(
                SSH_CONFIG.format(known_hosts=hostkeys_path,
                                  hostname=self.ssh_hostname,
                                  port=self.ssh_port))

        state = {
            'bmoweb_id': self.bmoweb_id,
            'bugzilla_url': self.bugzilla_url,
            'reviewboard_url': self.reviewboard_url,
            'rbweb_id': self.rbweb_id,
            'mercurial_url': self.mercurial_url,
            'admin_username': bugzilla.username,
            'admin_password': bugzilla.password,
            'ldap_uri': self.ldap_uri,
            'pulse_id': self.pulse_id,
            'pulse_host': self.pulse_host,
            'pulse_port': self.pulse_port,
            'autoland_url': self.autoland_url,
            'autoland_id': self.autoland_id,
            'hgrb_id': self.hgrb_id,
            'hgweb_url': self.hgweb_url,
            'hgweb_id': self.hgweb_id,
            'ssh_hostname': self.ssh_hostname,
            'ssh_port': self.ssh_port,
            'treestatus_url': self.treestatus_url,
            'treestatus_id': self.treestatus_id,
            'docker_env':
            {k: v
             for k, v in os.environ.items() if k.startswith('DOCKER')}
        }

        with open(self._state_path, 'wb') as fh:
            json.dump(state, fh, indent=2, sort_keys=True)

    def stop(self):
        """Stop all services associated with this MozReview instance."""
        self._docker.stop_bmo(self._name)
        self.started = False

        if WATCHMAN:
            with open(os.devnull, 'wb') as devnull:
                subprocess.call([
                    WATCHMAN, 'trigger-del', ROOT,
                    'mozreview-%s' % os.path.basename(self._path)
                ],
                                stdout=devnull,
                                stderr=subprocess.STDOUT)

    def refresh(self,
                verbose=False,
                refresh_reviewboard=False,
                autoland_only=False):
        """Refresh a running cluster with latest version of code.

        This only updates code from the v-c-t repo. Not all containers
        are currently updated.
        """
        with self._docker.vct_container(verbose=verbose) as vct_state:
            # We update rbweb by rsyncing state and running the refresh script.
            rsync_port = vct_state['NetworkSettings']['Ports']['873/tcp'][0][
                'HostPort']
            url = 'rsync://%s:%s/vct-mount/' % (self._docker.docker_hostname,
                                                rsync_port)

            def execute(name, cid, command):
                res = self._docker.execute(cid,
                                           command,
                                           stream=True,
                                           stderr=verbose,
                                           stdout=verbose)
                for msg in res:
                    if verbose:
                        msg = msg.rstrip().lstrip('\n')
                        for line in msg.splitlines():
                            if line != '':
                                print('%s> %s' % (name, line))

            def refresh(name, cid):
                execute(
                    name, cid,
                    ['/refresh', url, 'all' if refresh_reviewboard else ''])

            with futures.ThreadPoolExecutor(4) as e:
                if not autoland_only:
                    e.submit(refresh, 'rbweb', self.rbweb_id)
                    e.submit(refresh, 'hgrb', self.hgrb_id)
                    e.submit(execute, 'bmoweb', self.bmoweb_id,
                             ['/usr/bin/supervisorctl', 'restart', 'httpd'])
                e.submit(refresh, 'autoland', self.autoland_id)

    def start_autorefresh(self):
        """Enable auto refreshing of the cluster when changes are made.

        Watchman will be configured to start watching the source directory.
        When relevant files are changed, containers will be synchronized
        automatically.

        When enabled, this removes overhead from developers having to manually
        refresh state during development.
        """
        if not WATCHMAN:
            raise Exception('watchman binary not found')
        subprocess.check_call([WATCHMAN, 'watch-project', ROOT])
        name = 'mozreview-%s' % os.path.basename(self._path)

        expression = [
            'anyof',
            ['dirname', 'hgext/reviewboard'],
            ['dirname', 'pylib/mozreview'],
            ['dirname', 'pylib/reviewboardmods'],
            ['dirname', 'reviewboard-fork/djblets'],
            ['dirname', 'reviewboard-fork/reviewboard'],
        ]
        command = [
            '%s/scripts/watchman-refresh-wrapper' % ROOT, ROOT, self._path
        ]
        data = json.dumps([
            'trigger', ROOT, {
                'name': name,
                'chdir': ROOT,
                'expression': expression,
                'command': command,
                'append_files': True,
            }
        ])
        p = subprocess.Popen([WATCHMAN, '-j'], stdin=subprocess.PIPE)
        p.communicate(data)
        res = p.wait()
        if res != 0:
            raise Exception('error creating watchman trigger')

    def repo_urls(self, name):
        """Obtain the http:// and ssh:// URLs for a review repo."""
        http_url = '%s%s' % (self.mercurial_url, name)
        ssh_url = 'ssh://%s:%d/%s' % (self.ssh_hostname, self.ssh_port, name)

        return http_url, ssh_url

    def create_repository(self, name):
        http_url, ssh_url = self.repo_urls(name)

        rb = self.get_reviewboard()
        rbid = rb.add_repository(os.path.dirname(name) or name,
                                 http_url,
                                 bugzilla_url=self.bugzilla_url)

        self._docker.execute(self.hgrb_id, ['/create-repo', name, str(rbid)])

        if self.autoland_id:
            time.sleep(1)
            self._docker.execute(self.autoland_id, ['/clone-repo', name])

        return http_url, ssh_url, rbid

    def clone(self, repo, dest, username=None):
        """Clone and configure a review repository.

        The specified review repo will be cloned and configured such that it is
        bound to this MozReview instance. If a username is specified, all
        operations will be performed as that user.
        """
        http_url, ssh_url = self.repo_urls(repo)

        subprocess.check_call([self._hg, 'clone', http_url, dest], cwd='/')

        mrext = os.path.join(ROOT, 'testing', 'mozreview-repo.py')
        mrssh = os.path.join(ROOT, 'testing', 'mozreview-ssh')
        rbext = os.path.join(ROOT, 'hgext', 'reviewboard', 'client.py')

        with open(os.path.join(dest, '.hg', 'hgrc'), 'w') as fh:
            lines = [
                '[paths]',
                'default = %s' % http_url,
                'default-push = %s' % ssh_url,
                '',
                '[extensions]',
                'mozreview-repo = %s' % mrext,
                'reviewboard = %s' % rbext,
                '',
                '[ui]',
                'ssh = %s' % mrssh,
                '',
                # TODO use ircnick from the current user, if available.
                '[mozilla]',
                'ircnick = dummy',
                '',
                '[mozreview]',
                'home = %s' % self._path,
            ]

            if username:
                lines.append('username = %s' % username)

            fh.write('\n'.join(lines))
            fh.write('\n')

    def get_local_repository(self,
                             path,
                             ircnick=None,
                             bugzilla_username=None,
                             bugzilla_apikey=None):
        """Obtain a LocalMercurialRepository for the named server repository.

        Call this with the same argument passed to ``create_repository()``
        to obtain an object to interface with a local clone of that server
        repository.

        If bugzilla credentials are passed, they will be defined in the
        repository's hgrc.

        The repository is configured to be in deterministic mode. Therefore
        these repositories are suitable for use in tests.
        """
        localrepos = os.path.join(self._path, 'localrepos')
        try:
            os.mkdir(localrepos)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        local_path = os.path.join(localrepos, os.path.basename(path))

        http_url, ssh_url = self.repo_urls(path)

        # TODO make pushes via SSH work (it doesn't work outside of Mercurial
        # tests because dummy expects certain environment variables).
        return LocalMercurialRepository(self._path,
                                        self._hg,
                                        local_path,
                                        http_url,
                                        push_url=ssh_url,
                                        ircnick=ircnick,
                                        bugzilla_username=bugzilla_username,
                                        bugzilla_apikey=bugzilla_apikey)

    def create_user_api_key(self,
                            email,
                            sync_to_reviewboard=True,
                            description=''):
        """Creates an API key for the given user.

        This creates an API key in Bugzilla and then triggers the
        auth-delegation callback to register the key with Review Board. Note
        that this also logs the user in, although we don't record the session
        cookie anywhere so this shouldn't have an effect on subsequent
        interactions with Review Board.
        """
        api_key = self._docker.execute(self.bmoweb_id, [
            '/var/lib/bugzilla/bugzilla/scripts/issue-api-key.pl', email,
            description
        ],
                                       stdout=True).strip()

        assert len(api_key) == 40

        if not sync_to_reviewboard:
            return api_key

        # When running tests in parallel, the auth callback can time out.
        # Try up to 3 times before giving up.
        url = self.reviewboard_url + 'mozreview/bmo_auth_callback/'
        data = {'client_api_login': email, 'client_api_key': api_key}

        for i in range(3):
            response = requests.post(url, data=json.dumps(data))
            if response.status_code == 200:
                result = response.json()['result']
                break
        else:
            raise Exception('Failed to successfully run the BMO auth POST '
                            'callback.')

        params = {
            'client_api_login': email,
            'callback_result': result,
            'secret': 'mozreview',
        }
        cookies = {'bmo_auth_secret': params['secret']}

        for i in range(3):
            if requests.get(url, params=params,
                            cookies=cookies).status_code == 200:
                break
        else:
            raise Exception('Failed to successfully run the BMO auth GET '
                            'callback.')

        return api_key

    def create_user(self,
                    email,
                    password,
                    fullname,
                    bugzilla_groups=None,
                    uid=None,
                    username=None,
                    key_filename=None,
                    scm_level=None,
                    api_key=True,
                    bugzilla_email=None):
        """Create a new user.

        This will create a user in at least Bugzilla. If the ``uid`` argument
        is specified, an LDAP user will be created as well.

        ``email`` is the email address of the user.
        ``password`` is the plain text Bugzilla password.
        ``fullname`` is the full name of the user. This is stored in both
        Bugzilla and the system account for the user (if an LDAP user is being
        created).
        ``bugzilla_groups`` is an iterable of Bugzilla groups to add the user
        to.
        ``uid`` is the numeric UID for the created system/LDAP account.
        ``username`` is the UNIX username for this user. It defaults to the
        username part of the email address.
        ``key_filename`` is a path to an ssh key. This is used only if ``uid``
        is given. If ``uid`` is given but ``key_filename`` is not specified,
        the latter defaults to <mozreview path>/keys/<email>.
        ``scm_level`` defines the source code level access to grant to this
        user. Supported levels are ``1``, ``2``, and ``3``. If not specified,
        the user won't be able to push to any repos.
        ``bugzilla_email`` set the bugzillaEmail LDAP attribute to this instead
        of ``email``.
        """
        bugzilla_groups = bugzilla_groups or []

        b = self.get_bugzilla()

        if not username:
            username = email[0:email.index('@')]

        res = {
            'bugzilla': b.create_user(email, password, fullname),
        }

        for g in bugzilla_groups:
            b.add_user_to_group(email, g)

        if api_key:
            res['bugzilla']['api_key'] = self.create_user_api_key(
                email, description='mozreview')

        # Create an LDAP account as well.
        if uid:
            if key_filename is None:
                key_filename = os.path.join(self._path, 'keys', email)

            lr = self.get_ldap().create_user(email,
                                             username,
                                             uid,
                                             fullname,
                                             key_filename=key_filename,
                                             scm_level=scm_level,
                                             bugzilla_email=bugzilla_email)

            res.update(lr)

        credentials_path = os.path.join(self._path, 'credentials', email)
        with open(credentials_path, 'wb') as fh:
            fh.write(password)

        return res

    @property
    def _hg(self):
        for path in os.environ['PATH'].split(os.pathsep):
            hg = os.path.join(path, 'hg')
            if os.path.isfile(hg):
                return hg

        raise Exception('could not find hg executable')
class MozReview(object):
    """Interface to MozService service.

    This class can be used to create and control MozReview instances.
    """

    def __init__(self, path, web_image=None, hgrb_image=None,
                 ldap_image=None, pulse_image=None, rbweb_image=None,
                 autolanddb_image=None, autoland_image=None,
                 hgweb_image=None, treestatus_image=None):
        if not path:
            raise Exception('You must specify a path to create an instance')
        path = os.path.abspath(path)
        self._path = path

        self.started = False

        self.web_image = web_image
        self.hgrb_image = hgrb_image
        self.ldap_image = ldap_image
        self.pulse_image = pulse_image
        self.rbweb_image = rbweb_image
        self.autolanddb_image = autolanddb_image
        self.autoland_image = autoland_image
        self.hgweb_image = hgweb_image
        self.treestatus_image = treestatus_image

        self._name = os.path.dirname(path)

        if not os.path.exists(path):
            os.mkdir(path)

        keys_path = os.path.join(path, 'keys')
        try:
            os.mkdir(keys_path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        credentials_path = os.path.join(path, 'credentials')
        try:
            os.mkdir(credentials_path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        self._state_path = os.path.join(path, 'state.json')

        docker_state = os.path.join(path, '.dockerstate')

        self._docker_state = docker_state

        self.bugzilla_username = None
        self.bugzilla_password = None
        self.docker_env = {}

        if os.path.exists(self._state_path):
            with open(self._state_path, 'rb') as fh:
                state = json.load(fh)

                for k, v in state.items():
                    setattr(self, k, v)

        # Preserve Docker settings from last time.
        #
        # This was introduced to make watchman happy, as its triggers may not
        # inherit environment variables.
        for k, v in self.docker_env.items():
            os.environ[k] = v

        docker_url, tls = params_from_env(os.environ)
        self._docker = Docker(docker_state, docker_url, tls=tls)

        if not self._docker.is_alive():
            raise DockerNotAvailable('Docker is not available.')

    def get_bugzilla(self, username=None, password=None):
        username = username or self.bugzilla_username or '*****@*****.**'
        password = password or self.bugzilla_password or 'password'

        return Bugzilla(self.bugzilla_url, username=username, password=password)

    def get_reviewboard(self):
        """Obtain a MozReviewBoard instance tied to this MozReview instance."""
        return MozReviewBoard(self._docker, self.rbweb_id,
                              self.reviewboard_url,
                              bugzilla_url=self.bugzilla_url,
                              pulse_host=self.pulse_host,
                              pulse_port=self.pulse_port)

    def get_ldap(self):
        """Obtain an LDAP instance connected to the LDAP server in this instance."""
        return LDAP(self.ldap_uri, 'cn=admin,dc=mozilla', 'password')

    def start(self, bugzilla_port=None, reviewboard_port=None,
              mercurial_port=None, pulse_port=None, verbose=False,
              web_image=None, hgrb_image=None,
              ldap_image=None, ldap_port=None, pulse_image=None,
              rbweb_image=None, ssh_port=None,
              hgweb_image=None, hgweb_port=None,
              autolanddb_image=None, autoland_image=None, autoland_port=None,
              treestatus_image=None, treestatus_port=None, max_workers=None):
        """Start a MozReview instance."""
        if self.started:
            raise Exception('MozReview instance has already been started')

        if not bugzilla_port:
            bugzilla_port = get_available_port()
        if not reviewboard_port:
            reviewboard_port = get_available_port()
        if not mercurial_port:
            mercurial_port = get_available_port()
        if not ldap_port:
            ldap_port = get_available_port()
        if not ssh_port:
            ssh_port = get_available_port()
        if not pulse_port:
            pulse_port = get_available_port()
        if not autoland_port:
            autoland_port = get_available_port()
        if not hgweb_port:
            hgweb_port = get_available_port()
        if not treestatus_port:
            treestatus_port = get_available_port()

        web_image = web_image or self.web_image
        hgrb_image = hgrb_image or self.hgrb_image
        ldap_image = ldap_image or self.ldap_image
        pulse_image = pulse_image or self.pulse_image
        rbweb_image = rbweb_image or self.rbweb_image
        autolanddb_image = autolanddb_image or self.autolanddb_image
        autoland_image = autoland_image or self.autoland_image
        hgweb_image = hgweb_image or self.hgweb_image
        treestatus_image = treestatus_image or self.treestatus_image

        if not os.path.exists(os.path.join(ROOT, 'reviewboard-fork')):
            raise Exception('Failed to find reviewboard-fork. '
                            'Please run create-test-environment.')

        self.started = True
        mr_info = self._docker.start_mozreview(
                cluster=self._name,
                http_port=bugzilla_port,
                pulse_port=pulse_port,
                web_image=web_image,
                hgrb_image=hgrb_image,
                ldap_image=ldap_image,
                ldap_port=ldap_port,
                pulse_image=pulse_image,
                rbweb_image=rbweb_image,
                rbweb_port=reviewboard_port,
                ssh_port=ssh_port,
                hg_port=mercurial_port,
                autolanddb_image=autolanddb_image,
                autoland_image=autoland_image,
                autoland_port=autoland_port,
                hgweb_image=hgweb_image,
                hgweb_port=hgweb_port,
                treestatus_image=treestatus_image,
                treestatus_port=treestatus_port,
                max_workers=max_workers,
                verbose=verbose)

        self.bmoweb_id = mr_info['web_id']

        self.bugzilla_url = mr_info['bugzilla_url']
        bugzilla = self.get_bugzilla()

        self.reviewboard_url = mr_info['reviewboard_url']
        self.rbweb_id = mr_info['rbweb_id']

        self.autoland_id = mr_info['autoland_id']
        self.autoland_url = mr_info['autoland_url']
        self.pulse_id = mr_info['pulse_id']
        self.pulse_host = mr_info['pulse_host']
        self.pulse_port = mr_info['pulse_port']

        self.admin_username = bugzilla.username
        self.admin_password = bugzilla.password
        self.hg_rb_username = "******"
        self.hg_rb_email = "*****@*****.**"
        self.hg_rb_password = "******"
        self.ldap_uri = mr_info['ldap_uri']
        self.hgrb_id = mr_info['hgrb_id']
        self.ssh_hostname = mr_info['ssh_hostname']
        self.ssh_port = mr_info['ssh_port']
        self.mercurial_url = mr_info['mercurial_url']
        self.hgweb_id = mr_info['hgweb_id']
        self.hgweb_url = mr_info['hgweb_url']

        self.treestatus_id = mr_info['treestatus_id']
        self.treestatus_url = mr_info['treestatus_url']

        rb = self.get_reviewboard()

        # It is tempting to put the user creation inside the futures
        # block so it runs concurrently. However, there appeared to be
        # race conditions here. The hg_rb_username ("mozreview") user
        # was sometimes not getting created and this led to intermittent
        # test failures in test-auth.t.

        # Ensure admin user is present.
        rb.login_user(bugzilla.username, bugzilla.password)

        # Ensure the mozreview hg user is present and has privileges.
        # This has to occur after the admin user is logged in to avoid
        # race conditions with user IDs.
        rb.create_local_user(self.hg_rb_username, self.hg_rb_email,
                             self.hg_rb_password)

        with limited_threadpoolexecutor(7, max_workers) as e:
            # Ensure admin user had admin privileges.
            e.submit(rb.make_admin, bugzilla.username)

            # Ensure mozreview user has permissions for testing.
            e.submit(rb.grant_permission, self.hg_rb_username,
                     'Can change ldap assocation for all users')

            e.submit(rb.grant_permission, self.hg_rb_username,
                     'Can verify DiffSet legitimacy')

            e.submit(rb.grant_permission, self.hg_rb_username,
                     'Can enable or disable autoland for a repository')

            # Tell hgrb about URLs.
            e.submit(self._docker.execute, self.hgrb_id,
                     ['/set-urls', self.bugzilla_url, self.reviewboard_url])

            # Define site domain and hostname in rbweb. This is necessary so it
            # constructs self-referential URLs properly.
            e.submit(self._docker.execute, self.rbweb_id,
                     ['/set-site-url', self.reviewboard_url,
                      self.autoland_url, self.bugzilla_url])

            # Tell Bugzilla about Review Board URL.
            e.submit(self._docker.execute, mr_info['web_id'],
                     ['/set-urls', self.reviewboard_url])

        self.create_user_api_key(bugzilla.username, description='mozreview')

        hg_ssh_host_key = self._docker.get_file_content(
                mr_info['hgrb_id'],
                '/etc/ssh/ssh_host_rsa_key.pub').rstrip()
        key_type, key_key = hg_ssh_host_key.split()

        assert key_type == 'ssh-rsa'
        key = paramiko.rsakey.RSAKey(data=paramiko.py3compat.decodebytes(key_key))

        hostkeys_path = os.path.join(self._path, 'ssh-known-hosts')
        load_path = hostkeys_path if os.path.exists(hostkeys_path) else None
        hostkeys = paramiko.hostkeys.HostKeys(filename=load_path)
        hoststring = '[%s]:%d' % (mr_info['ssh_hostname'], mr_info['ssh_port'])
        hostkeys.add(hoststring, key_type, key)
        hostkeys.save(hostkeys_path)

        with open(os.path.join(self._path, 'ssh_config'), 'wb') as fh:
            fh.write(SSH_CONFIG.format(
                known_hosts=hostkeys_path,
                hostname=self.ssh_hostname,
                port=self.ssh_port))

        state = {
            'bmoweb_id': self.bmoweb_id,
            'bugzilla_url': self.bugzilla_url,
            'reviewboard_url': self.reviewboard_url,
            'rbweb_id': self.rbweb_id,
            'mercurial_url': self.mercurial_url,
            'admin_username': bugzilla.username,
            'admin_password': bugzilla.password,
            'ldap_uri': self.ldap_uri,
            'pulse_id': self.pulse_id,
            'pulse_host': self.pulse_host,
            'pulse_port': self.pulse_port,
            'autoland_url': self.autoland_url,
            'autoland_id': self.autoland_id,
            'hgrb_id': self.hgrb_id,
            'hgweb_url': self.hgweb_url,
            'hgweb_id': self.hgweb_id,
            'ssh_hostname': self.ssh_hostname,
            'ssh_port': self.ssh_port,
            'treestatus_url': self.treestatus_url,
            'treestatus_id': self.treestatus_id,
            'docker_env': {k: v for k, v in os.environ.items() if k.startswith('DOCKER')}
        }

        with open(self._state_path, 'wb') as fh:
            json.dump(state, fh, indent=2, sort_keys=True)

    def stop(self):
        """Stop all services associated with this MozReview instance."""
        self._docker.stop_bmo(self._name)
        self.started = False

        if WATCHMAN:
            with open(os.devnull, 'wb') as devnull:
                subprocess.call([WATCHMAN, 'trigger-del', ROOT,
                                 'mozreview-%s' % os.path.basename(self._path)],
                                stdout=devnull, stderr=subprocess.STDOUT)

    def refresh(self, verbose=False, refresh_reviewboard=False):
        """Refresh a running cluster with latest version of code.

        This only updates code from the v-c-t repo. Not all containers
        are currently updated.
        """
        with self._docker.vct_container(verbose=verbose) as vct_state:
            # We update rbweb by rsyncing state and running the refresh script.
            rsync_port = vct_state['NetworkSettings']['Ports']['873/tcp'][0]['HostPort']
            url = 'rsync://%s:%s/vct-mount/' % (self._docker.docker_hostname,
                                                rsync_port)

            def execute(name, cid, command):
                res = self._docker.execute(cid, command, stream=True,
                                           stderr=verbose, stdout=verbose)
                for msg in res:
                    if verbose:
                        msg = msg.rstrip().lstrip('\n')
                        for line in msg.splitlines():
                            if line != '':
                                print('%s> %s' % (name, line))

            def refresh(name, cid):
                execute(name, cid, ['/refresh', url,
                                    'all' if refresh_reviewboard else ''])

            with futures.ThreadPoolExecutor(4) as e:
                e.submit(refresh, 'rbweb', self.rbweb_id)
                e.submit(refresh, 'hgrb', self.hgrb_id)
                # TODO add hgweb support for refreshing.
                e.submit(execute, 'bmoweb', self.bmoweb_id,
                         ['/usr/bin/supervisorctl', 'restart', 'httpd'])

    def start_autorefresh(self):
        """Enable auto refreshing of the cluster when changes are made.

        Watchman will be configured to start watching the source directory.
        When relevant files are changed, containers will be synchronized
        automatically.

        When enabled, this removes overhead from developers having to manually
        refresh state during development.
        """
        if not WATCHMAN:
            raise Exception('watchman binary not found')
        subprocess.check_call([WATCHMAN, 'watch-project', ROOT])
        name = 'mozreview-%s' % os.path.basename(self._path)

        expression = [
            'anyof',
            ['dirname', 'hgext/reviewboard'],
            ['dirname', 'pylib/mozreview'],
            ['dirname', 'pylib/rbbz'],
            ['dirname', 'pylib/reviewboardmods'],
            ['dirname', 'reviewboard-fork/djblets'],
            ['dirname', 'reviewboard-fork/reviewboard'],
        ]
        command = ['%s/scripts/watchman-refresh-wrapper' % ROOT, ROOT,
                   self._path]
        data = json.dumps(['trigger', ROOT, {
            'name': name,
            'chdir': ROOT,
            'expression': expression,
            'command': command,
            'append_files': True,
        }])
        p = subprocess.Popen([WATCHMAN, '-j'], stdin=subprocess.PIPE)
        p.communicate(data)
        res = p.wait()
        if res != 0:
            raise Exception('error creating watchman trigger')

    def repo_urls(self, name):
        """Obtain the http:// and ssh:// URLs for a review repo."""
        http_url = '%s%s' % (self.mercurial_url, name)
        ssh_url = 'ssh://%s:%d/%s' % (self.ssh_hostname, self.ssh_port, name)

        return http_url, ssh_url

    def create_repository(self, name):
        http_url, ssh_url = self.repo_urls(name)

        rb = self.get_reviewboard()
        rbid = rb.add_repository(os.path.dirname(name) or name, http_url,
                                 bugzilla_url=self.bugzilla_url)

        self._docker.execute(self.hgrb_id,
                            ['/create-repo', name, str(rbid)])

        if self.autoland_id:
            time.sleep(1)
            self._docker.execute(self.autoland_id, ['/clone-repo', name])

        return http_url, ssh_url, rbid

    def clone(self, repo, dest, username=None):
        """Clone and configure a review repository.

        The specified review repo will be cloned and configured such that it is
        bound to this MozReview instance. If a username is specified, all
        operations will be performed as that user.
        """
        http_url, ssh_url = self.repo_urls(repo)

        subprocess.check_call([self._hg, 'clone', http_url, dest], cwd='/')

        mrext = os.path.join(ROOT, 'testing', 'mozreview-repo.py')
        mrssh = os.path.join(ROOT, 'testing', 'mozreview-ssh')
        rbext = os.path.join(ROOT, 'hgext', 'reviewboard', 'client.py')

        with open(os.path.join(dest, '.hg', 'hgrc'), 'w') as fh:
            lines = [
                '[paths]',
                'default = %s' % http_url,
                'default-push = %s' % ssh_url,
                '',
                '[extensions]',
                'mozreview-repo = %s' % mrext,
                'reviewboard = %s' % rbext,
                '',
                '[ui]',
                'ssh = %s' % mrssh,
                '',
                # TODO use ircnick from the current user, if available.
                '[mozilla]',
                'ircnick = dummy',
                '',
                '[mozreview]',
                'home = %s' % self._path,
            ]

            if username:
                lines.append('username = %s' % username)

            fh.write('\n'.join(lines))
            fh.write('\n')

    def get_local_repository(self, path, ircnick=None,
                             bugzilla_username=None,
                             bugzilla_apikey=None):
        """Obtain a LocalMercurialRepository for the named server repository.

        Call this with the same argument passed to ``create_repository()``
        to obtain an object to interface with a local clone of that server
        repository.

        If bugzilla credentials are passed, they will be defined in the
        repository's hgrc.

        The repository is configured to be in deterministic mode. Therefore
        these repositories are suitable for use in tests.
        """
        localrepos = os.path.join(self._path, 'localrepos')
        try:
            os.mkdir(localrepos)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        local_path = os.path.join(localrepos, os.path.basename(path))

        http_url, ssh_url = self.repo_urls(path)

        # TODO make pushes via SSH work (it doesn't work outside of Mercurial
        # tests because dummy expects certain environment variables).
        return LocalMercurialRepository(self._path, self._hg, local_path,
                                        http_url, push_url=ssh_url, ircnick=ircnick,
                                        bugzilla_username=bugzilla_username,
                                        bugzilla_apikey=bugzilla_apikey)

    def create_user_api_key(self, email, sync_to_reviewboard=True,
                            description=''):
        """Creates an API key for the given user.

        This creates an API key in Bugzilla and then triggers the
        auth-delegation callback to register the key with Review Board. Note
        that this also logs the user in, although we don't record the session
        cookie anywhere so this shouldn't have an effect on subsequent
        interactions with Review Board.
        """
        api_key = self._docker.execute(
            self.bmoweb_id,
            ['/var/lib/bugzilla/bugzilla/scripts/issue-api-key.pl',
             email, description], stdout=True).strip()

        assert len(api_key) == 40

        if not sync_to_reviewboard:
            return api_key

        # When running tests in parallel, the auth callback can time out.
        # Try up to 3 times before giving up.
        url = self.reviewboard_url + 'mozreview/bmo_auth_callback/'
        data = {'client_api_login': email, 'client_api_key': api_key}

        for i in range(3):
            response = requests.post(url, data=json.dumps(data))
            if response.status_code == 200:
                result = response.json()['result']
                break
        else:
            raise Exception('Failed to successfully run the BMO auth POST '
                            'callback.')

        params = {
            'client_api_login': email,
            'callback_result': result,
            'secret': 'mozreview',
        }
        cookies = {'bmo_auth_secret': params['secret']}

        for i in range(3):
            if requests.get(url, params=params,
                            cookies=cookies).status_code == 200:
                break
        else:
            raise Exception('Failed to successfully run the BMO auth GET '
                            'callback.')

        return api_key

    def create_user(self, email, password, fullname, bugzilla_groups=None,
                    uid=None, username=None, key_filename=None, scm_level=None,
                    api_key=True, bugzilla_email=None):
        """Create a new user.

        This will create a user in at least Bugzilla. If the ``uid`` argument
        is specified, an LDAP user will be created as well.

        ``email`` is the email address of the user.
        ``password`` is the plain text Bugzilla password.
        ``fullname`` is the full name of the user. This is stored in both
        Bugzilla and the system account for the user (if an LDAP user is being
        created).
        ``bugzilla_groups`` is an iterable of Bugzilla groups to add the user
        to.
        ``uid`` is the numeric UID for the created system/LDAP account.
        ``username`` is the UNIX username for this user. It defaults to the
        username part of the email address.
        ``key_filename`` is a path to an ssh key. This is used only if ``uid``
        is given. If ``uid`` is given but ``key_filename`` is not specified,
        the latter defaults to <mozreview path>/keys/<email>.
        ``scm_level`` defines the source code level access to grant to this
        user. Supported levels are ``1``, ``2``, and ``3``. If not specified,
        the user won't be able to push to any repos.
        ``bugzilla_email`` set the bugzillaEmail LDAP attribute to this instead
        of ``email``.
        """
        bugzilla_groups = bugzilla_groups or []

        b = self.get_bugzilla()

        if not username:
            username = email[0:email.index('@')]

        res = {
            'bugzilla': b.create_user(email, password, fullname),
        }

        for g in bugzilla_groups:
            b.add_user_to_group(email, g)

        if api_key:
            res['bugzilla']['api_key'] = self.create_user_api_key(
                email, description='mozreview')

        # Create an LDAP account as well.
        if uid:
            if key_filename is None:
                key_filename = os.path.join(self._path, 'keys', email)

            lr = self.get_ldap().create_user(email, username, uid,
                                             fullname,
                                             key_filename=key_filename,
                                             scm_level=scm_level,
                                             bugzilla_email=bugzilla_email)

            res.update(lr)

        credentials_path = os.path.join(self._path, 'credentials', email)
        with open(credentials_path, 'wb') as fh:
            fh.write(password)

        return res

    @property
    def _hg(self):
        for path in os.environ['PATH'].split(os.pathsep):
            hg = os.path.join(path, 'hg')
            if os.path.isfile(hg):
                return hg

        raise Exception('could not find hg executable')
class MozReview(object):
    """Interface to MozService service.

    This class can be used to create and control MozReview instances.
    """

    def __init__(self, path):
        if not path:
            raise Exception('You must specify a path to create an instance')
        path = os.path.abspath(path)
        self._path = path

        self._name = os.path.dirname(path)

        if not os.path.exists(path):
            os.mkdir(path)

        docker_state = os.path.join(path, 'docker-state.json')

        self._docker_state = docker_state

        docker_url, tls = params_from_env(os.environ)
        self._docker = Docker(docker_state, docker_url, tls=tls)

        if not self._docker.is_alive():
            raise Exception('Docker is not available.')

    def get_bugzilla(self, url, username='******', password='******'):
        return Bugzilla(url, username=username, password=password)

    def start(self, bugzilla_port=None, reviewboard_port=None,
            mercurial_port=None, verbose=False):
        """Start a MozReview instance."""
        if not bugzilla_port:
            bugzilla_port = get_available_port()
        if not reviewboard_port:
            reviewboard_port = get_available_port()
        if not mercurial_port:
            mercurial_port = get_available_port()

        db_image, web_image = self._docker.build_bmo(verbose=verbose)

        bugzilla_url = self._docker.start_bmo(cluster=self._name,
                hostname=None, http_port=bugzilla_port,
                db_image=db_image, web_image=web_image)[0]
        with open(self._bugzilla_url_path, 'wb') as fh:
            fh.write(bugzilla_url)

        bugzilla = self.get_bugzilla(bugzilla_url)

        rb = MozReviewBoard(self._path, bugzilla_url=bugzilla_url)
        rb.create()
        reviewboard_pid = rb.start(reviewboard_port)

        reviewboard_url = 'http://localhost:%s/' % reviewboard_port

        self.bugzilla_url = bugzilla_url
        self.reviewboard_url = reviewboard_url
        self.reviewboard_pid = reviewboard_pid
        self.admin_username = bugzilla.username
        self.admin_password = bugzilla.password

        mercurial_pid = self._start_mercurial_server(mercurial_port)

        self.mercurial_url = 'http://localhost:%s/' % mercurial_port
        with open(self._hg_url_path, 'w') as fh:
            fh.write(self.mercurial_url)

        self.mercurial_pid = mercurial_pid

    def stop(self):
        """Stop all services associated with this MozReview instance."""
        if os.path.exists(self._hg_pid_path):
            with open(self._hg_pid_path, 'rb') as fh:
                pid = int(fh.read().strip())
                kill(pid)

            os.unlink(self._hg_pid_path)

        rb = MozReviewBoard(self._path)
        rb.stop()

        self._docker.stop_bmo(self._name)

    def create_repository(self, path):
        with open(self._hg_url_path, 'rb') as fh:
            url = '%s%s' % (fh.read(), path)

        full_path = os.path.join(self._path, 'repos', path)

        env = dict(os.environ)
        env['HGRCPATH'] = '/dev/null'
        subprocess.check_call([self._hg, 'init', full_path], env=env)

        rb = MozReviewBoard(self._path)
        rbid = rb.add_repository(os.path.dirname(path), url)

        with open(os.path.join(full_path, '.hg', 'hgrc'), 'w') as fh:
            fh.write('\n'.join([
                '[reviewboard]',
                'repoid = %s' % rbid,
                '',
            ]))

        return url, rbid

    def create_user(self, email, password, fullname):
        with open(self._bugzilla_url_path, 'r') as fh:
            url = fh.read()

        b = self.get_bugzilla(url)
        return b.create_user(email, password, fullname)

    @property
    def _hg_pid_path(self):
        return os.path.join(self._path, 'hg.pid')

    @property
    def _hg_url_path(self):
        return os.path.join(self._path, 'hg.url')

    @property
    def _hg(self):
        return os.path.join(ROOT, 'venv', 'bin', 'hg')

    @property
    def _bugzilla_url_path(self):
        return os.path.join(self._path, 'bugzilla.url')

    def _start_mercurial_server(self, port):
        repos_path = os.path.join(self._path, 'repos')
        if not os.path.exists(repos_path):
            os.mkdir(repos_path)

        rb_ext_path = os.path.join(ROOT, 'hgext', 'reviewboard', 'server.py')
        dummyssh = os.path.join(ROOT, 'pylib', 'mercurial-support', 'dummyssh')

        global_hgrc = os.path.join(self._path, 'hgrc')
        with open(global_hgrc, 'w') as fh:
            fh.write('\n'.join([
                '[phases]',
                'publish = False',
                '',
                '[ui]',
                'ssh = python "%s"' % dummyssh,
                '',
                '[reviewboard]',
                'url = %s' % self.reviewboard_url,
                '',
                '[extensions]',
                'reviewboard = %s' % rb_ext_path,
                '',
                '[web]',
                'push_ssl = False',
                'allow_push = *',
                '',
                '[paths]',
                '/ = %s/**' % repos_path,
                '',
            ]))

        env = os.environ.copy()
        env['HGRCPATH'] = '/dev/null'
        env['HGENCODING'] = 'UTF-8'
        args = [
            self._hg,
            'serve',
            '-d',
            '-p', str(port),
            '--pid-file', self._hg_pid_path,
            '--web-conf', global_hgrc,
            '--accesslog', os.path.join(self._path, 'hg.access.log'),
            '--errorlog', os.path.join(self._path, 'hg.error.log'),
        ]
        # We execute from / so Mercurial doesn't pick up config files
        # from parent directories.
        subprocess.check_call(args, env=env, cwd='/')

        with open(self._hg_pid_path, 'rb') as fh:
            pid = fh.read().strip()

        return pid
class MozReview(object):
    """Interface to MozService service.

    This class can be used to create and control MozReview instances.
    """

    def __init__(
        self,
        path,
        web_image=None,
        db_image=None,
        hgrb_image=None,
        ldap_image=None,
        pulse_image=None,
        rbweb_image=None,
        autolanddb_image=None,
        autoland_image=None,
        hgweb_image=None,
        treestatus_image=None,
    ):
        if not path:
            raise Exception("You must specify a path to create an instance")
        path = os.path.abspath(path)
        self._path = path

        self.started = False

        self.db_image = db_image
        self.web_image = web_image
        self.hgrb_image = hgrb_image
        self.ldap_image = ldap_image
        self.pulse_image = pulse_image
        self.rbweb_image = rbweb_image
        self.autolanddb_image = autolanddb_image
        self.autoland_image = autoland_image
        self.hgweb_image = hgweb_image
        self.treestatus_image = treestatus_image

        self._name = os.path.dirname(path)

        if not os.path.exists(path):
            os.mkdir(path)

        keys_path = os.path.join(path, "keys")
        try:
            os.mkdir(keys_path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        credentials_path = os.path.join(path, "credentials")
        try:
            os.mkdir(credentials_path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        self._state_path = os.path.join(path, "state.json")

        docker_state = os.path.join(path, ".dockerstate")

        self._docker_state = docker_state

        self.bugzilla_username = None
        self.bugzilla_password = None
        self.docker_env = {}

        if os.path.exists(self._state_path):
            with open(self._state_path, "rb") as fh:
                state = json.load(fh)

                for k, v in state.items():
                    setattr(self, k, v)

        # Preserve Docker settings from last time.
        #
        # This was introduced to make watchman happy, as its triggers may not
        # inherit environment variables.
        for k, v in self.docker_env.items():
            os.environ[k] = v

        docker_url, tls = params_from_env(os.environ)
        self._docker = Docker(docker_state, docker_url, tls=tls)

        if not self._docker.is_alive():
            raise DockerNotAvailable("Docker is not available.")

    def get_bugzilla(self, username=None, password=None):
        username = username or self.bugzilla_username or "*****@*****.**"
        password = password or self.bugzilla_password or "password"

        return Bugzilla(self.bugzilla_url, username=username, password=password)

    def get_reviewboard(self):
        """Obtain a MozReviewBoard instance tied to this MozReview instance."""
        return MozReviewBoard(
            self._docker,
            self.rbweb_id,
            self.reviewboard_url,
            bugzilla_url=self.bugzilla_url,
            pulse_host=self.pulse_host,
            pulse_port=self.pulse_port,
        )

    def get_ldap(self):
        """Obtain an LDAP instance connected to the LDAP server in this instance."""
        return LDAP(self.ldap_uri, "cn=admin,dc=mozilla", "password")

    def start(
        self,
        bugzilla_port=None,
        reviewboard_port=None,
        mercurial_port=None,
        pulse_port=None,
        verbose=False,
        db_image=None,
        web_image=None,
        hgrb_image=None,
        ldap_image=None,
        ldap_port=None,
        pulse_image=None,
        rbweb_image=None,
        ssh_port=None,
        hgweb_image=None,
        hgweb_port=None,
        autolanddb_image=None,
        autoland_image=None,
        autoland_port=None,
        treestatus_image=None,
        treestatus_port=None,
    ):
        """Start a MozReview instance."""
        if self.started:
            raise Exception("MozReview instance has already been started")

        if not bugzilla_port:
            bugzilla_port = get_available_port()
        if not reviewboard_port:
            reviewboard_port = get_available_port()
        if not mercurial_port:
            mercurial_port = get_available_port()
        if not ldap_port:
            ldap_port = get_available_port()
        if not ssh_port:
            ssh_port = get_available_port()
        if not pulse_port:
            pulse_port = get_available_port()
        if not autoland_port:
            autoland_port = get_available_port()
        if not hgweb_port:
            hgweb_port = get_available_port()
        if not treestatus_port:
            treestatus_port = get_available_port()

        db_image = db_image or self.db_image
        web_image = web_image or self.web_image
        hgrb_image = hgrb_image or self.hgrb_image
        ldap_image = ldap_image or self.ldap_image
        pulse_image = pulse_image or self.pulse_image
        rbweb_image = rbweb_image or self.rbweb_image
        autolanddb_image = autolanddb_image or self.autolanddb_image
        autoland_image = autoland_image or self.autoland_image
        hgweb_image = hgweb_image or self.hgweb_image
        treestatus_image = treestatus_image or self.treestatus_image

        self.started = True
        mr_info = self._docker.start_mozreview(
            cluster=self._name,
            http_port=bugzilla_port,
            pulse_port=pulse_port,
            db_image=db_image,
            web_image=web_image,
            hgrb_image=hgrb_image,
            ldap_image=ldap_image,
            ldap_port=ldap_port,
            pulse_image=pulse_image,
            rbweb_image=rbweb_image,
            rbweb_port=reviewboard_port,
            ssh_port=ssh_port,
            hg_port=mercurial_port,
            autolanddb_image=autolanddb_image,
            autoland_image=autoland_image,
            autoland_port=autoland_port,
            hgweb_image=hgweb_image,
            hgweb_port=hgweb_port,
            treestatus_image=treestatus_image,
            treestatus_port=treestatus_port,
            verbose=verbose,
        )

        self.bmoweb_id = mr_info["web_id"]
        self.bmodb_id = mr_info["db_id"]

        self.bugzilla_url = mr_info["bugzilla_url"]
        bugzilla = self.get_bugzilla()

        self.reviewboard_url = mr_info["reviewboard_url"]
        self.rbweb_id = mr_info["rbweb_id"]

        self.autoland_id = mr_info["autoland_id"]
        self.autoland_url = mr_info["autoland_url"]
        self.pulse_id = mr_info["pulse_id"]
        self.pulse_host = mr_info["pulse_host"]
        self.pulse_port = mr_info["pulse_port"]

        self.admin_username = bugzilla.username
        self.admin_password = bugzilla.password
        self.hg_rb_username = "******"
        self.hg_rb_email = "*****@*****.**"
        self.hg_rb_password = "******"
        self.ldap_uri = mr_info["ldap_uri"]
        self.hgrb_id = mr_info["hgrb_id"]
        self.ssh_hostname = mr_info["ssh_hostname"]
        self.ssh_port = mr_info["ssh_port"]
        self.mercurial_url = mr_info["mercurial_url"]
        self.hgweb_id = mr_info["hgweb_id"]
        self.hgweb_url = mr_info["hgweb_url"]

        self.treestatus_id = mr_info["treestatus_id"]
        self.treestatus_url = mr_info["treestatus_url"]

        # Ensure admin user is present and has admin privileges.
        def make_users():
            rb = self.get_reviewboard()

            # Ensure admin user is present and has admin privileges.
            rb.login_user(bugzilla.username, bugzilla.password)
            rb.make_admin(bugzilla.username)

            # Ensure the MozReview hg user is present and has privileges.
            rb.create_local_user(self.hg_rb_username, self.hg_rb_email, self.hg_rb_password)
            rb.grant_permission(self.hg_rb_username, "Can change ldap assocation for all users")

        with futures.ThreadPoolExecutor(4) as e:
            e.submit(make_users)

            # Tell hgrb about URLs.
            e.submit(self._docker.execute, self.hgrb_id, ["/set-urls", self.bugzilla_url, self.reviewboard_url])

            # Define site domain and hostname in rbweb. This is necessary so it
            # constructs self-referential URLs properly.
            e.submit(
                self._docker.execute,
                self.rbweb_id,
                ["/set-site-url", self.reviewboard_url, self.autoland_url, self.bugzilla_url],
            )

            # Tell Bugzilla about Review Board URL.
            e.submit(self._docker.execute, mr_info["web_id"], ["/set-urls", self.reviewboard_url])

        self.create_user_api_key(bugzilla.username)

        hg_ssh_host_key = self._docker.get_file_content(mr_info["hgrb_id"], "/etc/ssh/ssh_host_rsa_key.pub").rstrip()
        key_type, key_key = hg_ssh_host_key.split()

        assert key_type == "ssh-rsa"
        key = paramiko.rsakey.RSAKey(data=paramiko.py3compat.decodebytes(key_key))

        hostkeys_path = os.path.join(self._path, "ssh-known-hosts")
        load_path = hostkeys_path if os.path.exists(hostkeys_path) else None
        hostkeys = paramiko.hostkeys.HostKeys(filename=load_path)
        hoststring = "[%s]:%d" % (mr_info["ssh_hostname"], mr_info["ssh_port"])
        hostkeys.add(hoststring, key_type, key)
        hostkeys.save(hostkeys_path)

        with open(os.path.join(self._path, "ssh_config"), "wb") as fh:
            fh.write(SSH_CONFIG.format(known_hosts=hostkeys_path, hostname=self.ssh_hostname, port=self.ssh_port))

        state = {
            "bmoweb_id": self.bmoweb_id,
            "bmodb_id": self.bmodb_id,
            "bugzilla_url": self.bugzilla_url,
            "reviewboard_url": self.reviewboard_url,
            "rbweb_id": self.rbweb_id,
            "mercurial_url": self.mercurial_url,
            "admin_username": bugzilla.username,
            "admin_password": bugzilla.password,
            "ldap_uri": self.ldap_uri,
            "pulse_id": self.pulse_id,
            "pulse_host": self.pulse_host,
            "pulse_port": self.pulse_port,
            "autoland_url": self.autoland_url,
            "autoland_id": self.autoland_id,
            "hgrb_id": self.hgrb_id,
            "hgweb_url": self.hgweb_url,
            "hgweb_id": self.hgweb_id,
            "ssh_hostname": self.ssh_hostname,
            "ssh_port": self.ssh_port,
            "treestatus_url": self.treestatus_url,
            "treestatus_id": self.treestatus_id,
            "docker_env": {k: v for k, v in os.environ.items() if k.startswith("DOCKER")},
        }

        with open(self._state_path, "wb") as fh:
            json.dump(state, fh, indent=2, sort_keys=True)

    def stop(self):
        """Stop all services associated with this MozReview instance."""
        self._docker.stop_bmo(self._name)
        self.started = False

        if WATCHMAN:
            with open(os.devnull, "wb") as devnull:
                subprocess.call(
                    [WATCHMAN, "trigger-del", ROOT, "mozreview-%s" % os.path.basename(self._path)],
                    stdout=devnull,
                    stderr=subprocess.STDOUT,
                )

    def refresh(self, verbose=False):
        """Refresh a running cluster with latest version of code.

        This only updates code from the v-c-t repo. Not all containers
        are currently updated.
        """
        with self._docker.vct_container(verbose=verbose) as vct_state:
            # We update rbweb by rsyncing state and running the refresh script.
            rsync_port = vct_state["NetworkSettings"]["Ports"]["873/tcp"][0]["HostPort"]
            url = "rsync://%s:%s/vct-mount/" % (self._docker.docker_hostname, rsync_port)

            def execute(name, cid, command):
                res = self._docker.execute(cid, command, stream=True, stderr=verbose, stdout=verbose)
                for msg in res:
                    if verbose:
                        print("%s> %s" % (name, msg), end="")

            def refresh(name, cid):
                execute(name, cid, ["/refresh", url])

            with futures.ThreadPoolExecutor(4) as e:
                e.submit(refresh, "rbweb", self.rbweb_id)
                e.submit(refresh, "hgrb", self.hgrb_id)
                # TODO add hgweb support for refreshing.
                e.submit(execute, "bmoweb", self.bmoweb_id, ["/usr/bin/supervisorctl", "restart", "httpd"])

    def start_autorefresh(self):
        """Enable auto refreshing of the cluster when changes are made.

        Watchman will be configured to start watching the source directory.
        When relevant files are changed, containers will be synchronized
        automatically.

        When enabled, this removes overhead from developers having to manually
        refresh state during development.
        """
        if not WATCHMAN:
            raise Exception("watchman binary not found")
        subprocess.check_call([WATCHMAN, "watch-project", ROOT])
        name = "mozreview-%s" % os.path.basename(self._path)

        data = json.dumps(
            [
                "trigger",
                ROOT,
                {
                    "name": name,
                    "chdir": ROOT,
                    "expression": [
                        "anyof",
                        ["dirname", "hgext/reviewboard"],
                        ["dirname", "pylib/mozreview"],
                        ["dirname", "pylib/rbbz"],
                        ["dirname", "reviewboardmods"],
                    ],
                    "command": ["%s/mozreview" % ROOT, "refresh", self._path],
                },
            ]
        )
        p = subprocess.Popen([WATCHMAN, "-j"], stdin=subprocess.PIPE)
        p.communicate(data)
        res = p.wait()
        if res != 0:
            raise Exception("error creating watchman trigger")

    def repo_urls(self, name):
        """Obtain the http:// and ssh:// URLs for a review repo."""
        http_url = "%s%s" % (self.mercurial_url, name)
        ssh_url = "ssh://%s:%d/%s" % (self.ssh_hostname, self.ssh_port, name)

        return http_url, ssh_url

    def create_repository(self, name):
        http_url, ssh_url = self.repo_urls(name)

        rb = self.get_reviewboard()
        rbid = rb.add_repository(os.path.dirname(name) or name, http_url, bugzilla_url=self.bugzilla_url)

        self._docker.execute(self.hgrb_id, ["/create-repo", name, str(rbid)])

        if self.autoland_id:
            time.sleep(1)
            self._docker.execute(self.autoland_id, ["/clone-repo", name])

        return http_url, ssh_url, rbid

    def clone(self, repo, dest, username=None):
        """Clone and configure a review repository.

        The specified review repo will be cloned and configured such that it is
        bound to this MozReview instance. If a username is specified, all
        operations will be performed as that user.
        """
        http_url, ssh_url = self.repo_urls(repo)

        subprocess.check_call([self._hg, "clone", http_url, dest], cwd="/")

        mrext = os.path.join(ROOT, "testing", "mozreview-repo.py")
        mrssh = os.path.join(ROOT, "testing", "mozreview-ssh")
        rbext = os.path.join(ROOT, "hgext", "reviewboard", "client.py")

        with open(os.path.join(dest, ".hg", "hgrc"), "w") as fh:
            lines = [
                "[paths]",
                "default = %s" % http_url,
                "default-push = %s" % ssh_url,
                "",
                "[extensions]",
                "mozreview-repo = %s" % mrext,
                "reviewboard = %s" % rbext,
                "",
                "[ui]",
                "ssh = %s" % mrssh,
                "",
                # TODO use ircnick from the current user, if available.
                "[mozilla]",
                "ircnick = dummy",
                "",
                "[mozreview]",
                "home = %s" % self._path,
            ]

            if username:
                lines.append("username = %s" % username)

            fh.write("\n".join(lines))
            fh.write("\n")

    def get_local_repository(self, path, ircnick=None, bugzilla_username=None, bugzilla_apikey=None):
        """Obtain a LocalMercurialRepository for the named server repository.

        Call this with the same argument passed to ``create_repository()``
        to obtain an object to interface with a local clone of that server
        repository.

        If bugzilla credentials are passed, they will be defined in the
        repository's hgrc.

        The repository is configured to be in deterministic mode. Therefore
        these repositories are suitable for use in tests.
        """
        localrepos = os.path.join(self._path, "localrepos")
        try:
            os.mkdir(localrepos)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        local_path = os.path.join(localrepos, os.path.basename(path))

        http_url, ssh_url = self.repo_urls(path)

        # TODO make pushes via SSH work (it doesn't work outside of Mercurial
        # tests because dummy expects certain environment variables).
        return LocalMercurialRepository(
            self._path,
            self._hg,
            local_path,
            http_url,
            push_url=ssh_url,
            ircnick=ircnick,
            bugzilla_username=bugzilla_username,
            bugzilla_apikey=bugzilla_apikey,
        )

    def create_user_api_key(self, email, sync_to_reviewboard=True):
        """Creates an API key for the given user.

        This creates an API key in Bugzilla and then triggers the
        auth-delegation callback to register the key with Review Board. Note
        that this also logs the user in, although we don't record the session
        cookie anywhere so this shouldn't have an effect on subsequent
        interactions with Review Board.
        """
        api_key = self._docker.execute(
            self.bmoweb_id, ["/var/lib/bugzilla/bugzilla/scripts/issue-api-key.pl", email], stdout=True
        ).strip()

        assert len(api_key) == 40

        if not sync_to_reviewboard:
            return api_key

        # When running tests in parallel, the auth callback can time out.
        # Try up to 3 times before giving up.
        url = self.reviewboard_url + "mozreview/bmo_auth_callback/"
        data = {"client_api_login": email, "client_api_key": api_key}

        for i in range(3):
            response = requests.post(url, data=json.dumps(data))
            if response.status_code == 200:
                result = response.json()["result"]
                break
        else:
            raise Exception("Failed to successfully run the BMO auth POST " "callback.")

        params = {"client_api_login": email, "callback_result": result}

        for i in range(3):
            if requests.get(url, params=params).status_code == 200:
                break
        else:
            raise Exception("Failed to successfully run the BMO auth GET " "callback.")

        return api_key

    def create_user(
        self,
        email,
        password,
        fullname,
        bugzilla_groups=None,
        uid=None,
        username=None,
        key_filename=None,
        scm_level=None,
        api_key=True,
    ):
        """Create a new user.

        This will create a user in at least Bugzilla. If the ``uid`` argument
        is specified, an LDAP user will be created as well.

        ``email`` is the email address of the user.
        ``password`` is the plain text Bugzilla password.
        ``fullname`` is the full name of the user. This is stored in both
        Bugzilla and the system account for the user (if an LDAP user is being
        created).
        ``bugzilla_groups`` is an iterable of Bugzilla groups to add the user
        to.
        ``uid`` is the numeric UID for the created system/LDAP account.
        ``username`` is the UNIX username for this user. It defaults to the
        username part of the email address.
        ``key_filename`` is a path to an ssh key. This is used only if ``uid``
        is given. If ``uid`` is given but ``key_filename`` is not specified,
        the latter defaults to <mozreview path>/keys/<email>.
        ``scm_level`` defines the source code level access to grant to this
        user. Supported levels are ``1``, ``2``, and ``3``. If not specified,
        the user won't be able to push to any repos.
        """
        bugzilla_groups = bugzilla_groups or []

        b = self.get_bugzilla()

        if not username:
            username = email[0 : email.index("@")]

        res = {"bugzilla": b.create_user(email, password, fullname)}

        for g in bugzilla_groups:
            b.add_user_to_group(email, g)

        if api_key:
            res["bugzilla"]["api_key"] = self.create_user_api_key(email)

        # Create an LDAP account as well.
        if uid:
            if key_filename is None:
                key_filename = os.path.join(self._path, "keys", email)

            lr = self.get_ldap().create_user(
                email, username, uid, fullname, key_filename=key_filename, scm_level=scm_level
            )

            res.update(lr)

        credentials_path = os.path.join(self._path, "credentials", email)
        with open(credentials_path, "wb") as fh:
            fh.write(password)

        return res

    @property
    def _hg(self):
        for path in os.environ["PATH"].split(os.pathsep):
            hg = os.path.join(path, "hg")
            if os.path.isfile(hg):
                return hg

        raise Exception("could not find hg executable")