def __init__(self, context):
from vcttesting.docker import Docker, params_from_env
from vcttesting.hgmo import HgCluster
if 'HGMO_STATE_FILE' not in os.environ:
print('Do not know where to store state.')
print(
'Set the HGMO_STATE_FILE environment variable and try again.')
sys.exit(1)
if 'DOCKER_STATE_FILE' not in os.environ:
print('Do not where to store Docker state.')
print(
'Set the DOCKER_STATE_FILE environment variable and try again.'
)
sys.exit(1)
docker_url, tls = params_from_env(os.environ)
docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls)
if not docker.is_alive():
print('Docker not available')
sys.exit(1)
self.c = HgCluster(
docker,
os.environ['HGMO_STATE_FILE'],
ldap_image=os.environ.get('DOCKER_LDAP_IMAGE'),
master_image=os.environ.get('DOCKER_HGMASTER_IMAGE'),
web_image=os.environ.get('DOCKER_HGWEB_IMAGE'),
pulse_image=os.environ.get('DOCKER_PULSE_IMAGE'))
def has_docker():
from vcttesting.docker import Docker, params_from_env
url, tls = params_from_env(os.environ)
tf = tempfile.NamedTemporaryFile()
tf.close()
d = Docker(tf.name, url, tls=tls)
return d.is_alive()
def has_docker():
if 'SKIP_DOCKER_TESTS' in os.environ:
return False
from vcttesting.docker import Docker, params_from_env
url, tls = params_from_env(os.environ)
tf = tempfile.NamedTemporaryFile()
tf.close()
d = Docker(tf.name, url, tls=tls)
return d.is_alive()
def has_docker():
if 'SKIP_DOCKER_TESTS' in os.environ:
return False
from vcttesting.docker import Docker, params_from_env
url, tls = params_from_env(os.environ)
tf = tempfile.NamedTemporaryFile()
tf.close()
d = Docker(tf.name, url, tls=tls)
return d.is_alive()
def __init__(self, context):
from vcttesting.docker import Docker, params_from_env
from vcttesting.hgmo import HgCluster
if 'DOCKER_STATE_FILE' not in os.environ:
print('Do not where to store Docker state.')
print('Set the DOCKER_STATE_FILE environment variable and try again.')
sys.exit(1)
docker_url, tls = params_from_env(os.environ)
docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls)
if not docker.is_alive():
print('Docker not available')
sys.exit(1)
self.c = HgCluster(docker)
def __init__(self, context):
if 'DOCKER_STATE_FILE' in os.environ:
state_file = os.environ['DOCKER_STATE_FILE']
# When running from Mercurial tests, use a per-test state file.
# We can't use HGTMP because it is shared across many tests. We
# use HGRCPATH as a base, since it is in a test-specific directory.
elif 'HGRCPATH' in os.environ:
state_file = os.path.join(os.path.dirname(os.environ['HGRCPATH']),
'.dockerstate')
else:
state_file = os.path.join(ROOT, '.dockerstate')
docker_url, tls = params_from_env(os.environ)
d = Docker(state_file, docker_url, tls=tls)
if not d.is_alive():
print('Docker is not available!')
sys.exit(1)
self.d = d
def __init__(self, context):
if 'DOCKER_STATE_FILE' in os.environ:
state_file = os.environ['DOCKER_STATE_FILE']
# When running from Mercurial tests, use a per-test state file.
# We can't use HGTMP because it is shared across many tests. We
# use HGRCPATH as a base, since it is in a test-specific directory.
elif 'HGRCPATH' in os.environ:
state_file = os.path.join(os.path.dirname(os.environ['HGRCPATH']),
'.dockerstate')
else:
state_file = os.path.join(ROOT, '.dockerstate')
docker_url, tls = params_from_env(os.environ)
d = Docker(state_file, docker_url, tls=tls)
if not d.is_alive():
print('Docker is not available!')
sys.exit(1)
self.d = d
def __init__(self, context):
from vcttesting.docker import Docker, params_from_env
from vcttesting.hgmo import HgCluster
if 'HGMO_STATE_FILE' not in os.environ:
print('Do not know where to store state.')
print('Set the HGMO_STATE_FILE environment variable and try again.')
sys.exit(1)
if 'DOCKER_STATE_FILE' not in os.environ:
print('Do not where to store Docker state.')
print('Set the DOCKER_STATE_FILE environment variable and try again.')
sys.exit(1)
docker_url, tls = params_from_env(os.environ)
docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls)
if not docker.is_alive():
print('Docker not available')
sys.exit(1)
self.c = HgCluster(docker, os.environ['HGMO_STATE_FILE'],
ldap_image=os.environ.get('DOCKER_LDAP_IMAGE'),
master_image=os.environ.get('DOCKER_HGMASTER_IMAGE'),
web_image=os.environ.get('DOCKER_HGWEB_IMAGE'))
class MozReview(object):
"""Interface to MozService service.
This class can be used to create and control MozReview instances.
"""
def __init__(self,
path,
web_image=None,
hgrb_image=None,
ldap_image=None,
pulse_image=None,
rbweb_image=None,
autolanddb_image=None,
autoland_image=None,
hgweb_image=None,
treestatus_image=None):
if not path:
raise Exception('You must specify a path to create an instance')
path = os.path.abspath(path)
self._path = path
self.started = False
self.web_image = web_image
self.hgrb_image = hgrb_image
self.ldap_image = ldap_image
self.pulse_image = pulse_image
self.rbweb_image = rbweb_image
self.autolanddb_image = autolanddb_image
self.autoland_image = autoland_image
self.hgweb_image = hgweb_image
self.treestatus_image = treestatus_image
self._name = os.path.dirname(path)
if not os.path.exists(path):
os.mkdir(path)
keys_path = os.path.join(path, 'keys')
try:
os.mkdir(keys_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
credentials_path = os.path.join(path, 'credentials')
try:
os.mkdir(credentials_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
self._state_path = os.path.join(path, 'state.json')
docker_state = os.path.join(path, '.dockerstate')
self._docker_state = docker_state
self.bugzilla_username = None
self.bugzilla_password = None
self.docker_env = {}
if os.path.exists(self._state_path):
with open(self._state_path, 'rb') as fh:
state = json.load(fh)
for k, v in state.items():
setattr(self, k, v)
# Preserve Docker settings from last time.
#
# This was introduced to make watchman happy, as its triggers may not
# inherit environment variables.
for k, v in self.docker_env.items():
os.environ[k] = v
docker_url, tls = params_from_env(os.environ)
self._docker = Docker(docker_state, docker_url, tls=tls)
if not self._docker.is_alive():
raise DockerNotAvailable('Docker is not available.')
def get_bugzilla(self, username=None, password=None):
username = username or self.bugzilla_username or '*****@*****.**'
password = password or self.bugzilla_password or 'password'
return Bugzilla(self.bugzilla_url,
username=username,
password=password)
def get_reviewboard(self):
"""Obtain a MozReviewBoard instance tied to this MozReview instance."""
return MozReviewBoard(self._docker,
self.rbweb_id,
self.reviewboard_url,
bugzilla_url=self.bugzilla_url,
pulse_host=self.pulse_host,
pulse_port=self.pulse_port)
def get_ldap(self):
"""Obtain an LDAP instance connected to the LDAP server in this instance."""
return LDAP(self.ldap_uri, 'cn=admin,dc=mozilla', 'password')
def start(self,
bugzilla_port=None,
reviewboard_port=None,
mercurial_port=None,
pulse_port=None,
verbose=False,
web_image=None,
hgrb_image=None,
ldap_image=None,
ldap_port=None,
pulse_image=None,
rbweb_image=None,
ssh_port=None,
hgweb_image=None,
hgweb_port=None,
autolanddb_image=None,
autoland_image=None,
autoland_port=None,
treestatus_image=None,
treestatus_port=None,
max_workers=None):
"""Start a MozReview instance."""
if self.started:
raise Exception('MozReview instance has already been started')
if not bugzilla_port:
bugzilla_port = get_available_port()
if not reviewboard_port:
reviewboard_port = get_available_port()
if not mercurial_port:
mercurial_port = get_available_port()
if not ldap_port:
ldap_port = get_available_port()
if not ssh_port:
ssh_port = get_available_port()
if not pulse_port:
pulse_port = get_available_port()
if not autoland_port:
autoland_port = get_available_port()
if not hgweb_port:
hgweb_port = get_available_port()
if not treestatus_port:
treestatus_port = get_available_port()
web_image = web_image or self.web_image
hgrb_image = hgrb_image or self.hgrb_image
ldap_image = ldap_image or self.ldap_image
pulse_image = pulse_image or self.pulse_image
rbweb_image = rbweb_image or self.rbweb_image
autolanddb_image = autolanddb_image or self.autolanddb_image
autoland_image = autoland_image or self.autoland_image
hgweb_image = hgweb_image or self.hgweb_image
treestatus_image = treestatus_image or self.treestatus_image
if not os.path.exists(os.path.join(ROOT, 'reviewboard-fork')):
raise Exception('Failed to find reviewboard-fork. '
'Please run create-test-environment.')
self.started = True
mr_info = self._docker.start_mozreview(
cluster=self._name,
http_port=bugzilla_port,
pulse_port=pulse_port,
web_image=web_image,
hgrb_image=hgrb_image,
ldap_image=ldap_image,
ldap_port=ldap_port,
pulse_image=pulse_image,
rbweb_image=rbweb_image,
rbweb_port=reviewboard_port,
ssh_port=ssh_port,
hg_port=mercurial_port,
autolanddb_image=autolanddb_image,
autoland_image=autoland_image,
autoland_port=autoland_port,
hgweb_image=hgweb_image,
hgweb_port=hgweb_port,
treestatus_image=treestatus_image,
treestatus_port=treestatus_port,
max_workers=max_workers,
verbose=verbose)
self.bmoweb_id = mr_info['web_id']
self.bugzilla_url = mr_info['bugzilla_url']
bugzilla = self.get_bugzilla()
self.reviewboard_url = mr_info['reviewboard_url']
self.rbweb_id = mr_info['rbweb_id']
self.autoland_id = mr_info['autoland_id']
self.autoland_url = mr_info['autoland_url']
self.pulse_id = mr_info['pulse_id']
self.pulse_host = mr_info['pulse_host']
self.pulse_port = mr_info['pulse_port']
self.admin_username = bugzilla.username
self.admin_password = bugzilla.password
self.hg_rb_username = "******"
self.hg_rb_email = "*****@*****.**"
self.hg_rb_password = "******"
self.ldap_uri = mr_info['ldap_uri']
self.hgrb_id = mr_info['hgrb_id']
self.ssh_hostname = mr_info['ssh_hostname']
self.ssh_port = mr_info['ssh_port']
self.mercurial_url = mr_info['mercurial_url']
self.hgweb_id = mr_info['hgweb_id']
self.hgweb_url = mr_info['hgweb_url']
self.treestatus_id = mr_info['treestatus_id']
self.treestatus_url = mr_info['treestatus_url']
rb = self.get_reviewboard()
# It is tempting to put the user creation inside the futures
# block so it runs concurrently. However, there appeared to be
# race conditions here. The hg_rb_username ("mozreview") user
# was sometimes not getting created and this led to intermittent
# test failures in test-auth.t.
# Ensure admin user is present.
rb.login_user(bugzilla.username, bugzilla.password)
# Ensure the mozreview hg user is present and has privileges.
# This has to occur after the admin user is logged in to avoid
# race conditions with user IDs.
rb.create_local_user(self.hg_rb_username, self.hg_rb_email,
self.hg_rb_password)
with limited_threadpoolexecutor(7, max_workers) as e:
# Ensure admin user had admin privileges.
e.submit(rb.make_admin, bugzilla.username)
# Ensure mozreview user has permissions for testing.
e.submit(rb.grant_permission, self.hg_rb_username,
'Can change ldap assocation for all users')
e.submit(rb.grant_permission, self.hg_rb_username,
'Can verify DiffSet legitimacy')
e.submit(rb.grant_permission, self.hg_rb_username,
'Can enable or disable autoland for a repository')
# Tell hgrb about URLs.
e.submit(self._docker.execute, self.hgrb_id,
['/set-urls', self.bugzilla_url, self.reviewboard_url])
# Define site domain and hostname in rbweb. This is necessary so it
# constructs self-referential URLs properly.
e.submit(self._docker.execute, self.rbweb_id, [
'/set-site-url', self.reviewboard_url, self.autoland_url,
self.bugzilla_url
])
# Tell Bugzilla about Review Board URL.
e.submit(self._docker.execute, mr_info['web_id'],
['/set-urls', self.reviewboard_url])
self.create_user_api_key(bugzilla.username, description='mozreview')
with futures.ThreadPoolExecutor(2) as e:
f_ssh_ed25519_key = e.submit(
self._docker.get_file_content, mr_info['hgrb_id'],
'/etc/mercurial/ssh/ssh_host_ed25519_key.pub')
f_ssh_rsa_key = e.submit(
self._docker.get_file_content, mr_info['hgrb_id'],
'/etc/mercurial/ssh/ssh_host_rsa_key.pub')
ssh_ed25519_key = f_ssh_ed25519_key.result().split()[0:2]
ssh_rsa_key = f_ssh_rsa_key.result().split()[0:2]
hostkeys_path = os.path.join(self._path, 'ssh-known-hosts')
hoststring = '[%s]:%d' % (mr_info['ssh_hostname'], mr_info['ssh_port'])
with open(hostkeys_path, 'wb') as fh:
fh.write('%s %s %s\n' %
(hoststring, ssh_ed25519_key[0], ssh_ed25519_key[1]))
fh.write('%s %s %s\n' %
(hoststring, ssh_rsa_key[0], ssh_rsa_key[1]))
with open(os.path.join(self._path, 'ssh_config'), 'wb') as fh:
fh.write(
SSH_CONFIG.format(known_hosts=hostkeys_path,
hostname=self.ssh_hostname,
port=self.ssh_port))
state = {
'bmoweb_id': self.bmoweb_id,
'bugzilla_url': self.bugzilla_url,
'reviewboard_url': self.reviewboard_url,
'rbweb_id': self.rbweb_id,
'mercurial_url': self.mercurial_url,
'admin_username': bugzilla.username,
'admin_password': bugzilla.password,
'ldap_uri': self.ldap_uri,
'pulse_id': self.pulse_id,
'pulse_host': self.pulse_host,
'pulse_port': self.pulse_port,
'autoland_url': self.autoland_url,
'autoland_id': self.autoland_id,
'hgrb_id': self.hgrb_id,
'hgweb_url': self.hgweb_url,
'hgweb_id': self.hgweb_id,
'ssh_hostname': self.ssh_hostname,
'ssh_port': self.ssh_port,
'treestatus_url': self.treestatus_url,
'treestatus_id': self.treestatus_id,
'docker_env':
{k: v
for k, v in os.environ.items() if k.startswith('DOCKER')}
}
with open(self._state_path, 'wb') as fh:
json.dump(state, fh, indent=2, sort_keys=True)
def stop(self):
"""Stop all services associated with this MozReview instance."""
self._docker.stop_bmo(self._name)
self.started = False
if WATCHMAN:
with open(os.devnull, 'wb') as devnull:
subprocess.call([
WATCHMAN, 'trigger-del', ROOT,
'mozreview-%s' % os.path.basename(self._path)
],
stdout=devnull,
stderr=subprocess.STDOUT)
def refresh(self,
verbose=False,
refresh_reviewboard=False,
autoland_only=False):
"""Refresh a running cluster with latest version of code.
This only updates code from the v-c-t repo. Not all containers
are currently updated.
"""
with self._docker.vct_container(verbose=verbose) as vct_state:
# We update rbweb by rsyncing state and running the refresh script.
rsync_port = vct_state['NetworkSettings']['Ports']['873/tcp'][0][
'HostPort']
url = 'rsync://%s:%s/vct-mount/' % (self._docker.docker_hostname,
rsync_port)
def execute(name, cid, command):
res = self._docker.execute(cid,
command,
stream=True,
stderr=verbose,
stdout=verbose)
for msg in res:
if verbose:
msg = msg.rstrip().lstrip('\n')
for line in msg.splitlines():
if line != '':
print('%s> %s' % (name, line))
def refresh(name, cid):
execute(
name, cid,
['/refresh', url, 'all' if refresh_reviewboard else ''])
with futures.ThreadPoolExecutor(4) as e:
if not autoland_only:
e.submit(refresh, 'rbweb', self.rbweb_id)
e.submit(refresh, 'hgrb', self.hgrb_id)
e.submit(execute, 'bmoweb', self.bmoweb_id,
['/usr/bin/supervisorctl', 'restart', 'httpd'])
e.submit(refresh, 'autoland', self.autoland_id)
def start_autorefresh(self):
"""Enable auto refreshing of the cluster when changes are made.
Watchman will be configured to start watching the source directory.
When relevant files are changed, containers will be synchronized
automatically.
When enabled, this removes overhead from developers having to manually
refresh state during development.
"""
if not WATCHMAN:
raise Exception('watchman binary not found')
subprocess.check_call([WATCHMAN, 'watch-project', ROOT])
name = 'mozreview-%s' % os.path.basename(self._path)
expression = [
'anyof',
['dirname', 'hgext/reviewboard'],
['dirname', 'pylib/mozreview'],
['dirname', 'pylib/reviewboardmods'],
['dirname', 'reviewboard-fork/djblets'],
['dirname', 'reviewboard-fork/reviewboard'],
]
command = [
'%s/scripts/watchman-refresh-wrapper' % ROOT, ROOT, self._path
]
data = json.dumps([
'trigger', ROOT, {
'name': name,
'chdir': ROOT,
'expression': expression,
'command': command,
'append_files': True,
}
])
p = subprocess.Popen([WATCHMAN, '-j'], stdin=subprocess.PIPE)
p.communicate(data)
res = p.wait()
if res != 0:
raise Exception('error creating watchman trigger')
def repo_urls(self, name):
"""Obtain the http:// and ssh:// URLs for a review repo."""
http_url = '%s%s' % (self.mercurial_url, name)
ssh_url = 'ssh://%s:%d/%s' % (self.ssh_hostname, self.ssh_port, name)
return http_url, ssh_url
def create_repository(self, name):
http_url, ssh_url = self.repo_urls(name)
rb = self.get_reviewboard()
rbid = rb.add_repository(os.path.dirname(name) or name,
http_url,
bugzilla_url=self.bugzilla_url)
self._docker.execute(self.hgrb_id, ['/create-repo', name, str(rbid)])
if self.autoland_id:
time.sleep(1)
self._docker.execute(self.autoland_id, ['/clone-repo', name])
return http_url, ssh_url, rbid
def clone(self, repo, dest, username=None):
"""Clone and configure a review repository.
The specified review repo will be cloned and configured such that it is
bound to this MozReview instance. If a username is specified, all
operations will be performed as that user.
"""
http_url, ssh_url = self.repo_urls(repo)
subprocess.check_call([self._hg, 'clone', http_url, dest], cwd='/')
mrext = os.path.join(ROOT, 'testing', 'mozreview-repo.py')
mrssh = os.path.join(ROOT, 'testing', 'mozreview-ssh')
rbext = os.path.join(ROOT, 'hgext', 'reviewboard', 'client.py')
with open(os.path.join(dest, '.hg', 'hgrc'), 'w') as fh:
lines = [
'[paths]',
'default = %s' % http_url,
'default-push = %s' % ssh_url,
'',
'[extensions]',
'mozreview-repo = %s' % mrext,
'reviewboard = %s' % rbext,
'',
'[ui]',
'ssh = %s' % mrssh,
'',
# TODO use ircnick from the current user, if available.
'[mozilla]',
'ircnick = dummy',
'',
'[mozreview]',
'home = %s' % self._path,
]
if username:
lines.append('username = %s' % username)
fh.write('\n'.join(lines))
fh.write('\n')
def get_local_repository(self,
path,
ircnick=None,
bugzilla_username=None,
bugzilla_apikey=None):
"""Obtain a LocalMercurialRepository for the named server repository.
Call this with the same argument passed to ``create_repository()``
to obtain an object to interface with a local clone of that server
repository.
If bugzilla credentials are passed, they will be defined in the
repository's hgrc.
The repository is configured to be in deterministic mode. Therefore
these repositories are suitable for use in tests.
"""
localrepos = os.path.join(self._path, 'localrepos')
try:
os.mkdir(localrepos)
except OSError as e:
if e.errno != errno.EEXIST:
raise
local_path = os.path.join(localrepos, os.path.basename(path))
http_url, ssh_url = self.repo_urls(path)
# TODO make pushes via SSH work (it doesn't work outside of Mercurial
# tests because dummy expects certain environment variables).
return LocalMercurialRepository(self._path,
self._hg,
local_path,
http_url,
push_url=ssh_url,
ircnick=ircnick,
bugzilla_username=bugzilla_username,
bugzilla_apikey=bugzilla_apikey)
def create_user_api_key(self,
email,
sync_to_reviewboard=True,
description=''):
"""Creates an API key for the given user.
This creates an API key in Bugzilla and then triggers the
auth-delegation callback to register the key with Review Board. Note
that this also logs the user in, although we don't record the session
cookie anywhere so this shouldn't have an effect on subsequent
interactions with Review Board.
"""
api_key = self._docker.execute(self.bmoweb_id, [
'/var/lib/bugzilla/bugzilla/scripts/issue-api-key.pl', email,
description
],
stdout=True).strip()
assert len(api_key) == 40
if not sync_to_reviewboard:
return api_key
# When running tests in parallel, the auth callback can time out.
# Try up to 3 times before giving up.
url = self.reviewboard_url + 'mozreview/bmo_auth_callback/'
data = {'client_api_login': email, 'client_api_key': api_key}
for i in range(3):
response = requests.post(url, data=json.dumps(data))
if response.status_code == 200:
result = response.json()['result']
break
else:
raise Exception('Failed to successfully run the BMO auth POST '
'callback.')
params = {
'client_api_login': email,
'callback_result': result,
'secret': 'mozreview',
}
cookies = {'bmo_auth_secret': params['secret']}
for i in range(3):
if requests.get(url, params=params,
cookies=cookies).status_code == 200:
break
else:
raise Exception('Failed to successfully run the BMO auth GET '
'callback.')
return api_key
def create_user(self,
email,
password,
fullname,
bugzilla_groups=None,
uid=None,
username=None,
key_filename=None,
scm_level=None,
api_key=True,
bugzilla_email=None):
"""Create a new user.
This will create a user in at least Bugzilla. If the ``uid`` argument
is specified, an LDAP user will be created as well.
``email`` is the email address of the user.
``password`` is the plain text Bugzilla password.
``fullname`` is the full name of the user. This is stored in both
Bugzilla and the system account for the user (if an LDAP user is being
created).
``bugzilla_groups`` is an iterable of Bugzilla groups to add the user
to.
``uid`` is the numeric UID for the created system/LDAP account.
``username`` is the UNIX username for this user. It defaults to the
username part of the email address.
``key_filename`` is a path to an ssh key. This is used only if ``uid``
is given. If ``uid`` is given but ``key_filename`` is not specified,
the latter defaults to <mozreview path>/keys/<email>.
``scm_level`` defines the source code level access to grant to this
user. Supported levels are ``1``, ``2``, and ``3``. If not specified,
the user won't be able to push to any repos.
``bugzilla_email`` set the bugzillaEmail LDAP attribute to this instead
of ``email``.
"""
bugzilla_groups = bugzilla_groups or []
b = self.get_bugzilla()
if not username:
username = email[0:email.index('@')]
res = {
'bugzilla': b.create_user(email, password, fullname),
}
for g in bugzilla_groups:
b.add_user_to_group(email, g)
if api_key:
res['bugzilla']['api_key'] = self.create_user_api_key(
email, description='mozreview')
# Create an LDAP account as well.
if uid:
if key_filename is None:
key_filename = os.path.join(self._path, 'keys', email)
lr = self.get_ldap().create_user(email,
username,
uid,
fullname,
key_filename=key_filename,
scm_level=scm_level,
bugzilla_email=bugzilla_email)
res.update(lr)
credentials_path = os.path.join(self._path, 'credentials', email)
with open(credentials_path, 'wb') as fh:
fh.write(password)
return res
@property
def _hg(self):
for path in os.environ['PATH'].split(os.pathsep):
hg = os.path.join(path, 'hg')
if os.path.isfile(hg):
return hg
raise Exception('could not find hg executable')
class MozReview(object):
"""Interface to MozService service.
This class can be used to create and control MozReview instances.
"""
def __init__(self, path, web_image=None, hgrb_image=None,
ldap_image=None, pulse_image=None, rbweb_image=None,
autolanddb_image=None, autoland_image=None,
hgweb_image=None, treestatus_image=None):
if not path:
raise Exception('You must specify a path to create an instance')
path = os.path.abspath(path)
self._path = path
self.started = False
self.web_image = web_image
self.hgrb_image = hgrb_image
self.ldap_image = ldap_image
self.pulse_image = pulse_image
self.rbweb_image = rbweb_image
self.autolanddb_image = autolanddb_image
self.autoland_image = autoland_image
self.hgweb_image = hgweb_image
self.treestatus_image = treestatus_image
self._name = os.path.dirname(path)
if not os.path.exists(path):
os.mkdir(path)
keys_path = os.path.join(path, 'keys')
try:
os.mkdir(keys_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
credentials_path = os.path.join(path, 'credentials')
try:
os.mkdir(credentials_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
self._state_path = os.path.join(path, 'state.json')
docker_state = os.path.join(path, '.dockerstate')
self._docker_state = docker_state
self.bugzilla_username = None
self.bugzilla_password = None
self.docker_env = {}
if os.path.exists(self._state_path):
with open(self._state_path, 'rb') as fh:
state = json.load(fh)
for k, v in state.items():
setattr(self, k, v)
# Preserve Docker settings from last time.
#
# This was introduced to make watchman happy, as its triggers may not
# inherit environment variables.
for k, v in self.docker_env.items():
os.environ[k] = v
docker_url, tls = params_from_env(os.environ)
self._docker = Docker(docker_state, docker_url, tls=tls)
if not self._docker.is_alive():
raise DockerNotAvailable('Docker is not available.')
def get_bugzilla(self, username=None, password=None):
username = username or self.bugzilla_username or '*****@*****.**'
password = password or self.bugzilla_password or 'password'
return Bugzilla(self.bugzilla_url, username=username, password=password)
def get_reviewboard(self):
"""Obtain a MozReviewBoard instance tied to this MozReview instance."""
return MozReviewBoard(self._docker, self.rbweb_id,
self.reviewboard_url,
bugzilla_url=self.bugzilla_url,
pulse_host=self.pulse_host,
pulse_port=self.pulse_port)
def get_ldap(self):
"""Obtain an LDAP instance connected to the LDAP server in this instance."""
return LDAP(self.ldap_uri, 'cn=admin,dc=mozilla', 'password')
def start(self, bugzilla_port=None, reviewboard_port=None,
mercurial_port=None, pulse_port=None, verbose=False,
web_image=None, hgrb_image=None,
ldap_image=None, ldap_port=None, pulse_image=None,
rbweb_image=None, ssh_port=None,
hgweb_image=None, hgweb_port=None,
autolanddb_image=None, autoland_image=None, autoland_port=None,
treestatus_image=None, treestatus_port=None, max_workers=None):
"""Start a MozReview instance."""
if self.started:
raise Exception('MozReview instance has already been started')
if not bugzilla_port:
bugzilla_port = get_available_port()
if not reviewboard_port:
reviewboard_port = get_available_port()
if not mercurial_port:
mercurial_port = get_available_port()
if not ldap_port:
ldap_port = get_available_port()
if not ssh_port:
ssh_port = get_available_port()
if not pulse_port:
pulse_port = get_available_port()
if not autoland_port:
autoland_port = get_available_port()
if not hgweb_port:
hgweb_port = get_available_port()
if not treestatus_port:
treestatus_port = get_available_port()
web_image = web_image or self.web_image
hgrb_image = hgrb_image or self.hgrb_image
ldap_image = ldap_image or self.ldap_image
pulse_image = pulse_image or self.pulse_image
rbweb_image = rbweb_image or self.rbweb_image
autolanddb_image = autolanddb_image or self.autolanddb_image
autoland_image = autoland_image or self.autoland_image
hgweb_image = hgweb_image or self.hgweb_image
treestatus_image = treestatus_image or self.treestatus_image
if not os.path.exists(os.path.join(ROOT, 'reviewboard-fork')):
raise Exception('Failed to find reviewboard-fork. '
'Please run create-test-environment.')
self.started = True
mr_info = self._docker.start_mozreview(
cluster=self._name,
http_port=bugzilla_port,
pulse_port=pulse_port,
web_image=web_image,
hgrb_image=hgrb_image,
ldap_image=ldap_image,
ldap_port=ldap_port,
pulse_image=pulse_image,
rbweb_image=rbweb_image,
rbweb_port=reviewboard_port,
ssh_port=ssh_port,
hg_port=mercurial_port,
autolanddb_image=autolanddb_image,
autoland_image=autoland_image,
autoland_port=autoland_port,
hgweb_image=hgweb_image,
hgweb_port=hgweb_port,
treestatus_image=treestatus_image,
treestatus_port=treestatus_port,
max_workers=max_workers,
verbose=verbose)
self.bmoweb_id = mr_info['web_id']
self.bugzilla_url = mr_info['bugzilla_url']
bugzilla = self.get_bugzilla()
self.reviewboard_url = mr_info['reviewboard_url']
self.rbweb_id = mr_info['rbweb_id']
self.autoland_id = mr_info['autoland_id']
self.autoland_url = mr_info['autoland_url']
self.pulse_id = mr_info['pulse_id']
self.pulse_host = mr_info['pulse_host']
self.pulse_port = mr_info['pulse_port']
self.admin_username = bugzilla.username
self.admin_password = bugzilla.password
self.hg_rb_username = "******"
self.hg_rb_email = "*****@*****.**"
self.hg_rb_password = "******"
self.ldap_uri = mr_info['ldap_uri']
self.hgrb_id = mr_info['hgrb_id']
self.ssh_hostname = mr_info['ssh_hostname']
self.ssh_port = mr_info['ssh_port']
self.mercurial_url = mr_info['mercurial_url']
self.hgweb_id = mr_info['hgweb_id']
self.hgweb_url = mr_info['hgweb_url']
self.treestatus_id = mr_info['treestatus_id']
self.treestatus_url = mr_info['treestatus_url']
rb = self.get_reviewboard()
# It is tempting to put the user creation inside the futures
# block so it runs concurrently. However, there appeared to be
# race conditions here. The hg_rb_username ("mozreview") user
# was sometimes not getting created and this led to intermittent
# test failures in test-auth.t.
# Ensure admin user is present.
rb.login_user(bugzilla.username, bugzilla.password)
# Ensure the mozreview hg user is present and has privileges.
# This has to occur after the admin user is logged in to avoid
# race conditions with user IDs.
rb.create_local_user(self.hg_rb_username, self.hg_rb_email,
self.hg_rb_password)
with limited_threadpoolexecutor(7, max_workers) as e:
# Ensure admin user had admin privileges.
e.submit(rb.make_admin, bugzilla.username)
# Ensure mozreview user has permissions for testing.
e.submit(rb.grant_permission, self.hg_rb_username,
'Can change ldap assocation for all users')
e.submit(rb.grant_permission, self.hg_rb_username,
'Can verify DiffSet legitimacy')
e.submit(rb.grant_permission, self.hg_rb_username,
'Can enable or disable autoland for a repository')
# Tell hgrb about URLs.
e.submit(self._docker.execute, self.hgrb_id,
['/set-urls', self.bugzilla_url, self.reviewboard_url])
# Define site domain and hostname in rbweb. This is necessary so it
# constructs self-referential URLs properly.
e.submit(self._docker.execute, self.rbweb_id,
['/set-site-url', self.reviewboard_url,
self.autoland_url, self.bugzilla_url])
# Tell Bugzilla about Review Board URL.
e.submit(self._docker.execute, mr_info['web_id'],
['/set-urls', self.reviewboard_url])
self.create_user_api_key(bugzilla.username, description='mozreview')
hg_ssh_host_key = self._docker.get_file_content(
mr_info['hgrb_id'],
'/etc/ssh/ssh_host_rsa_key.pub').rstrip()
key_type, key_key = hg_ssh_host_key.split()
assert key_type == 'ssh-rsa'
key = paramiko.rsakey.RSAKey(data=paramiko.py3compat.decodebytes(key_key))
hostkeys_path = os.path.join(self._path, 'ssh-known-hosts')
load_path = hostkeys_path if os.path.exists(hostkeys_path) else None
hostkeys = paramiko.hostkeys.HostKeys(filename=load_path)
hoststring = '[%s]:%d' % (mr_info['ssh_hostname'], mr_info['ssh_port'])
hostkeys.add(hoststring, key_type, key)
hostkeys.save(hostkeys_path)
with open(os.path.join(self._path, 'ssh_config'), 'wb') as fh:
fh.write(SSH_CONFIG.format(
known_hosts=hostkeys_path,
hostname=self.ssh_hostname,
port=self.ssh_port))
state = {
'bmoweb_id': self.bmoweb_id,
'bugzilla_url': self.bugzilla_url,
'reviewboard_url': self.reviewboard_url,
'rbweb_id': self.rbweb_id,
'mercurial_url': self.mercurial_url,
'admin_username': bugzilla.username,
'admin_password': bugzilla.password,
'ldap_uri': self.ldap_uri,
'pulse_id': self.pulse_id,
'pulse_host': self.pulse_host,
'pulse_port': self.pulse_port,
'autoland_url': self.autoland_url,
'autoland_id': self.autoland_id,
'hgrb_id': self.hgrb_id,
'hgweb_url': self.hgweb_url,
'hgweb_id': self.hgweb_id,
'ssh_hostname': self.ssh_hostname,
'ssh_port': self.ssh_port,
'treestatus_url': self.treestatus_url,
'treestatus_id': self.treestatus_id,
'docker_env': {k: v for k, v in os.environ.items() if k.startswith('DOCKER')}
}
with open(self._state_path, 'wb') as fh:
json.dump(state, fh, indent=2, sort_keys=True)
def stop(self):
"""Stop all services associated with this MozReview instance."""
self._docker.stop_bmo(self._name)
self.started = False
if WATCHMAN:
with open(os.devnull, 'wb') as devnull:
subprocess.call([WATCHMAN, 'trigger-del', ROOT,
'mozreview-%s' % os.path.basename(self._path)],
stdout=devnull, stderr=subprocess.STDOUT)
def refresh(self, verbose=False, refresh_reviewboard=False):
"""Refresh a running cluster with latest version of code.
This only updates code from the v-c-t repo. Not all containers
are currently updated.
"""
with self._docker.vct_container(verbose=verbose) as vct_state:
# We update rbweb by rsyncing state and running the refresh script.
rsync_port = vct_state['NetworkSettings']['Ports']['873/tcp'][0]['HostPort']
url = 'rsync://%s:%s/vct-mount/' % (self._docker.docker_hostname,
rsync_port)
def execute(name, cid, command):
res = self._docker.execute(cid, command, stream=True,
stderr=verbose, stdout=verbose)
for msg in res:
if verbose:
msg = msg.rstrip().lstrip('\n')
for line in msg.splitlines():
if line != '':
print('%s> %s' % (name, line))
def refresh(name, cid):
execute(name, cid, ['/refresh', url,
'all' if refresh_reviewboard else ''])
with futures.ThreadPoolExecutor(4) as e:
e.submit(refresh, 'rbweb', self.rbweb_id)
e.submit(refresh, 'hgrb', self.hgrb_id)
# TODO add hgweb support for refreshing.
e.submit(execute, 'bmoweb', self.bmoweb_id,
['/usr/bin/supervisorctl', 'restart', 'httpd'])
def start_autorefresh(self):
"""Enable auto refreshing of the cluster when changes are made.
Watchman will be configured to start watching the source directory.
When relevant files are changed, containers will be synchronized
automatically.
When enabled, this removes overhead from developers having to manually
refresh state during development.
"""
if not WATCHMAN:
raise Exception('watchman binary not found')
subprocess.check_call([WATCHMAN, 'watch-project', ROOT])
name = 'mozreview-%s' % os.path.basename(self._path)
expression = [
'anyof',
['dirname', 'hgext/reviewboard'],
['dirname', 'pylib/mozreview'],
['dirname', 'pylib/rbbz'],
['dirname', 'pylib/reviewboardmods'],
['dirname', 'reviewboard-fork/djblets'],
['dirname', 'reviewboard-fork/reviewboard'],
]
command = ['%s/scripts/watchman-refresh-wrapper' % ROOT, ROOT,
self._path]
data = json.dumps(['trigger', ROOT, {
'name': name,
'chdir': ROOT,
'expression': expression,
'command': command,
'append_files': True,
}])
p = subprocess.Popen([WATCHMAN, '-j'], stdin=subprocess.PIPE)
p.communicate(data)
res = p.wait()
if res != 0:
raise Exception('error creating watchman trigger')
def repo_urls(self, name):
"""Obtain the http:// and ssh:// URLs for a review repo."""
http_url = '%s%s' % (self.mercurial_url, name)
ssh_url = 'ssh://%s:%d/%s' % (self.ssh_hostname, self.ssh_port, name)
return http_url, ssh_url
def create_repository(self, name):
http_url, ssh_url = self.repo_urls(name)
rb = self.get_reviewboard()
rbid = rb.add_repository(os.path.dirname(name) or name, http_url,
bugzilla_url=self.bugzilla_url)
self._docker.execute(self.hgrb_id,
['/create-repo', name, str(rbid)])
if self.autoland_id:
time.sleep(1)
self._docker.execute(self.autoland_id, ['/clone-repo', name])
return http_url, ssh_url, rbid
def clone(self, repo, dest, username=None):
"""Clone and configure a review repository.
The specified review repo will be cloned and configured such that it is
bound to this MozReview instance. If a username is specified, all
operations will be performed as that user.
"""
http_url, ssh_url = self.repo_urls(repo)
subprocess.check_call([self._hg, 'clone', http_url, dest], cwd='/')
mrext = os.path.join(ROOT, 'testing', 'mozreview-repo.py')
mrssh = os.path.join(ROOT, 'testing', 'mozreview-ssh')
rbext = os.path.join(ROOT, 'hgext', 'reviewboard', 'client.py')
with open(os.path.join(dest, '.hg', 'hgrc'), 'w') as fh:
lines = [
'[paths]',
'default = %s' % http_url,
'default-push = %s' % ssh_url,
'',
'[extensions]',
'mozreview-repo = %s' % mrext,
'reviewboard = %s' % rbext,
'',
'[ui]',
'ssh = %s' % mrssh,
'',
# TODO use ircnick from the current user, if available.
'[mozilla]',
'ircnick = dummy',
'',
'[mozreview]',
'home = %s' % self._path,
]
if username:
lines.append('username = %s' % username)
fh.write('\n'.join(lines))
fh.write('\n')
def get_local_repository(self, path, ircnick=None,
bugzilla_username=None,
bugzilla_apikey=None):
"""Obtain a LocalMercurialRepository for the named server repository.
Call this with the same argument passed to ``create_repository()``
to obtain an object to interface with a local clone of that server
repository.
If bugzilla credentials are passed, they will be defined in the
repository's hgrc.
The repository is configured to be in deterministic mode. Therefore
these repositories are suitable for use in tests.
"""
localrepos = os.path.join(self._path, 'localrepos')
try:
os.mkdir(localrepos)
except OSError as e:
if e.errno != errno.EEXIST:
raise
local_path = os.path.join(localrepos, os.path.basename(path))
http_url, ssh_url = self.repo_urls(path)
# TODO make pushes via SSH work (it doesn't work outside of Mercurial
# tests because dummy expects certain environment variables).
return LocalMercurialRepository(self._path, self._hg, local_path,
http_url, push_url=ssh_url, ircnick=ircnick,
bugzilla_username=bugzilla_username,
bugzilla_apikey=bugzilla_apikey)
def create_user_api_key(self, email, sync_to_reviewboard=True,
description=''):
"""Creates an API key for the given user.
This creates an API key in Bugzilla and then triggers the
auth-delegation callback to register the key with Review Board. Note
that this also logs the user in, although we don't record the session
cookie anywhere so this shouldn't have an effect on subsequent
interactions with Review Board.
"""
api_key = self._docker.execute(
self.bmoweb_id,
['/var/lib/bugzilla/bugzilla/scripts/issue-api-key.pl',
email, description], stdout=True).strip()
assert len(api_key) == 40
if not sync_to_reviewboard:
return api_key
# When running tests in parallel, the auth callback can time out.
# Try up to 3 times before giving up.
url = self.reviewboard_url + 'mozreview/bmo_auth_callback/'
data = {'client_api_login': email, 'client_api_key': api_key}
for i in range(3):
response = requests.post(url, data=json.dumps(data))
if response.status_code == 200:
result = response.json()['result']
break
else:
raise Exception('Failed to successfully run the BMO auth POST '
'callback.')
params = {
'client_api_login': email,
'callback_result': result,
'secret': 'mozreview',
}
cookies = {'bmo_auth_secret': params['secret']}
for i in range(3):
if requests.get(url, params=params,
cookies=cookies).status_code == 200:
break
else:
raise Exception('Failed to successfully run the BMO auth GET '
'callback.')
return api_key
def create_user(self, email, password, fullname, bugzilla_groups=None,
uid=None, username=None, key_filename=None, scm_level=None,
api_key=True, bugzilla_email=None):
"""Create a new user.
This will create a user in at least Bugzilla. If the ``uid`` argument
is specified, an LDAP user will be created as well.
``email`` is the email address of the user.
``password`` is the plain text Bugzilla password.
``fullname`` is the full name of the user. This is stored in both
Bugzilla and the system account for the user (if an LDAP user is being
created).
``bugzilla_groups`` is an iterable of Bugzilla groups to add the user
to.
``uid`` is the numeric UID for the created system/LDAP account.
``username`` is the UNIX username for this user. It defaults to the
username part of the email address.
``key_filename`` is a path to an ssh key. This is used only if ``uid``
is given. If ``uid`` is given but ``key_filename`` is not specified,
the latter defaults to <mozreview path>/keys/<email>.
``scm_level`` defines the source code level access to grant to this
user. Supported levels are ``1``, ``2``, and ``3``. If not specified,
the user won't be able to push to any repos.
``bugzilla_email`` set the bugzillaEmail LDAP attribute to this instead
of ``email``.
"""
bugzilla_groups = bugzilla_groups or []
b = self.get_bugzilla()
if not username:
username = email[0:email.index('@')]
res = {
'bugzilla': b.create_user(email, password, fullname),
}
for g in bugzilla_groups:
b.add_user_to_group(email, g)
if api_key:
res['bugzilla']['api_key'] = self.create_user_api_key(
email, description='mozreview')
# Create an LDAP account as well.
if uid:
if key_filename is None:
key_filename = os.path.join(self._path, 'keys', email)
lr = self.get_ldap().create_user(email, username, uid,
fullname,
key_filename=key_filename,
scm_level=scm_level,
bugzilla_email=bugzilla_email)
res.update(lr)
credentials_path = os.path.join(self._path, 'credentials', email)
with open(credentials_path, 'wb') as fh:
fh.write(password)
return res
@property
def _hg(self):
for path in os.environ['PATH'].split(os.pathsep):
hg = os.path.join(path, 'hg')
if os.path.isfile(hg):
return hg
raise Exception('could not find hg executable')
class MozReview(object):
"""Interface to MozService service.
This class can be used to create and control MozReview instances.
"""
def __init__(self, path):
if not path:
raise Exception('You must specify a path to create an instance')
path = os.path.abspath(path)
self._path = path
self._name = os.path.dirname(path)
if not os.path.exists(path):
os.mkdir(path)
docker_state = os.path.join(path, 'docker-state.json')
self._docker_state = docker_state
docker_url, tls = params_from_env(os.environ)
self._docker = Docker(docker_state, docker_url, tls=tls)
if not self._docker.is_alive():
raise Exception('Docker is not available.')
def get_bugzilla(self, url, username='******', password='******'):
return Bugzilla(url, username=username, password=password)
def start(self, bugzilla_port=None, reviewboard_port=None,
mercurial_port=None, verbose=False):
"""Start a MozReview instance."""
if not bugzilla_port:
bugzilla_port = get_available_port()
if not reviewboard_port:
reviewboard_port = get_available_port()
if not mercurial_port:
mercurial_port = get_available_port()
db_image, web_image = self._docker.build_bmo(verbose=verbose)
bugzilla_url = self._docker.start_bmo(cluster=self._name,
hostname=None, http_port=bugzilla_port,
db_image=db_image, web_image=web_image)[0]
with open(self._bugzilla_url_path, 'wb') as fh:
fh.write(bugzilla_url)
bugzilla = self.get_bugzilla(bugzilla_url)
rb = MozReviewBoard(self._path, bugzilla_url=bugzilla_url)
rb.create()
reviewboard_pid = rb.start(reviewboard_port)
reviewboard_url = 'http://localhost:%s/' % reviewboard_port
self.bugzilla_url = bugzilla_url
self.reviewboard_url = reviewboard_url
self.reviewboard_pid = reviewboard_pid
self.admin_username = bugzilla.username
self.admin_password = bugzilla.password
mercurial_pid = self._start_mercurial_server(mercurial_port)
self.mercurial_url = 'http://localhost:%s/' % mercurial_port
with open(self._hg_url_path, 'w') as fh:
fh.write(self.mercurial_url)
self.mercurial_pid = mercurial_pid
def stop(self):
"""Stop all services associated with this MozReview instance."""
if os.path.exists(self._hg_pid_path):
with open(self._hg_pid_path, 'rb') as fh:
pid = int(fh.read().strip())
kill(pid)
os.unlink(self._hg_pid_path)
rb = MozReviewBoard(self._path)
rb.stop()
self._docker.stop_bmo(self._name)
def create_repository(self, path):
with open(self._hg_url_path, 'rb') as fh:
url = '%s%s' % (fh.read(), path)
full_path = os.path.join(self._path, 'repos', path)
env = dict(os.environ)
env['HGRCPATH'] = '/dev/null'
subprocess.check_call([self._hg, 'init', full_path], env=env)
rb = MozReviewBoard(self._path)
rbid = rb.add_repository(os.path.dirname(path), url)
with open(os.path.join(full_path, '.hg', 'hgrc'), 'w') as fh:
fh.write('\n'.join([
'[reviewboard]',
'repoid = %s' % rbid,
'',
]))
return url, rbid
def create_user(self, email, password, fullname):
with open(self._bugzilla_url_path, 'r') as fh:
url = fh.read()
b = self.get_bugzilla(url)
return b.create_user(email, password, fullname)
@property
def _hg_pid_path(self):
return os.path.join(self._path, 'hg.pid')
@property
def _hg_url_path(self):
return os.path.join(self._path, 'hg.url')
@property
def _hg(self):
return os.path.join(ROOT, 'venv', 'bin', 'hg')
@property
def _bugzilla_url_path(self):
return os.path.join(self._path, 'bugzilla.url')
def _start_mercurial_server(self, port):
repos_path = os.path.join(self._path, 'repos')
if not os.path.exists(repos_path):
os.mkdir(repos_path)
rb_ext_path = os.path.join(ROOT, 'hgext', 'reviewboard', 'server.py')
dummyssh = os.path.join(ROOT, 'pylib', 'mercurial-support', 'dummyssh')
global_hgrc = os.path.join(self._path, 'hgrc')
with open(global_hgrc, 'w') as fh:
fh.write('\n'.join([
'[phases]',
'publish = False',
'',
'[ui]',
'ssh = python "%s"' % dummyssh,
'',
'[reviewboard]',
'url = %s' % self.reviewboard_url,
'',
'[extensions]',
'reviewboard = %s' % rb_ext_path,
'',
'[web]',
'push_ssl = False',
'allow_push = *',
'',
'[paths]',
'/ = %s/**' % repos_path,
'',
]))
env = os.environ.copy()
env['HGRCPATH'] = '/dev/null'
env['HGENCODING'] = 'UTF-8'
args = [
self._hg,
'serve',
'-d',
'-p', str(port),
'--pid-file', self._hg_pid_path,
'--web-conf', global_hgrc,
'--accesslog', os.path.join(self._path, 'hg.access.log'),
'--errorlog', os.path.join(self._path, 'hg.error.log'),
]
# We execute from / so Mercurial doesn't pick up config files
# from parent directories.
subprocess.check_call(args, env=env, cwd='/')
with open(self._hg_pid_path, 'rb') as fh:
pid = fh.read().strip()
return pid
class MozReview(object):
"""Interface to MozService service.
This class can be used to create and control MozReview instances.
"""
def __init__(
self,
path,
web_image=None,
db_image=None,
hgrb_image=None,
ldap_image=None,
pulse_image=None,
rbweb_image=None,
autolanddb_image=None,
autoland_image=None,
hgweb_image=None,
treestatus_image=None,
):
if not path:
raise Exception("You must specify a path to create an instance")
path = os.path.abspath(path)
self._path = path
self.started = False
self.db_image = db_image
self.web_image = web_image
self.hgrb_image = hgrb_image
self.ldap_image = ldap_image
self.pulse_image = pulse_image
self.rbweb_image = rbweb_image
self.autolanddb_image = autolanddb_image
self.autoland_image = autoland_image
self.hgweb_image = hgweb_image
self.treestatus_image = treestatus_image
self._name = os.path.dirname(path)
if not os.path.exists(path):
os.mkdir(path)
keys_path = os.path.join(path, "keys")
try:
os.mkdir(keys_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
credentials_path = os.path.join(path, "credentials")
try:
os.mkdir(credentials_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
self._state_path = os.path.join(path, "state.json")
docker_state = os.path.join(path, ".dockerstate")
self._docker_state = docker_state
self.bugzilla_username = None
self.bugzilla_password = None
self.docker_env = {}
if os.path.exists(self._state_path):
with open(self._state_path, "rb") as fh:
state = json.load(fh)
for k, v in state.items():
setattr(self, k, v)
# Preserve Docker settings from last time.
#
# This was introduced to make watchman happy, as its triggers may not
# inherit environment variables.
for k, v in self.docker_env.items():
os.environ[k] = v
docker_url, tls = params_from_env(os.environ)
self._docker = Docker(docker_state, docker_url, tls=tls)
if not self._docker.is_alive():
raise DockerNotAvailable("Docker is not available.")
def get_bugzilla(self, username=None, password=None):
username = username or self.bugzilla_username or "*****@*****.**"
password = password or self.bugzilla_password or "password"
return Bugzilla(self.bugzilla_url, username=username, password=password)
def get_reviewboard(self):
"""Obtain a MozReviewBoard instance tied to this MozReview instance."""
return MozReviewBoard(
self._docker,
self.rbweb_id,
self.reviewboard_url,
bugzilla_url=self.bugzilla_url,
pulse_host=self.pulse_host,
pulse_port=self.pulse_port,
)
def get_ldap(self):
"""Obtain an LDAP instance connected to the LDAP server in this instance."""
return LDAP(self.ldap_uri, "cn=admin,dc=mozilla", "password")
def start(
self,
bugzilla_port=None,
reviewboard_port=None,
mercurial_port=None,
pulse_port=None,
verbose=False,
db_image=None,
web_image=None,
hgrb_image=None,
ldap_image=None,
ldap_port=None,
pulse_image=None,
rbweb_image=None,
ssh_port=None,
hgweb_image=None,
hgweb_port=None,
autolanddb_image=None,
autoland_image=None,
autoland_port=None,
treestatus_image=None,
treestatus_port=None,
):
"""Start a MozReview instance."""
if self.started:
raise Exception("MozReview instance has already been started")
if not bugzilla_port:
bugzilla_port = get_available_port()
if not reviewboard_port:
reviewboard_port = get_available_port()
if not mercurial_port:
mercurial_port = get_available_port()
if not ldap_port:
ldap_port = get_available_port()
if not ssh_port:
ssh_port = get_available_port()
if not pulse_port:
pulse_port = get_available_port()
if not autoland_port:
autoland_port = get_available_port()
if not hgweb_port:
hgweb_port = get_available_port()
if not treestatus_port:
treestatus_port = get_available_port()
db_image = db_image or self.db_image
web_image = web_image or self.web_image
hgrb_image = hgrb_image or self.hgrb_image
ldap_image = ldap_image or self.ldap_image
pulse_image = pulse_image or self.pulse_image
rbweb_image = rbweb_image or self.rbweb_image
autolanddb_image = autolanddb_image or self.autolanddb_image
autoland_image = autoland_image or self.autoland_image
hgweb_image = hgweb_image or self.hgweb_image
treestatus_image = treestatus_image or self.treestatus_image
self.started = True
mr_info = self._docker.start_mozreview(
cluster=self._name,
http_port=bugzilla_port,
pulse_port=pulse_port,
db_image=db_image,
web_image=web_image,
hgrb_image=hgrb_image,
ldap_image=ldap_image,
ldap_port=ldap_port,
pulse_image=pulse_image,
rbweb_image=rbweb_image,
rbweb_port=reviewboard_port,
ssh_port=ssh_port,
hg_port=mercurial_port,
autolanddb_image=autolanddb_image,
autoland_image=autoland_image,
autoland_port=autoland_port,
hgweb_image=hgweb_image,
hgweb_port=hgweb_port,
treestatus_image=treestatus_image,
treestatus_port=treestatus_port,
verbose=verbose,
)
self.bmoweb_id = mr_info["web_id"]
self.bmodb_id = mr_info["db_id"]
self.bugzilla_url = mr_info["bugzilla_url"]
bugzilla = self.get_bugzilla()
self.reviewboard_url = mr_info["reviewboard_url"]
self.rbweb_id = mr_info["rbweb_id"]
self.autoland_id = mr_info["autoland_id"]
self.autoland_url = mr_info["autoland_url"]
self.pulse_id = mr_info["pulse_id"]
self.pulse_host = mr_info["pulse_host"]
self.pulse_port = mr_info["pulse_port"]
self.admin_username = bugzilla.username
self.admin_password = bugzilla.password
self.hg_rb_username = "******"
self.hg_rb_email = "*****@*****.**"
self.hg_rb_password = "******"
self.ldap_uri = mr_info["ldap_uri"]
self.hgrb_id = mr_info["hgrb_id"]
self.ssh_hostname = mr_info["ssh_hostname"]
self.ssh_port = mr_info["ssh_port"]
self.mercurial_url = mr_info["mercurial_url"]
self.hgweb_id = mr_info["hgweb_id"]
self.hgweb_url = mr_info["hgweb_url"]
self.treestatus_id = mr_info["treestatus_id"]
self.treestatus_url = mr_info["treestatus_url"]
# Ensure admin user is present and has admin privileges.
def make_users():
rb = self.get_reviewboard()
# Ensure admin user is present and has admin privileges.
rb.login_user(bugzilla.username, bugzilla.password)
rb.make_admin(bugzilla.username)
# Ensure the MozReview hg user is present and has privileges.
rb.create_local_user(self.hg_rb_username, self.hg_rb_email, self.hg_rb_password)
rb.grant_permission(self.hg_rb_username, "Can change ldap assocation for all users")
with futures.ThreadPoolExecutor(4) as e:
e.submit(make_users)
# Tell hgrb about URLs.
e.submit(self._docker.execute, self.hgrb_id, ["/set-urls", self.bugzilla_url, self.reviewboard_url])
# Define site domain and hostname in rbweb. This is necessary so it
# constructs self-referential URLs properly.
e.submit(
self._docker.execute,
self.rbweb_id,
["/set-site-url", self.reviewboard_url, self.autoland_url, self.bugzilla_url],
)
# Tell Bugzilla about Review Board URL.
e.submit(self._docker.execute, mr_info["web_id"], ["/set-urls", self.reviewboard_url])
self.create_user_api_key(bugzilla.username)
hg_ssh_host_key = self._docker.get_file_content(mr_info["hgrb_id"], "/etc/ssh/ssh_host_rsa_key.pub").rstrip()
key_type, key_key = hg_ssh_host_key.split()
assert key_type == "ssh-rsa"
key = paramiko.rsakey.RSAKey(data=paramiko.py3compat.decodebytes(key_key))
hostkeys_path = os.path.join(self._path, "ssh-known-hosts")
load_path = hostkeys_path if os.path.exists(hostkeys_path) else None
hostkeys = paramiko.hostkeys.HostKeys(filename=load_path)
hoststring = "[%s]:%d" % (mr_info["ssh_hostname"], mr_info["ssh_port"])
hostkeys.add(hoststring, key_type, key)
hostkeys.save(hostkeys_path)
with open(os.path.join(self._path, "ssh_config"), "wb") as fh:
fh.write(SSH_CONFIG.format(known_hosts=hostkeys_path, hostname=self.ssh_hostname, port=self.ssh_port))
state = {
"bmoweb_id": self.bmoweb_id,
"bmodb_id": self.bmodb_id,
"bugzilla_url": self.bugzilla_url,
"reviewboard_url": self.reviewboard_url,
"rbweb_id": self.rbweb_id,
"mercurial_url": self.mercurial_url,
"admin_username": bugzilla.username,
"admin_password": bugzilla.password,
"ldap_uri": self.ldap_uri,
"pulse_id": self.pulse_id,
"pulse_host": self.pulse_host,
"pulse_port": self.pulse_port,
"autoland_url": self.autoland_url,
"autoland_id": self.autoland_id,
"hgrb_id": self.hgrb_id,
"hgweb_url": self.hgweb_url,
"hgweb_id": self.hgweb_id,
"ssh_hostname": self.ssh_hostname,
"ssh_port": self.ssh_port,
"treestatus_url": self.treestatus_url,
"treestatus_id": self.treestatus_id,
"docker_env": {k: v for k, v in os.environ.items() if k.startswith("DOCKER")},
}
with open(self._state_path, "wb") as fh:
json.dump(state, fh, indent=2, sort_keys=True)
def stop(self):
"""Stop all services associated with this MozReview instance."""
self._docker.stop_bmo(self._name)
self.started = False
if WATCHMAN:
with open(os.devnull, "wb") as devnull:
subprocess.call(
[WATCHMAN, "trigger-del", ROOT, "mozreview-%s" % os.path.basename(self._path)],
stdout=devnull,
stderr=subprocess.STDOUT,
)
def refresh(self, verbose=False):
"""Refresh a running cluster with latest version of code.
This only updates code from the v-c-t repo. Not all containers
are currently updated.
"""
with self._docker.vct_container(verbose=verbose) as vct_state:
# We update rbweb by rsyncing state and running the refresh script.
rsync_port = vct_state["NetworkSettings"]["Ports"]["873/tcp"][0]["HostPort"]
url = "rsync://%s:%s/vct-mount/" % (self._docker.docker_hostname, rsync_port)
def execute(name, cid, command):
res = self._docker.execute(cid, command, stream=True, stderr=verbose, stdout=verbose)
for msg in res:
if verbose:
print("%s> %s" % (name, msg), end="")
def refresh(name, cid):
execute(name, cid, ["/refresh", url])
with futures.ThreadPoolExecutor(4) as e:
e.submit(refresh, "rbweb", self.rbweb_id)
e.submit(refresh, "hgrb", self.hgrb_id)
# TODO add hgweb support for refreshing.
e.submit(execute, "bmoweb", self.bmoweb_id, ["/usr/bin/supervisorctl", "restart", "httpd"])
def start_autorefresh(self):
"""Enable auto refreshing of the cluster when changes are made.
Watchman will be configured to start watching the source directory.
When relevant files are changed, containers will be synchronized
automatically.
When enabled, this removes overhead from developers having to manually
refresh state during development.
"""
if not WATCHMAN:
raise Exception("watchman binary not found")
subprocess.check_call([WATCHMAN, "watch-project", ROOT])
name = "mozreview-%s" % os.path.basename(self._path)
data = json.dumps(
[
"trigger",
ROOT,
{
"name": name,
"chdir": ROOT,
"expression": [
"anyof",
["dirname", "hgext/reviewboard"],
["dirname", "pylib/mozreview"],
["dirname", "pylib/rbbz"],
["dirname", "reviewboardmods"],
],
"command": ["%s/mozreview" % ROOT, "refresh", self._path],
},
]
)
p = subprocess.Popen([WATCHMAN, "-j"], stdin=subprocess.PIPE)
p.communicate(data)
res = p.wait()
if res != 0:
raise Exception("error creating watchman trigger")
def repo_urls(self, name):
"""Obtain the http:// and ssh:// URLs for a review repo."""
http_url = "%s%s" % (self.mercurial_url, name)
ssh_url = "ssh://%s:%d/%s" % (self.ssh_hostname, self.ssh_port, name)
return http_url, ssh_url
def create_repository(self, name):
http_url, ssh_url = self.repo_urls(name)
rb = self.get_reviewboard()
rbid = rb.add_repository(os.path.dirname(name) or name, http_url, bugzilla_url=self.bugzilla_url)
self._docker.execute(self.hgrb_id, ["/create-repo", name, str(rbid)])
if self.autoland_id:
time.sleep(1)
self._docker.execute(self.autoland_id, ["/clone-repo", name])
return http_url, ssh_url, rbid
def clone(self, repo, dest, username=None):
"""Clone and configure a review repository.
The specified review repo will be cloned and configured such that it is
bound to this MozReview instance. If a username is specified, all
operations will be performed as that user.
"""
http_url, ssh_url = self.repo_urls(repo)
subprocess.check_call([self._hg, "clone", http_url, dest], cwd="/")
mrext = os.path.join(ROOT, "testing", "mozreview-repo.py")
mrssh = os.path.join(ROOT, "testing", "mozreview-ssh")
rbext = os.path.join(ROOT, "hgext", "reviewboard", "client.py")
with open(os.path.join(dest, ".hg", "hgrc"), "w") as fh:
lines = [
"[paths]",
"default = %s" % http_url,
"default-push = %s" % ssh_url,
"",
"[extensions]",
"mozreview-repo = %s" % mrext,
"reviewboard = %s" % rbext,
"",
"[ui]",
"ssh = %s" % mrssh,
"",
# TODO use ircnick from the current user, if available.
"[mozilla]",
"ircnick = dummy",
"",
"[mozreview]",
"home = %s" % self._path,
]
if username:
lines.append("username = %s" % username)
fh.write("\n".join(lines))
fh.write("\n")
def get_local_repository(self, path, ircnick=None, bugzilla_username=None, bugzilla_apikey=None):
"""Obtain a LocalMercurialRepository for the named server repository.
Call this with the same argument passed to ``create_repository()``
to obtain an object to interface with a local clone of that server
repository.
If bugzilla credentials are passed, they will be defined in the
repository's hgrc.
The repository is configured to be in deterministic mode. Therefore
these repositories are suitable for use in tests.
"""
localrepos = os.path.join(self._path, "localrepos")
try:
os.mkdir(localrepos)
except OSError as e:
if e.errno != errno.EEXIST:
raise
local_path = os.path.join(localrepos, os.path.basename(path))
http_url, ssh_url = self.repo_urls(path)
# TODO make pushes via SSH work (it doesn't work outside of Mercurial
# tests because dummy expects certain environment variables).
return LocalMercurialRepository(
self._path,
self._hg,
local_path,
http_url,
push_url=ssh_url,
ircnick=ircnick,
bugzilla_username=bugzilla_username,
bugzilla_apikey=bugzilla_apikey,
)
def create_user_api_key(self, email, sync_to_reviewboard=True):
"""Creates an API key for the given user.
This creates an API key in Bugzilla and then triggers the
auth-delegation callback to register the key with Review Board. Note
that this also logs the user in, although we don't record the session
cookie anywhere so this shouldn't have an effect on subsequent
interactions with Review Board.
"""
api_key = self._docker.execute(
self.bmoweb_id, ["/var/lib/bugzilla/bugzilla/scripts/issue-api-key.pl", email], stdout=True
).strip()
assert len(api_key) == 40
if not sync_to_reviewboard:
return api_key
# When running tests in parallel, the auth callback can time out.
# Try up to 3 times before giving up.
url = self.reviewboard_url + "mozreview/bmo_auth_callback/"
data = {"client_api_login": email, "client_api_key": api_key}
for i in range(3):
response = requests.post(url, data=json.dumps(data))
if response.status_code == 200:
result = response.json()["result"]
break
else:
raise Exception("Failed to successfully run the BMO auth POST " "callback.")
params = {"client_api_login": email, "callback_result": result}
for i in range(3):
if requests.get(url, params=params).status_code == 200:
break
else:
raise Exception("Failed to successfully run the BMO auth GET " "callback.")
return api_key
def create_user(
self,
email,
password,
fullname,
bugzilla_groups=None,
uid=None,
username=None,
key_filename=None,
scm_level=None,
api_key=True,
):
"""Create a new user.
This will create a user in at least Bugzilla. If the ``uid`` argument
is specified, an LDAP user will be created as well.
``email`` is the email address of the user.
``password`` is the plain text Bugzilla password.
``fullname`` is the full name of the user. This is stored in both
Bugzilla and the system account for the user (if an LDAP user is being
created).
``bugzilla_groups`` is an iterable of Bugzilla groups to add the user
to.
``uid`` is the numeric UID for the created system/LDAP account.
``username`` is the UNIX username for this user. It defaults to the
username part of the email address.
``key_filename`` is a path to an ssh key. This is used only if ``uid``
is given. If ``uid`` is given but ``key_filename`` is not specified,
the latter defaults to <mozreview path>/keys/<email>.
``scm_level`` defines the source code level access to grant to this
user. Supported levels are ``1``, ``2``, and ``3``. If not specified,
the user won't be able to push to any repos.
"""
bugzilla_groups = bugzilla_groups or []
b = self.get_bugzilla()
if not username:
username = email[0 : email.index("@")]
res = {"bugzilla": b.create_user(email, password, fullname)}
for g in bugzilla_groups:
b.add_user_to_group(email, g)
if api_key:
res["bugzilla"]["api_key"] = self.create_user_api_key(email)
# Create an LDAP account as well.
if uid:
if key_filename is None:
key_filename = os.path.join(self._path, "keys", email)
lr = self.get_ldap().create_user(
email, username, uid, fullname, key_filename=key_filename, scm_level=scm_level
)
res.update(lr)
credentials_path = os.path.join(self._path, "credentials", email)
with open(credentials_path, "wb") as fh:
fh.write(password)
return res
@property
def _hg(self):
for path in os.environ["PATH"].split(os.pathsep):
hg = os.path.join(path, "hg")
if os.path.isfile(hg):
return hg
raise Exception("could not find hg executable")