def tahoe_venv(request, reactor):
"""
A virtualenv for our Tahoe install, letting us install a different
one from the Tahoe we depend on.
"""
tahoe_env = request.config.getoption("tahoe_tox_env")
print("creating venv")
out_protocol = _DumpOutputProtocol(None)
reactor.spawnProcess(
out_protocol,
sys.executable,
("python", "-m", "tox", "--notest", "-e", tahoe_env),
env=environ,
)
pytest_twisted.blockon(out_protocol.done)
output = subprocess.check_output(
[sys.executable, "-m", "tox", "-e", request.config.getoption("tahoe_tox_env"), "--showconfig"],
env=environ,
)
parser = ConfigParser(strict=False)
parser.read_string(output.decode("utf-8"))
venv_dir = parser.get("testenv:{}".format(tahoe_env), 'envdir')
return VirtualEnv(FilePath(venv_dir))
def chutney(reactor, temp_dir):
chutney_dir = join(temp_dir, 'chutney')
mkdir(chutney_dir)
# TODO:
# check for 'tor' binary explicitly and emit a "skip" if we can't
# find it
# XXX yuck! should add a setup.py to chutney so we can at least
# "pip install <path to tarball>" and/or depend on chutney in "pip
# install -e .[dev]" (i.e. in the 'dev' extra)
#
# https://trac.torproject.org/projects/tor/ticket/20343
proto = _DumpOutputProtocol(None)
reactor.spawnProcess(
proto,
'git',
(
'git',
'clone',
'--depth=1',
'https://git.torproject.org/chutney.git',
chutney_dir,
),
env=environ,
)
pytest_twisted.blockon(proto.done)
return chutney_dir
def admin_user():
"""
Fixture that creates a random admin user and deletes the user afterwards.
Useful if the test wishes to alter permissions to test failure cases. Default
permissions is complete access to the "/" vhost.
Returns:
The username of the new administrator. The password is "guest".
"""
# Create a user with no permissions
username = str(uuid.uuid4())
url = "{base}users/{user}".format(base=HTTP_API, user=username)
body = {"username": username, "password": "******", "tags": "administrator"}
deferred_resp = treq.put(url, json=body, auth=HTTP_AUTH, timeout=3)
@pytest_twisted.inlineCallbacks
def cp(resp):
assert resp.code == 201
url = "{base}permissions/%2F/{user}".format(base=HTTP_API,
user=username)
body = {"configure": ".*", "write": ".*", "read": ".*"}
resp = yield treq.put(url, json=body, auth=HTTP_AUTH, timeout=3)
assert resp.code == 201
deferred_resp.addCallbacks(cp, cp)
pytest_twisted.blockon(deferred_resp)
yield username
# Cleanup
deferred_resp = treq.delete(url, auth=HTTP_AUTH, timeout=3)
pytest_twisted.blockon(deferred_resp)
def setup_function(function):
"""Ensure each test starts with a fresh Service and configuration."""
config.conf = config.LazyConfig()
config.conf["client_properties"]["app"] = function.__name__
if api._twisted_service:
pytest_twisted.blockon(api._twisted_service.stopService())
api._twisted_service = None
def chutney(reactor, temp_dir):
chutney_dir = join(temp_dir, 'chutney')
mkdir(chutney_dir)
# TODO:
# check for 'tor' binary explicitly and emit a "skip" if we can't
# find it
# XXX yuck! should add a setup.py to chutney so we can at least
# "pip install <path to tarball>" and/or depend on chutney in "pip
# install -e .[dev]" (i.e. in the 'dev' extra)
#
# https://trac.torproject.org/projects/tor/ticket/20343
proto = _DumpOutputProtocol(None)
reactor.spawnProcess(
proto,
'/usr/bin/git',
(
'/usr/bin/git', 'clone', '--depth=1',
'https://git.torproject.org/chutney.git',
chutney_dir,
)
)
pytest_twisted.blockon(proto.done)
return chutney_dir
def _cleanup_service_process(process, exited, action):
"""
Terminate the given process with a kill signal (SIGKILL on POSIX,
TerminateProcess on Windows).
:param process: The `IProcessTransport` representing the process.
:param exited: A `Deferred` which fires when the process has exited.
:return: After the process has exited.
"""
try:
with action.context():
def report(m):
log_message(message_type="integration:cleanup", message=m)
print(m)
if process.pid is not None:
report("signaling {} with TERM".format(process.pid))
process.signalProcess('TERM')
report("signaled, blocking on exit")
pytest_twisted.blockon(exited)
report("exited, goodbye")
except ProcessExitedAlready:
pass
def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes,
request):
process = pytest_twisted.blockon(
_create_node(
reactor,
request,
temp_dir,
introducer_furl,
flog_gatherer,
"alice",
web_port="tcp:9980:interface=localhost",
storage=False,
# We're going to kill this ourselves, so no need for finalizer to
# do it:
finalize=False,
))
await_client_ready(process)
# 1. Create a new RW directory cap:
cli(process, "create-alias", "test")
rwcap = loads(cli(process, "list-aliases", "--json"))["test"]["readwrite"]
# 2. Enable SFTP on the node:
host_ssh_key_path = join(process.node_dir, "private", "ssh_host_rsa_key")
accounts_path = join(process.node_dir, "private", "accounts")
with open(join(process.node_dir, "tahoe.cfg"), "a") as f:
f.write("""\
[sftpd]
enabled = true
port = tcp:8022:interface=127.0.0.1
host_pubkey_file = {ssh_key_path}.pub
host_privkey_file = {ssh_key_path}
accounts.file = {accounts_path}
""".format(ssh_key_path=host_ssh_key_path, accounts_path=accounts_path))
generate_ssh_key(host_ssh_key_path)
# 3. Add a SFTP access file with username/password and SSH key auth.
# The client SSH key path is typically going to be somewhere else (~/.ssh,
# typically), but for convenience sake for testing we'll put it inside node.
client_ssh_key_path = join(process.node_dir, "private",
"ssh_client_rsa_key")
generate_ssh_key(client_ssh_key_path)
# Pub key format is "ssh-rsa <thekey> <username>". We want the key.
ssh_public_key = open(client_ssh_key_path +
".pub").read().strip().split()[1]
with open(accounts_path, "w") as f:
f.write("""\
alice password {rwcap}
alice2 ssh-rsa {ssh_public_key} {rwcap}
""".format(rwcap=rwcap, ssh_public_key=ssh_public_key))
# 4. Restart the node with new SFTP config.
process.kill()
pytest_twisted.blockon(_run_node(reactor, process.node_dir, request, None))
await_client_ready(process)
return process
def flog_gatherer(reactor, base_dir, tahoe_venv, flog_binary, request):
if not request.config.getoption("gather_foolscap_logs"):
return ""
with start_task(action_type=u"integration:flog_gatherer"):
out_protocol = _CollectOutputProtocol()
gather_dir = join(base_dir, 'flog_gather')
reactor.spawnProcess(
out_protocol,
flog_binary,
(
u'flogtool', u'create-gatherer',
u'--location', u'tcp:localhost:3117',
u'--port', u'3117',
gather_dir,
)
)
pytest_twisted.blockon(out_protocol.done)
magic_text = "Gatherer waiting at"
executable = str(tahoe_venv.bin('twistd'))
args = (
'twistd', '--nodaemon', '--python',
join(gather_dir, 'gatherer.tac'),
)
action_fields = {
"action_type": u"integration:flog-gatherer:service",
}
pytest_twisted.blockon(
run_service(reactor, request, action_fields, magic_text, executable, args, cwd=gather_dir)
)
def cleanup():
flog_file = 'integration.flog_dump'
flog_protocol = _DumpOutputProtocol(open(flog_file, 'w'))
flogs = [x for x in listdir(gather_dir) if x.endswith('.flog')]
print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file))
reactor.spawnProcess(
flog_protocol,
flog_binary,
(
'flogtool', 'dump', join(gather_dir, flogs[0])
),
)
print("Waiting for flogtool to complete")
try:
pytest_twisted.blockon(flog_protocol.done)
except ProcessTerminated as e:
print("flogtool exited unexpectedly: {}".format(str(e)))
print("Flogtool completed")
request.addfinalizer(cleanup)
with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f:
furl = f.read().strip()
return furl
def tor_introducer(reactor, temp_dir, flog_gatherer, request):
config = '''
[node]
nickname = introducer_tor
web.port = 4561
log_gatherer.furl = {log_furl}
'''.format(log_furl=flog_gatherer)
intro_dir = join(temp_dir, 'introducer_tor')
print("making introducer", intro_dir)
if not exists(intro_dir):
mkdir(intro_dir)
done_proto = _ProcessExitedProtocol()
_tahoe_runner_optional_coverage(
done_proto,
reactor,
request,
(
'create-introducer',
'--tor-control-port',
'tcp:localhost:8010',
'--listen=tor',
intro_dir,
),
)
pytest_twisted.blockon(done_proto.done)
# over-write the config file with our stuff
with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
f.write(config)
# on windows, "tahoe start" means: run forever in the foreground,
# but on linux it means daemonize. "tahoe run" is consistent
# between platforms.
protocol = _MagicTextProtocol('introducer running')
transport = _tahoe_runner_optional_coverage(
protocol,
reactor,
request,
(
'run',
intro_dir,
),
)
def cleanup():
try:
transport.signalProcess('TERM')
pytest_twisted.blockon(protocol.exited)
except ProcessExitedAlready:
pass
request.addfinalizer(cleanup)
pytest_twisted.blockon(protocol.magic_seen)
return transport
def sleep(timeout):
"""
Sleep for the given amount of time, letting the pytest-twisted reactor run.
This can only be called from the main pytest greenlet, not from the reactor
greenlet.
"""
from twisted.internet import reactor as _reactor
pytest_twisted.blockon(twisted_sleep(_reactor, timeout))
def test_allocate_channel_no_more_channels(self):
"""Assert a pika NoFreeChannels exception turns into a fedora_messaging exception."""
self.protocol.channel = mock.Mock(
name="channel", side_effect=pika.exceptions.NoFreeChannels())
d = self.protocol._allocate_channel()
d.addCallback(pytest.fail, "Expected a NoFreeChannels exception")
d.addErrback(lambda f: f.trap(NoFreeChannels))
pytest_twisted.blockon(d)
def flog_gatherer(reactor, temp_dir, flog_binary, request):
out_protocol = _CollectOutputProtocol()
gather_dir = join(temp_dir, 'flog_gather')
reactor.spawnProcess(out_protocol, flog_binary, (
'flogtool',
'create-gatherer',
'--location',
'tcp:localhost:3117',
'--port',
'3117',
gather_dir,
))
pytest_twisted.blockon(out_protocol.done)
twistd_protocol = _MagicTextProtocol("Gatherer waiting at")
twistd_process = reactor.spawnProcess(
twistd_protocol,
which('twistd')[0],
(
'twistd',
'--nodaemon',
'--python',
join(gather_dir, 'gatherer.tac'),
),
path=gather_dir,
)
pytest_twisted.blockon(twistd_protocol.magic_seen)
def cleanup():
_cleanup_tahoe_process(twistd_process, twistd_protocol.exited)
flog_file = mktemp('.flog_dump')
flog_protocol = _DumpOutputProtocol(open(flog_file, 'w'))
flog_dir = join(temp_dir, 'flog_gather')
flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')]
print("Dumping {} flogtool logfiles to '{}'".format(
len(flogs), flog_file))
reactor.spawnProcess(
flog_protocol,
flog_binary,
('flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0])),
)
print("Waiting for flogtool to complete")
try:
pytest_twisted.blockon(flog_protocol.done)
except ProcessTerminated as e:
print("flogtool exited unexpectedly: {}".format(str(e)))
print("Flogtool completed")
request.addfinalizer(cleanup)
with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f:
furl = f.read().strip()
return furl
def tor_introducer(reactor, temp_dir, flog_gatherer, request):
config = '''
[node]
nickname = introducer_tor
web.port = 4561
log_gatherer.furl = {log_furl}
'''.format(log_furl=flog_gatherer)
intro_dir = join(temp_dir, 'introducer_tor')
print("making introducer", intro_dir)
if not exists(intro_dir):
mkdir(intro_dir)
done_proto = _ProcessExitedProtocol()
reactor.spawnProcess(
done_proto,
sys.executable,
(
sys.executable, '-m', 'allmydata.scripts.runner',
'create-introducer',
'--tor-control-port', 'tcp:localhost:8010',
'--listen=tor',
intro_dir,
),
)
pytest_twisted.blockon(done_proto.done)
# over-write the config file with our stuff
with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
f.write(config)
# on windows, "tahoe start" means: run forever in the foreground,
# but on linux it means daemonize. "tahoe run" is consistent
# between platforms.
protocol = _MagicTextProtocol('introducer running')
process = reactor.spawnProcess(
protocol,
sys.executable,
(
sys.executable, '-m', 'allmydata.scripts.runner',
'run',
intro_dir,
),
)
def cleanup():
try:
process.signalProcess('TERM')
pytest_twisted.blockon(protocol.exited)
except ProcessExitedAlready:
pass
request.addfinalizer(cleanup)
pytest_twisted.blockon(protocol.magic_seen)
return process
def flog_gatherer(reactor, temp_dir, flog_binary, request):
out_protocol = _CollectOutputProtocol()
gather_dir = join(temp_dir, 'flog_gather')
reactor.spawnProcess(
out_protocol,
flog_binary,
(
'flogtool', 'create-gatherer',
'--location', 'tcp:localhost:3117',
'--port', '3117',
gather_dir,
)
)
pytest_twisted.blockon(out_protocol.done)
twistd_protocol = _MagicTextProtocol("Gatherer waiting at")
twistd_process = reactor.spawnProcess(
twistd_protocol,
which('twistd')[0],
(
'twistd', '--nodaemon', '--python',
join(gather_dir, 'gatherer.tac'),
),
path=gather_dir,
)
pytest_twisted.blockon(twistd_protocol.magic_seen)
def cleanup():
_cleanup_twistd_process(twistd_process, twistd_protocol.exited)
flog_file = mktemp('.flog_dump')
flog_protocol = _DumpOutputProtocol(open(flog_file, 'w'))
flog_dir = join(temp_dir, 'flog_gather')
flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')]
print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file))
reactor.spawnProcess(
flog_protocol,
flog_binary,
(
'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0])
),
)
print("Waiting for flogtool to complete")
try:
pytest_twisted.blockon(flog_protocol.done)
except ProcessTerminated as e:
print("flogtool exited unexpectedly: {}".format(str(e)))
print("Flogtool completed")
request.addfinalizer(cleanup)
with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f:
furl = f.read().strip()
return furl
def i2p_introducer(reactor, temp_dir, flog_gatherer, request):
config = '''
[node]
nickname = introducer_i2p
web.port = 4561
log_gatherer.furl = {log_furl}
'''.format(log_furl=flog_gatherer)
intro_dir = join(temp_dir, 'introducer_i2p')
print("making introducer", intro_dir)
if not exists(intro_dir):
mkdir(intro_dir)
done_proto = util._ProcessExitedProtocol()
util._tahoe_runner_optional_coverage(
done_proto,
reactor,
request,
(
'create-introducer',
'--listen=i2p',
intro_dir,
),
)
pytest_twisted.blockon(done_proto.done)
# over-write the config file with our stuff
with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
f.write(config)
# "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
# "start" command.
protocol = util._MagicTextProtocol('introducer running')
transport = util._tahoe_runner_optional_coverage(
protocol,
reactor,
request,
(
'run',
intro_dir,
),
)
def cleanup():
try:
transport.signalProcess('TERM')
util.block_with_timeout(protocol.exited, reactor)
except ProcessExitedAlready:
pass
request.addfinalizer(cleanup)
pytest_twisted.blockon(protocol.magic_seen)
return transport
def enabled_plugins(started_deluge_client):
started_deluge_client.core.set_config({"pre_allocate_storage": True})
pytest_twisted.blockon(started_deluge_client.core.enable_plugin("Label"))
pytest_twisted.blockon(
started_deluge_client.core.enable_plugin("Waterwheel"))
setup_translation()
yield
label_plugin = component.get("CorePlugin.Label")
for label in label_plugin.get_labels():
label_plugin.remove(label)
def introducer(reactor, temp_dir, flog_gatherer, request):
config = '''
[node]
nickname = introducer0
web.port = 4560
log_gatherer.furl = {log_furl}
'''.format(log_furl=flog_gatherer)
intro_dir = join(temp_dir, 'introducer')
print("making introducer", intro_dir)
if not exists(intro_dir):
mkdir(intro_dir)
done_proto = _ProcessExitedProtocol()
reactor.spawnProcess(
done_proto,
sys.executable,
(
sys.executable,
'-m',
'allmydata.scripts.runner',
'create-introducer',
'--listen=tcp',
'--hostname=localhost',
intro_dir,
),
)
pytest_twisted.blockon(done_proto.done)
# over-write the config file with our stuff
with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
f.write(config)
# on windows, "tahoe start" means: run forever in the foreground,
# but on linux it means daemonize. "tahoe run" is consistent
# between platforms.
protocol = _MagicTextProtocol('introducer running')
process = reactor.spawnProcess(
protocol,
sys.executable,
(
sys.executable,
'-m',
'allmydata.scripts.runner',
'run',
intro_dir,
),
)
request.addfinalizer(
partial(_cleanup_twistd_process, process, protocol.exited))
pytest_twisted.blockon(protocol.magic_seen)
return process
def device(factory_device):
factory_device.bus.transmit = True
factory_device.cyclic_send_signal(
path=('ProcessToInverter', 'CommandModeControl', 'Enable'),
period=0.1,
)
try:
yield factory_device
finally:
pytest_twisted.blockon(factory_device.set_access_level(level=0), )
factory_device.bus.transmit = False
factory_device.cancel_all_cyclic_sends()
def cleanup():
print("Tearing down Chutney Tor network")
proto = _CollectOutputProtocol()
reactor.spawnProcess(
proto,
sys.executable,
(
sys.executable, '-m', 'chutney.TorNet', 'stop',
join(chutney_dir, 'networks', 'basic'),
),
path=join(chutney_dir),
env={"PYTHONPATH": join(chutney_dir, "lib")},
)
pytest_twisted.blockon(proto.done)
def cleanup():
print("Tearing down Chutney Tor network")
proto = _CollectOutputProtocol()
reactor.spawnProcess(
proto,
sys.executable,
(
sys.executable, '-m', 'chutney.TorNet', 'stop',
join(chutney_dir, 'networks', 'basic'),
),
path=join(chutney_dir),
env=env,
)
pytest_twisted.blockon(proto.done)
def cleanup():
try:
pytest_twisted.blockon(
self.management.session.call(
u"crossbarfabriccenter.remote.node.stop_worker",
self.node_oid,
name,
))
except Exception:
# this can fail if e.g. ALL the tests are stopping and
# other pytest shutdown code runs before this does --
# probably only CancelledError needs to be caught
# here.
pass
def introducer(reactor, temp_dir, flog_gatherer, request):
config = '''
[node]
nickname = introducer0
web.port = 4560
log_gatherer.furl = {log_furl}
'''.format(log_furl=flog_gatherer)
intro_dir = join(temp_dir, 'introducer')
print("making introducer", intro_dir)
if not exists(intro_dir):
mkdir(intro_dir)
done_proto = _ProcessExitedProtocol()
_tahoe_runner_optional_coverage(
done_proto,
reactor,
request,
(
'create-introducer',
'--listen=tcp',
'--hostname=localhost',
intro_dir,
),
)
pytest_twisted.blockon(done_proto.done)
# over-write the config file with our stuff
with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
f.write(config)
# "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
# "start" command.
protocol = _MagicTextProtocol('introducer running')
transport = _tahoe_runner_optional_coverage(
protocol,
reactor,
request,
(
'run',
intro_dir,
),
)
request.addfinalizer(
partial(_cleanup_tahoe_process, transport, protocol.exited))
pytest_twisted.blockon(protocol.magic_seen)
return TahoeProcess(transport, intro_dir)
def alice_invite(reactor, alice, temp_dir, request):
node_dir = join(temp_dir, 'alice')
with start_action(action_type=u"integration:alice:magic_folder:create"):
# FIXME XXX by the time we see "client running" in the logs, the
# storage servers aren't "really" ready to roll yet (uploads fairly
# consistently fail if we don't hack in this pause...)
import time
time.sleep(5)
proto = _CollectOutputProtocol()
reactor.spawnProcess(proto, sys.executable, [
sys.executable,
'-m',
'allmydata.scripts.runner',
'magic-folder',
'create',
'--poll-interval',
'2',
'--basedir',
node_dir,
'magik:',
'alice',
join(temp_dir, 'magic-alice'),
])
pytest_twisted.blockon(proto.done)
with start_action(
action_type=u"integration:alice:magic_folder:invite") as a:
proto = _CollectOutputProtocol()
reactor.spawnProcess(proto, sys.executable, [
sys.executable,
'-m',
'allmydata.scripts.runner',
'magic-folder',
'invite',
'--basedir',
node_dir,
'magik:',
'bob',
])
pytest_twisted.blockon(proto.done)
invite = proto.output.getvalue()
a.add_success_fields(invite=invite)
with start_action(action_type=u"integration:alice:magic_folder:restart"):
# before magic-folder works, we have to stop and restart (this is
# crappy for the tests -- can we fix it in magic-folder?)
try:
alice.signalProcess('TERM')
pytest_twisted.blockon(alice.exited)
except ProcessExitedAlready:
pass
with start_action(
action_type=u"integration:alice:magic_folder:magic-text"):
magic_text = 'Completed initial Magic Folder scan successfully'
pytest_twisted.blockon(
_run_node(reactor, node_dir, request, magic_text))
return invite
def test_open_server():
server = RestServer(access_config=OpenAccessConfig())
d = server.start()
def return_server(results, srv):
return srv if results else None
def failed(reason):
return reason
d.addCallback(return_server, server)
d.addErrback(failed)
yield pytest_twisted.blockon(d)
d = server.stop()
pytest_twisted.blockon(d)
def test_stopTrying(self):
# The _client_ready deferred must errback when we stop trying to
# reconnect.
self.factory._client_ready.addCallbacks(
self.fail, lambda f: f.trap(pika.exceptions.AMQPConnectionError))
self.factory.stopTrying()
return pytest_twisted.blockon(self.factory._client_ready)
def test_passive_declares(self):
"""Assert queues and exchanges are created passively when configured so."""
factory = FedoraMessagingFactory(
None,
queues=[{
"queue": "my_queue"
}],
exchanges=[{
"exchange": "my_exchange"
}],
)
factory.buildProtocol(None)
mock_channel = mock.Mock()
factory.client._allocate_channel = mock.Mock(return_value=mock_channel)
def _check(_):
mock_channel.queue_declare.assert_called_once_with(
queue="my_queue", passive=True)
mock_channel.exchange_declare.assert_called_once_with(
exchange="my_exchange", passive=True)
with mock.patch.dict(config.conf, {"passive_declares": True}):
d = factory._on_client_ready()
d.addCallback(_check)
return pytest_twisted.blockon(d)
def test_resumeProducing(self):
# Check the resumeProducing method.
self.protocol._running = False
callback = mock.Mock()
self.protocol._consumers = {
"testqueue1":
Consumer("consumer1", "testqueue1", callback,
self.protocol._channel),
"testqueue2":
Consumer("consumer2", "testqueue2", callback,
self.protocol._channel),
"testqueue3":
Consumer("consumer3", "testqueue3", callback,
self.protocol._channel),
}
self.protocol._read = mock.Mock(
side_effect=lambda _, __: defer.succeed(None))
d = self.protocol.resumeProducing()
def _check(_):
self.assertTrue(self.protocol._running)
self.assertEqual(self.protocol._channel.basic_consume.call_count,
3)
called_queues = [
kw["queue"] for arg, kw in
self.protocol._channel.basic_consume.call_args_list
]
self.assertEqual(sorted(called_queues),
sorted(self.protocol._consumers.keys()))
self.assertEqual(self.protocol._read.call_count, 3)
d.addCallback(_check)
return pytest_twisted.blockon(d)
def test_when_connected_unexpected_failure(self):
"""Assert when_connected errbacks when the connection fails."""
class DummyError(Exception):
pass
def _get_protocol(*a, **kw):
protocol = mock.Mock(name="protocol mock")
# Fail immediately
protocol.ready = defer.fail(DummyError())
return protocol
def _check(f):
f.trap(DummyError)
# Make sure a new Deferred has been generated for when_connected()
new_d = self.factory.when_connected()
assert new_d.called is False
assert new_d != connected_d
self.factory.protocol = _get_protocol
connected_d = self.factory.when_connected()
connected_d.addCallbacks(
lambda r: ValueError("This should fail but I got: {!r}".format(r)),
_check)
with mock.patch(
"fedora_messaging.twisted.factory._std_log") as mock_log:
self.factory.buildProtocol(None)
mock_log.error.assert_called()
last_log_call_args = mock_log.error.call_args_list[-1][0]
assert last_log_call_args[0] == (
"The connection failed with an unexpected exception; please report this bug: %s"
)
assert last_log_call_args[1].startswith(
"Traceback (most recent call last):")
return pytest_twisted.blockon(connected_d)
def test_when_connected(self):
"""Assert whenConnected returns the current client once _client_ready fires"""
self.factory.client = mock.Mock()
self.factory._client_ready.callback(None)
d = self.factory.whenConnected()
d.addCallback(
lambda client: self.assertEqual(self.factory.client, client))
return pytest_twisted.blockon(d)
def build(self, monkeypatch):
# To create this fixture file:
# cbs call getBuild 24284 \
# --json-output > txkoji/tests/fixtures/calls/getBuild.json
monkeypatch.setattr('txkoji.connection.Proxy', FakeProxy)
koji = Connection('mykoji')
d = koji.getBuild(24284)
return pytest_twisted.blockon(d)
def tasks(self, monkeypatch):
# To create this fixture file:
# cbs call listTasks --kwargs="{'opts': {'owner': 144, 'method': 'build', 'decode': True}}" --json-output > txkoji/tests/fixtures/calls/listTasks.json # NOQA: E501
monkeypatch.setattr('txkoji.connection.Proxy', FakeProxy)
koji = Connection('mykoji')
opts = {'owner': 144, 'method': 'build'}
d = koji.listTasks(opts)
return pytest_twisted.blockon(d)
def task(self, monkeypatch):
# To create this fixture file:
# cbs call getTaskInfo 291929 --kwargs="{'request': True}" \
# --json-output > txkoji/tests/fixtures/calls/getTaskInfo.json
monkeypatch.setattr('txkoji.connection.Proxy', FakeProxy)
koji = Connection('mykoji')
d = koji.getTaskInfo(291929)
return pytest_twisted.blockon(d)
def channels(self, monkeypatch):
# To create this fixture file:
# cbs call listChannels \
# --json-output > txkoji/tests/fixtures/calls/listChannels.json
monkeypatch.setattr('txkoji.connection.Proxy', FakeProxy)
koji = Connection('mykoji')
d = koji.listChannels()
return pytest_twisted.blockon(d)
def chutney(reactor, temp_dir):
chutney_dir = join(temp_dir, 'chutney')
mkdir(chutney_dir)
# TODO:
# check for 'tor' binary explicitly and emit a "skip" if we can't
# find it
# XXX yuck! should add a setup.py to chutney so we can at least
# "pip install <path to tarball>" and/or depend on chutney in "pip
# install -e .[dev]" (i.e. in the 'dev' extra)
#
# https://trac.torproject.org/projects/tor/ticket/20343
proto = _DumpOutputProtocol(None)
reactor.spawnProcess(
proto,
'git',
(
'git', 'clone',
'https://git.torproject.org/chutney.git',
chutney_dir,
),
env=environ,
)
pytest_twisted.blockon(proto.done)
# XXX: Here we reset Chutney to the last revision known to work
# with Python 2, as a workaround for Chutney moving to Python 3.
# When this is no longer necessary, we will have to drop this and
# add '--depth=1' back to the above 'git clone' subprocess.
proto = _DumpOutputProtocol(None)
reactor.spawnProcess(
proto,
'git',
(
'git', '-C', chutney_dir,
'reset', '--hard',
'99bd06c7554b9113af8c0877b6eca4ceb95dcbaa'
),
env=environ,
)
pytest_twisted.blockon(proto.done)
return chutney_dir
def magic_folder(reactor, alice_invite, alice, bob, temp_dir, request):
print("pairing magic-folder")
bob_dir = join(temp_dir, 'bob')
proto = _CollectOutputProtocol()
reactor.spawnProcess(
proto,
sys.executable,
[
sys.executable, '-m', 'allmydata.scripts.runner',
'magic-folder', 'join',
'--poll-interval', '2',
'--basedir', bob_dir,
alice_invite,
join(temp_dir, 'magic-bob'),
]
)
pytest_twisted.blockon(proto.done)
# before magic-folder works, we have to stop and restart (this is
# crappy for the tests -- can we fix it in magic-folder?)
try:
print("Sending TERM to Bob")
bob.signalProcess('TERM')
pytest_twisted.blockon(bob.exited)
except ProcessExitedAlready:
pass
magic_text = 'Completed initial Magic Folder scan successfully'
pytest_twisted.blockon(_run_node(reactor, bob_dir, request, magic_text))
return (join(temp_dir, 'magic-alice'), join(temp_dir, 'magic-bob'))
def cleanup():
_cleanup_twistd_process(twistd_process, twistd_protocol.exited)
flog_file = mktemp('.flog_dump')
flog_protocol = _DumpOutputProtocol(open(flog_file, 'w'))
flog_dir = join(temp_dir, 'flog_gather')
flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')]
print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file))
reactor.spawnProcess(
flog_protocol,
flog_binary,
(
'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0])
),
)
print("Waiting for flogtool to complete")
try:
pytest_twisted.blockon(flog_protocol.done)
except ProcessTerminated as e:
print("flogtool exited unexpectedly: {}".format(str(e)))
print("Flogtool completed")
def alice_invite(reactor, alice, temp_dir, request):
node_dir = join(temp_dir, 'alice')
with start_action(action_type=u"integration:alice:magic_folder:create"):
# FIXME XXX by the time we see "client running" in the logs, the
# storage servers aren't "really" ready to roll yet (uploads fairly
# consistently fail if we don't hack in this pause...)
import time ; time.sleep(5)
proto = _CollectOutputProtocol()
reactor.spawnProcess(
proto,
sys.executable,
[
sys.executable, '-m', 'allmydata.scripts.runner',
'magic-folder', 'create',
'--poll-interval', '2',
'--basedir', node_dir, 'magik:', 'alice',
join(temp_dir, 'magic-alice'),
]
)
pytest_twisted.blockon(proto.done)
with start_action(action_type=u"integration:alice:magic_folder:invite") as a:
proto = _CollectOutputProtocol()
reactor.spawnProcess(
proto,
sys.executable,
[
sys.executable, '-m', 'allmydata.scripts.runner',
'magic-folder', 'invite',
'--basedir', node_dir, 'magik:', 'bob',
]
)
pytest_twisted.blockon(proto.done)
invite = proto.output.getvalue()
a.add_success_fields(invite=invite)
with start_action(action_type=u"integration:alice:magic_folder:restart"):
# before magic-folder works, we have to stop and restart (this is
# crappy for the tests -- can we fix it in magic-folder?)
try:
alice.signalProcess('TERM')
pytest_twisted.blockon(alice.exited)
except ProcessExitedAlready:
pass
with start_action(action_type=u"integration:alice:magic_folder:magic-text"):
magic_text = 'Completed initial Magic Folder scan successfully'
pytest_twisted.blockon(_run_node(reactor, node_dir, request, magic_text))
return invite
def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):
try:
mkdir(join(temp_dir, 'magic-bob'))
except OSError:
pass
process = pytest_twisted.blockon(
_create_node(
reactor, request, temp_dir, introducer_furl, flog_gatherer, "bob",
web_port="tcp:9981:interface=localhost",
storage=False,
)
)
return process
def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request):
nodes = []
# start all 5 nodes in parallel
for x in range(5):
name = 'node{}'.format(x)
# tub_port = 9900 + x
nodes.append(
pytest_twisted.blockon(
_create_node(
reactor, request, temp_dir, introducer_furl, flog_gatherer, name,
web_port=None, storage=True,
)
)
)
#nodes = pytest_twisted.blockon(DeferredList(nodes))
return nodes
def tor_network(reactor, temp_dir, chutney, request):
# this is the actual "chutney" script at the root of a chutney checkout
chutney_dir = chutney
chut = join(chutney_dir, 'chutney')
# now, as per Chutney's README, we have to create the network
# ./chutney configure networks/basic
# ./chutney start networks/basic
proto = _DumpOutputProtocol(None)
reactor.spawnProcess(
proto,
sys.executable,
(
sys.executable, '-m', 'chutney.TorNet', 'configure',
join(chutney_dir, 'networks', 'basic'),
),
path=join(chutney_dir),
env={"PYTHONPATH": join(chutney_dir, "lib")},
)
pytest_twisted.blockon(proto.done)
proto = _DumpOutputProtocol(None)
reactor.spawnProcess(
proto,
sys.executable,
(
sys.executable, '-m', 'chutney.TorNet', 'start',
join(chutney_dir, 'networks', 'basic'),
),
path=join(chutney_dir),
env={"PYTHONPATH": join(chutney_dir, "lib")},
)
pytest_twisted.blockon(proto.done)
# print some useful stuff
proto = _CollectOutputProtocol()
reactor.spawnProcess(
proto,
sys.executable,
(
sys.executable, '-m', 'chutney.TorNet', 'status',
join(chutney_dir, 'networks', 'basic'),
),
path=join(chutney_dir),
env={"PYTHONPATH": join(chutney_dir, "lib")},
)
try:
pytest_twisted.blockon(proto.done)
except ProcessTerminated:
print("Chutney.TorNet status failed (continuing):")
print(proto.output.getvalue())
def cleanup():
print("Tearing down Chutney Tor network")
proto = _CollectOutputProtocol()
reactor.spawnProcess(
proto,
sys.executable,
(
sys.executable, '-m', 'chutney.TorNet', 'stop',
join(chutney_dir, 'networks', 'basic'),
),
path=join(chutney_dir),
env={"PYTHONPATH": join(chutney_dir, "lib")},
)
pytest_twisted.blockon(proto.done)
request.addfinalizer(cleanup)
return chut
def cleanup():
try:
process.signalProcess('TERM')
pytest_twisted.blockon(protocol.exited)
except ProcessExitedAlready:
pass
def test_that_threads_are_different(self):
self.use_same_thread = False
d = self.request_same_or_different_thread_thread()
d.addCallback(lambda _: self.assert_that_threads_are_different)
return pytest_twisted.blockon(d)
