def magic_folder(reactor, alice_invite, alice, bob, temp_dir, request):
print("pairing magic-folder")
bob_dir = join(temp_dir, 'bob')
proto = _CollectOutputProtocol()
_tahoe_runner_optional_coverage(proto, reactor, request, [
'magic-folder',
'join',
'--poll-interval',
'1',
'--basedir',
bob_dir,
alice_invite,
join(temp_dir, 'magic-bob'),
])
pytest_twisted.blockon(proto.done)
# before magic-folder works, we have to stop and restart (this is
# crappy for the tests -- can we fix it in magic-folder?)
try:
print("Sending TERM to Bob")
bob.transport.signalProcess('TERM')
pytest_twisted.blockon(bob.transport.exited)
except ProcessExitedAlready:
pass
magic_text = 'Completed initial Magic Folder scan successfully'
pytest_twisted.blockon(_run_node(reactor, bob_dir, request, magic_text))
await_client_ready(bob)
return (join(temp_dir, 'magic-alice'), join(temp_dir, 'magic-bob'))
def magic_folder(reactor, alice_invite, alice, bob, temp_dir, request):
print("pairing magic-folder")
bob_dir = join(temp_dir, 'bob')
proto = _CollectOutputProtocol()
reactor.spawnProcess(proto, sys.executable, [
sys.executable,
'-m',
'allmydata.scripts.runner',
'magic-folder',
'join',
'--poll-interval',
'2',
'--basedir',
bob_dir,
alice_invite,
join(temp_dir, 'magic-bob'),
])
pytest_twisted.blockon(proto.done)
# before magic-folder works, we have to stop and restart (this is
# crappy for the tests -- can we fix it in magic-folder?)
try:
print("Sending TERM to Bob")
bob.signalProcess('TERM')
pytest_twisted.blockon(bob.exited)
except ProcessExitedAlready:
pass
magic_text = 'Completed initial Magic Folder scan successfully'
pytest_twisted.blockon(_run_node(reactor, bob_dir, request, magic_text))
return (join(temp_dir, 'magic-alice'), join(temp_dir, 'magic-bob'))
def magic_folder(reactor, alice_invite, alice, bob, temp_dir, request):
print("pairing magic-folder")
bob_dir = join(temp_dir, 'bob')
proto = _CollectOutputProtocol()
transport = reactor.spawnProcess(
proto,
sys.executable,
[
sys.executable, '-m', 'allmydata.scripts.runner',
'magic-folder', 'join',
'--poll-interval', '2',
'--basedir', bob_dir,
alice_invite,
join(temp_dir, 'magic-bob'),
]
)
pytest.blockon(proto.done)
# before magic-folder works, we have to stop and restart (this is
# crappy for the tests -- can we fix it in magic-folder?)
try:
print("Sending TERM to Bob")
bob.signalProcess('TERM')
pytest.blockon(bob.exited)
except ProcessExitedAlready:
pass
magic_text = 'Completed initial Magic Folder scan successfully'
pytest.blockon(_run_node(reactor, bob_dir, request, magic_text))
return (join(temp_dir, 'magic-alice'), join(temp_dir, 'magic-bob'))
def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes,
request):
process = pytest_twisted.blockon(
_create_node(
reactor,
request,
temp_dir,
introducer_furl,
flog_gatherer,
"alice",
web_port="tcp:9980:interface=localhost",
storage=False,
# We're going to kill this ourselves, so no need for finalizer to
# do it:
finalize=False,
))
await_client_ready(process)
# 1. Create a new RW directory cap:
cli(process, "create-alias", "test")
rwcap = loads(cli(process, "list-aliases", "--json"))["test"]["readwrite"]
# 2. Enable SFTP on the node:
host_ssh_key_path = join(process.node_dir, "private", "ssh_host_rsa_key")
accounts_path = join(process.node_dir, "private", "accounts")
with open(join(process.node_dir, "tahoe.cfg"), "a") as f:
f.write("""\
[sftpd]
enabled = true
port = tcp:8022:interface=127.0.0.1
host_pubkey_file = {ssh_key_path}.pub
host_privkey_file = {ssh_key_path}
accounts.file = {accounts_path}
""".format(ssh_key_path=host_ssh_key_path, accounts_path=accounts_path))
generate_ssh_key(host_ssh_key_path)
# 3. Add a SFTP access file with username/password and SSH key auth.
# The client SSH key path is typically going to be somewhere else (~/.ssh,
# typically), but for convenience sake for testing we'll put it inside node.
client_ssh_key_path = join(process.node_dir, "private",
"ssh_client_rsa_key")
generate_ssh_key(client_ssh_key_path)
# Pub key format is "ssh-rsa <thekey> <username>". We want the key.
ssh_public_key = open(client_ssh_key_path +
".pub").read().strip().split()[1]
with open(accounts_path, "w") as f:
f.write("""\
alice password {rwcap}
alice2 ssh-rsa {ssh_public_key} {rwcap}
""".format(rwcap=rwcap, ssh_public_key=ssh_public_key))
# 4. Restart the node with new SFTP config.
process.kill()
pytest_twisted.blockon(_run_node(reactor, process.node_dir, request, None))
await_client_ready(process)
return process
def alice_invite(reactor, alice, temp_dir, request):
node_dir = join(temp_dir, 'alice')
with start_action(action_type=u"integration:alice:magic_folder:create"):
# FIXME XXX by the time we see "client running" in the logs, the
# storage servers aren't "really" ready to roll yet (uploads fairly
# consistently fail if we don't hack in this pause...)
import time
time.sleep(5)
proto = _CollectOutputProtocol()
reactor.spawnProcess(proto, sys.executable, [
sys.executable,
'-m',
'allmydata.scripts.runner',
'magic-folder',
'create',
'--poll-interval',
'2',
'--basedir',
node_dir,
'magik:',
'alice',
join(temp_dir, 'magic-alice'),
])
pytest_twisted.blockon(proto.done)
with start_action(
action_type=u"integration:alice:magic_folder:invite") as a:
proto = _CollectOutputProtocol()
reactor.spawnProcess(proto, sys.executable, [
sys.executable,
'-m',
'allmydata.scripts.runner',
'magic-folder',
'invite',
'--basedir',
node_dir,
'magik:',
'bob',
])
pytest_twisted.blockon(proto.done)
invite = proto.output.getvalue()
a.add_success_fields(invite=invite)
with start_action(action_type=u"integration:alice:magic_folder:restart"):
# before magic-folder works, we have to stop and restart (this is
# crappy for the tests -- can we fix it in magic-folder?)
try:
alice.signalProcess('TERM')
pytest_twisted.blockon(alice.exited)
except ProcessExitedAlready:
pass
with start_action(
action_type=u"integration:alice:magic_folder:magic-text"):
magic_text = 'Completed initial Magic Folder scan successfully'
pytest_twisted.blockon(
_run_node(reactor, node_dir, request, magic_text))
return invite
def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl):
node_dir = join(temp_dir, name)
web_port = "tcp:{}:interface=localhost".format(control_port + 2000)
if True:
print("creating", node_dir)
mkdir(node_dir)
proto = util._DumpOutputProtocol(None)
reactor.spawnProcess(
proto,
sys.executable,
(
sys.executable, '-m', 'allmydata.scripts.runner',
'create-node',
'--nickname', name,
'--introducer', introducer_furl,
'--hide-ip',
'--tor-control-port', 'tcp:localhost:{}'.format(control_port),
'--listen', 'tor',
node_dir,
)
)
yield proto.done
with open(join(node_dir, 'tahoe.cfg'), 'w') as f:
f.write('''
[node]
nickname = %(name)s
web.port = %(web_port)s
web.static = public_html
log_gatherer.furl = %(log_furl)s
[tor]
control.port = tcp:localhost:%(control_port)d
onion.external_port = 3457
onion.local_port = %(local_port)d
onion = true
onion.private_key_file = private/tor_onion.privkey
[client]
# Which services should this client connect to?
introducer.furl = %(furl)s
shares.needed = 1
shares.happy = 1
shares.total = 2
''' % {
'name': name,
'furl': introducer_furl,
'web_port': web_port,
'log_furl': flog_gatherer,
'control_port': control_port,
'local_port': control_port + 1000,
})
print("running")
yield util._run_node(reactor, node_dir, request, None)
print("okay, launched")
def test_alice_adds_files_while_bob_is_offline(reactor, request, temp_dir,
magic_folder):
"""
Alice can add new files to a magic folder while Bob is offline. When Bob
comes back online his copy is updated to reflect the new files.
"""
alice_magic_dir, bob_magic_dir = magic_folder
alice_node_dir = join(temp_dir, "alice")
bob_node_dir = join(temp_dir, "bob")
# Take Bob offline.
yield util.cli(request, reactor, bob_node_dir, "stop")
# Create a couple files in Alice's local directory.
some_files = list((name * 3) + ".added-while-offline" for name in "xyz")
for name in some_files:
with open(join(alice_magic_dir, name), "w") as f:
f.write(name + " some content")
good = False
for i in range(15):
status = yield util.magic_folder_cli(request, reactor, alice_node_dir,
"status")
good = status.count(".added-while-offline (36 B): good, version=0"
) == len(some_files) * 2
if good:
# We saw each file as having a local good state and a remote good
# state. That means we're ready to involve Bob.
break
else:
time.sleep(1.0)
assert good, ("Timed out waiting for good Alice state. Last status:\n{}".
format(status))
# Start Bob up again
magic_text = 'Completed initial Magic Folder scan successfully'
yield util._run_node(reactor, bob_node_dir, request, magic_text)
yield util.await_files_exist(
list(join(bob_magic_dir, name) for name in some_files),
await_all=True,
)
# Let it settle. It would be nicer to have a readable status output we
# could query. Parsing the current text format is more than I want to
# deal with right now.
time.sleep(1.0)
conflict_files = list(name + ".conflict" for name in some_files)
assert all(
list(not exists(join(bob_magic_dir, name))
for name in conflict_files), )
def alice_invite(reactor, alice, temp_dir, request):
node_dir = join(temp_dir, 'alice')
with start_action(action_type=u"integration:alice:magic_folder:create"):
# FIXME XXX by the time we see "client running" in the logs, the
# storage servers aren't "really" ready to roll yet (uploads fairly
# consistently fail if we don't hack in this pause...)
import time ; time.sleep(5)
proto = _CollectOutputProtocol()
reactor.spawnProcess(
proto,
sys.executable,
[
sys.executable, '-m', 'allmydata.scripts.runner',
'magic-folder', 'create',
'--poll-interval', '2',
'--basedir', node_dir, 'magik:', 'alice',
join(temp_dir, 'magic-alice'),
]
)
pytest_twisted.blockon(proto.done)
with start_action(action_type=u"integration:alice:magic_folder:invite") as a:
proto = _CollectOutputProtocol()
reactor.spawnProcess(
proto,
sys.executable,
[
sys.executable, '-m', 'allmydata.scripts.runner',
'magic-folder', 'invite',
'--basedir', node_dir, 'magik:', 'bob',
]
)
pytest_twisted.blockon(proto.done)
invite = proto.output.getvalue()
a.add_success_fields(invite=invite)
with start_action(action_type=u"integration:alice:magic_folder:restart"):
# before magic-folder works, we have to stop and restart (this is
# crappy for the tests -- can we fix it in magic-folder?)
try:
alice.signalProcess('TERM')
pytest_twisted.blockon(alice.exited)
except ProcessExitedAlready:
pass
with start_action(action_type=u"integration:alice:magic_folder:magic-text"):
magic_text = 'Completed initial Magic Folder scan successfully'
pytest_twisted.blockon(_run_node(reactor, node_dir, request, magic_text))
return invite
def test_edmond_uploads_then_restarts(reactor, request, temp_dir,
introducer_furl, flog_gatherer,
storage_nodes):
"""
ticket 2880: if a magic-folder client uploads something, then
re-starts a spurious .backup file should not appear
"""
edmond_dir = join(temp_dir, 'edmond')
edmond = yield util._create_node(
reactor,
request,
temp_dir,
introducer_furl,
flog_gatherer,
"edmond",
web_port="tcp:9985:interface=localhost",
storage=False,
)
magic_folder = join(temp_dir, 'magic-edmond')
mkdir(magic_folder)
created = False
# create a magic-folder
# (how can we know that the grid is ready?)
for _ in range(10): # try 10 times
try:
proto = util._CollectOutputProtocol()
transport = reactor.spawnProcess(proto, sys.executable, [
sys.executable,
'-m',
'allmydata.scripts.runner',
'magic-folder',
'create',
'--poll-interval',
'2',
'--basedir',
edmond_dir,
'magik:',
'edmond_magic',
magic_folder,
])
yield proto.done
created = True
break
except Exception as e:
print("failed to create magic-folder: {}".format(e))
time.sleep(1)
assert created, "Didn't create a magic-folder"
# to actually-start the magic-folder we have to re-start
edmond.signalProcess('TERM')
yield edmond._protocol.exited
time.sleep(1)
edmond = yield util._run_node(
reactor, edmond._node_dir, request,
'Completed initial Magic Folder scan successfully')
# add a thing to the magic-folder
with open(join(magic_folder, "its_a_file"), "w") as f:
f.write("edmond wrote this")
# fixme, do status-update attempts in a loop below
time.sleep(5)
# let it upload; poll the HTTP magic-folder status API until it is
# uploaded
from allmydata.scripts.magic_folder_cli import _get_json_for_fragment
with open(join(edmond_dir, u'private', u'api_auth_token'), 'rb') as f:
token = f.read()
uploaded = False
for _ in range(10):
options = {
"node-url": open(join(edmond_dir, u'node.url'),
'r').read().strip(),
}
try:
magic_data = _get_json_for_fragment(options,
'magic_folder?t=json',
method='POST',
post_args=dict(
t='json',
name='default',
token=token,
))
for mf in magic_data:
if mf['status'] == u'success' and mf['path'] == u'its_a_file':
uploaded = True
break
except Exception as e:
time.sleep(1)
assert uploaded, "expected to upload 'its_a_file'"
# re-starting edmond right now would "normally" trigger the 2880 bug
# kill edmond
edmond.signalProcess('TERM')
yield edmond._protocol.exited
time.sleep(1)
edmond = yield util._run_node(
reactor, edmond._node_dir, request,
'Completed initial Magic Folder scan successfully')
# XXX how can we say for sure if we've waited long enough? look at
# tail of logs for magic-folder ... somethingsomething?
print("waiting 20 seconds to see if a .backup appears")
for _ in range(20):
assert exists(join(magic_folder, "its_a_file"))
assert not exists(join(magic_folder, "its_a_file.backup"))
time.sleep(1)
def _create_anonymous_node(reactor, name, control_port, request, temp_dir,
flog_gatherer, tor_network, introducer_furl):
node_dir = join(temp_dir, name)
web_port = "tcp:{}:interface=localhost".format(control_port + 2000)
if True:
print("creating", node_dir)
mkdir(node_dir)
proto = util._DumpOutputProtocol(None)
reactor.spawnProcess(proto, sys.executable, (
sys.executable,
'-m',
'allmydata.scripts.runner',
'create-node',
'--nickname',
name,
'--introducer',
introducer_furl,
'--hide-ip',
'--tor-control-port',
'tcp:localhost:{}'.format(control_port),
'--listen',
'tor',
node_dir,
))
yield proto.done
with open(join(node_dir, 'tahoe.cfg'), 'w') as f:
f.write(
'''
[node]
nickname = %(name)s
web.port = %(web_port)s
web.static = public_html
log_gatherer.furl = %(log_furl)s
[tor]
control.port = tcp:localhost:%(control_port)d
onion.external_port = 3457
onion.local_port = %(local_port)d
onion = true
onion.private_key_file = private/tor_onion.privkey
[client]
# Which services should this client connect to?
introducer.furl = %(furl)s
shares.needed = 1
shares.happy = 1
shares.total = 2
''' % {
'name': name,
'furl': introducer_furl,
'web_port': web_port,
'log_furl': flog_gatherer,
'control_port': control_port,
'local_port': control_port + 1000,
})
print("running")
yield util._run_node(reactor, node_dir, request, None)
print("okay, launched")
def test_alice_adds_files_while_bob_is_offline(reactor, request, temp_dir, magic_folder):
"""
Alice can add new files to a magic folder while Bob is offline. When Bob
comes back online his copy is updated to reflect the new files.
"""
alice_magic_dir, bob_magic_dir = magic_folder
alice_node_dir = join(temp_dir, "alice")
bob_node_dir = join(temp_dir, "bob")
# Take Bob offline.
yield util.cli(reactor, bob_node_dir, "stop")
# Create a couple files in Alice's local directory.
some_files = list(
(name * 3) + ".added-while-offline"
for name
in "xyz"
)
for name in some_files:
with open(join(alice_magic_dir, name), "w") as f:
f.write(name + " some content")
good = False
for i in range(15):
status = yield util.magic_folder_cli(reactor, alice_node_dir, "status")
good = status.count(".added-while-offline (36 B): good, version=0") == len(some_files) * 2
if good:
# We saw each file as having a local good state and a remote good
# state. That means we're ready to involve Bob.
break
else:
time.sleep(1.0)
assert good, (
"Timed out waiting for good Alice state. Last status:\n{}".format(status)
)
# Start Bob up again
magic_text = 'Completed initial Magic Folder scan successfully'
yield util._run_node(reactor, bob_node_dir, request, magic_text)
yield util.await_files_exist(
list(
join(bob_magic_dir, name)
for name
in some_files
),
await_all=True,
)
# Let it settle. It would be nicer to have a readable status output we
# could query. Parsing the current text format is more than I want to
# deal with right now.
time.sleep(1.0)
conflict_files = list(name + ".conflict" for name in some_files)
assert all(
list(
not exists(join(bob_magic_dir, name))
for name
in conflict_files
),
)
def test_edmond_uploads_then_restarts(reactor, request, temp_dir, introducer_furl, flog_gatherer, storage_nodes):
"""
ticket 2880: if a magic-folder client uploads something, then
re-starts a spurious .backup file should not appear
"""
edmond_dir = join(temp_dir, 'edmond')
edmond = yield util._create_node(
reactor, request, temp_dir, introducer_furl, flog_gatherer,
"edmond", web_port="tcp:9985:interface=localhost",
storage=False,
)
magic_folder = join(temp_dir, 'magic-edmond')
mkdir(magic_folder)
created = False
# create a magic-folder
# (how can we know that the grid is ready?)
for _ in range(10): # try 10 times
try:
proto = util._CollectOutputProtocol()
transport = reactor.spawnProcess(
proto,
sys.executable,
[
sys.executable, '-m', 'allmydata.scripts.runner',
'magic-folder', 'create',
'--poll-interval', '2',
'--basedir', edmond_dir,
'magik:',
'edmond_magic',
magic_folder,
]
)
yield proto.done
created = True
break
except Exception as e:
print("failed to create magic-folder: {}".format(e))
time.sleep(1)
assert created, "Didn't create a magic-folder"
# to actually-start the magic-folder we have to re-start
edmond.signalProcess('TERM')
yield edmond._protocol.exited
time.sleep(1)
edmond = yield util._run_node(reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully')
# add a thing to the magic-folder
with open(join(magic_folder, "its_a_file"), "w") as f:
f.write("edmond wrote this")
# fixme, do status-update attempts in a loop below
time.sleep(5)
# let it upload; poll the HTTP magic-folder status API until it is
# uploaded
from allmydata.scripts.magic_folder_cli import _get_json_for_fragment
with open(join(edmond_dir, u'private', u'api_auth_token'), 'rb') as f:
token = f.read()
uploaded = False
for _ in range(10):
options = {
"node-url": open(join(edmond_dir, u'node.url'), 'r').read().strip(),
}
try:
magic_data = _get_json_for_fragment(
options,
'magic_folder?t=json',
method='POST',
post_args=dict(
t='json',
name='default',
token=token,
)
)
for mf in magic_data:
if mf['status'] == u'success' and mf['path'] == u'its_a_file':
uploaded = True
break
except Exception as e:
time.sleep(1)
assert uploaded, "expected to upload 'its_a_file'"
# re-starting edmond right now would "normally" trigger the 2880 bug
# kill edmond
edmond.signalProcess('TERM')
yield edmond._protocol.exited
time.sleep(1)
edmond = yield util._run_node(reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully')
# XXX how can we say for sure if we've waited long enough? look at
# tail of logs for magic-folder ... somethingsomething?
print("waiting 20 seconds to see if a .backup appears")
for _ in range(20):
assert exists(join(magic_folder, "its_a_file"))
assert not exists(join(magic_folder, "its_a_file.backup"))
time.sleep(1)
