def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):
edna = yield util._create_node(
reactor, request, temp_dir, introducer_furl, flog_gatherer, "edna",
web_port="tcp:9983:interface=localhost",
storage=False,
needed=3,
happy=7,
total=10,
)
util.await_client_ready(edna)
node_dir = join(temp_dir, 'edna')
# upload a file, which should fail because we have don't have 7
# storage servers (but happiness is set to 7)
proto = util._CollectOutputProtocol()
reactor.spawnProcess(
proto,
sys.executable,
[
sys.executable, '-m', 'allmydata.scripts.runner',
'-d', node_dir,
'put', __file__,
]
)
try:
yield proto.done
assert False, "should raise exception"
except Exception as e:
assert isinstance(e, ProcessTerminated)
output = proto.output.getvalue()
assert "shares could be placed on only" in output
def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes,
request):
process = pytest_twisted.blockon(
_create_node(
reactor,
request,
temp_dir,
introducer_furl,
flog_gatherer,
"alice",
web_port="tcp:9980:interface=localhost",
storage=False,
# We're going to kill this ourselves, so no need for finalizer to
# do it:
finalize=False,
))
await_client_ready(process)
# 1. Create a new RW directory cap:
cli(process, "create-alias", "test")
rwcap = loads(cli(process, "list-aliases", "--json"))["test"]["readwrite"]
# 2. Enable SFTP on the node:
host_ssh_key_path = join(process.node_dir, "private", "ssh_host_rsa_key")
accounts_path = join(process.node_dir, "private", "accounts")
with open(join(process.node_dir, "tahoe.cfg"), "a") as f:
f.write("""\
[sftpd]
enabled = true
port = tcp:8022:interface=127.0.0.1
host_pubkey_file = {ssh_key_path}.pub
host_privkey_file = {ssh_key_path}
accounts.file = {accounts_path}
""".format(ssh_key_path=host_ssh_key_path, accounts_path=accounts_path))
generate_ssh_key(host_ssh_key_path)
# 3. Add a SFTP access file with username/password and SSH key auth.
# The client SSH key path is typically going to be somewhere else (~/.ssh,
# typically), but for convenience sake for testing we'll put it inside node.
client_ssh_key_path = join(process.node_dir, "private",
"ssh_client_rsa_key")
generate_ssh_key(client_ssh_key_path)
# Pub key format is "ssh-rsa <thekey> <username>". We want the key.
ssh_public_key = open(client_ssh_key_path +
".pub").read().strip().split()[1]
with open(accounts_path, "w") as f:
f.write("""\
alice password {rwcap}
alice2 ssh-rsa {ssh_public_key} {rwcap}
""".format(rwcap=rwcap, ssh_public_key=ssh_public_key))
# 4. Restart the node with new SFTP config.
process.kill()
pytest_twisted.blockon(_run_node(reactor, process.node_dir, request, None))
await_client_ready(process)
return process
def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):
process = pytest_twisted.blockon(
_create_node(
reactor, request, temp_dir, introducer_furl, flog_gatherer, "bob",
web_port="tcp:9981:interface=localhost",
storage=False,
)
)
await_client_ready(process)
return process
def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):
try:
mkdir(join(temp_dir, 'magic-bob'))
except OSError:
pass
process = pytest.blockon(
_create_node(
reactor, request, temp_dir, introducer_furl, flog_gatherer, "bob",
web_port="tcp:9981:interface=localhost",
storage=False,
)
)
return process
def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):
try:
mkdir(join(temp_dir, 'magic-bob'))
except OSError:
pass
process = pytest.blockon(
_create_node(
reactor, request, temp_dir, introducer_furl, flog_gatherer, "bob",
web_port="tcp:9981:interface=localhost",
storage=False,
)
)
return process
def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request):
nodes = []
# start all 5 nodes in parallel
for x in range(5):
name = 'node{}'.format(x)
# tub_port = 9900 + x
nodes.append(
pytest.blockon(
_create_node(
reactor, request, temp_dir, introducer_furl, flog_gatherer, name,
web_port=None, storage=True,
)
)
)
#nodes = pytest.blockon(DeferredList(nodes))
return nodes
def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request):
nodes = []
# start all 5 nodes in parallel
for x in range(5):
name = 'node{}'.format(x)
# tub_port = 9900 + x
nodes.append(
pytest.blockon(
_create_node(
reactor, request, temp_dir, introducer_furl, flog_gatherer, name,
web_port=None, storage=True,
)
)
)
#nodes = pytest.blockon(DeferredList(nodes))
return nodes
def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request):
nodes_d = []
# start all 5 nodes in parallel
for x in range(5):
name = 'node{}'.format(x)
web_port= 9990 + x
nodes_d.append(
_create_node(
reactor, request, temp_dir, introducer_furl, flog_gatherer, name,
web_port="tcp:{}:interface=localhost".format(web_port),
storage=True,
)
)
nodes_status = pytest_twisted.blockon(DeferredList(nodes_d))
nodes = []
for ok, process in nodes_status:
assert ok, "Storage node creation failed: {}".format(process)
nodes.append(process)
return nodes
def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer,
storage_nodes, request):
# hmm, for some reason this still gets storage enabled ...
process = yield util._create_node(
reactor,
request,
temp_dir,
introducer_furl,
flog_gatherer,
"edna",
web_port="tcp:9983:interface=localhost",
storage=False,
needed=3,
happy=10,
total=10,
)
node_dir = join(temp_dir, 'edna')
print("waiting 5 seconds unil we're maybe ready")
yield task.deferLater(reactor, 5, lambda: None)
# upload a file, which should fail because we have don't have 7
# storage servers (but happiness is set to 7)
proto = util._CollectOutputProtocol()
transport = reactor.spawnProcess(proto, sys.executable, [
sys.executable,
'-m',
'allmydata.scripts.runner',
'-d',
node_dir,
'put',
__file__,
])
try:
yield proto.done
assert False, "should raise exception"
except Exception as e:
assert isinstance(e, ProcessTerminated)
output = proto.output.getvalue()
assert "shares could be placed on only" in output
def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes,
request):
try:
mkdir(join(temp_dir, 'magic-alice'))
except OSError:
pass
process = pytest_twisted.blockon(
_create_node(
reactor,
request,
temp_dir,
introducer_furl,
flog_gatherer,
"alice",
web_port="tcp:9980:interface=localhost",
storage=False,
))
await_client_ready(process)
return process
def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):
yield util._create_node(
reactor, request, temp_dir, introducer_furl, flog_gatherer, "edna",
web_port="tcp:9983:interface=localhost",
storage=False,
needed=3,
happy=7,
total=10,
)
node_dir = join(temp_dir, 'edna')
print("waiting 10 seconds unil we're maybe ready")
yield task.deferLater(reactor, 10, lambda: None)
# upload a file, which should fail because we have don't have 7
# storage servers (but happiness is set to 7)
proto = util._CollectOutputProtocol()
reactor.spawnProcess(
proto,
sys.executable,
[
sys.executable, '-m', 'allmydata.scripts.runner',
'-d', node_dir,
'put', __file__,
]
)
try:
yield proto.done
assert False, "should raise exception"
except Exception as e:
assert isinstance(e, ProcessTerminated)
output = proto.output.getvalue()
assert "shares could be placed on only" in output
def test_edmond_uploads_then_restarts(reactor, request, temp_dir,
introducer_furl, flog_gatherer,
storage_nodes):
"""
ticket 2880: if a magic-folder client uploads something, then
re-starts a spurious .backup file should not appear
"""
edmond_dir = join(temp_dir, 'edmond')
edmond = yield util._create_node(
reactor,
request,
temp_dir,
introducer_furl,
flog_gatherer,
"edmond",
web_port="tcp:9985:interface=localhost",
storage=False,
)
magic_folder = join(temp_dir, 'magic-edmond')
mkdir(magic_folder)
created = False
# create a magic-folder
# (how can we know that the grid is ready?)
for _ in range(10): # try 10 times
try:
proto = util._CollectOutputProtocol()
transport = reactor.spawnProcess(proto, sys.executable, [
sys.executable,
'-m',
'allmydata.scripts.runner',
'magic-folder',
'create',
'--poll-interval',
'2',
'--basedir',
edmond_dir,
'magik:',
'edmond_magic',
magic_folder,
])
yield proto.done
created = True
break
except Exception as e:
print("failed to create magic-folder: {}".format(e))
time.sleep(1)
assert created, "Didn't create a magic-folder"
# to actually-start the magic-folder we have to re-start
edmond.signalProcess('TERM')
yield edmond._protocol.exited
time.sleep(1)
edmond = yield util._run_node(
reactor, edmond._node_dir, request,
'Completed initial Magic Folder scan successfully')
# add a thing to the magic-folder
with open(join(magic_folder, "its_a_file"), "w") as f:
f.write("edmond wrote this")
# fixme, do status-update attempts in a loop below
time.sleep(5)
# let it upload; poll the HTTP magic-folder status API until it is
# uploaded
from allmydata.scripts.magic_folder_cli import _get_json_for_fragment
with open(join(edmond_dir, u'private', u'api_auth_token'), 'rb') as f:
token = f.read()
uploaded = False
for _ in range(10):
options = {
"node-url": open(join(edmond_dir, u'node.url'),
'r').read().strip(),
}
try:
magic_data = _get_json_for_fragment(options,
'magic_folder?t=json',
method='POST',
post_args=dict(
t='json',
name='default',
token=token,
))
for mf in magic_data:
if mf['status'] == u'success' and mf['path'] == u'its_a_file':
uploaded = True
break
except Exception as e:
time.sleep(1)
assert uploaded, "expected to upload 'its_a_file'"
# re-starting edmond right now would "normally" trigger the 2880 bug
# kill edmond
edmond.signalProcess('TERM')
yield edmond._protocol.exited
time.sleep(1)
edmond = yield util._run_node(
reactor, edmond._node_dir, request,
'Completed initial Magic Folder scan successfully')
# XXX how can we say for sure if we've waited long enough? look at
# tail of logs for magic-folder ... somethingsomething?
print("waiting 20 seconds to see if a .backup appears")
for _ in range(20):
assert exists(join(magic_folder, "its_a_file"))
assert not exists(join(magic_folder, "its_a_file.backup"))
time.sleep(1)
def test_edmond_uploads_then_restarts(reactor, request, temp_dir, introducer_furl, flog_gatherer, storage_nodes):
"""
ticket 2880: if a magic-folder client uploads something, then
re-starts a spurious .backup file should not appear
"""
edmond_dir = join(temp_dir, 'edmond')
edmond = yield util._create_node(
reactor, request, temp_dir, introducer_furl, flog_gatherer,
"edmond", web_port="tcp:9985:interface=localhost",
storage=False,
)
magic_folder = join(temp_dir, 'magic-edmond')
mkdir(magic_folder)
created = False
# create a magic-folder
# (how can we know that the grid is ready?)
for _ in range(10): # try 10 times
try:
proto = util._CollectOutputProtocol()
transport = reactor.spawnProcess(
proto,
sys.executable,
[
sys.executable, '-m', 'allmydata.scripts.runner',
'magic-folder', 'create',
'--poll-interval', '2',
'--basedir', edmond_dir,
'magik:',
'edmond_magic',
magic_folder,
]
)
yield proto.done
created = True
break
except Exception as e:
print("failed to create magic-folder: {}".format(e))
time.sleep(1)
assert created, "Didn't create a magic-folder"
# to actually-start the magic-folder we have to re-start
edmond.signalProcess('TERM')
yield edmond._protocol.exited
time.sleep(1)
edmond = yield util._run_node(reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully')
# add a thing to the magic-folder
with open(join(magic_folder, "its_a_file"), "w") as f:
f.write("edmond wrote this")
# fixme, do status-update attempts in a loop below
time.sleep(5)
# let it upload; poll the HTTP magic-folder status API until it is
# uploaded
from allmydata.scripts.magic_folder_cli import _get_json_for_fragment
with open(join(edmond_dir, u'private', u'api_auth_token'), 'rb') as f:
token = f.read()
uploaded = False
for _ in range(10):
options = {
"node-url": open(join(edmond_dir, u'node.url'), 'r').read().strip(),
}
try:
magic_data = _get_json_for_fragment(
options,
'magic_folder?t=json',
method='POST',
post_args=dict(
t='json',
name='default',
token=token,
)
)
for mf in magic_data:
if mf['status'] == u'success' and mf['path'] == u'its_a_file':
uploaded = True
break
except Exception as e:
time.sleep(1)
assert uploaded, "expected to upload 'its_a_file'"
# re-starting edmond right now would "normally" trigger the 2880 bug
# kill edmond
edmond.signalProcess('TERM')
yield edmond._protocol.exited
time.sleep(1)
edmond = yield util._run_node(reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully')
# XXX how can we say for sure if we've waited long enough? look at
# tail of logs for magic-folder ... somethingsomething?
print("waiting 20 seconds to see if a .backup appears")
for _ in range(20):
assert exists(join(magic_folder, "its_a_file"))
assert not exists(join(magic_folder, "its_a_file.backup"))
time.sleep(1)
def create(
cls,
reactor,
request,
temp_dir,
introducer_furl,
flog_gatherer,
name,
tahoe_web_port,
magic_folder_web_port,
storage,
):
"""
Launch the two processes and return a new ``MagicFolderEnabledNode``
referencing them.
Note this depends on pytest/Twisted integration for magical blocking.
:param reactor: The reactor to use to launch the processes.
:param request: The pytest request object to use for cleanup.
:param bytes temp_dir: A directory beneath which to place the
Tahoe-LAFS node.
:param bytes introducer_furl: The introducer fURL to configure the new
Tahoe-LAFS node with.
:param bytes flog_gatherer: The flog gatherer fURL to configure the
new Tahoe-LAFS node with.
:param bytes name: A nickname to assign the new Tahoe-LAFS node.
:param bytes tahoe_web_port: An endpoint description of the web port
for the new Tahoe-LAFS node to listen on.
:param bytes magic_folder_web_port: An endpoint description of the web
port for the new magic-folder process to listen on.
:param bool storage: True if the node should offer storage, False
otherwise.
"""
# Make the Tahoe-LAFS node process
tahoe = pytest_twisted.blockon(
_create_node(
reactor,
request,
temp_dir,
introducer_furl,
flog_gatherer,
name,
tahoe_web_port,
storage,
needed=1,
happy=1,
total=1,
))
await_client_ready(tahoe)
# Make the magic folder process.
magic_folder = pytest_twisted.blockon(
_run_magic_folder(
reactor,
request,
temp_dir,
name,
magic_folder_web_port,
), )
return cls(
reactor,
request,
temp_dir,
name,
tahoe,
magic_folder,
magic_folder_web_port,
)