def magic_folder(reactor, alice_invite, alice, bob, temp_dir, request): print("pairing magic-folder") bob_dir = join(temp_dir, 'bob') proto = _CollectOutputProtocol() _tahoe_runner_optional_coverage(proto, reactor, request, [ 'magic-folder', 'join', '--poll-interval', '1', '--basedir', bob_dir, alice_invite, join(temp_dir, 'magic-bob'), ]) pytest_twisted.blockon(proto.done) # before magic-folder works, we have to stop and restart (this is # crappy for the tests -- can we fix it in magic-folder?) try: print("Sending TERM to Bob") bob.transport.signalProcess('TERM') pytest_twisted.blockon(bob.transport.exited) except ProcessExitedAlready: pass magic_text = 'Completed initial Magic Folder scan successfully' pytest_twisted.blockon(_run_node(reactor, bob_dir, request, magic_text)) await_client_ready(bob) return (join(temp_dir, 'magic-alice'), join(temp_dir, 'magic-bob'))
def magic_folder(reactor, alice_invite, alice, bob, temp_dir, request): print("pairing magic-folder") bob_dir = join(temp_dir, 'bob') proto = _CollectOutputProtocol() reactor.spawnProcess(proto, sys.executable, [ sys.executable, '-m', 'allmydata.scripts.runner', 'magic-folder', 'join', '--poll-interval', '2', '--basedir', bob_dir, alice_invite, join(temp_dir, 'magic-bob'), ]) pytest_twisted.blockon(proto.done) # before magic-folder works, we have to stop and restart (this is # crappy for the tests -- can we fix it in magic-folder?) try: print("Sending TERM to Bob") bob.signalProcess('TERM') pytest_twisted.blockon(bob.exited) except ProcessExitedAlready: pass magic_text = 'Completed initial Magic Folder scan successfully' pytest_twisted.blockon(_run_node(reactor, bob_dir, request, magic_text)) return (join(temp_dir, 'magic-alice'), join(temp_dir, 'magic-bob'))
def magic_folder(reactor, alice_invite, alice, bob, temp_dir, request): print("pairing magic-folder") bob_dir = join(temp_dir, 'bob') proto = _CollectOutputProtocol() transport = reactor.spawnProcess( proto, sys.executable, [ sys.executable, '-m', 'allmydata.scripts.runner', 'magic-folder', 'join', '--poll-interval', '2', '--basedir', bob_dir, alice_invite, join(temp_dir, 'magic-bob'), ] ) pytest.blockon(proto.done) # before magic-folder works, we have to stop and restart (this is # crappy for the tests -- can we fix it in magic-folder?) try: print("Sending TERM to Bob") bob.signalProcess('TERM') pytest.blockon(bob.exited) except ProcessExitedAlready: pass magic_text = 'Completed initial Magic Folder scan successfully' pytest.blockon(_run_node(reactor, bob_dir, request, magic_text)) return (join(temp_dir, 'magic-alice'), join(temp_dir, 'magic-bob'))
def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request): process = pytest_twisted.blockon( _create_node( reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice", web_port="tcp:9980:interface=localhost", storage=False, # We're going to kill this ourselves, so no need for finalizer to # do it: finalize=False, )) await_client_ready(process) # 1. Create a new RW directory cap: cli(process, "create-alias", "test") rwcap = loads(cli(process, "list-aliases", "--json"))["test"]["readwrite"] # 2. Enable SFTP on the node: host_ssh_key_path = join(process.node_dir, "private", "ssh_host_rsa_key") accounts_path = join(process.node_dir, "private", "accounts") with open(join(process.node_dir, "tahoe.cfg"), "a") as f: f.write("""\ [sftpd] enabled = true port = tcp:8022:interface=127.0.0.1 host_pubkey_file = {ssh_key_path}.pub host_privkey_file = {ssh_key_path} accounts.file = {accounts_path} """.format(ssh_key_path=host_ssh_key_path, accounts_path=accounts_path)) generate_ssh_key(host_ssh_key_path) # 3. Add a SFTP access file with username/password and SSH key auth. # The client SSH key path is typically going to be somewhere else (~/.ssh, # typically), but for convenience sake for testing we'll put it inside node. client_ssh_key_path = join(process.node_dir, "private", "ssh_client_rsa_key") generate_ssh_key(client_ssh_key_path) # Pub key format is "ssh-rsa <thekey> <username>". We want the key. ssh_public_key = open(client_ssh_key_path + ".pub").read().strip().split()[1] with open(accounts_path, "w") as f: f.write("""\ alice password {rwcap} alice2 ssh-rsa {ssh_public_key} {rwcap} """.format(rwcap=rwcap, ssh_public_key=ssh_public_key)) # 4. Restart the node with new SFTP config. process.kill() pytest_twisted.blockon(_run_node(reactor, process.node_dir, request, None)) await_client_ready(process) return process
def alice_invite(reactor, alice, temp_dir, request): node_dir = join(temp_dir, 'alice') with start_action(action_type=u"integration:alice:magic_folder:create"): # FIXME XXX by the time we see "client running" in the logs, the # storage servers aren't "really" ready to roll yet (uploads fairly # consistently fail if we don't hack in this pause...) import time time.sleep(5) proto = _CollectOutputProtocol() reactor.spawnProcess(proto, sys.executable, [ sys.executable, '-m', 'allmydata.scripts.runner', 'magic-folder', 'create', '--poll-interval', '2', '--basedir', node_dir, 'magik:', 'alice', join(temp_dir, 'magic-alice'), ]) pytest_twisted.blockon(proto.done) with start_action( action_type=u"integration:alice:magic_folder:invite") as a: proto = _CollectOutputProtocol() reactor.spawnProcess(proto, sys.executable, [ sys.executable, '-m', 'allmydata.scripts.runner', 'magic-folder', 'invite', '--basedir', node_dir, 'magik:', 'bob', ]) pytest_twisted.blockon(proto.done) invite = proto.output.getvalue() a.add_success_fields(invite=invite) with start_action(action_type=u"integration:alice:magic_folder:restart"): # before magic-folder works, we have to stop and restart (this is # crappy for the tests -- can we fix it in magic-folder?) try: alice.signalProcess('TERM') pytest_twisted.blockon(alice.exited) except ProcessExitedAlready: pass with start_action( action_type=u"integration:alice:magic_folder:magic-text"): magic_text = 'Completed initial Magic Folder scan successfully' pytest_twisted.blockon( _run_node(reactor, node_dir, request, magic_text)) return invite
def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl): node_dir = join(temp_dir, name) web_port = "tcp:{}:interface=localhost".format(control_port + 2000) if True: print("creating", node_dir) mkdir(node_dir) proto = util._DumpOutputProtocol(None) reactor.spawnProcess( proto, sys.executable, ( sys.executable, '-m', 'allmydata.scripts.runner', 'create-node', '--nickname', name, '--introducer', introducer_furl, '--hide-ip', '--tor-control-port', 'tcp:localhost:{}'.format(control_port), '--listen', 'tor', node_dir, ) ) yield proto.done with open(join(node_dir, 'tahoe.cfg'), 'w') as f: f.write(''' [node] nickname = %(name)s web.port = %(web_port)s web.static = public_html log_gatherer.furl = %(log_furl)s [tor] control.port = tcp:localhost:%(control_port)d onion.external_port = 3457 onion.local_port = %(local_port)d onion = true onion.private_key_file = private/tor_onion.privkey [client] # Which services should this client connect to? introducer.furl = %(furl)s shares.needed = 1 shares.happy = 1 shares.total = 2 ''' % { 'name': name, 'furl': introducer_furl, 'web_port': web_port, 'log_furl': flog_gatherer, 'control_port': control_port, 'local_port': control_port + 1000, }) print("running") yield util._run_node(reactor, node_dir, request, None) print("okay, launched")
def test_alice_adds_files_while_bob_is_offline(reactor, request, temp_dir, magic_folder): """ Alice can add new files to a magic folder while Bob is offline. When Bob comes back online his copy is updated to reflect the new files. """ alice_magic_dir, bob_magic_dir = magic_folder alice_node_dir = join(temp_dir, "alice") bob_node_dir = join(temp_dir, "bob") # Take Bob offline. yield util.cli(request, reactor, bob_node_dir, "stop") # Create a couple files in Alice's local directory. some_files = list((name * 3) + ".added-while-offline" for name in "xyz") for name in some_files: with open(join(alice_magic_dir, name), "w") as f: f.write(name + " some content") good = False for i in range(15): status = yield util.magic_folder_cli(request, reactor, alice_node_dir, "status") good = status.count(".added-while-offline (36 B): good, version=0" ) == len(some_files) * 2 if good: # We saw each file as having a local good state and a remote good # state. That means we're ready to involve Bob. break else: time.sleep(1.0) assert good, ("Timed out waiting for good Alice state. Last status:\n{}". format(status)) # Start Bob up again magic_text = 'Completed initial Magic Folder scan successfully' yield util._run_node(reactor, bob_node_dir, request, magic_text) yield util.await_files_exist( list(join(bob_magic_dir, name) for name in some_files), await_all=True, ) # Let it settle. It would be nicer to have a readable status output we # could query. Parsing the current text format is more than I want to # deal with right now. time.sleep(1.0) conflict_files = list(name + ".conflict" for name in some_files) assert all( list(not exists(join(bob_magic_dir, name)) for name in conflict_files), )
def alice_invite(reactor, alice, temp_dir, request): node_dir = join(temp_dir, 'alice') with start_action(action_type=u"integration:alice:magic_folder:create"): # FIXME XXX by the time we see "client running" in the logs, the # storage servers aren't "really" ready to roll yet (uploads fairly # consistently fail if we don't hack in this pause...) import time ; time.sleep(5) proto = _CollectOutputProtocol() reactor.spawnProcess( proto, sys.executable, [ sys.executable, '-m', 'allmydata.scripts.runner', 'magic-folder', 'create', '--poll-interval', '2', '--basedir', node_dir, 'magik:', 'alice', join(temp_dir, 'magic-alice'), ] ) pytest_twisted.blockon(proto.done) with start_action(action_type=u"integration:alice:magic_folder:invite") as a: proto = _CollectOutputProtocol() reactor.spawnProcess( proto, sys.executable, [ sys.executable, '-m', 'allmydata.scripts.runner', 'magic-folder', 'invite', '--basedir', node_dir, 'magik:', 'bob', ] ) pytest_twisted.blockon(proto.done) invite = proto.output.getvalue() a.add_success_fields(invite=invite) with start_action(action_type=u"integration:alice:magic_folder:restart"): # before magic-folder works, we have to stop and restart (this is # crappy for the tests -- can we fix it in magic-folder?) try: alice.signalProcess('TERM') pytest_twisted.blockon(alice.exited) except ProcessExitedAlready: pass with start_action(action_type=u"integration:alice:magic_folder:magic-text"): magic_text = 'Completed initial Magic Folder scan successfully' pytest_twisted.blockon(_run_node(reactor, node_dir, request, magic_text)) return invite
def test_edmond_uploads_then_restarts(reactor, request, temp_dir, introducer_furl, flog_gatherer, storage_nodes): """ ticket 2880: if a magic-folder client uploads something, then re-starts a spurious .backup file should not appear """ edmond_dir = join(temp_dir, 'edmond') edmond = yield util._create_node( reactor, request, temp_dir, introducer_furl, flog_gatherer, "edmond", web_port="tcp:9985:interface=localhost", storage=False, ) magic_folder = join(temp_dir, 'magic-edmond') mkdir(magic_folder) created = False # create a magic-folder # (how can we know that the grid is ready?) for _ in range(10): # try 10 times try: proto = util._CollectOutputProtocol() transport = reactor.spawnProcess(proto, sys.executable, [ sys.executable, '-m', 'allmydata.scripts.runner', 'magic-folder', 'create', '--poll-interval', '2', '--basedir', edmond_dir, 'magik:', 'edmond_magic', magic_folder, ]) yield proto.done created = True break except Exception as e: print("failed to create magic-folder: {}".format(e)) time.sleep(1) assert created, "Didn't create a magic-folder" # to actually-start the magic-folder we have to re-start edmond.signalProcess('TERM') yield edmond._protocol.exited time.sleep(1) edmond = yield util._run_node( reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully') # add a thing to the magic-folder with open(join(magic_folder, "its_a_file"), "w") as f: f.write("edmond wrote this") # fixme, do status-update attempts in a loop below time.sleep(5) # let it upload; poll the HTTP magic-folder status API until it is # uploaded from allmydata.scripts.magic_folder_cli import _get_json_for_fragment with open(join(edmond_dir, u'private', u'api_auth_token'), 'rb') as f: token = f.read() uploaded = False for _ in range(10): options = { "node-url": open(join(edmond_dir, u'node.url'), 'r').read().strip(), } try: magic_data = _get_json_for_fragment(options, 'magic_folder?t=json', method='POST', post_args=dict( t='json', name='default', token=token, )) for mf in magic_data: if mf['status'] == u'success' and mf['path'] == u'its_a_file': uploaded = True break except Exception as e: time.sleep(1) assert uploaded, "expected to upload 'its_a_file'" # re-starting edmond right now would "normally" trigger the 2880 bug # kill edmond edmond.signalProcess('TERM') yield edmond._protocol.exited time.sleep(1) edmond = yield util._run_node( reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully') # XXX how can we say for sure if we've waited long enough? look at # tail of logs for magic-folder ... somethingsomething? print("waiting 20 seconds to see if a .backup appears") for _ in range(20): assert exists(join(magic_folder, "its_a_file")) assert not exists(join(magic_folder, "its_a_file.backup")) time.sleep(1)
def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl): node_dir = join(temp_dir, name) web_port = "tcp:{}:interface=localhost".format(control_port + 2000) if True: print("creating", node_dir) mkdir(node_dir) proto = util._DumpOutputProtocol(None) reactor.spawnProcess(proto, sys.executable, ( sys.executable, '-m', 'allmydata.scripts.runner', 'create-node', '--nickname', name, '--introducer', introducer_furl, '--hide-ip', '--tor-control-port', 'tcp:localhost:{}'.format(control_port), '--listen', 'tor', node_dir, )) yield proto.done with open(join(node_dir, 'tahoe.cfg'), 'w') as f: f.write( ''' [node] nickname = %(name)s web.port = %(web_port)s web.static = public_html log_gatherer.furl = %(log_furl)s [tor] control.port = tcp:localhost:%(control_port)d onion.external_port = 3457 onion.local_port = %(local_port)d onion = true onion.private_key_file = private/tor_onion.privkey [client] # Which services should this client connect to? introducer.furl = %(furl)s shares.needed = 1 shares.happy = 1 shares.total = 2 ''' % { 'name': name, 'furl': introducer_furl, 'web_port': web_port, 'log_furl': flog_gatherer, 'control_port': control_port, 'local_port': control_port + 1000, }) print("running") yield util._run_node(reactor, node_dir, request, None) print("okay, launched")
def test_alice_adds_files_while_bob_is_offline(reactor, request, temp_dir, magic_folder): """ Alice can add new files to a magic folder while Bob is offline. When Bob comes back online his copy is updated to reflect the new files. """ alice_magic_dir, bob_magic_dir = magic_folder alice_node_dir = join(temp_dir, "alice") bob_node_dir = join(temp_dir, "bob") # Take Bob offline. yield util.cli(reactor, bob_node_dir, "stop") # Create a couple files in Alice's local directory. some_files = list( (name * 3) + ".added-while-offline" for name in "xyz" ) for name in some_files: with open(join(alice_magic_dir, name), "w") as f: f.write(name + " some content") good = False for i in range(15): status = yield util.magic_folder_cli(reactor, alice_node_dir, "status") good = status.count(".added-while-offline (36 B): good, version=0") == len(some_files) * 2 if good: # We saw each file as having a local good state and a remote good # state. That means we're ready to involve Bob. break else: time.sleep(1.0) assert good, ( "Timed out waiting for good Alice state. Last status:\n{}".format(status) ) # Start Bob up again magic_text = 'Completed initial Magic Folder scan successfully' yield util._run_node(reactor, bob_node_dir, request, magic_text) yield util.await_files_exist( list( join(bob_magic_dir, name) for name in some_files ), await_all=True, ) # Let it settle. It would be nicer to have a readable status output we # could query. Parsing the current text format is more than I want to # deal with right now. time.sleep(1.0) conflict_files = list(name + ".conflict" for name in some_files) assert all( list( not exists(join(bob_magic_dir, name)) for name in conflict_files ), )
def test_edmond_uploads_then_restarts(reactor, request, temp_dir, introducer_furl, flog_gatherer, storage_nodes): """ ticket 2880: if a magic-folder client uploads something, then re-starts a spurious .backup file should not appear """ edmond_dir = join(temp_dir, 'edmond') edmond = yield util._create_node( reactor, request, temp_dir, introducer_furl, flog_gatherer, "edmond", web_port="tcp:9985:interface=localhost", storage=False, ) magic_folder = join(temp_dir, 'magic-edmond') mkdir(magic_folder) created = False # create a magic-folder # (how can we know that the grid is ready?) for _ in range(10): # try 10 times try: proto = util._CollectOutputProtocol() transport = reactor.spawnProcess( proto, sys.executable, [ sys.executable, '-m', 'allmydata.scripts.runner', 'magic-folder', 'create', '--poll-interval', '2', '--basedir', edmond_dir, 'magik:', 'edmond_magic', magic_folder, ] ) yield proto.done created = True break except Exception as e: print("failed to create magic-folder: {}".format(e)) time.sleep(1) assert created, "Didn't create a magic-folder" # to actually-start the magic-folder we have to re-start edmond.signalProcess('TERM') yield edmond._protocol.exited time.sleep(1) edmond = yield util._run_node(reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully') # add a thing to the magic-folder with open(join(magic_folder, "its_a_file"), "w") as f: f.write("edmond wrote this") # fixme, do status-update attempts in a loop below time.sleep(5) # let it upload; poll the HTTP magic-folder status API until it is # uploaded from allmydata.scripts.magic_folder_cli import _get_json_for_fragment with open(join(edmond_dir, u'private', u'api_auth_token'), 'rb') as f: token = f.read() uploaded = False for _ in range(10): options = { "node-url": open(join(edmond_dir, u'node.url'), 'r').read().strip(), } try: magic_data = _get_json_for_fragment( options, 'magic_folder?t=json', method='POST', post_args=dict( t='json', name='default', token=token, ) ) for mf in magic_data: if mf['status'] == u'success' and mf['path'] == u'its_a_file': uploaded = True break except Exception as e: time.sleep(1) assert uploaded, "expected to upload 'its_a_file'" # re-starting edmond right now would "normally" trigger the 2880 bug # kill edmond edmond.signalProcess('TERM') yield edmond._protocol.exited time.sleep(1) edmond = yield util._run_node(reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully') # XXX how can we say for sure if we've waited long enough? look at # tail of logs for magic-folder ... somethingsomething? print("waiting 20 seconds to see if a .backup appears") for _ in range(20): assert exists(join(magic_folder, "its_a_file")) assert not exists(join(magic_folder, "its_a_file.backup")) time.sleep(1)