Example #1
0
def alice_invite(reactor, alice, temp_dir, request):
    node_dir = join(temp_dir, 'alice')

    with start_action(action_type=u"integration:alice:magic_folder:create"):
        # FIXME XXX by the time we see "client running" in the logs, the
        # storage servers aren't "really" ready to roll yet (uploads fairly
        # consistently fail if we don't hack in this pause...)
        import time ; time.sleep(5)
        proto = _CollectOutputProtocol()
        reactor.spawnProcess(
            proto,
            sys.executable,
            [
                sys.executable, '-m', 'allmydata.scripts.runner',
                'magic-folder', 'create',
                '--poll-interval', '2',
                '--basedir', node_dir, 'magik:', 'alice',
                join(temp_dir, 'magic-alice'),
            ]
        )
        pytest_twisted.blockon(proto.done)

    with start_action(action_type=u"integration:alice:magic_folder:invite") as a:
        proto = _CollectOutputProtocol()
        reactor.spawnProcess(
            proto,
            sys.executable,
            [
                sys.executable, '-m', 'allmydata.scripts.runner',
                'magic-folder', 'invite',
                '--basedir', node_dir, 'magik:', 'bob',
            ]
        )
        pytest_twisted.blockon(proto.done)
        invite = proto.output.getvalue()
        a.add_success_fields(invite=invite)

    with start_action(action_type=u"integration:alice:magic_folder:restart"):
        # before magic-folder works, we have to stop and restart (this is
        # crappy for the tests -- can we fix it in magic-folder?)
        try:
            alice.signalProcess('TERM')
            pytest_twisted.blockon(alice.exited)
        except ProcessExitedAlready:
            pass
        with start_action(action_type=u"integration:alice:magic_folder:magic-text"):
            magic_text = 'Completed initial Magic Folder scan successfully'
            pytest_twisted.blockon(_run_node(reactor, node_dir, request, magic_text))
    return invite
Example #2
0
def magic_folder(reactor, alice_invite, alice, bob, temp_dir, request):
    print("pairing magic-folder")
    bob_dir = join(temp_dir, 'bob')
    proto = _CollectOutputProtocol()
    transport = reactor.spawnProcess(
        proto,
        sys.executable,
        [
            sys.executable, '-m', 'allmydata.scripts.runner',
            'magic-folder', 'join',
            '--poll-interval', '2',
            '--basedir', bob_dir,
            alice_invite,
            join(temp_dir, 'magic-bob'),
        ]
    )
    pytest.blockon(proto.done)

    # before magic-folder works, we have to stop and restart (this is
    # crappy for the tests -- can we fix it in magic-folder?)
    try:
        print("Sending TERM to Bob")
        bob.signalProcess('TERM')
        pytest.blockon(bob.exited)
    except ProcessExitedAlready:
        pass

    magic_text = 'Completed initial Magic Folder scan successfully'
    pytest.blockon(_run_node(reactor, bob_dir, request, magic_text))
    return (join(temp_dir, 'magic-alice'), join(temp_dir, 'magic-bob'))
Example #3
0
def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl):
    yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
    yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
    # ensure both nodes are connected to "a grid" by uploading
    # something via carol, and retrieve it using dave.
    gold_path = join(temp_dir, "gold")
    with open(gold_path, "w") as f:
        f.write(
            "The object-capability model is a computer security model. A "
            "capability describes a transferable right to perform one (or "
            "more) operations on a given object."
        )
    # XXX could use treq or similar to POST these to their respective
    # WUIs instead ...

    proto = util._CollectOutputProtocol()
    reactor.spawnProcess(
        proto,
        sys.executable,
        (
            sys.executable, '-m', 'allmydata.scripts.runner',
            '-d', join(temp_dir, 'carol'),
            'put', gold_path,
        )
    )
    yield proto.done
    cap = proto.output.getvalue().strip().split()[-1]
    print("TEH CAP!", cap)

    proto = util._CollectOutputProtocol()
    reactor.spawnProcess(
        proto,
        sys.executable,
        (
            sys.executable, '-m', 'allmydata.scripts.runner',
            '-d', join(temp_dir, 'dave'),
            'get', cap,
        )
    )
    yield proto.done

    dave_got = proto.output.getvalue().strip()
    assert dave_got == open(gold_path, 'r').read().strip()
Example #4
0
def flog_gatherer(reactor, temp_dir, flog_binary, request):
    out_protocol = _CollectOutputProtocol()
    gather_dir = join(temp_dir, 'flog_gather')
    reactor.spawnProcess(
        out_protocol,
        flog_binary,
        (
            'flogtool', 'create-gatherer',
            '--location', 'tcp:localhost:3117',
            '--port', '3117',
            gather_dir,
        )
    )
    pytest_twisted.blockon(out_protocol.done)

    twistd_protocol = _MagicTextProtocol("Gatherer waiting at")
    twistd_process = reactor.spawnProcess(
        twistd_protocol,
        which('twistd')[0],
        (
            'twistd', '--nodaemon', '--python',
            join(gather_dir, 'gatherer.tac'),
        ),
        path=gather_dir,
    )
    pytest_twisted.blockon(twistd_protocol.magic_seen)

    def cleanup():
        _cleanup_twistd_process(twistd_process, twistd_protocol.exited)

        flog_file = mktemp('.flog_dump')
        flog_protocol = _DumpOutputProtocol(open(flog_file, 'w'))
        flog_dir = join(temp_dir, 'flog_gather')
        flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')]

        print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file))
        reactor.spawnProcess(
            flog_protocol,
            flog_binary,
            (
                'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0])
            ),
        )
        print("Waiting for flogtool to complete")
        try:
            pytest_twisted.blockon(flog_protocol.done)
        except ProcessTerminated as e:
            print("flogtool exited unexpectedly: {}".format(str(e)))
        print("Flogtool completed")

    request.addfinalizer(cleanup)

    with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f:
        furl = f.read().strip()
    return furl
Example #5
0
 def cleanup():
     print("Tearing down Chutney Tor network")
     proto = _CollectOutputProtocol()
     reactor.spawnProcess(
         proto,
         sys.executable,
         (
             sys.executable, '-m', 'chutney.TorNet', 'stop',
             join(chutney_dir, 'networks', 'basic'),
         ),
         path=join(chutney_dir),
         env={"PYTHONPATH": join(chutney_dir, "lib")},
     )
     pytest.blockon(proto.done)
def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):

    yield util._create_node(
        reactor, request, temp_dir, introducer_furl, flog_gatherer, "edna",
        web_port="tcp:9983:interface=localhost",
        storage=False,
        needed=3,
        happy=7,
        total=10,
    )


    node_dir = join(temp_dir, 'edna')

    print("waiting 10 seconds unil we're maybe ready")
    yield task.deferLater(reactor, 10, lambda: None)

    # upload a file, which should fail because we have don't have 7
    # storage servers (but happiness is set to 7)
    proto = util._CollectOutputProtocol()
    reactor.spawnProcess(
        proto,
        sys.executable,
        [
            sys.executable, '-m', 'allmydata.scripts.runner',
            '-d', node_dir,
            'put', __file__,
        ]
    )
    try:
        yield proto.done
        assert False, "should raise exception"
    except Exception as e:
        assert isinstance(e, ProcessTerminated)

    output = proto.output.getvalue()
    assert "shares could be placed on only" in output
Example #7
0
def tor_network(reactor, temp_dir, chutney, request):
    # this is the actual "chutney" script at the root of a chutney checkout
    chutney_dir = chutney
    chut = join(chutney_dir, 'chutney')

    # now, as per Chutney's README, we have to create the network
    # ./chutney configure networks/basic
    # ./chutney start networks/basic

    proto = _DumpOutputProtocol(None)
    reactor.spawnProcess(
        proto,
        sys.executable,
        (
            sys.executable, '-m', 'chutney.TorNet', 'configure',
            join(chutney_dir, 'networks', 'basic'),
        ),
        path=join(chutney_dir),
        env={"PYTHONPATH": join(chutney_dir, "lib")},
    )
    pytest.blockon(proto.done)

    proto = _DumpOutputProtocol(None)
    reactor.spawnProcess(
        proto,
        sys.executable,
        (
            sys.executable, '-m', 'chutney.TorNet', 'start',
            join(chutney_dir, 'networks', 'basic'),
        ),
        path=join(chutney_dir),
        env={"PYTHONPATH": join(chutney_dir, "lib")},
    )
    pytest.blockon(proto.done)

    # print some useful stuff
    proto = _CollectOutputProtocol()
    reactor.spawnProcess(
        proto,
        sys.executable,
        (
            sys.executable, '-m', 'chutney.TorNet', 'status',
            join(chutney_dir, 'networks', 'basic'),
        ),
        path=join(chutney_dir),
        env={"PYTHONPATH": join(chutney_dir, "lib")},
    )
    pytest.blockon(proto.done)

    def cleanup():
        print("Tearing down Chutney Tor network")
        proto = _CollectOutputProtocol()
        reactor.spawnProcess(
            proto,
            sys.executable,
            (
                sys.executable, '-m', 'chutney.TorNet', 'stop',
                join(chutney_dir, 'networks', 'basic'),
            ),
            path=join(chutney_dir),
            env={"PYTHONPATH": join(chutney_dir, "lib")},
        )
        pytest.blockon(proto.done)
    request.addfinalizer(cleanup)

    return chut
def test_edmond_uploads_then_restarts(reactor, request, temp_dir, introducer_furl, flog_gatherer, storage_nodes):
    """
    ticket 2880: if a magic-folder client uploads something, then
    re-starts a spurious .backup file should not appear
    """

    edmond_dir = join(temp_dir, 'edmond')
    edmond = yield util._create_node(
        reactor, request, temp_dir, introducer_furl, flog_gatherer,
        "edmond", web_port="tcp:9985:interface=localhost",
        storage=False,
    )


    magic_folder = join(temp_dir, 'magic-edmond')
    mkdir(magic_folder)
    created = False
    # create a magic-folder
    # (how can we know that the grid is ready?)
    for _ in range(10):  # try 10 times
        try:
            proto = util._CollectOutputProtocol()
            transport = reactor.spawnProcess(
                proto,
                sys.executable,
                [
                    sys.executable, '-m', 'allmydata.scripts.runner',
                    'magic-folder', 'create',
                    '--poll-interval', '2',
                    '--basedir', edmond_dir,
                    'magik:',
                    'edmond_magic',
                    magic_folder,
                ]
            )
            yield proto.done
            created = True
            break
        except Exception as e:
            print("failed to create magic-folder: {}".format(e))
            time.sleep(1)

    assert created, "Didn't create a magic-folder"

    # to actually-start the magic-folder we have to re-start
    edmond.signalProcess('TERM')
    yield edmond._protocol.exited
    time.sleep(1)
    edmond = yield util._run_node(reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully')

    # add a thing to the magic-folder
    with open(join(magic_folder, "its_a_file"), "w") as f:
        f.write("edmond wrote this")

    # fixme, do status-update attempts in a loop below
    time.sleep(5)

    # let it upload; poll the HTTP magic-folder status API until it is
    # uploaded
    from allmydata.scripts.magic_folder_cli import _get_json_for_fragment

    with open(join(edmond_dir, u'private', u'api_auth_token'), 'rb') as f:
        token = f.read()

    uploaded = False
    for _ in range(10):
        options = {
            "node-url": open(join(edmond_dir, u'node.url'), 'r').read().strip(),
        }
        try:
            magic_data = _get_json_for_fragment(
                options,
                'magic_folder?t=json',
                method='POST',
                post_args=dict(
                    t='json',
                    name='default',
                    token=token,
                )
            )
            for mf in magic_data:
                if mf['status'] == u'success' and mf['path'] == u'its_a_file':
                    uploaded = True
                    break
        except Exception as e:
            time.sleep(1)

    assert uploaded, "expected to upload 'its_a_file'"

    # re-starting edmond right now would "normally" trigger the 2880 bug

    # kill edmond
    edmond.signalProcess('TERM')
    yield edmond._protocol.exited
    time.sleep(1)
    edmond = yield util._run_node(reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully')

    # XXX how can we say for sure if we've waited long enough? look at
    # tail of logs for magic-folder ... somethingsomething?
    print("waiting 20 seconds to see if a .backup appears")
    for _ in range(20):
        assert exists(join(magic_folder, "its_a_file"))
        assert not exists(join(magic_folder, "its_a_file.backup"))
        time.sleep(1)