def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):

    edna = yield util._create_node(
        reactor, request, temp_dir, introducer_furl, flog_gatherer, "edna",
        web_port="tcp:9983:interface=localhost",
        storage=False,
        needed=3,
        happy=7,
        total=10,
    )
    util.await_client_ready(edna)

    node_dir = join(temp_dir, 'edna')

    # upload a file, which should fail because we have don't have 7
    # storage servers (but happiness is set to 7)
    proto = util._CollectOutputProtocol()
    reactor.spawnProcess(
        proto,
        sys.executable,
        [
            sys.executable, '-m', 'allmydata.scripts.runner',
            '-d', node_dir,
            'put', __file__,
        ]
    )
    try:
        yield proto.done
        assert False, "should raise exception"
    except Exception as e:
        assert isinstance(e, ProcessTerminated)

    output = proto.output.getvalue()
    assert "shares could be placed on only" in output
예제 #2
0
def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes,
          request):
    process = pytest_twisted.blockon(
        _create_node(
            reactor,
            request,
            temp_dir,
            introducer_furl,
            flog_gatherer,
            "alice",
            web_port="tcp:9980:interface=localhost",
            storage=False,
            # We're going to kill this ourselves, so no need for finalizer to
            # do it:
            finalize=False,
        ))
    await_client_ready(process)

    # 1. Create a new RW directory cap:
    cli(process, "create-alias", "test")
    rwcap = loads(cli(process, "list-aliases", "--json"))["test"]["readwrite"]

    # 2. Enable SFTP on the node:
    host_ssh_key_path = join(process.node_dir, "private", "ssh_host_rsa_key")
    accounts_path = join(process.node_dir, "private", "accounts")
    with open(join(process.node_dir, "tahoe.cfg"), "a") as f:
        f.write("""\
[sftpd]
enabled = true
port = tcp:8022:interface=127.0.0.1
host_pubkey_file = {ssh_key_path}.pub
host_privkey_file = {ssh_key_path}
accounts.file = {accounts_path}
""".format(ssh_key_path=host_ssh_key_path, accounts_path=accounts_path))
    generate_ssh_key(host_ssh_key_path)

    # 3. Add a SFTP access file with username/password and SSH key auth.

    # The client SSH key path is typically going to be somewhere else (~/.ssh,
    # typically), but for convenience sake for testing we'll put it inside node.
    client_ssh_key_path = join(process.node_dir, "private",
                               "ssh_client_rsa_key")
    generate_ssh_key(client_ssh_key_path)
    # Pub key format is "ssh-rsa <thekey> <username>". We want the key.
    ssh_public_key = open(client_ssh_key_path +
                          ".pub").read().strip().split()[1]
    with open(accounts_path, "w") as f:
        f.write("""\
alice password {rwcap}

alice2 ssh-rsa {ssh_public_key} {rwcap}
""".format(rwcap=rwcap, ssh_public_key=ssh_public_key))

    # 4. Restart the node with new SFTP config.
    process.kill()
    pytest_twisted.blockon(_run_node(reactor, process.node_dir, request, None))

    await_client_ready(process)
    return process
예제 #3
0
def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):
    process = pytest_twisted.blockon(
        _create_node(
            reactor, request, temp_dir, introducer_furl, flog_gatherer, "bob",
            web_port="tcp:9981:interface=localhost",
            storage=False,
        )
    )
    await_client_ready(process)
    return process
예제 #4
0
def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):
    try:
        mkdir(join(temp_dir, 'magic-bob'))
    except OSError:
        pass

    process = pytest.blockon(
        _create_node(
            reactor, request, temp_dir, introducer_furl, flog_gatherer, "bob",
            web_port="tcp:9981:interface=localhost",
            storage=False,
        )
    )
    return process
예제 #5
0
def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):
    try:
        mkdir(join(temp_dir, 'magic-bob'))
    except OSError:
        pass

    process = pytest.blockon(
        _create_node(
            reactor, request, temp_dir, introducer_furl, flog_gatherer, "bob",
            web_port="tcp:9981:interface=localhost",
            storage=False,
        )
    )
    return process
예제 #6
0
def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request):
    nodes = []
    # start all 5 nodes in parallel
    for x in range(5):
        name = 'node{}'.format(x)
        # tub_port = 9900 + x
        nodes.append(
            pytest.blockon(
                _create_node(
                    reactor, request, temp_dir, introducer_furl, flog_gatherer, name,
                    web_port=None, storage=True,
                )
            )
        )
    #nodes = pytest.blockon(DeferredList(nodes))
    return nodes
예제 #7
0
def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request):
    nodes = []
    # start all 5 nodes in parallel
    for x in range(5):
        name = 'node{}'.format(x)
        # tub_port = 9900 + x
        nodes.append(
            pytest.blockon(
                _create_node(
                    reactor, request, temp_dir, introducer_furl, flog_gatherer, name,
                    web_port=None, storage=True,
                )
            )
        )
    #nodes = pytest.blockon(DeferredList(nodes))
    return nodes
예제 #8
0
def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request):
    nodes_d = []
    # start all 5 nodes in parallel
    for x in range(5):
        name = 'node{}'.format(x)
        web_port=  9990 + x
        nodes_d.append(
            _create_node(
                reactor, request, temp_dir, introducer_furl, flog_gatherer, name,
                web_port="tcp:{}:interface=localhost".format(web_port),
                storage=True,
            )
        )
    nodes_status = pytest_twisted.blockon(DeferredList(nodes_d))
    nodes = []
    for ok, process in nodes_status:
        assert ok, "Storage node creation failed: {}".format(process)
        nodes.append(process)
    return nodes
예제 #9
0
def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer,
                          storage_nodes, request):

    # hmm, for some reason this still gets storage enabled ...
    process = yield util._create_node(
        reactor,
        request,
        temp_dir,
        introducer_furl,
        flog_gatherer,
        "edna",
        web_port="tcp:9983:interface=localhost",
        storage=False,
        needed=3,
        happy=10,
        total=10,
    )

    node_dir = join(temp_dir, 'edna')

    print("waiting 5 seconds unil we're maybe ready")
    yield task.deferLater(reactor, 5, lambda: None)

    # upload a file, which should fail because we have don't have 7
    # storage servers (but happiness is set to 7)
    proto = util._CollectOutputProtocol()
    transport = reactor.spawnProcess(proto, sys.executable, [
        sys.executable,
        '-m',
        'allmydata.scripts.runner',
        '-d',
        node_dir,
        'put',
        __file__,
    ])
    try:
        yield proto.done
        assert False, "should raise exception"
    except Exception as e:
        assert isinstance(e, ProcessTerminated)

    output = proto.output.getvalue()
    assert "shares could be placed on only" in output
예제 #10
0
def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes,
          request):
    try:
        mkdir(join(temp_dir, 'magic-alice'))
    except OSError:
        pass

    process = pytest_twisted.blockon(
        _create_node(
            reactor,
            request,
            temp_dir,
            introducer_furl,
            flog_gatherer,
            "alice",
            web_port="tcp:9980:interface=localhost",
            storage=False,
        ))
    await_client_ready(process)
    return process
def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):

    yield util._create_node(
        reactor, request, temp_dir, introducer_furl, flog_gatherer, "edna",
        web_port="tcp:9983:interface=localhost",
        storage=False,
        needed=3,
        happy=7,
        total=10,
    )


    node_dir = join(temp_dir, 'edna')

    print("waiting 10 seconds unil we're maybe ready")
    yield task.deferLater(reactor, 10, lambda: None)

    # upload a file, which should fail because we have don't have 7
    # storage servers (but happiness is set to 7)
    proto = util._CollectOutputProtocol()
    reactor.spawnProcess(
        proto,
        sys.executable,
        [
            sys.executable, '-m', 'allmydata.scripts.runner',
            '-d', node_dir,
            'put', __file__,
        ]
    )
    try:
        yield proto.done
        assert False, "should raise exception"
    except Exception as e:
        assert isinstance(e, ProcessTerminated)

    output = proto.output.getvalue()
    assert "shares could be placed on only" in output
예제 #12
0
def test_edmond_uploads_then_restarts(reactor, request, temp_dir,
                                      introducer_furl, flog_gatherer,
                                      storage_nodes):
    """
    ticket 2880: if a magic-folder client uploads something, then
    re-starts a spurious .backup file should not appear
    """

    edmond_dir = join(temp_dir, 'edmond')
    edmond = yield util._create_node(
        reactor,
        request,
        temp_dir,
        introducer_furl,
        flog_gatherer,
        "edmond",
        web_port="tcp:9985:interface=localhost",
        storage=False,
    )

    magic_folder = join(temp_dir, 'magic-edmond')
    mkdir(magic_folder)
    created = False
    # create a magic-folder
    # (how can we know that the grid is ready?)
    for _ in range(10):  # try 10 times
        try:
            proto = util._CollectOutputProtocol()
            transport = reactor.spawnProcess(proto, sys.executable, [
                sys.executable,
                '-m',
                'allmydata.scripts.runner',
                'magic-folder',
                'create',
                '--poll-interval',
                '2',
                '--basedir',
                edmond_dir,
                'magik:',
                'edmond_magic',
                magic_folder,
            ])
            yield proto.done
            created = True
            break
        except Exception as e:
            print("failed to create magic-folder: {}".format(e))
            time.sleep(1)

    assert created, "Didn't create a magic-folder"

    # to actually-start the magic-folder we have to re-start
    edmond.signalProcess('TERM')
    yield edmond._protocol.exited
    time.sleep(1)
    edmond = yield util._run_node(
        reactor, edmond._node_dir, request,
        'Completed initial Magic Folder scan successfully')

    # add a thing to the magic-folder
    with open(join(magic_folder, "its_a_file"), "w") as f:
        f.write("edmond wrote this")

    # fixme, do status-update attempts in a loop below
    time.sleep(5)

    # let it upload; poll the HTTP magic-folder status API until it is
    # uploaded
    from allmydata.scripts.magic_folder_cli import _get_json_for_fragment

    with open(join(edmond_dir, u'private', u'api_auth_token'), 'rb') as f:
        token = f.read()

    uploaded = False
    for _ in range(10):
        options = {
            "node-url": open(join(edmond_dir, u'node.url'),
                             'r').read().strip(),
        }
        try:
            magic_data = _get_json_for_fragment(options,
                                                'magic_folder?t=json',
                                                method='POST',
                                                post_args=dict(
                                                    t='json',
                                                    name='default',
                                                    token=token,
                                                ))
            for mf in magic_data:
                if mf['status'] == u'success' and mf['path'] == u'its_a_file':
                    uploaded = True
                    break
        except Exception as e:
            time.sleep(1)

    assert uploaded, "expected to upload 'its_a_file'"

    # re-starting edmond right now would "normally" trigger the 2880 bug

    # kill edmond
    edmond.signalProcess('TERM')
    yield edmond._protocol.exited
    time.sleep(1)
    edmond = yield util._run_node(
        reactor, edmond._node_dir, request,
        'Completed initial Magic Folder scan successfully')

    # XXX how can we say for sure if we've waited long enough? look at
    # tail of logs for magic-folder ... somethingsomething?
    print("waiting 20 seconds to see if a .backup appears")
    for _ in range(20):
        assert exists(join(magic_folder, "its_a_file"))
        assert not exists(join(magic_folder, "its_a_file.backup"))
        time.sleep(1)
예제 #13
0
def test_edmond_uploads_then_restarts(reactor, request, temp_dir, introducer_furl, flog_gatherer, storage_nodes):
    """
    ticket 2880: if a magic-folder client uploads something, then
    re-starts a spurious .backup file should not appear
    """

    edmond_dir = join(temp_dir, 'edmond')
    edmond = yield util._create_node(
        reactor, request, temp_dir, introducer_furl, flog_gatherer,
        "edmond", web_port="tcp:9985:interface=localhost",
        storage=False,
    )


    magic_folder = join(temp_dir, 'magic-edmond')
    mkdir(magic_folder)
    created = False
    # create a magic-folder
    # (how can we know that the grid is ready?)
    for _ in range(10):  # try 10 times
        try:
            proto = util._CollectOutputProtocol()
            transport = reactor.spawnProcess(
                proto,
                sys.executable,
                [
                    sys.executable, '-m', 'allmydata.scripts.runner',
                    'magic-folder', 'create',
                    '--poll-interval', '2',
                    '--basedir', edmond_dir,
                    'magik:',
                    'edmond_magic',
                    magic_folder,
                ]
            )
            yield proto.done
            created = True
            break
        except Exception as e:
            print("failed to create magic-folder: {}".format(e))
            time.sleep(1)

    assert created, "Didn't create a magic-folder"

    # to actually-start the magic-folder we have to re-start
    edmond.signalProcess('TERM')
    yield edmond._protocol.exited
    time.sleep(1)
    edmond = yield util._run_node(reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully')

    # add a thing to the magic-folder
    with open(join(magic_folder, "its_a_file"), "w") as f:
        f.write("edmond wrote this")

    # fixme, do status-update attempts in a loop below
    time.sleep(5)

    # let it upload; poll the HTTP magic-folder status API until it is
    # uploaded
    from allmydata.scripts.magic_folder_cli import _get_json_for_fragment

    with open(join(edmond_dir, u'private', u'api_auth_token'), 'rb') as f:
        token = f.read()

    uploaded = False
    for _ in range(10):
        options = {
            "node-url": open(join(edmond_dir, u'node.url'), 'r').read().strip(),
        }
        try:
            magic_data = _get_json_for_fragment(
                options,
                'magic_folder?t=json',
                method='POST',
                post_args=dict(
                    t='json',
                    name='default',
                    token=token,
                )
            )
            for mf in magic_data:
                if mf['status'] == u'success' and mf['path'] == u'its_a_file':
                    uploaded = True
                    break
        except Exception as e:
            time.sleep(1)

    assert uploaded, "expected to upload 'its_a_file'"

    # re-starting edmond right now would "normally" trigger the 2880 bug

    # kill edmond
    edmond.signalProcess('TERM')
    yield edmond._protocol.exited
    time.sleep(1)
    edmond = yield util._run_node(reactor, edmond._node_dir, request, 'Completed initial Magic Folder scan successfully')

    # XXX how can we say for sure if we've waited long enough? look at
    # tail of logs for magic-folder ... somethingsomething?
    print("waiting 20 seconds to see if a .backup appears")
    for _ in range(20):
        assert exists(join(magic_folder, "its_a_file"))
        assert not exists(join(magic_folder, "its_a_file.backup"))
        time.sleep(1)
예제 #14
0
    def create(
        cls,
        reactor,
        request,
        temp_dir,
        introducer_furl,
        flog_gatherer,
        name,
        tahoe_web_port,
        magic_folder_web_port,
        storage,
    ):
        """
        Launch the two processes and return a new ``MagicFolderEnabledNode``
        referencing them.

        Note this depends on pytest/Twisted integration for magical blocking.

        :param reactor: The reactor to use to launch the processes.
        :param request: The pytest request object to use for cleanup.
        :param bytes temp_dir: A directory beneath which to place the
            Tahoe-LAFS node.
        :param bytes introducer_furl: The introducer fURL to configure the new
            Tahoe-LAFS node with.
        :param bytes flog_gatherer: The flog gatherer fURL to configure the
            new Tahoe-LAFS node with.
        :param bytes name: A nickname to assign the new Tahoe-LAFS node.
        :param bytes tahoe_web_port: An endpoint description of the web port
            for the new Tahoe-LAFS node to listen on.
        :param bytes magic_folder_web_port: An endpoint description of the web
            port for the new magic-folder process to listen on.
        :param bool storage: True if the node should offer storage, False
            otherwise.
        """
        # Make the Tahoe-LAFS node process
        tahoe = pytest_twisted.blockon(
            _create_node(
                reactor,
                request,
                temp_dir,
                introducer_furl,
                flog_gatherer,
                name,
                tahoe_web_port,
                storage,
                needed=1,
                happy=1,
                total=1,
            ))
        await_client_ready(tahoe)

        # Make the magic folder process.
        magic_folder = pytest_twisted.blockon(
            _run_magic_folder(
                reactor,
                request,
                temp_dir,
                name,
                magic_folder_web_port,
            ), )
        return cls(
            reactor,
            request,
            temp_dir,
            name,
            tahoe,
            magic_folder,
            magic_folder_web_port,
        )