def url_for_string(req, url_string):
"""
Construct a universal URL using the given URL string.
:param IRequest req: The request being served. If ``redir_to`` is not
absolute then this is used to determine the net location of this
server and the resulting URL is made to point at it.
:param bytes url_string: A byte string giving a universal or absolute URL.
:return DecodedURL: An absolute URL based on this server's net location
and the given URL string.
"""
url = DecodedURL.from_text(url_string.decode("utf-8"))
if not url.host:
root = req.URLPath()
netloc = root.netloc.split(b":", 1)
if len(netloc) == 1:
host = netloc
port = None
else:
host = netloc[0]
port = int(netloc[1])
url = url.replace(
scheme=root.scheme.decode("ascii"),
host=host.decode("ascii"),
port=port,
)
return url
def uri(self):
# type: () -> DecodedURL
request = self._request
# This code borrows from t.w.server.Request._prePathURL.
if request.isSecure():
scheme = u"https"
else:
scheme = u"http"
netloc = nativeString(request.getRequestHostname())
port = request.getHost().port
if request.isSecure():
default = 443
else:
default = 80
if port != default:
netloc += u":{}".format(port)
path = nativeString(request.uri)
if path and path[0] == u"/":
path = path[1:]
return DecodedURL.fromText(u"{}://{}/{}".format(scheme, netloc, path))
def requestFromBytes(data: bytes = b"") -> FrozenHTTPRequest:
return FrozenHTTPRequest(
method="GET",
uri=DecodedURL.fromText("https://twistedmatrix.com/"),
headers=FrozenHTTPHeaders(rawHeaders=()),
body=data,
)
def uri(self):
# type: () -> DecodedURL
request = self._request
# This code borrows from t.w.server.Request._prePathURL.
if request.isSecure():
scheme = "https"
else:
scheme = "http"
netloc = nativeString(request.getRequestHostname())
port = request.getHost().port
if request.isSecure():
default = 443
else:
default = 80
if port != default:
netloc += ":{}".format(port)
path = nativeString(request.uri)
if path and path[0] == "/":
path = path[1:]
return DecodedURL.fromText("{}://{}/{}".format(scheme, netloc, path))
def test_decodedurl(self):
"""
If the decorated method returns a ``DecodedURL`` then a redirect to that
location is rendered into the response.
"""
loc = u"http://example.invalid/foo?bar=baz"
resource = StaticResource(DecodedURL.from_text(loc))
self.assertThat(
render(resource, {}),
succeeded(
MatchesPredicate(
lambda value: assert_soup_has_tag_with_attributes(
self,
BeautifulSoup(value),
"meta",
{
"http-equiv": "refresh",
"content": "0;URL={}".format(loc.encode("ascii")),
},
)
# The assertion will raise if it has a problem, otherwise
# return None. Turn the None into something
# MatchesPredicate recognizes as success.
or True,
"did not find meta refresh tag in %r",
), ),
)
def test_add_snapshot_no_folder(self):
"""
An error results using /v1/snapshot API on non-existent
folder.
"""
local_path = FilePath(self.mktemp())
local_path.makedirs()
root = create_fake_tahoe_root()
tahoe_client = create_tahoe_client(
DecodedURL.from_text(u"http://invalid./"),
create_tahoe_treq_client(root),
)
treq = treq_for_folders(
Clock(),
FilePath(self.mktemp()),
AUTH_TOKEN,
{},
start_folder_services=False,
tahoe_client=tahoe_client,
)
self.assertThat(
authorized_request(
treq,
AUTH_TOKEN,
b"POST",
self.url.child("a-folder-that-doesnt-exist").child('snapshot'),
), succeeded(matches_response(code_matcher=Equals(NOT_FOUND), ), ))
def from_nurl(cls,
nurl: DecodedURL,
reactor,
persistent: bool = True) -> StorageClient:
"""
Create a ``StorageClient`` for the given NURL.
``persistent`` indicates whether to use persistent HTTP connections.
"""
assert nurl.fragment == "v=1"
assert nurl.scheme == "pb"
swissnum = nurl.path[0].encode("ascii")
certificate_hash = nurl.user.encode("ascii")
treq_client = HTTPClient(
Agent(
reactor,
_StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash),
pool=HTTPConnectionPool(reactor, persistent=persistent),
))
https_url = DecodedURL().replace(scheme="https",
host=nurl.host,
port=nurl.port)
return cls(https_url, swissnum, treq_client)
def uri(self) -> DecodedURL:
request = self._request
# This code borrows from t.w.server.Request._prePathURL.
if request.isSecure():
scheme = "https"
else:
scheme = "http"
netloc = nativeString(request.getRequestHostname())
port = request.getHost().port
if request.isSecure():
default = 443
else:
default = 80
if port != default:
netloc += f":{port}"
path = nativeString(request.uri)
if path and path[0] == "/":
path = path[1:]
return DecodedURL.fromText(f"{scheme}://{netloc}/{path}")
def __init__(self, temp, author, upload_dircap, root=None):
"""
:param FilePath temp: A path where the fixture may write whatever it
likes.
:param LocalAuthor author: The author which will be used to sign
snapshots the ``RemoteSnapshotCreator`` creates.
:param bytes upload_dircap: The Tahoe-LAFS capability for a writeable
directory into which new snapshots will be linked.
:param IResource root: The root resource for the fake Tahoe-LAFS HTTP
API hierarchy. The default is one created by
``create_fake_tahoe_root``.
"""
if root is None:
root = create_fake_tahoe_root()
self.temp = temp
self.author = author
self.upload_dircap = upload_dircap
self.root = root
self.http_client = create_tahoe_treq_client(self.root)
self.tahoe_client = create_tahoe_client(
DecodedURL.from_text(u"http://example.com"),
self.http_client,
)
class TestApiMonitor(AsyncTestCase):
"""
Tests related to 'magic-folder-api monitor'
"""
url = DecodedURL.from_text(u"http://invalid./v1/")
def setUp(self):
super(TestApiMonitor, self).setUp()
self.magic_config = FilePath(self.mktemp())
self.global_config = create_testing_configuration(
self.magic_config,
FilePath(u"/no/tahoe/node-directory"),
)
self.reactor = MemoryReactorClockResolver()
self.pumper = create_pumper()
self.service = WebSocketStatusService(
self.reactor,
self.global_config,
)
self.factory = StatusFactory(self.service)
self.agent = create_memory_agent(
self.reactor,
self.pumper,
lambda: self.factory.buildProtocol(None)
)
return self.pumper.start()
def tearDown(self):
super(TestApiMonitor, self).tearDown()
return self.pumper.stop()
@inline_callbacks
def test_once(self):
"""
Output a single status message with --once option
"""
stdout = StringIO()
stderr = StringIO()
yield dispatch_magic_folder_api_command(
["--config", self.magic_config.path, "monitor",
"--once",
],
stdout=stdout,
stderr=stderr,
websocket_agent=self.agent,
config=self.global_config,
)
self.pumper._flush()
self.assertThat(
json.loads(stdout.getvalue()),
Equals({
'state': {
'folders': {},
'synchronizing': False,
}
}),
)
def test_add_participant_personal_dmd_non_dir(self, author, folder_name,
personal_dmd):
"""
When a new Personal DMD is passed that is not a directory
capability an error is produced.
"""
local_path = FilePath(self.mktemp())
local_path.makedirs()
folder_config = magic_folder_config(
create_local_author(author),
FilePath(self.mktemp()),
local_path,
)
root = create_fake_tahoe_root()
# put our Collective DMD into the fake root
root._uri.data[folder_config["collective-dircap"]] = dumps([
u"dirnode",
{
u"children": {
author: format_filenode(folder_config["upload-dircap"]),
},
},
])
tahoe_client = create_tahoe_client(
DecodedURL.from_text(u"http://invalid./"),
create_tahoe_treq_client(root),
)
treq = treq_for_folders(
Clock(),
FilePath(self.mktemp()),
AUTH_TOKEN,
{
folder_name: folder_config,
},
start_folder_services=False,
tahoe_client=tahoe_client,
)
# add a participant using the API
self.assertThat(
authorized_request(
treq, AUTH_TOKEN, b"POST",
self.url.child(folder_name, "participants"),
dumps({
"author": {
"name": "kelly"
},
"personal_dmd": personal_dmd,
}).encode("utf8")),
succeeded(
matches_response(
code_matcher=Equals(BAD_REQUEST),
body_matcher=AfterPreprocessing(
loads,
Equals({
"reason":
"personal_dmd must be a directory-capability"
})))))
def main(reactor):
url = (
DecodedURL.from_text(u"https://httpbin.org").child(
u"get") # add path /get
.add(u"foo", u"&") # add query ?foo=%26
)
print(url.to_text())
return treq.get(url).addCallback(print_response)
def decoded_urls(draw): # pragma: no cover
# type: (DrawCallable) -> DecodedURL
"""
A strategy which generates L{DecodedURL}s.
Call the L{EncodedURL.to_uri} method on each URL to get an HTTP
protocol-friendly URI.
"""
return DecodedURL(draw(encoded_urls()))
def test_authorized(self, auth_token, child_segments, content):
"""
If the correct bearer token is not given in the **Authorization** header
of the request then the response code is UNAUTHORIZED.
:param bytes auth_token: A bearer token which, when presented, should
authorize access to the resource.
:param [unicode] child_segments: Additional path segments to add to the
request path beneath **v1**.
:param bytes content: The bytes we expect to see on a successful
request.
"""
def get_auth_token():
return auth_token
# Since we don't want to exercise any real magic-folder application
# logic we'll just magic up the child resource being requested.
branch = Data(
content,
b"application/binary",
)
segments_remaining = child_segments[:]
while segments_remaining:
name = segments_remaining.pop()
resource = Resource()
resource.putChild(name.encode("utf-8"), branch)
branch = resource
root = magic_folder_resource(
get_auth_token,
v1_resource=branch,
)
treq = StubTreq(root)
url = DecodedURL.from_text(u"http://example.invalid./v1").child(
*child_segments)
encoded_url = url_to_bytes(url)
# A request with no token at all or the wrong token should receive an
# unauthorized response.
headers = {
b"Authorization": u"Bearer {}".format(auth_token).encode("ascii"),
}
self.assertThat(
treq.get(
encoded_url,
headers=headers,
),
succeeded(
matches_response(
code_matcher=Equals(OK),
body_matcher=Equals(content),
), ),
)
def setUp(self):
super(CustomHTTPServerTests, self).setUp()
# Could be a fixture, but will only be used in this test class so not
# going to bother:
self._http_server = TestApp()
self.client = StorageClient(
DecodedURL.from_text("http://127.0.0.1"),
SWISSNUM_FOR_TEST,
treq=StubTreq(self._http_server._app.resource()),
)
def setup_example(self):
self.root = create_fake_tahoe_root()
self.http_client = create_tahoe_treq_client(self.root)
self.tahoe_client = create_tahoe_client(
DecodedURL.from_text(u"http://example.com"),
self.http_client,
)
self.alice = create_local_author(u"alice")
self.stash_dir = FilePath(mktemp())
self.stash_dir.makedirs() # 'trial' will delete this when done
def magic_folder_invite(node_directory, alias, nickname, treq):
"""
Invite a user identified by the nickname to a folder owned by the alias
:param unicode node_directory: The root of the Tahoe-LAFS node.
:param unicode alias: The alias of the folder to which the invitation is
being generated.
:param unicode nickname: The nickname of the invitee.
:param HTTPClient treq: An ``HTTPClient`` or similar object to use to make
the queries.
:return Deferred[unicode]: A secret invitation code.
"""
aliases = get_aliases(node_directory)[alias]
nodeurl = get_node_url(node_directory)
node_url = DecodedURL.from_text(unicode(nodeurl, 'utf-8'))
# create an unlinked directory and get the dmd write-cap
dmd_write_cap = yield tahoe_mkdir(node_url, treq)
# derive a dmd read-only cap from it.
dmd_readonly_cap = uri.from_string(
dmd_write_cap).get_readonly().to_string()
if dmd_readonly_cap is None:
raise Exception("failed to diminish dmd write cap")
# Now, we need to create a link to the nickname from inside the
# collective to this read-cap. For that we will need to know
# the write-cap of the collective (which is stored by the private/aliases
# file in the node_directory) so that a link can be created inside it
# to the .
# To do that, we use tahoe ln dmd_read_cap <collective-write-cap>/<alias>
magic_write_cap = get_aliases(node_directory)[alias]
magic_readonly_cap = uri.from_string(
magic_write_cap).get_readonly().to_string()
# tahoe ln CLIENT_READCAP COLLECTIVE_WRITECAP/NICKNAME
from_file = unicode(dmd_readonly_cap, 'utf-8')
to_file = u"%s/%s" % (unicode(magic_write_cap, 'utf-8'), nickname)
try:
yield tahoe_mv(node_url, aliases, from_file, to_file, treq)
except Exception:
raise
# return invite code, which is:
# magic_readonly_cap + INVITE_SEPARATOR + dmd_write_cap
invite_code = "{}{}{}".format(magic_readonly_cap, INVITE_SEPARATOR,
dmd_write_cap)
returnValue(invite_code)
def tahoe_client_url(self):
"""
The twisted client-string describing how we will connect to the
Tahoe LAFS client we will use.
"""
with self.database:
cursor = self.database.cursor()
cursor.execute("SELECT tahoe_node_directory FROM config")
node_dir = FilePath(cursor.fetchone()[0])
with node_dir.child("node.url").open("rt") as f:
return DecodedURL.from_text(f.read().strip().decode("utf8"))
def test_download_no_arg(self):
"""
Error if we GET from "/uri" with no ?uri= query-arg
"""
http_client = create_tahoe_treq_client()
uri = DecodedURL.from_text(u"http://example.com/uri/")
resp = http_client.get(uri.to_uri().to_text())
self.assertThat(resp, succeeded(MatchesStructure(code=Equals(400))))
def _setUp(self):
self.clock = Clock()
self.tempdir = self.useFixture(TempDir())
self.storage_server = StorageServer(self.tempdir.path,
b"\x00" * 20,
clock=self.clock)
self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST)
self.client = StorageClient(
DecodedURL.from_text("http://127.0.0.1"),
SWISSNUM_FOR_TEST,
treq=StubTreq(self.http_server.get_resource()),
)
def redirect_to(self, req):
"""
:param allmydata.webish.MyRequest req:
"""
ophandle = get_arg(req, "ophandle").decode("utf-8")
assert ophandle
here = DecodedURL.from_text(str(URLPath.fromRequest(req)))
target = here.click(u"/").child(u"operations", ophandle)
output = get_arg(req, "output")
if output:
target = target.add(u"output", output.decode("utf-8"))
return target
def test_request_uri_decodedurl(self):
"""
A URL may be passed as a `hyperlink.DecodedURL` object. It is converted
to bytes when passed to the underlying agent.
"""
url = DecodedURL.from_text(u"https://example.org/foo")
self.client.request("GET", url)
self.agent.request.assert_called_once_with(
b"GET", b"https://example.org/foo",
Headers({b"accept-encoding": [b"gzip"]}),
None,
)
def test_download_missing(self):
"""
Error if we download a capability that doesn't exist
"""
http_client = create_tahoe_treq_client()
cap_gen = capability_generator("URI:CHK:")
uri = DecodedURL.from_text(u"http://example.com/uri?uri={}".format(
next(cap_gen)))
resp = http_client.get(uri.to_uri().to_text())
self.assertThat(resp, succeeded(MatchesStructure(code=Equals(500))))
def test_bad_authentication(self):
"""
If the wrong swissnum is used, an ``Unauthorized`` response code is
returned.
"""
client = StorageClientGeneral(
StorageClient(
DecodedURL.from_text("http://127.0.0.1"),
b"something wrong",
treq=StubTreq(self.http.http_server.get_resource()),
))
with assert_fails_with_http_code(self, http.UNAUTHORIZED):
result_of(client.get_version())
def test_download_missing(self):
"""
If a capability is requested for which the stored cyphertext cannot be
located, **GET /uri?uri=CAP** returns a GONE response code.
"""
http_client = create_tahoe_treq_client()
cap_gen = capability_generator("URI:CHK:")
uri = DecodedURL.from_text(u"http://example.com/uri?uri={}".format(
next(cap_gen)))
resp = http_client.get(uri.to_uri().to_text())
self.assertThat(resp, succeeded(MatchesStructure(code=Equals(GONE), )))
def test_initInvalidBodyType(self) -> None:
"""
L{FrozenHTTPRequest} raises L{TypeError} when given a body of an
unknown type.
"""
e = self.assertRaises(
TypeError,
FrozenHTTPRequest,
method="GET",
uri=DecodedURL.fromText("https://twistedmatrix.com/"),
headers=FrozenHTTPHeaders(rawHeaders=()),
body=object(),
)
self.assertEqual(str(e), "body must be bytes or IFount")
def tahoe_create_alias(node_directory, alias, treq):
# mkdir+add_alias
nodeurl = get_node_url(node_directory)
try:
node_url = DecodedURL.from_text(unicode(nodeurl, 'utf-8'))
new_uri = yield tahoe_mkdir(node_url, treq)
except Exception:
raise
_add_alias(node_directory, alias, new_uri)
returnValue(0)
def status(folder_name, node_directory, treq):
"""
Retrieve information about the current state of a named magic folder.
:param unicode folder_name: The name of the magic folder about which to
return information.
:param FilePath node_directory: The path to the Tahoe-LAFS node directory
which owns the magic folder in question.
:return Deferred[Status]: A Deferred which fires with information about
the magic folder.
"""
magic_folders = load_magic_folders(node_directory.path)
token = node_directory.descendant([u"private", u"api_auth_token"]).getContent()
node_root_url = DecodedURL.from_text(
node_directory.child(u"node.url").getContent().decode("ascii").strip(),
)
magic_root_url = DecodedURL.from_text(
node_directory.child(u"magic-folder.url").getContent().decode("ascii").strip(),
)
try:
folder_config = magic_folders[folder_name]
except KeyError:
raise BadFolderName(node_directory, folder_name)
return status_from_folder_config(
folder_name,
folder_config[u"upload_dircap"].decode("ascii"),
folder_config[u"collective_dircap"].decode("ascii"),
node_root_url,
magic_root_url,
token,
treq,
)
def test_load_db(self):
"""
``load_global_configuration`` can read the global configuration written by
``create_global_configuration``.
"""
create_global_configuration(self.temp, u"tcp:1234", self.node_dir,
u"tcp:localhost:1234")
config = load_global_configuration(self.temp)
self.assertThat(
config,
MatchesStructure(
api_endpoint=Equals(u"tcp:1234"),
tahoe_client_url=Equals(
DecodedURL.from_text(u"http://127.0.0.1:9876/")),
))
def build_nurl(listening_port: IListeningPort) -> DecodedURL:
nurl = DecodedURL().replace(
fragment="v=1", # how we know this NURL is HTTP-based (i.e. not Foolscap)
host=hostname,
port=listening_port.getHost().port,
path=(str(server._swissnum, "ascii"),),
userinfo=(
str(
get_spki_hash(load_pem_x509_certificate(cert_path.getContent())),
"ascii",
),
),
scheme="pb",
)
return nurl
def test_request_uri_hyperlink_params(self):
"""
The *params* argument augments an instance of `hyperlink.DecodedURL`
passed as the *url* parameter, just as if it were a string.
"""
self.client.request(
method="GET",
url=DecodedURL.from_text(u"http://č.net"),
params={"foo": "bar"},
)
self.agent.request.assert_called_once_with(
b"GET", b"http://xn--bea.net/?foo=bar",
Headers({b"accept-encoding": [b"gzip"]}),
None,
)