def test_index_json(alice):
"""
we can download the index file as json
"""
data = util.web_get(alice, u"", params={u"t": u"json"})
# it should be valid json
json.loads(data)
def _get_config_via_wormhole(config):
out = config.stdout
print("Opening wormhole with code '{}'".format(config['join']), file=out)
relay_url = config.parent['wormhole-server']
print("Connecting to '{}'".format(relay_url), file=out)
wh = config.parent.wormhole.create(
appid=config.parent['wormhole-invite-appid'],
relay_url=relay_url,
reactor=reactor,
)
code = str(config['join'])
wh.set_code(code)
yield wh.get_welcome()
print("Connected to wormhole server", file=out)
intro = {
u"abilities": {
"client-v1": {},
}
}
wh.send_message(json.dumps_bytes(intro))
server_intro = yield wh.get_message()
server_intro = json.loads(server_intro)
print(" received server introduction", file=out)
if u'abilities' not in server_intro:
raise RuntimeError(" Expected 'abilities' in server introduction")
if u'server-v1' not in server_intro['abilities']:
raise RuntimeError(" Expected 'server-v1' in server abilities")
remote_data = yield wh.get_message()
print(" received configuration", file=out)
defer.returnValue(json.loads(remote_data))
def check_backupdb_file(self, childpath):
if not self.backupdb:
return True, None
use_timestamps = not self.options["ignore-timestamps"]
r = self.backupdb.check_file(childpath, use_timestamps)
if not r.was_uploaded():
return True, r
if not r.should_check():
# the file was uploaded or checked recently, so we can just use
# it
return False, r
# we must check the file before using the results
filecap = r.was_uploaded()
self.verboseprint("checking %s" % quote_output(filecap))
nodeurl = self.options['node-url']
checkurl = nodeurl + "uri/%s?t=check&output=JSON" % url_quote(filecap)
self._files_checked += 1
resp = do_http("POST", checkurl)
if resp.status != 200:
# can't check, so we must assume it's bad
return True, r
cr = json.loads(resp.read())
healthy = cr["results"]["healthy"]
if not healthy:
# must upload
return True, r
# file is healthy, no need to upload
r.did_check_healthy(cr)
return False, r
def check_backupdb_directory(self, compare_contents):
if not self.backupdb:
return True, None
r = self.backupdb.check_directory(compare_contents)
if not r.was_created():
return True, r
if not r.should_check():
# the file was uploaded or checked recently, so we can just use
# it
return False, r
# we must check the directory before re-using it
dircap = r.was_created()
self.verboseprint("checking %s" % quote_output(dircap))
nodeurl = self.options['node-url']
checkurl = nodeurl + "uri/%s?t=check&output=JSON" % url_quote(dircap)
self._directories_checked += 1
resp = do_http("POST", checkurl)
if resp.status != 200:
# can't check, so we must assume it's bad
return True, r
cr = json.loads(resp.read())
healthy = cr["results"]["healthy"]
if not healthy:
# must create
return True, r
# directory is healthy, no need to upload
r.did_check_healthy(cr)
return False, r
def _send_config_via_wormhole(options, config):
out = options.stdout
err = options.stderr
relay_url = options.parent['wormhole-server']
print("Connecting to '{}'...".format(relay_url), file=out)
wh = options.parent.wormhole.create(
appid=options.parent['wormhole-invite-appid'],
relay_url=relay_url,
reactor=reactor,
)
yield wh.get_welcome()
print("Connected to wormhole server", file=out)
# must call allocate_code before get_code will ever succeed
wh.allocate_code()
code = yield wh.get_code()
print("Invite Code for client: {}".format(code), file=out)
wh.send_message(json.dumps_bytes({u"abilities": {
u"server-v1": {},
}}))
client_intro = yield wh.get_message()
print(" received client introduction", file=out)
client_intro = json.loads(client_intro)
if not u'abilities' in client_intro:
print("No 'abilities' from client", file=err)
defer.returnValue(1)
if not u'client-v1' in client_intro[u'abilities']:
print("No 'client-v1' in abilities from client", file=err)
defer.returnValue(1)
print(" transmitting configuration", file=out)
wh.send_message(json.dumps_bytes(config))
yield wh.close()
def _unpack_contents(self, data):
# the directory is serialized as a list of netstrings, one per child.
# Each child is serialized as a list of four netstrings: (name, ro_uri,
# rwcapdata, metadata), in which the name, ro_uri, metadata are in
# cleartext. The 'name' is UTF-8 encoded, and should be normalized to NFC.
# The rwcapdata is formatted as:
# pack("16ss32s", iv, AES(H(writekey+iv), plaintext_rw_uri), mac)
assert isinstance(data, bytes), (repr(data), type(data))
# an empty directory is serialized as an empty string
if data == b"":
return AuxValueDict()
writeable = not self.is_readonly()
mutable = self.is_mutable()
children = AuxValueDict()
position = 0
while position < len(data):
entries, position = split_netstring(data, 1, position)
entry = entries[0]
(namex_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
if not mutable and len(rwcapdata) > 0:
raise ValueError("the rwcapdata field of a dirnode in an immutable directory was not empty")
# A name containing characters that are unassigned in one version of Unicode might
# not be normalized wrt a later version. See the note in section 'Normalization Stability'
# at <http://unicode.org/policies/stability_policy.html>.
# Therefore we normalize names going both in and out of directories.
name = normalize(namex_utf8.decode("utf-8"))
rw_uri = b""
if writeable:
rw_uri = self._decrypt_rwcapdata(rwcapdata)
# Since the encryption uses CTR mode, it currently leaks the length of the
# plaintext rw_uri -- and therefore whether it is present, i.e. whether the
# dirnode is writeable (ticket #925). By stripping trailing spaces in
# Tahoe >= 1.6.0, we may make it easier for future versions to plug this leak.
# ro_uri is treated in the same way for consistency.
# rw_uri and ro_uri will be either None or a non-empty string.
rw_uri = rw_uri.rstrip(b' ') or None
ro_uri = ro_uri.rstrip(b' ') or None
try:
child = self._create_and_validate_node(rw_uri, ro_uri, name)
if mutable or child.is_allowed_in_immutable_directory():
metadata = json.loads(metadata_s)
assert isinstance(metadata, dict)
children[name] = (child, metadata)
children.set_with_aux(name, (child, metadata), auxilliary=entry)
else:
log.msg(format="mutable cap for child %(name)s unpacked from an immutable directory",
name=quote_output(name, encoding='utf-8'),
facility="tahoe.webish", level=log.UNUSUAL)
except CapConstraintError as e:
log.msg(format="unmet constraint on cap for child %(name)s unpacked from a directory:\n"
"%(message)s", message=e.args[0], name=quote_output(name, encoding='utf-8'),
facility="tahoe.webish", level=log.UNUSUAL)
return children
def test_encode_bytes(self):
"""BytesJSONEncoder can encode bytes."""
data = {
b"hello": [1, b"cd"],
}
expected = {
u"hello": [1, u"cd"],
}
# Bytes get passed through as if they were UTF-8 Unicode:
encoded = jsonbytes.dumps(data)
self.assertEqual(json.loads(encoded), expected)
self.assertEqual(jsonbytes.loads(encoded), expected)
def test_storage_info_json(storage_nodes):
"""
retrieve and confirm /storage?t=json URI for one storage node
"""
storage0 = storage_nodes[0]
resp = requests.get(
util.node_url(storage0.node_dir, u"storage"),
params={u"t": u"json"},
)
data = json.loads(resp.content)
assert data[u"stats"][u"storage_server.reserved_space"] == 1000000000
def init_from_grid(self, writecap, readcap):
self.writecap = writecap
self.readcap = readcap
bestcap = writecap or readcap
url = self.nodeurl + "uri/%s" % url_quote(bestcap)
resp = do_http("GET", url + "?t=json")
if resp.status != 200:
raise HTTPError("Error examining target directory", resp)
parsed = json.loads(resp.read())
nodetype, d = parsed
assert nodetype == "dirnode"
self.mutable = d.get("mutable", False) # older nodes don't provide it
self.children_d = dict([(str(name), value)
for (name, value) in d["children"].items()])
self.children = None
def unsign_from_foolscap(ann_t):
(msg, sig_vs, claimed_key_vs) = ann_t
if not sig_vs or not claimed_key_vs:
raise UnknownKeyError("only signed announcements recognized")
if not sig_vs.startswith(b"v0-"):
raise UnknownKeyError("only v0- signatures recognized")
if not claimed_key_vs.startswith(b"v0-"):
raise UnknownKeyError("only v0- keys recognized")
claimed_key = ed25519.verifying_key_from_string(b"pub-" + claimed_key_vs)
sig_bytes = base32.a2b(remove_prefix(sig_vs, b"v0-"))
ed25519.verify_signature(claimed_key, sig_bytes, msg)
key_vs = claimed_key_vs
ann = json.loads(msg.decode("utf-8"))
return (ann, key_vs)
def test_encode_bytes(self):
"""BytesJSONEncoder can encode bytes.
Bytes are presumed to be UTF-8 encoded.
"""
snowman = u"def\N{SNOWMAN}\uFF00"
data = {
b"hello": [1, b"cd", {b"abc": [123, snowman.encode("utf-8")]}],
}
expected = {
u"hello": [1, u"cd", {u"abc": [123, snowman]}],
}
# Bytes get passed through as if they were UTF-8 Unicode:
encoded = jsonbytes.dumps(data)
self.assertEqual(json.loads(encoded), expected)
self.assertEqual(jsonbytes.loads(encoded), expected)
def test_introducer_info(introducer):
"""
retrieve and confirm /introducer URI for the introducer
"""
resp = requests.get(
util.node_url(introducer.node_dir, u""),
)
assert b"Introducer" in resp.content
resp = requests.get(
util.node_url(introducer.node_dir, u""),
params={u"t": u"json"},
)
data = json.loads(resp.content)
assert "announcement_summary" in data
assert "subscription_summary" in data
def get_target_info(self, destination_spec):
precondition(isinstance(destination_spec, str), destination_spec)
rootcap, path_utf8 = get_alias(self.aliases, destination_spec, None)
path = path_utf8.decode("utf-8")
if rootcap == DefaultAliasMarker:
# no alias, so this is a local file
pathname = abspath_expanduser_unicode(path)
if not os.path.exists(pathname):
t = LocalMissingTarget(pathname)
elif os.path.isdir(pathname):
t = LocalDirectoryTarget(self.progress, pathname)
else:
# TODO: should this be _assert? what happens if the target is
# a special file?
assert os.path.isfile(pathname), pathname
t = LocalFileTarget(pathname) # non-empty
else:
# this is a tahoe object
url = self.nodeurl + "uri/%s" % url_quote(rootcap)
if path:
url += "/" + escape_path(path)
resp = do_http("GET", url + "?t=json")
if resp.status == 404:
# doesn't exist yet
t = TahoeMissingTarget(url)
elif resp.status == 200:
parsed = json.loads(resp.read())
nodetype, d = parsed
if nodetype == "dirnode":
t = TahoeDirectoryTarget(self.nodeurl, self.cache,
self.progress)
t.init_from_parsed(parsed)
else:
writecap = to_bytes(d.get("rw_uri"))
readcap = to_bytes(d.get("ro_uri"))
mutable = d.get("mutable", False)
t = TahoeFileTarget(self.nodeurl, mutable, writecap,
readcap, url)
else:
raise HTTPError(
"Error examining target %s" %
quote_output(destination_spec), resp)
return t
def test_deep_stats(alice):
"""
create a directory, do deep-stats on it and prove the /operations/
URIs work
"""
resp = requests.post(
util.node_url(alice.node_dir, "uri"),
params={
"format": "sdmf",
"t": "mkdir",
"redirect_to_result": "true",
},
)
assert resp.status_code >= 200 and resp.status_code < 300
# when creating a directory, we'll be re-directed to a URL
# containing our writecap..
uri = url_unquote(resp.url)
assert 'URI:DIR2:' in uri
dircap = uri[uri.find("URI:DIR2:"):].rstrip('/')
dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(url_quote(dircap)))
# POST a file into this directory
FILE_CONTENTS = u"a file in a directory"
resp = requests.post(
dircap_uri,
data={
u"t": u"upload",
},
files={
u"file": FILE_CONTENTS,
},
)
resp.raise_for_status()
# confirm the file is in the directory
resp = requests.get(
dircap_uri,
params={
u"t": u"json",
},
)
d = json.loads(resp.content)
k, data = d
assert k == u"dirnode"
assert len(data['children']) == 1
k, child = list(data['children'].values())[0]
assert k == u"filenode"
assert child['size'] == len(FILE_CONTENTS)
# perform deep-stats on it...
resp = requests.post(
dircap_uri,
data={
u"t": u"start-deep-stats",
u"ophandle": u"something_random",
},
)
assert resp.status_code >= 200 and resp.status_code < 300
# confirm we get information from the op .. after its done
tries = 10
while tries > 0:
tries -= 1
resp = requests.get(
util.node_url(alice.node_dir, u"operations/something_random"),
)
d = json.loads(resp.content)
if d['size-literal-files'] == len(FILE_CONTENTS):
print("stats completed successfully")
break
else:
print("{} != {}; waiting".format(d['size-literal-files'], len(FILE_CONTENTS)))
time.sleep(.5)
def test_directory_deep_check(alice):
"""
use deep-check and confirm the result pages work
"""
# create a directory
resp = requests.post(
util.node_url(alice.node_dir, u"uri"),
params={
u"t": u"mkdir",
u"redirect_to_result": u"true",
}
)
# get json information about our directory
dircap_url = resp.url
resp = requests.get(
dircap_url,
params={u"t": u"json"},
)
# Just verify it is valid JSON.
json.loads(resp.content)
# upload a file of pangrams into the directory
FILE_CONTENTS = u"Sphinx of black quartz, judge my vow.\n" * (2048*10)
resp = requests.post(
dircap_url,
params={
u"t": u"upload",
u"upload-chk": u"upload-chk",
},
files={
u"file": FILE_CONTENTS,
}
)
cap0 = resp.content
print("Uploaded data0, cap={}".format(cap0))
# a different pangram
FILE_CONTENTS = u"The five boxing wizards jump quickly.\n" * (2048*10)
resp = requests.post(
dircap_url,
params={
u"t": u"upload",
u"upload-chk": u"upload-chk",
},
files={
u"file": FILE_CONTENTS,
}
)
cap1 = resp.content
print("Uploaded data1, cap={}".format(cap1))
resp = requests.get(
util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap0))),
params={u"t": u"info"},
)
def check_repair_data(checkdata):
assert checkdata["healthy"] is True
assert checkdata["count-happiness"] == 4
assert checkdata["count-good-share-hosts"] == 4
assert checkdata["count-shares-good"] == 4
assert checkdata["count-corrupt-shares"] == 0
assert checkdata["list-corrupt-shares"] == []
# do a "check" (once for HTML, then with JSON for easier asserts)
resp = requests.post(
dircap_url,
params={
u"t": u"check",
u"return_to": u".",
u"verify": u"true",
}
)
resp = requests.post(
dircap_url,
params={
u"t": u"check",
u"return_to": u".",
u"verify": u"true",
u"output": u"JSON",
}
)
check_repair_data(json.loads(resp.content)["results"])
# "check and repair"
resp = requests.post(
dircap_url,
params={
u"t": u"check",
u"return_to": u".",
u"verify": u"true",
u"repair": u"true",
}
)
resp = requests.post(
dircap_url,
params={
u"t": u"check",
u"return_to": u".",
u"verify": u"true",
u"repair": u"true",
u"output": u"JSON",
}
)
check_repair_data(json.loads(resp.content)["post-repair-results"]["results"])
# start a "deep check and repair"
resp = requests.post(
dircap_url,
params={
u"t": u"start-deep-check",
u"return_to": u".",
u"verify": u"on",
u"repair": u"on",
u"output": u"JSON",
u"ophandle": u"deadbeef",
}
)
deepcheck_uri = resp.url
data = json.loads(resp.content)
tries = 10
while not data['finished'] and tries > 0:
tries -= 1
time.sleep(0.5)
print("deep-check not finished, reloading")
resp = requests.get(deepcheck_uri, params={u"output": "JSON"})
data = json.loads(resp.content)
print("deep-check finished")
assert data[u"stats"][u"count-immutable-files"] == 1
assert data[u"stats"][u"count-literal-files"] == 0
assert data[u"stats"][u"largest-immutable-file"] == 778240
assert data[u"count-objects-checked"] == 2
# also get the HTML version
resp = requests.post(
dircap_url,
params={
u"t": u"start-deep-check",
u"return_to": u".",
u"verify": u"on",
u"repair": u"on",
u"ophandle": u"definitely_random",
}
)
deepcheck_uri = resp.url
# if the operations isn't done, there's an <H2> tag with the
# reload link; otherwise there's only an <H1> tag..wait up to 5
# seconds for this to respond properly.
for _ in range(5):
resp = requests.get(deepcheck_uri)
dom = BeautifulSoup(resp.content, "html5lib")
if dom.h1 and u'Results' in str(dom.h1.string):
break
if dom.h2 and dom.h2.a and u"Reload" in str(dom.h2.a.string):
dom = None
time.sleep(1)
assert dom is not None, "Operation never completed"
def get_source_info(self, source_spec):
"""
This turns an argv string into a (Local|Tahoe)(File|Directory)Source.
"""
precondition(isinstance(source_spec, str), source_spec)
rootcap, path_utf8 = get_alias(self.aliases, source_spec, None)
path = path_utf8.decode("utf-8")
# any trailing slash is removed in abspath_expanduser_unicode(), so
# make a note of it here, to throw an error later
had_trailing_slash = path.endswith("/")
if rootcap == DefaultAliasMarker:
# no alias, so this is a local file
pathname = abspath_expanduser_unicode(path)
name = os.path.basename(pathname)
if not os.path.exists(pathname):
raise MissingSourceError(source_spec,
quotefn=quote_local_unicode_path)
if os.path.isdir(pathname):
t = LocalDirectorySource(self.progress, pathname, name)
else:
if had_trailing_slash:
raise FilenameWithTrailingSlashError(
source_spec, quotefn=quote_local_unicode_path)
if not os.path.isfile(pathname):
raise WeirdSourceError(pathname)
t = LocalFileSource(pathname, name) # non-empty
else:
# this is a tahoe object
url = self.nodeurl + "uri/%s" % url_quote(rootcap)
name = None
if path:
if path.endswith("/"):
path = path[:-1]
url += "/" + escape_path(path)
last_slash = path.rfind(u"/")
name = path
if last_slash != -1:
name = path[last_slash + 1:]
resp = do_http("GET", url + "?t=json")
if resp.status == 404:
raise MissingSourceError(source_spec)
elif resp.status != 200:
raise HTTPError(
"Error examining source %s" % quote_output(source_spec),
resp)
parsed = json.loads(resp.read())
nodetype, d = parsed
if nodetype == "dirnode":
t = TahoeDirectorySource(self.nodeurl, self.cache,
self.progress, name)
t.init_from_parsed(parsed)
else:
if had_trailing_slash:
raise FilenameWithTrailingSlashError(source_spec)
writecap = to_bytes(d.get("rw_uri"))
readcap = to_bytes(d.get("ro_uri"))
mutable = d.get("mutable",
False) # older nodes don't provide it
t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap,
name)
return t
def test_status(alice):
"""
confirm we get something sensible from /status and the various sub-types
"""
# upload a file
# (because of the nature of the integration-tests, we can only
# assert things about "our" file because we don't know what other
# operations may have happened in the grid before our test runs).
FILE_CONTENTS = u"all the Important Data of alice\n" * 1200
resp = requests.put(
util.node_url(alice.node_dir, u"uri"),
data=FILE_CONTENTS,
)
cap = resp.text.strip()
print("Uploaded data, cap={}".format(cap))
resp = requests.get(
util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap))),
)
print("Downloaded {} bytes of data".format(len(resp.content)))
assert str(resp.content, "ascii") == FILE_CONTENTS
resp = requests.get(
util.node_url(alice.node_dir, "status"),
)
dom = html5lib.parse(resp.content)
hrefs = [
a.get('href')
for a in dom.iter(u'{http://www.w3.org/1999/xhtml}a')
]
found_upload = False
found_download = False
for href in hrefs:
if href == u"/" or not href:
continue
resp = requests.get(util.node_url(alice.node_dir, href))
if href.startswith(u"/status/up"):
assert b"File Upload Status" in resp.content
if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content:
found_upload = True
elif href.startswith(u"/status/down"):
assert b"File Download Status" in resp.content
if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content:
found_download = True
# download the specialized event information
resp = requests.get(
util.node_url(alice.node_dir, u"{}/event_json".format(href)),
)
js = json.loads(resp.content)
# there's usually just one "read" operation, but this can handle many ..
total_bytes = sum([st['bytes_returned'] for st in js['read']], 0)
assert total_bytes == len(FILE_CONTENTS)
assert found_upload, "Failed to find the file we uploaded in the status-page"
assert found_download, "Failed to find the file we downloaded in the status-page"
