def test_index_json(alice): """ we can download the index file as json """ data = util.web_get(alice, u"", params={u"t": u"json"}) # it should be valid json json.loads(data)
def _get_config_via_wormhole(config): out = config.stdout print("Opening wormhole with code '{}'".format(config['join']), file=out) relay_url = config.parent['wormhole-server'] print("Connecting to '{}'".format(relay_url), file=out) wh = config.parent.wormhole.create( appid=config.parent['wormhole-invite-appid'], relay_url=relay_url, reactor=reactor, ) code = str(config['join']) wh.set_code(code) yield wh.get_welcome() print("Connected to wormhole server", file=out) intro = { u"abilities": { "client-v1": {}, } } wh.send_message(json.dumps_bytes(intro)) server_intro = yield wh.get_message() server_intro = json.loads(server_intro) print(" received server introduction", file=out) if u'abilities' not in server_intro: raise RuntimeError(" Expected 'abilities' in server introduction") if u'server-v1' not in server_intro['abilities']: raise RuntimeError(" Expected 'server-v1' in server abilities") remote_data = yield wh.get_message() print(" received configuration", file=out) defer.returnValue(json.loads(remote_data))
def check_backupdb_file(self, childpath): if not self.backupdb: return True, None use_timestamps = not self.options["ignore-timestamps"] r = self.backupdb.check_file(childpath, use_timestamps) if not r.was_uploaded(): return True, r if not r.should_check(): # the file was uploaded or checked recently, so we can just use # it return False, r # we must check the file before using the results filecap = r.was_uploaded() self.verboseprint("checking %s" % quote_output(filecap)) nodeurl = self.options['node-url'] checkurl = nodeurl + "uri/%s?t=check&output=JSON" % url_quote(filecap) self._files_checked += 1 resp = do_http("POST", checkurl) if resp.status != 200: # can't check, so we must assume it's bad return True, r cr = json.loads(resp.read()) healthy = cr["results"]["healthy"] if not healthy: # must upload return True, r # file is healthy, no need to upload r.did_check_healthy(cr) return False, r
def check_backupdb_directory(self, compare_contents): if not self.backupdb: return True, None r = self.backupdb.check_directory(compare_contents) if not r.was_created(): return True, r if not r.should_check(): # the file was uploaded or checked recently, so we can just use # it return False, r # we must check the directory before re-using it dircap = r.was_created() self.verboseprint("checking %s" % quote_output(dircap)) nodeurl = self.options['node-url'] checkurl = nodeurl + "uri/%s?t=check&output=JSON" % url_quote(dircap) self._directories_checked += 1 resp = do_http("POST", checkurl) if resp.status != 200: # can't check, so we must assume it's bad return True, r cr = json.loads(resp.read()) healthy = cr["results"]["healthy"] if not healthy: # must create return True, r # directory is healthy, no need to upload r.did_check_healthy(cr) return False, r
def _send_config_via_wormhole(options, config): out = options.stdout err = options.stderr relay_url = options.parent['wormhole-server'] print("Connecting to '{}'...".format(relay_url), file=out) wh = options.parent.wormhole.create( appid=options.parent['wormhole-invite-appid'], relay_url=relay_url, reactor=reactor, ) yield wh.get_welcome() print("Connected to wormhole server", file=out) # must call allocate_code before get_code will ever succeed wh.allocate_code() code = yield wh.get_code() print("Invite Code for client: {}".format(code), file=out) wh.send_message(json.dumps_bytes({u"abilities": { u"server-v1": {}, }})) client_intro = yield wh.get_message() print(" received client introduction", file=out) client_intro = json.loads(client_intro) if not u'abilities' in client_intro: print("No 'abilities' from client", file=err) defer.returnValue(1) if not u'client-v1' in client_intro[u'abilities']: print("No 'client-v1' in abilities from client", file=err) defer.returnValue(1) print(" transmitting configuration", file=out) wh.send_message(json.dumps_bytes(config)) yield wh.close()
def _unpack_contents(self, data): # the directory is serialized as a list of netstrings, one per child. # Each child is serialized as a list of four netstrings: (name, ro_uri, # rwcapdata, metadata), in which the name, ro_uri, metadata are in # cleartext. The 'name' is UTF-8 encoded, and should be normalized to NFC. # The rwcapdata is formatted as: # pack("16ss32s", iv, AES(H(writekey+iv), plaintext_rw_uri), mac) assert isinstance(data, bytes), (repr(data), type(data)) # an empty directory is serialized as an empty string if data == b"": return AuxValueDict() writeable = not self.is_readonly() mutable = self.is_mutable() children = AuxValueDict() position = 0 while position < len(data): entries, position = split_netstring(data, 1, position) entry = entries[0] (namex_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4) if not mutable and len(rwcapdata) > 0: raise ValueError("the rwcapdata field of a dirnode in an immutable directory was not empty") # A name containing characters that are unassigned in one version of Unicode might # not be normalized wrt a later version. See the note in section 'Normalization Stability' # at <http://unicode.org/policies/stability_policy.html>. # Therefore we normalize names going both in and out of directories. name = normalize(namex_utf8.decode("utf-8")) rw_uri = b"" if writeable: rw_uri = self._decrypt_rwcapdata(rwcapdata) # Since the encryption uses CTR mode, it currently leaks the length of the # plaintext rw_uri -- and therefore whether it is present, i.e. whether the # dirnode is writeable (ticket #925). By stripping trailing spaces in # Tahoe >= 1.6.0, we may make it easier for future versions to plug this leak. # ro_uri is treated in the same way for consistency. # rw_uri and ro_uri will be either None or a non-empty string. rw_uri = rw_uri.rstrip(b' ') or None ro_uri = ro_uri.rstrip(b' ') or None try: child = self._create_and_validate_node(rw_uri, ro_uri, name) if mutable or child.is_allowed_in_immutable_directory(): metadata = json.loads(metadata_s) assert isinstance(metadata, dict) children[name] = (child, metadata) children.set_with_aux(name, (child, metadata), auxilliary=entry) else: log.msg(format="mutable cap for child %(name)s unpacked from an immutable directory", name=quote_output(name, encoding='utf-8'), facility="tahoe.webish", level=log.UNUSUAL) except CapConstraintError as e: log.msg(format="unmet constraint on cap for child %(name)s unpacked from a directory:\n" "%(message)s", message=e.args[0], name=quote_output(name, encoding='utf-8'), facility="tahoe.webish", level=log.UNUSUAL) return children
def test_encode_bytes(self): """BytesJSONEncoder can encode bytes.""" data = { b"hello": [1, b"cd"], } expected = { u"hello": [1, u"cd"], } # Bytes get passed through as if they were UTF-8 Unicode: encoded = jsonbytes.dumps(data) self.assertEqual(json.loads(encoded), expected) self.assertEqual(jsonbytes.loads(encoded), expected)
def test_storage_info_json(storage_nodes): """ retrieve and confirm /storage?t=json URI for one storage node """ storage0 = storage_nodes[0] resp = requests.get( util.node_url(storage0.node_dir, u"storage"), params={u"t": u"json"}, ) data = json.loads(resp.content) assert data[u"stats"][u"storage_server.reserved_space"] == 1000000000
def init_from_grid(self, writecap, readcap): self.writecap = writecap self.readcap = readcap bestcap = writecap or readcap url = self.nodeurl + "uri/%s" % url_quote(bestcap) resp = do_http("GET", url + "?t=json") if resp.status != 200: raise HTTPError("Error examining target directory", resp) parsed = json.loads(resp.read()) nodetype, d = parsed assert nodetype == "dirnode" self.mutable = d.get("mutable", False) # older nodes don't provide it self.children_d = dict([(str(name), value) for (name, value) in d["children"].items()]) self.children = None
def unsign_from_foolscap(ann_t): (msg, sig_vs, claimed_key_vs) = ann_t if not sig_vs or not claimed_key_vs: raise UnknownKeyError("only signed announcements recognized") if not sig_vs.startswith(b"v0-"): raise UnknownKeyError("only v0- signatures recognized") if not claimed_key_vs.startswith(b"v0-"): raise UnknownKeyError("only v0- keys recognized") claimed_key = ed25519.verifying_key_from_string(b"pub-" + claimed_key_vs) sig_bytes = base32.a2b(remove_prefix(sig_vs, b"v0-")) ed25519.verify_signature(claimed_key, sig_bytes, msg) key_vs = claimed_key_vs ann = json.loads(msg.decode("utf-8")) return (ann, key_vs)
def test_encode_bytes(self): """BytesJSONEncoder can encode bytes. Bytes are presumed to be UTF-8 encoded. """ snowman = u"def\N{SNOWMAN}\uFF00" data = { b"hello": [1, b"cd", {b"abc": [123, snowman.encode("utf-8")]}], } expected = { u"hello": [1, u"cd", {u"abc": [123, snowman]}], } # Bytes get passed through as if they were UTF-8 Unicode: encoded = jsonbytes.dumps(data) self.assertEqual(json.loads(encoded), expected) self.assertEqual(jsonbytes.loads(encoded), expected)
def test_introducer_info(introducer): """ retrieve and confirm /introducer URI for the introducer """ resp = requests.get( util.node_url(introducer.node_dir, u""), ) assert b"Introducer" in resp.content resp = requests.get( util.node_url(introducer.node_dir, u""), params={u"t": u"json"}, ) data = json.loads(resp.content) assert "announcement_summary" in data assert "subscription_summary" in data
def get_target_info(self, destination_spec): precondition(isinstance(destination_spec, str), destination_spec) rootcap, path_utf8 = get_alias(self.aliases, destination_spec, None) path = path_utf8.decode("utf-8") if rootcap == DefaultAliasMarker: # no alias, so this is a local file pathname = abspath_expanduser_unicode(path) if not os.path.exists(pathname): t = LocalMissingTarget(pathname) elif os.path.isdir(pathname): t = LocalDirectoryTarget(self.progress, pathname) else: # TODO: should this be _assert? what happens if the target is # a special file? assert os.path.isfile(pathname), pathname t = LocalFileTarget(pathname) # non-empty else: # this is a tahoe object url = self.nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) resp = do_http("GET", url + "?t=json") if resp.status == 404: # doesn't exist yet t = TahoeMissingTarget(url) elif resp.status == 200: parsed = json.loads(resp.read()) nodetype, d = parsed if nodetype == "dirnode": t = TahoeDirectoryTarget(self.nodeurl, self.cache, self.progress) t.init_from_parsed(parsed) else: writecap = to_bytes(d.get("rw_uri")) readcap = to_bytes(d.get("ro_uri")) mutable = d.get("mutable", False) t = TahoeFileTarget(self.nodeurl, mutable, writecap, readcap, url) else: raise HTTPError( "Error examining target %s" % quote_output(destination_spec), resp) return t
def test_deep_stats(alice): """ create a directory, do deep-stats on it and prove the /operations/ URIs work """ resp = requests.post( util.node_url(alice.node_dir, "uri"), params={ "format": "sdmf", "t": "mkdir", "redirect_to_result": "true", }, ) assert resp.status_code >= 200 and resp.status_code < 300 # when creating a directory, we'll be re-directed to a URL # containing our writecap.. uri = url_unquote(resp.url) assert 'URI:DIR2:' in uri dircap = uri[uri.find("URI:DIR2:"):].rstrip('/') dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(url_quote(dircap))) # POST a file into this directory FILE_CONTENTS = u"a file in a directory" resp = requests.post( dircap_uri, data={ u"t": u"upload", }, files={ u"file": FILE_CONTENTS, }, ) resp.raise_for_status() # confirm the file is in the directory resp = requests.get( dircap_uri, params={ u"t": u"json", }, ) d = json.loads(resp.content) k, data = d assert k == u"dirnode" assert len(data['children']) == 1 k, child = list(data['children'].values())[0] assert k == u"filenode" assert child['size'] == len(FILE_CONTENTS) # perform deep-stats on it... resp = requests.post( dircap_uri, data={ u"t": u"start-deep-stats", u"ophandle": u"something_random", }, ) assert resp.status_code >= 200 and resp.status_code < 300 # confirm we get information from the op .. after its done tries = 10 while tries > 0: tries -= 1 resp = requests.get( util.node_url(alice.node_dir, u"operations/something_random"), ) d = json.loads(resp.content) if d['size-literal-files'] == len(FILE_CONTENTS): print("stats completed successfully") break else: print("{} != {}; waiting".format(d['size-literal-files'], len(FILE_CONTENTS))) time.sleep(.5)
def test_directory_deep_check(alice): """ use deep-check and confirm the result pages work """ # create a directory resp = requests.post( util.node_url(alice.node_dir, u"uri"), params={ u"t": u"mkdir", u"redirect_to_result": u"true", } ) # get json information about our directory dircap_url = resp.url resp = requests.get( dircap_url, params={u"t": u"json"}, ) # Just verify it is valid JSON. json.loads(resp.content) # upload a file of pangrams into the directory FILE_CONTENTS = u"Sphinx of black quartz, judge my vow.\n" * (2048*10) resp = requests.post( dircap_url, params={ u"t": u"upload", u"upload-chk": u"upload-chk", }, files={ u"file": FILE_CONTENTS, } ) cap0 = resp.content print("Uploaded data0, cap={}".format(cap0)) # a different pangram FILE_CONTENTS = u"The five boxing wizards jump quickly.\n" * (2048*10) resp = requests.post( dircap_url, params={ u"t": u"upload", u"upload-chk": u"upload-chk", }, files={ u"file": FILE_CONTENTS, } ) cap1 = resp.content print("Uploaded data1, cap={}".format(cap1)) resp = requests.get( util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap0))), params={u"t": u"info"}, ) def check_repair_data(checkdata): assert checkdata["healthy"] is True assert checkdata["count-happiness"] == 4 assert checkdata["count-good-share-hosts"] == 4 assert checkdata["count-shares-good"] == 4 assert checkdata["count-corrupt-shares"] == 0 assert checkdata["list-corrupt-shares"] == [] # do a "check" (once for HTML, then with JSON for easier asserts) resp = requests.post( dircap_url, params={ u"t": u"check", u"return_to": u".", u"verify": u"true", } ) resp = requests.post( dircap_url, params={ u"t": u"check", u"return_to": u".", u"verify": u"true", u"output": u"JSON", } ) check_repair_data(json.loads(resp.content)["results"]) # "check and repair" resp = requests.post( dircap_url, params={ u"t": u"check", u"return_to": u".", u"verify": u"true", u"repair": u"true", } ) resp = requests.post( dircap_url, params={ u"t": u"check", u"return_to": u".", u"verify": u"true", u"repair": u"true", u"output": u"JSON", } ) check_repair_data(json.loads(resp.content)["post-repair-results"]["results"]) # start a "deep check and repair" resp = requests.post( dircap_url, params={ u"t": u"start-deep-check", u"return_to": u".", u"verify": u"on", u"repair": u"on", u"output": u"JSON", u"ophandle": u"deadbeef", } ) deepcheck_uri = resp.url data = json.loads(resp.content) tries = 10 while not data['finished'] and tries > 0: tries -= 1 time.sleep(0.5) print("deep-check not finished, reloading") resp = requests.get(deepcheck_uri, params={u"output": "JSON"}) data = json.loads(resp.content) print("deep-check finished") assert data[u"stats"][u"count-immutable-files"] == 1 assert data[u"stats"][u"count-literal-files"] == 0 assert data[u"stats"][u"largest-immutable-file"] == 778240 assert data[u"count-objects-checked"] == 2 # also get the HTML version resp = requests.post( dircap_url, params={ u"t": u"start-deep-check", u"return_to": u".", u"verify": u"on", u"repair": u"on", u"ophandle": u"definitely_random", } ) deepcheck_uri = resp.url # if the operations isn't done, there's an <H2> tag with the # reload link; otherwise there's only an <H1> tag..wait up to 5 # seconds for this to respond properly. for _ in range(5): resp = requests.get(deepcheck_uri) dom = BeautifulSoup(resp.content, "html5lib") if dom.h1 and u'Results' in str(dom.h1.string): break if dom.h2 and dom.h2.a and u"Reload" in str(dom.h2.a.string): dom = None time.sleep(1) assert dom is not None, "Operation never completed"
def get_source_info(self, source_spec): """ This turns an argv string into a (Local|Tahoe)(File|Directory)Source. """ precondition(isinstance(source_spec, str), source_spec) rootcap, path_utf8 = get_alias(self.aliases, source_spec, None) path = path_utf8.decode("utf-8") # any trailing slash is removed in abspath_expanduser_unicode(), so # make a note of it here, to throw an error later had_trailing_slash = path.endswith("/") if rootcap == DefaultAliasMarker: # no alias, so this is a local file pathname = abspath_expanduser_unicode(path) name = os.path.basename(pathname) if not os.path.exists(pathname): raise MissingSourceError(source_spec, quotefn=quote_local_unicode_path) if os.path.isdir(pathname): t = LocalDirectorySource(self.progress, pathname, name) else: if had_trailing_slash: raise FilenameWithTrailingSlashError( source_spec, quotefn=quote_local_unicode_path) if not os.path.isfile(pathname): raise WeirdSourceError(pathname) t = LocalFileSource(pathname, name) # non-empty else: # this is a tahoe object url = self.nodeurl + "uri/%s" % url_quote(rootcap) name = None if path: if path.endswith("/"): path = path[:-1] url += "/" + escape_path(path) last_slash = path.rfind(u"/") name = path if last_slash != -1: name = path[last_slash + 1:] resp = do_http("GET", url + "?t=json") if resp.status == 404: raise MissingSourceError(source_spec) elif resp.status != 200: raise HTTPError( "Error examining source %s" % quote_output(source_spec), resp) parsed = json.loads(resp.read()) nodetype, d = parsed if nodetype == "dirnode": t = TahoeDirectorySource(self.nodeurl, self.cache, self.progress, name) t.init_from_parsed(parsed) else: if had_trailing_slash: raise FilenameWithTrailingSlashError(source_spec) writecap = to_bytes(d.get("rw_uri")) readcap = to_bytes(d.get("ro_uri")) mutable = d.get("mutable", False) # older nodes don't provide it t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap, name) return t
def test_status(alice): """ confirm we get something sensible from /status and the various sub-types """ # upload a file # (because of the nature of the integration-tests, we can only # assert things about "our" file because we don't know what other # operations may have happened in the grid before our test runs). FILE_CONTENTS = u"all the Important Data of alice\n" * 1200 resp = requests.put( util.node_url(alice.node_dir, u"uri"), data=FILE_CONTENTS, ) cap = resp.text.strip() print("Uploaded data, cap={}".format(cap)) resp = requests.get( util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap))), ) print("Downloaded {} bytes of data".format(len(resp.content))) assert str(resp.content, "ascii") == FILE_CONTENTS resp = requests.get( util.node_url(alice.node_dir, "status"), ) dom = html5lib.parse(resp.content) hrefs = [ a.get('href') for a in dom.iter(u'{http://www.w3.org/1999/xhtml}a') ] found_upload = False found_download = False for href in hrefs: if href == u"/" or not href: continue resp = requests.get(util.node_url(alice.node_dir, href)) if href.startswith(u"/status/up"): assert b"File Upload Status" in resp.content if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content: found_upload = True elif href.startswith(u"/status/down"): assert b"File Download Status" in resp.content if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content: found_download = True # download the specialized event information resp = requests.get( util.node_url(alice.node_dir, u"{}/event_json".format(href)), ) js = json.loads(resp.content) # there's usually just one "read" operation, but this can handle many .. total_bytes = sum([st['bytes_returned'] for st in js['read']], 0) assert total_bytes == len(FILE_CONTENTS) assert found_upload, "Failed to find the file we uploaded in the status-page" assert found_download, "Failed to find the file we downloaded in the status-page"