def test_status(alice): """ confirm we get something sensible from /status and the various sub-types """ # upload a file # (because of the nature of the integration-tests, we can only # assert things about "our" file because we don't know what other # operations may have happened in the grid before our test runs). FILE_CONTENTS = u"all the Important Data of alice\n" * 1200 resp = requests.put( util.node_url(alice.node_dir, u"uri"), data=FILE_CONTENTS, ) cap = resp.text.strip() print("Uploaded data, cap={}".format(cap)) resp = requests.get( util.node_url(alice.node_dir, u"uri/{}".format(urllib2.quote(cap))), ) print("Downloaded {} bytes of data".format(len(resp.content))) assert resp.content == FILE_CONTENTS resp = requests.get(util.node_url(alice.node_dir, "status"), ) dom = html5lib.parse(resp.content) hrefs = [ a.get('href') for a in dom.iter(u'{http://www.w3.org/1999/xhtml}a') ] found_upload = False found_download = False for href in hrefs: if href.startswith(u"/") or not href: continue resp = requests.get( util.node_url(alice.node_dir, u"status/{}".format(href)), ) if href.startswith(u'up'): assert "File Upload Status" in resp.content if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content: found_upload = True elif href.startswith(u'down'): assert "File Download Status" in resp.content if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content: found_download = True # download the specialized event information resp = requests.get( util.node_url(alice.node_dir, u"status/{}/event_json".format(href)), ) js = json.loads(resp.content) # there's usually just one "read" operation, but this can handle many .. total_bytes = sum([st['bytes_returned'] for st in js['read']], 0) assert total_bytes == len(FILE_CONTENTS) assert found_upload, "Failed to find the file we uploaded in the status-page" assert found_download, "Failed to find the file we downloaded in the status-page"
def test_introducer_info(introducer): """ retrieve and confirm /introducer URI for the introducer """ resp = requests.get(util.node_url(introducer.node_dir, u""), ) assert "Introducer" in resp.content resp = requests.get( util.node_url(introducer.node_dir, u""), params={u"t": u"json"}, ) data = json.loads(resp.content) assert "announcement_summary" in data assert "subscription_summary" in data
def test_storage_info(storage_nodes): """ retrieve and confirm /storage URI for one storage node """ storage0 = storage_nodes[0] requests.get(util.node_url(storage0.node_dir, u"storage"), )
def test_helper_status(storage_nodes): """ successfully GET the /helper_status page """ url = util.node_url(storage_nodes[0].node_dir, "helper_status") resp = requests.get(url) assert resp.status_code >= 200 and resp.status_code < 300 dom = BeautifulSoup(resp.content, "html5lib") assert unicode(dom.h1.string) == u"Helper Status"
def test_storage_info_json(storage_nodes): """ retrieve and confirm /storage?t=json URI for one storage node """ storage0 = storage_nodes[0] resp = requests.get( util.node_url(storage0.node_dir, u"storage"), params={u"t": u"json"}, ) data = json.loads(resp.content) assert data[u"stats"][u"storage_server.reserved_space"] == 1000000000
def test_put(alice): """ use PUT to create a file """ FILE_CONTENTS = b"added via PUT" * 20 resp = requests.put( util.node_url(alice.node_dir, u"uri"), data=FILE_CONTENTS, ) cap = allmydata.uri.from_string(resp.text.strip().encode('ascii')) cfg = alice.get_config() assert isinstance(cap, allmydata.uri.CHKFileURI) assert cap.size == len(FILE_CONTENTS) assert cap.total_shares == int(cfg.get_config("client", "shares.total")) assert cap.needed_shares == int(cfg.get_config("client", "shares.needed"))
def test_mkdir_with_children(alice): """ create a directory using ?t=mkdir-with-children """ # create a file to put in our directory FILE_CONTENTS = u"some file contents\n" * 500 resp = requests.put( util.node_url(alice.node_dir, u"uri"), data=FILE_CONTENTS, ) filecap = resp.content.strip() # create a (sub) directory to put in our directory resp = requests.post(util.node_url(alice.node_dir, u"uri"), params={ u"t": u"mkdir", }) # (we need both the read-write and read-only URIs I guess) dircap = resp.content dircap_obj = allmydata.uri.from_string(dircap) dircap_ro = dircap_obj.get_readonly().to_string() # create json information about our directory meta = { "a_file": [ "filenode", { "ro_uri": filecap, "metadata": { "ctime": 1202777696.7564139, "mtime": 1202777696.7564139, "tahoe": { "linkcrtime": 1202777696.7564139, "linkmotime": 1202777696.7564139 } } } ], "some_subdir": [ "dirnode", { "rw_uri": dircap, "ro_uri": dircap_ro, "metadata": { "ctime": 1202778102.7589991, "mtime": 1202778111.2160511, "tahoe": { "linkcrtime": 1202777696.7564139, "linkmotime": 1202777696.7564139 } } } ] } # create a new directory with one file and one sub-dir (all-at-once) resp = util.web_post( alice, u"uri", params={u"t": "mkdir-with-children"}, data=json.dumps(meta), ) assert resp.startswith("URI:DIR2") cap = allmydata.uri.from_string(resp) assert isinstance(cap, allmydata.uri.DirectoryURI)
def test_directory_deep_check(alice): """ use deep-check and confirm the result pages work """ # create a directory resp = requests.post(util.node_url(alice.node_dir, u"uri"), params={ u"t": u"mkdir", u"redirect_to_result": u"true", }) # get json information about our directory dircap_url = resp.url resp = requests.get( dircap_url, params={u"t": u"json"}, ) dir_meta = json.loads(resp.content) # upload a file of pangrams into the directory FILE_CONTENTS = u"Sphinx of black quartz, judge my vow.\n" * (2048 * 10) resp = requests.post(dircap_url, params={ u"t": u"upload", u"upload-chk": u"upload-chk", }, files={ u"file": FILE_CONTENTS, }) cap0 = resp.content print("Uploaded data0, cap={}".format(cap0)) # a different pangram FILE_CONTENTS = u"The five boxing wizards jump quickly.\n" * (2048 * 10) resp = requests.post(dircap_url, params={ u"t": u"upload", u"upload-chk": u"upload-chk", }, files={ u"file": FILE_CONTENTS, }) cap1 = resp.content print("Uploaded data1, cap={}".format(cap1)) resp = requests.get( util.node_url(alice.node_dir, u"uri/{}".format(urllib2.quote(cap0))), params={u"t": u"info"}, ) def check_repair_data(checkdata): assert checkdata["healthy"] is True assert checkdata["count-happiness"] == 4 assert checkdata["count-good-share-hosts"] == 4 assert checkdata["count-shares-good"] == 4 assert checkdata["count-corrupt-shares"] == 0 assert checkdata["list-corrupt-shares"] == [] # do a "check" (once for HTML, then with JSON for easier asserts) resp = requests.post(dircap_url, params={ u"t": u"check", u"return_to": u".", u"verify": u"true", }) resp = requests.post(dircap_url, params={ u"t": u"check", u"return_to": u".", u"verify": u"true", u"output": u"JSON", }) check_repair_data(json.loads(resp.content)["results"]) # "check and repair" resp = requests.post(dircap_url, params={ u"t": u"check", u"return_to": u".", u"verify": u"true", u"repair": u"true", }) resp = requests.post(dircap_url, params={ u"t": u"check", u"return_to": u".", u"verify": u"true", u"repair": u"true", u"output": u"JSON", }) check_repair_data( json.loads(resp.content)["post-repair-results"]["results"]) # start a "deep check and repair" resp = requests.post(dircap_url, params={ u"t": u"start-deep-check", u"return_to": u".", u"verify": u"on", u"repair": u"on", u"output": u"JSON", u"ophandle": u"deadbeef", }) deepcheck_uri = resp.url data = json.loads(resp.content) tries = 10 while not data['finished'] and tries > 0: tries -= 1 time.sleep(0.5) print("deep-check not finished, reloading") resp = requests.get(deepcheck_uri, params={u"output": "JSON"}) data = json.loads(resp.content) print("deep-check finished") assert data[u"stats"][u"count-immutable-files"] == 1 assert data[u"stats"][u"count-literal-files"] == 0 assert data[u"stats"][u"largest-immutable-file"] == 778240 assert data[u"count-objects-checked"] == 2 # also get the HTML version resp = requests.post(dircap_url, params={ u"t": u"start-deep-check", u"return_to": u".", u"verify": u"on", u"repair": u"on", u"ophandle": u"definitely_random", }) deepcheck_uri = resp.url # if the operations isn't done, there's an <H2> tag with the # reload link; otherwise there's only an <H1> tag..wait up to 5 # seconds for this to respond properly. for _ in range(5): resp = requests.get(deepcheck_uri) dom = BeautifulSoup(resp.content, "html5lib") if dom.h1 and u'Results' in unicode(dom.h1.string): break if dom.h2 and dom.h2.a and u"Reload" in unicode(dom.h2.a.string): dom = None time.sleep(1) assert dom is not None, "Operation never completed"
def test_deep_stats(alice): """ create a directory, do deep-stats on it and prove the /operations/ URIs work """ resp = requests.post( util.node_url(alice.node_dir, "uri"), params={ "format": "sdmf", "t": "mkdir", "redirect_to_result": "true", }, ) assert resp.status_code >= 200 and resp.status_code < 300 # when creating a directory, we'll be re-directed to a URL # containing our writecap.. uri = urllib2.unquote(resp.url) assert 'URI:DIR2:' in uri dircap = uri[uri.find("URI:DIR2:"):].rstrip('/') dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(urllib2.quote(dircap))) # POST a file into this directory FILE_CONTENTS = u"a file in a directory" resp = requests.post( dircap_uri, data={ u"t": u"upload", u"when_done": u".", }, files={ u"file": FILE_CONTENTS, }, ) # confirm the file is in the directory resp = requests.get( dircap_uri, params={ u"t": u"json", }, ) d = json.loads(resp.content) k, data = d assert k == u"dirnode" assert len(data['children']) == 1 k, child = data['children'].values()[0] assert k == u"filenode" assert child['size'] == len(FILE_CONTENTS) # perform deep-stats on it... resp = requests.post( dircap_uri, data={ u"t": u"start-deep-stats", u"ophandle": u"something_random", }, ) assert resp.status_code >= 200 and resp.status_code < 300 # confirm we get information from the op .. after its done tries = 10 while tries > 0: tries -= 1 resp = requests.get( util.node_url(alice.node_dir, u"operations/something_random"), ) d = json.loads(resp.content) if d['size-literal-files'] == len(FILE_CONTENTS): print("stats completed successfully") break else: print("{} != {}; waiting".format(d['size-literal-files'], len(FILE_CONTENTS))) time.sleep(.5)