def populate(self, recurse): if self.children is not None: return self.children = {} for i, (name, data) in enumerate(self.children_d.items()): self.progressfunc("examining %d of %d" % (i, len(self.children_d))) if data[0] == "filenode": mutable = data[1].get("mutable", False) writecap = to_str(data[1].get("rw_uri")) readcap = to_str(data[1].get("ro_uri")) self.children[name] = TahoeFileSource(self.nodeurl, mutable, writecap, readcap) elif data[0] == "dirnode": writecap = to_str(data[1].get("rw_uri")) readcap = to_str(data[1].get("ro_uri")) if writecap and writecap in self.cache: child = self.cache[writecap] elif readcap and readcap in self.cache: child = self.cache[readcap] else: child = TahoeDirectorySource(self.nodeurl, self.cache, self.progressfunc) child.init_from_grid(writecap, readcap) if writecap: self.cache[writecap] = child if readcap: self.cache[readcap] = child if recurse: child.populate(True) self.children[name] = child else: # TODO: there should be an option to skip unknown nodes. raise TahoeError("Cannot copy unknown nodes (ticket #839). " "You probably need to use a later version of " "Tahoe-LAFS to copy this directory.")
def init_from_parsed(self, parsed): nodetype, d = parsed self.writecap = to_str(d.get("rw_uri")) self.readcap = to_str(d.get("ro_uri")) self.mutable = d.get("mutable", False) # older nodes don't provide it self.children_d = dict([(unicode(name), value) for (name, value) in d["children"].iteritems()]) self.children = None
def populate(self, recurse): if self.children is not None: return self.children = {} for i,(name, data) in enumerate(self.children_d.items()): self.progressfunc("examining %d of %d" % (i+1, len(self.children_d))) if data[0] == "filenode": mutable = data[1].get("mutable", False) writecap = to_str(data[1].get("rw_uri")) readcap = to_str(data[1].get("ro_uri")) self.children[name] = TahoeFileSource(self.nodeurl, mutable, writecap, readcap, name) elif data[0] == "dirnode": writecap = to_str(data[1].get("rw_uri")) readcap = to_str(data[1].get("ro_uri")) if writecap and writecap in self.cache: child = self.cache[writecap] elif readcap and readcap in self.cache: child = self.cache[readcap] else: child = TahoeDirectorySource(self.nodeurl, self.cache, self.progressfunc, name) child.init_from_grid(writecap, readcap) if writecap: self.cache[writecap] = child if readcap: self.cache[readcap] = child if recurse: child.populate(recurse=True) self.children[name] = child else: # TODO: there should be an option to skip unknown nodes. raise TahoeError("Cannot copy unknown nodes (ticket #839). " "You probably need to use a later version of " "Tahoe-LAFS to copy this directory.")
def test_to_str(self): self.failUnlessReallyEqual(to_str("foo"), "foo") self.failUnlessReallyEqual(to_str("lumi\xc3\xa8re"), "lumi\xc3\xa8re") self.failUnlessReallyEqual( to_str("\xFF"), "\xFF") # passes through invalid UTF-8 -- is this what we want? self.failUnlessReallyEqual(to_str(u"lumi\u00E8re"), "lumi\xc3\xa8re") self.failUnlessReallyEqual(to_str(None), None)
def get_source_info(self, source_spec): """ This turns an argv string into a (Local|Tahoe)(File|Directory)Source. """ precondition(isinstance(source_spec, unicode), source_spec) rootcap, path_utf8 = get_alias(self.aliases, source_spec, None) path = path_utf8.decode("utf-8") if rootcap == DefaultAliasMarker: # no alias, so this is a local file pathname = abspath_expanduser_unicode(path) name = os.path.basename(pathname) if not os.path.exists(pathname): raise MissingSourceError(source_spec, quotefn=quote_local_unicode_path) if os.path.isdir(pathname): t = LocalDirectorySource(self.progress, pathname, name) else: assert os.path.isfile(pathname) t = LocalFileSource(pathname, name) # non-empty else: # this is a tahoe object url = self.nodeurl + "uri/%s" % urllib.quote(rootcap) name = None if path: url += "/" + escape_path(path) last_slash = path.rfind(u"/") name = path if last_slash != -1: name = path[last_slash + 1:] resp = do_http("GET", url + "?t=json") if resp.status == 404: raise MissingSourceError(source_spec) elif resp.status != 200: raise HTTPError( "Error examining source %s" % quote_output(source_spec), resp) parsed = simplejson.loads(resp.read()) nodetype, d = parsed if nodetype == "dirnode": t = TahoeDirectorySource(self.nodeurl, self.cache, self.progress, name) t.init_from_parsed(parsed) else: writecap = to_str(d.get("rw_uri")) readcap = to_str(d.get("ro_uri")) mutable = d.get("mutable", False) # older nodes don't provide it last_slash = source_spec.rfind(u"/") if last_slash != -1: # TODO: this looks funny and redundant with the 'name' # assignment above. cf #2329 name = source_spec[last_slash + 1:] t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap, name) return t
def get_source_info(self, source_spec): """ This turns an argv string into a (Local|Tahoe)(File|Directory)Source. """ precondition(isinstance(source_spec, unicode), source_spec) rootcap, path_utf8 = get_alias(self.aliases, source_spec, None) path = path_utf8.decode("utf-8") # any trailing slash is removed in abspath_expanduser_unicode(), so # make a note of it here, to throw an error later had_trailing_slash = path.endswith("/") if rootcap == DefaultAliasMarker: # no alias, so this is a local file pathname = abspath_expanduser_unicode(path) name = os.path.basename(pathname) if not os.path.exists(pathname): raise MissingSourceError(source_spec, quotefn=quote_local_unicode_path) if os.path.isdir(pathname): t = LocalDirectorySource(self.progress, pathname, name) else: if had_trailing_slash: raise FilenameWithTrailingSlashError(source_spec, quotefn=quote_local_unicode_path) if not os.path.isfile(pathname): raise WeirdSourceError(pathname) t = LocalFileSource(pathname, name) # non-empty else: # this is a tahoe object url = self.nodeurl + "uri/%s" % urllib.quote(rootcap) name = None if path: if path.endswith("/"): path = path[:-1] url += "/" + escape_path(path) last_slash = path.rfind(u"/") name = path if last_slash != -1: name = path[last_slash+1:] resp = do_http("GET", url + "?t=json") if resp.status == 404: raise MissingSourceError(source_spec) elif resp.status != 200: raise HTTPError("Error examining source %s" % quote_output(source_spec), resp) parsed = json.loads(resp.read()) nodetype, d = parsed if nodetype == "dirnode": t = TahoeDirectorySource(self.nodeurl, self.cache, self.progress, name) t.init_from_parsed(parsed) else: if had_trailing_slash: raise FilenameWithTrailingSlashError(source_spec) writecap = to_str(d.get("rw_uri")) readcap = to_str(d.get("ro_uri")) mutable = d.get("mutable", False) # older nodes don't provide it t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap, name) return t
def get_source_info(self, source_spec): """ This turns an argv string into a (Local|Tahoe)(File|Directory)Source. """ precondition(isinstance(source_spec, unicode), source_spec) rootcap, path_utf8 = get_alias(self.aliases, source_spec, None) path = path_utf8.decode("utf-8") # any trailing slash is removed in abspath_expanduser_unicode(), so # make a note of it here, to throw an error later had_trailing_slash = path.endswith("/") if rootcap == DefaultAliasMarker: # no alias, so this is a local file pathname = abspath_expanduser_unicode(path) name = os.path.basename(pathname) if not os.path.exists(pathname): raise MissingSourceError(source_spec, quotefn=quote_local_unicode_path) if os.path.isdir(pathname): t = LocalDirectorySource(self.progress, pathname, name) else: if had_trailing_slash: raise FilenameWithTrailingSlashError(source_spec, quotefn=quote_local_unicode_path) if not os.path.isfile(pathname): raise WeirdSourceError(pathname) t = LocalFileSource(pathname, name) # non-empty else: # this is a tahoe object url = self.nodeurl + "uri/%s" % urllib.quote(rootcap) name = None if path: if path.endswith("/"): path = path[:-1] url += "/" + escape_path(path) last_slash = path.rfind(u"/") name = path if last_slash != -1: name = path[last_slash+1:] resp = do_http("GET", url + "?t=json") if resp.status == 404: raise MissingSourceError(source_spec) elif resp.status != 200: raise HTTPError("Error examining source %s" % quote_output(source_spec), resp) parsed = simplejson.loads(resp.read()) nodetype, d = parsed if nodetype == "dirnode": t = TahoeDirectorySource(self.nodeurl, self.cache, self.progress, name) t.init_from_parsed(parsed) else: if had_trailing_slash: raise FilenameWithTrailingSlashError(source_spec) writecap = to_str(d.get("rw_uri")) readcap = to_str(d.get("ro_uri")) mutable = d.get("mutable", False) # older nodes don't provide it t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap, name) return t
def init_from_parsed(self, parsed): nodetype, d = parsed self.writecap = to_str(d.get("rw_uri")) self.readcap = to_str(d.get("ro_uri")) self.mutable = d.get("mutable", False) # older nodes don't provide it self.children_d = dict([(unicode(name), value) for (name, value) in d["children"].iteritems() ]) self.children = None
def get_source_info(self, source_spec): """ This turns an argv string into a (Local|Tahoe)(File|Directory)Source. """ precondition(isinstance(source_spec, unicode), source_spec) rootcap, path_utf8 = get_alias(self.aliases, source_spec, None) path = path_utf8.decode("utf-8") if rootcap == DefaultAliasMarker: # no alias, so this is a local file pathname = abspath_expanduser_unicode(path) name = os.path.basename(pathname) if not os.path.exists(pathname): raise MissingSourceError(source_spec, quotefn=quote_local_unicode_path) if os.path.isdir(pathname): t = LocalDirectorySource(self.progress, pathname, name) else: assert os.path.isfile(pathname) t = LocalFileSource(pathname, name) # non-empty else: # this is a tahoe object url = self.nodeurl + "uri/%s" % urllib.quote(rootcap) name = None if path: url += "/" + escape_path(path) last_slash = path.rfind(u"/") name = path if last_slash != -1: name = path[last_slash+1:] resp = do_http("GET", url + "?t=json") if resp.status == 404: raise MissingSourceError(source_spec) elif resp.status != 200: raise HTTPError("Error examining source %s" % quote_output(source_spec), resp) parsed = simplejson.loads(resp.read()) nodetype, d = parsed if nodetype == "dirnode": t = TahoeDirectorySource(self.nodeurl, self.cache, self.progress, name) t.init_from_parsed(parsed) else: writecap = to_str(d.get("rw_uri")) readcap = to_str(d.get("ro_uri")) mutable = d.get("mutable", False) # older nodes don't provide it last_slash = source_spec.rfind(u"/") if last_slash != -1: # TODO: this looks funny and redundant with the 'name' # assignment above. cf #2329 name = source_spec[last_slash+1:] t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap, name) return t
def _get_test_txt_uris((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = json.loads(out) self.failUnlessEqual(filetype, "filenode") self.failUnless(data['mutable']) self.failUnlessIn("rw_uri", data) self.rw_uri = to_str(data["rw_uri"]) self.failUnlessIn("ro_uri", data) self.ro_uri = to_str(data["ro_uri"])
def _check_json((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = simplejson.loads(out) self.failUnlessEqual(filetype, "filenode") self.failUnless(data['mutable']) self.failUnlessIn("ro_uri", data) self.failUnlessEqual(to_str(data["ro_uri"]), self.ro_uri) self.failUnlessIn("rw_uri", data) self.failUnlessEqual(to_str(data["rw_uri"]), self.rw_uri)
def _get_test_txt_uris((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = simplejson.loads(out) self.failUnlessEqual(filetype, "filenode") self.failUnless(data['mutable']) self.failUnlessIn("rw_uri", data) self.rw_uri = to_str(data["rw_uri"]) self.failUnlessIn("ro_uri", data) self.ro_uri = to_str(data["ro_uri"])
def _check_json((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = json.loads(out) self.failUnlessEqual(filetype, "filenode") self.failUnless(data['mutable']) self.failUnlessIn("ro_uri", data) self.failUnlessEqual(to_str(data["ro_uri"]), self.ro_uri) self.failUnlessIn("rw_uri", data) self.failUnlessEqual(to_str(data["rw_uri"]), self.rw_uri)
def _process_file_json(args, fn): (rc, out, err) = args self.failUnlessEqual(rc, 0) filetype, data = json.loads(out) self.failUnlessEqual(filetype, "filenode") if "mutable" in fn: self.failUnless(data['mutable']) self.failUnlessIn("rw_uri", data) self.failUnlessEqual(to_str(data["rw_uri"]), self.childuris[fn]) else: self.failIf(data['mutable']) self.failUnlessIn("ro_uri", data) self.failIfEqual(to_str(data["ro_uri"]), self.childuris[fn])
def _check2(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) data = json.loads(out) self.failUnlessReallyEqual(to_str(data["summary"]), "Healthy") self.failUnlessReallyEqual(data["results"]["healthy"], True)
def _check2(args): (rc, out, err) = args self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) data = json.loads(out) self.failUnlessReallyEqual(to_str(data["summary"]), "Healthy") self.failUnlessReallyEqual(data["results"]["healthy"], True)
def _process_directory_json(args): (rc, out, err) = args self.failUnlessEqual(rc, 0) filetype, data = json.loads(out) self.failUnlessEqual(filetype, "dirnode") self.failUnless(data['mutable']) self.failUnlessIn("children", data) children = data['children'] # Store the URIs for later use. self.childuris = {} for k in ["mutable1", "mutable2", "imm1", "imm2"]: self.failUnlessIn(k, children) childtype, childdata = children[k] self.failUnlessEqual(childtype, "filenode") if "mutable" in k: self.failUnless(childdata['mutable']) self.failUnlessIn("rw_uri", childdata) uri_key = "rw_uri" else: self.failIf(childdata['mutable']) self.failUnlessIn("ro_uri", childdata) uri_key = "ro_uri" self.childuris[k] = to_str(childdata[uri_key])
def _process_imm2_json((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = simplejson.loads(out) self.failUnlessEqual(filetype, "filenode") self.failIf(data['mutable']) self.failUnlessIn("ro_uri", data) self.failUnlessEqual(to_str(data["ro_uri"]), self.childuris["imm2"])
def _process_imm2_json((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = json.loads(out) self.failUnlessEqual(filetype, "filenode") self.failIf(data['mutable']) self.failUnlessIn("ro_uri", data) self.failUnlessEqual(to_str(data["ro_uri"]), self.childuris["imm2"])
def get_source_info(self, source_spec): rootcap, path = get_alias(self.aliases, source_spec, None) if rootcap == DefaultAliasMarker: # no alias, so this is a local file pathname = abspath_expanduser_unicode(path.decode('utf-8')) name = os.path.basename(pathname) if not os.path.exists(pathname): raise MissingSourceError(source_spec) if os.path.isdir(pathname): t = LocalDirectorySource(self.progress, pathname) else: assert os.path.isfile(pathname) t = LocalFileSource(pathname) # non-empty else: # this is a tahoe object url = self.nodeurl + "uri/%s" % urllib.quote(rootcap) name = None if path: url += "/" + escape_path(path) last_slash = path.rfind("/") name = path if last_slash: name = path[last_slash + 1:] resp = do_http("GET", url + "?t=json") if resp.status == 404: raise MissingSourceError(source_spec) elif resp.status != 200: raise HTTPError( "Error examining source %s" % quote_output(source_spec), resp) parsed = simplejson.loads(resp.read()) nodetype, d = parsed if nodetype == "dirnode": t = TahoeDirectorySource(self.nodeurl, self.cache, self.progress) t.init_from_parsed(parsed) else: writecap = to_str(d.get("rw_uri")) readcap = to_str(d.get("ro_uri")) mutable = d.get("mutable", False) # older nodes don't provide it if source_spec.rfind('/') != -1: name = source_spec[source_spec.rfind('/') + 1:] t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap) return name, t
def _process_test_json((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = json.loads(out) self.failUnlessEqual(filetype, "filenode") self.failUnless(data['mutable']) self.failUnlessIn("ro_uri", data) self._test_read_uri = to_str(data["ro_uri"])
def convert_children_json(nodemaker, children_json): """I convert the JSON output of GET?t=json into the dict-of-nodes input to both dirnode.create_subdirectory() and client.create_directory(initial_children=). This is used by t=mkdir-with-children and t=mkdir-immutable""" children = {} if children_json: data = json.loads(children_json) for (namex, (ctype, propdict)) in data.iteritems(): namex = unicode(namex) writecap = to_str(propdict.get("rw_uri")) readcap = to_str(propdict.get("ro_uri")) metadata = propdict.get("metadata", {}) # name= argument is just for error reporting childnode = nodemaker.create_from_cap(writecap, readcap, name=namex) children[namex] = (childnode, metadata) return children
def _process_test_json((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = simplejson.loads(out) self.failUnlessEqual(filetype, "filenode") self.failUnless(data['mutable']) self.failUnlessIn("ro_uri", data) self._test_read_uri = to_str(data["ro_uri"])
def convert_children_json(nodemaker, children_json): """I convert the JSON output of GET?t=json into the dict-of-nodes input to both dirnode.create_subdirectory() and client.create_directory(initial_children=). This is used by t=mkdir-with-children and t=mkdir-immutable""" children = {} if children_json: data = simplejson.loads(children_json) for (namex, (ctype, propdict)) in data.iteritems(): namex = unicode(namex) writecap = to_str(propdict.get("rw_uri")) readcap = to_str(propdict.get("ro_uri")) metadata = propdict.get("metadata", {}) # name= argument is just for error reporting childnode = nodemaker.create_from_cap(writecap, readcap, name=namex) children[namex] = (childnode, metadata) return children
def get_source_info(self, source_spec): rootcap, path = get_alias(self.aliases, source_spec, None) if rootcap == DefaultAliasMarker: # no alias, so this is a local file pathname = abspath_expanduser_unicode(path.decode('utf-8')) name = os.path.basename(pathname) if not os.path.exists(pathname): raise MissingSourceError(source_spec) if os.path.isdir(pathname): t = LocalDirectorySource(self.progress, pathname) else: assert os.path.isfile(pathname) t = LocalFileSource(pathname) # non-empty else: # this is a tahoe object url = self.nodeurl + "uri/%s" % urllib.quote(rootcap) name = None if path: url += "/" + escape_path(path) last_slash = path.rfind("/") name = path if last_slash: name = path[last_slash+1:] resp = do_http("GET", url + "?t=json") if resp.status == 404: raise MissingSourceError(source_spec) elif resp.status != 200: raise HTTPError("Error examining source %s" % quote_output(source_spec), resp) parsed = simplejson.loads(resp.read()) nodetype, d = parsed if nodetype == "dirnode": t = TahoeDirectorySource(self.nodeurl, self.cache, self.progress) t.init_from_parsed(parsed) else: writecap = to_str(d.get("rw_uri")) readcap = to_str(d.get("ro_uri")) mutable = d.get("mutable", False) # older nodes don't provide it if source_spec.rfind('/') != -1: name = source_spec[source_spec.rfind('/')+1:] t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap) return name, t
def get_target_info(self, destination_spec): precondition(isinstance(destination_spec, unicode), destination_spec) rootcap, path_utf8 = get_alias(self.aliases, destination_spec, None) path = path_utf8.decode("utf-8") if rootcap == DefaultAliasMarker: # no alias, so this is a local file pathname = abspath_expanduser_unicode(path) if not os.path.exists(pathname): t = LocalMissingTarget(pathname) elif os.path.isdir(pathname): t = LocalDirectoryTarget(self.progress, pathname) else: # TODO: should this be _assert? what happens if the target is # a special file? assert os.path.isfile(pathname), pathname t = LocalFileTarget(pathname) # non-empty else: # this is a tahoe object url = self.nodeurl + "uri/%s" % urllib.quote(rootcap) if path: url += "/" + escape_path(path) resp = do_http("GET", url + "?t=json") if resp.status == 404: # doesn't exist yet t = TahoeMissingTarget(url) elif resp.status == 200: parsed = simplejson.loads(resp.read()) nodetype, d = parsed if nodetype == "dirnode": t = TahoeDirectoryTarget(self.nodeurl, self.cache, self.progress) t.init_from_parsed(parsed) else: writecap = to_str(d.get("rw_uri")) readcap = to_str(d.get("ro_uri")) mutable = d.get("mutable", False) t = TahoeFileTarget(self.nodeurl, mutable, writecap, readcap, url) else: raise HTTPError( "Error examining target %s" % quote_output(destination_spec), resp) return t
def get_target_info(self, destination_spec): precondition(isinstance(destination_spec, unicode), destination_spec) rootcap, path_utf8 = get_alias(self.aliases, destination_spec, None) path = path_utf8.decode("utf-8") if rootcap == DefaultAliasMarker: # no alias, so this is a local file pathname = abspath_expanduser_unicode(path) if not os.path.exists(pathname): t = LocalMissingTarget(pathname) elif os.path.isdir(pathname): t = LocalDirectoryTarget(self.progress, pathname) else: # TODO: should this be _assert? what happens if the target is # a special file? assert os.path.isfile(pathname), pathname t = LocalFileTarget(pathname) # non-empty else: # this is a tahoe object url = self.nodeurl + "uri/%s" % urllib.quote(rootcap) if path: url += "/" + escape_path(path) resp = do_http("GET", url + "?t=json") if resp.status == 404: # doesn't exist yet t = TahoeMissingTarget(url) elif resp.status == 200: parsed = json.loads(resp.read()) nodetype, d = parsed if nodetype == "dirnode": t = TahoeDirectoryTarget(self.nodeurl, self.cache, self.progress) t.init_from_parsed(parsed) else: writecap = to_str(d.get("rw_uri")) readcap = to_str(d.get("ro_uri")) mutable = d.get("mutable", False) t = TahoeFileTarget(self.nodeurl, mutable, writecap, readcap, url) else: raise HTTPError("Error examining target %s" % quote_output(destination_spec), resp) return t
def check_directory(self, contents): """I will tell you if a new directory needs to be created for a given set of directory contents, or if I know of an existing (immutable) directory that can be used instead. 'contents' should be a dictionary that maps from child name (a single unicode string) to immutable childcap (filecap or dircap). I return a DirectoryResult object, synchronously. If r.was_created() returns False, you should create the directory (with t=mkdir-immutable). When you are finished, call r.did_create(dircap) so I can update my database. If was_created() returns a dircap, you might be able to avoid the mkdir. Call r.should_check(), and if it says False, you can skip the mkdir and use the dircap returned by was_created(). If should_check() returns True, you should perform a check operation on the dircap returned by was_created(). If the check indicates the directory is healthy, please call r.did_check_healthy(checker_results) so I can update the database, using the de-JSONized response from the webapi t=check call for 'checker_results'. If the check indicates the directory is not healthy, please repair or re-create the directory and call r.did_create(dircap) when you're done. """ now = time.time() entries = [] for name in contents: entries.append([name.encode("utf-8"), contents[name]]) entries.sort() data = "".join([ netstring(name_utf8) + netstring(cap) for (name_utf8, cap) in entries ]) dirhash = backupdb_dirhash(data) dirhash_s = base32.b2a(dirhash) c = self.cursor c.execute( "SELECT dircap, last_checked" " FROM directories WHERE dirhash=?", (dirhash_s, )) row = c.fetchone() if not row: return DirectoryResult(self, dirhash_s, None, False) (dircap, last_checked) = row age = now - last_checked probability = ((age - self.NO_CHECK_BEFORE) / (self.ALWAYS_CHECK_AFTER - self.NO_CHECK_BEFORE)) probability = min(max(probability, 0.0), 1.0) should_check = bool(random.random() < probability) return DirectoryResult(self, dirhash_s, to_str(dircap), should_check)
def _POST_relink(self, req): charset = get_arg(req, "_charset", "utf-8") replace = parse_replace_arg(get_arg(req, "replace", "true")) from_name = get_arg(req, "from_name") if from_name is not None: from_name = from_name.strip() from_name = from_name.decode(charset) assert isinstance(from_name, unicode) else: raise WebError("from_name= is required") to_name = get_arg(req, "to_name") if to_name is not None: to_name = to_name.strip() to_name = to_name.decode(charset) assert isinstance(to_name, unicode) else: to_name = from_name # Disallow slashes in both from_name and to_name, that would only # cause confusion. if "/" in from_name: raise WebError("from_name= may not contain a slash", http.BAD_REQUEST) if "/" in to_name: raise WebError("to_name= may not contain a slash", http.BAD_REQUEST) to_dir = get_arg(req, "to_dir") if to_dir is not None and to_dir != self.node.get_write_uri(): to_dir = to_dir.strip() to_dir = to_dir.decode(charset) assert isinstance(to_dir, unicode) to_path = to_dir.split(u"/") to_root = self.client.nodemaker.create_from_cap(to_str(to_path[0])) if not IDirectoryNode.providedBy(to_root): raise WebError("to_dir is not a directory", http.BAD_REQUEST) d = to_root.get_child_at_path(to_path[1:]) else: d = defer.succeed(self.node) def _got_new_parent(new_parent): if not IDirectoryNode.providedBy(new_parent): raise WebError("to_dir is not a directory", http.BAD_REQUEST) return self.node.move_child_to(from_name, new_parent, to_name, replace) d.addCallback(_got_new_parent) d.addCallback(lambda res: "thing moved") return d
def mkdir(contents, options): kids = dict([(childname, (contents[childname][0], { "ro_uri": contents[childname][1], "metadata": contents[childname][2], })) for childname in contents]) body = json.dumps(kids).encode("utf-8") url = options['node-url'] + "uri?t=mkdir-immutable" resp = do_http("POST", url, body) if resp.status < 200 or resp.status >= 300: raise HTTPError("Error during mkdir", resp) dircap = to_str(resp.read().strip()) return dircap
def parseArgs(self, invite_code, local_dir): BasedirOptions.parseArgs(self) try: if int(self['poll-interval']) <= 0: raise ValueError("should be positive") except ValueError: raise usage.UsageError( "--poll-interval must be a positive integer" ) # Expand the path relative to the current directory of the CLI command, not the node. self.local_dir = None if local_dir is None else argv_to_abspath(local_dir, long_path=False) self.invite_code = to_str(argv_to_unicode(invite_code))
def parseArgs(self, invite_code, local_dir): BasedirOptions.parseArgs(self) try: if int(self['poll-interval']) <= 0: raise ValueError("should be positive") except ValueError: raise usage.UsageError( "--poll-interval must be a positive integer" ) # Expand the path relative to the current directory of the CLI command, not the node. self.local_dir = None if local_dir is None else argv_to_abspath(local_dir, long_path=False) self.invite_code = to_str(argv_to_unicode(invite_code))
def _POST_relink(self, req): charset = get_arg(req, "_charset", "utf-8") replace = parse_replace_arg(get_arg(req, "replace", "true")) from_name = get_arg(req, "from_name") if from_name is not None: from_name = from_name.strip() from_name = from_name.decode(charset) assert isinstance(from_name, unicode) else: raise WebError("from_name= is required") to_name = get_arg(req, "to_name") if to_name is not None: to_name = to_name.strip() to_name = to_name.decode(charset) assert isinstance(to_name, unicode) else: to_name = from_name # Disallow slashes in both from_name and to_name, that would only # cause confusion. if "/" in from_name: raise WebError("from_name= may not contain a slash", http.BAD_REQUEST) if "/" in to_name: raise WebError("to_name= may not contain a slash", http.BAD_REQUEST) to_dir = get_arg(req, "to_dir") if to_dir is not None and to_dir != self.node.get_write_uri(): to_dir = to_dir.strip() to_dir = to_dir.decode(charset) assert isinstance(to_dir, unicode) to_path = to_dir.split(u"/") to_root = self.client.nodemaker.create_from_cap(to_str(to_path[0])) if not IDirectoryNode.providedBy(to_root): raise WebError("to_dir is not a directory", http.BAD_REQUEST) d = to_root.get_child_at_path(to_path[1:]) else: d = defer.succeed(self.node) def _got_new_parent(new_parent): if not IDirectoryNode.providedBy(new_parent): raise WebError("to_dir is not a directory", http.BAD_REQUEST) return self.node.move_child_to(from_name, new_parent, to_name, replace) d.addCallback(_got_new_parent) d.addCallback(lambda res: "thing moved") return d
def check_directory(self, contents): """I will tell you if a new directory needs to be created for a given set of directory contents, or if I know of an existing (immutable) directory that can be used instead. 'contents' should be a dictionary that maps from child name (a single unicode string) to immutable childcap (filecap or dircap). I return a DirectoryResult object, synchronously. If r.was_created() returns False, you should create the directory (with t=mkdir-immutable). When you are finished, call r.did_create(dircap) so I can update my database. If was_created() returns a dircap, you might be able to avoid the mkdir. Call r.should_check(), and if it says False, you can skip the mkdir and use the dircap returned by was_created(). If should_check() returns True, you should perform a check operation on the dircap returned by was_created(). If the check indicates the directory is healthy, please call r.did_check_healthy(checker_results) so I can update the database, using the de-JSONized response from the webapi t=check call for 'checker_results'. If the check indicates the directory is not healthy, please repair or re-create the directory and call r.did_create(dircap) when you're done. """ now = time.time() entries = [] for name in contents: entries.append( [name.encode("utf-8"), contents[name]] ) entries.sort() data = "".join([netstring(name_utf8)+netstring(cap) for (name_utf8,cap) in entries]) dirhash = backupdb_dirhash(data) dirhash_s = base32.b2a(dirhash) c = self.cursor c.execute("SELECT dircap, last_checked" " FROM directories WHERE dirhash=?", (dirhash_s,)) row = c.fetchone() if not row: return DirectoryResult(self, dirhash_s, None, False) (dircap, last_checked) = row age = now - last_checked probability = ((age - self.NO_CHECK_BEFORE) / (self.ALWAYS_CHECK_AFTER - self.NO_CHECK_BEFORE)) probability = min(max(probability, 0.0), 1.0) should_check = bool(random.random() < probability) return DirectoryResult(self, dirhash_s, to_str(dircap), should_check)
def _process_tahoe_json((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = json.loads(out) self.failUnlessEqual(filetype, "dirnode") self.failUnlessIn("children", data) kiddata = data['children'] self.failUnlessIn("test_file.txt", kiddata) testtype, testdata = kiddata['test_file.txt'] self.failUnlessEqual(testtype, "filenode") self.failUnless(testdata['mutable']) self.failUnlessIn("ro_uri", testdata) self.failUnlessEqual(to_str(testdata["ro_uri"]), self._test_read_uri) self.failIfIn("rw_uri", testdata)
def _process_tahoe_json((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = simplejson.loads(out) self.failUnlessEqual(filetype, "dirnode") self.failUnlessIn("children", data) kiddata = data['children'] self.failUnlessIn("test_file.txt", kiddata) testtype, testdata = kiddata['test_file.txt'] self.failUnlessEqual(testtype, "filenode") self.failUnless(testdata['mutable']) self.failUnlessIn("ro_uri", testdata) self.failUnlessEqual(to_str(testdata["ro_uri"]), self._test_read_uri) self.failIfIn("rw_uri", testdata)
def mkdir(contents, options): kids = dict([ (childname, (contents[childname][0], {"ro_uri": contents[childname][1], "metadata": contents[childname][2], })) for childname in contents ]) body = simplejson.dumps(kids).encode("utf-8") url = options['node-url'] + "uri?t=mkdir-immutable" resp = do_http("POST", url, body) if resp.status < 200 or resp.status >= 300: raise HTTPError("Error during mkdir", resp) dircap = to_str(resp.read().strip()) return dircap
def _got_testdir_json((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = simplejson.loads(out) self.failUnlessEqual(filetype, "dirnode") self.failUnlessIn("children", data) childdata = data['children'] self.failUnlessIn("file2.txt", childdata) file2type, file2data = childdata['file2.txt'] self.failUnlessEqual(file2type, "filenode") self.failUnless(file2data['mutable']) self.failUnlessIn("ro_uri", file2data) self.failUnlessEqual(to_str(file2data["ro_uri"]), self._test_read_uri) self.failIfIn("rw_uri", file2data)
def _got_testdir_json((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = json.loads(out) self.failUnlessEqual(filetype, "dirnode") self.failUnlessIn("children", data) childdata = data['children'] self.failUnlessIn("file2.txt", childdata) file2type, file2data = childdata['file2.txt'] self.failUnlessEqual(file2type, "filenode") self.failUnless(file2data['mutable']) self.failUnlessIn("ro_uri", file2data) self.failUnlessEqual(to_str(file2data["ro_uri"]), self._test_read_uri) self.failIfIn("rw_uri", file2data)
def parseArgs(self, invite_code, local_dir): super(JoinOptions, self).parseArgs() try: if int(self['poll-interval']) <= 0: raise ValueError("should be positive") except ValueError: raise usage.UsageError( "--poll-interval must be a positive integer" ) self.local_dir = FilePath(local_dir) if not self.local_dir.exists(): raise usage.UsageError( "'{}' doesn't exist".format(local_dir) ) if not self.local_dir.isdir(): raise usage.UsageError( "'{}' isn't a directory".format(local_dir) ) self.invite_code = to_str(argv_to_unicode(invite_code))
def parseArgs(self, invite_code, local_dir): super(JoinOptions, self).parseArgs() try: if int(self['poll-interval']) <= 0: raise ValueError("should be positive") except ValueError: raise usage.UsageError( "--poll-interval must be a positive integer") # Expand the path relative to the current directory of the CLI command, not the node. self.local_dir = None if local_dir is None else argv_to_abspath( local_dir, long_path=False) self.invite_code = to_str(argv_to_unicode(invite_code)) if self['author'] is None: self['author'] = os.environ.get('USERNAME', os.environ.get('USER', None)) if self['author'] is None: raise usage.UsageError( "--author not provided and no USERNAME environment-variable" )
def test_to_str(self): self.failUnlessReallyEqual(to_str("foo"), "foo") self.failUnlessReallyEqual(to_str("lumi\xc3\xa8re"), "lumi\xc3\xa8re") self.failUnlessReallyEqual(to_str("\xFF"), "\xFF") # passes through invalid UTF-8 -- is this what we want? self.failUnlessReallyEqual(to_str(u"lumi\u00E8re"), "lumi\xc3\xa8re") self.failUnlessReallyEqual(to_str(None), None)
def dump_MDMF_share(m, length, options): from allmydata.mutable.layout import MDMFSlotReadProxy from allmydata.util import base32, hashutil from allmydata.uri import MDMFVerifierURI from allmydata.util.encodingutil import quote_output, to_str offset = m.DATA_OFFSET out = options.stdout f = open(options['filename'], "rb") storage_index = None shnum = 0 class ShareDumper(MDMFSlotReadProxy): def _read(self, readvs, force_remote=False, queue=False): data = [] for (where, length) in readvs: f.seek(offset + where) data.append(f.read(length)) return defer.succeed({shnum: data}) p = ShareDumper(None, storage_index, shnum) def extract(func): stash = [] # these methods return Deferreds, but we happen to know that they run # synchronously when not actually talking to a remote server d = func() d.addCallback(stash.append) return stash[0] verinfo = extract(p.get_verinfo) encprivkey = extract(p.get_encprivkey) signature = extract(p.get_signature) pubkey = extract(p.get_verification_key) block_hash_tree = extract(p.get_blockhashes) share_hash_chain = extract(p.get_sharehashes) f.close() (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix, offsets) = verinfo print(" MDMF contents:", file=out) print(" seqnum: %d" % seqnum, file=out) print(" root_hash: %s" % base32.b2a(root_hash), file=out) #print >>out, " IV: %s" % base32.b2a(IV) print(" required_shares: %d" % k, file=out) print(" total_shares: %d" % N, file=out) print(" segsize: %d" % segsize, file=out) print(" datalen: %d" % datalen, file=out) print(" enc_privkey: %d bytes" % len(encprivkey), file=out) print(" pubkey: %d bytes" % len(pubkey), file=out) print(" signature: %d bytes" % len(signature), file=out) share_hash_ids = ",".join( [str(hid) for hid in sorted(share_hash_chain.keys())]) print(" share_hash_chain: %s" % share_hash_ids, file=out) print(" block_hash_tree: %d nodes" % len(block_hash_tree), file=out) # the storage index isn't stored in the share itself, so we depend upon # knowing the parent directory name to get it pieces = options['filename'].split(os.sep) if len(pieces) >= 2: piece = to_str(pieces[-2]) if base32.could_be_base32_encoded(piece): storage_index = base32.a2b(piece) fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey) u = MDMFVerifierURI(storage_index, fingerprint) verify_cap = u.to_string() print(" verify-cap:", quote_output(verify_cap, quotemarks=False), file=out) if options['offsets']: # NOTE: this offset-calculation code is fragile, and needs to be # merged with MutableShareFile's internals. print(file=out) print(" Section Offsets:", file=out) def printoffset(name, value, shift=0): print("%s%.20s: %s (0x%x)" % (" " * shift, name, value, value), file=out) printoffset("first lease", m.HEADER_SIZE, 2) printoffset("share data", m.DATA_OFFSET, 2) o_seqnum = m.DATA_OFFSET + struct.calcsize(">B") printoffset("seqnum", o_seqnum, 4) o_root_hash = m.DATA_OFFSET + struct.calcsize(">BQ") printoffset("root_hash", o_root_hash, 4) for k in [ "enc_privkey", "share_hash_chain", "signature", "verification_key", "verification_key_end", "share_data", "block_hash_tree", "EOF" ]: name = { "share_data": "block data", "verification_key": "pubkey", "verification_key_end": "end of pubkey", "EOF": "end of share data" }.get(k, k) offset = m.DATA_OFFSET + offsets[k] printoffset(name, offset, 4) f = open(options['filename'], "rb") printoffset("extra leases", m._read_extra_lease_offset(f) + 4, 2) f.close() print(file=out)
def list(options): nodeurl = options['node-url'] aliases = options.aliases where = options.where stdout = options.stdout stderr = options.stderr if not nodeurl.endswith("/"): nodeurl += "/" if where.endswith("/"): where = where[:-1] try: rootcap, path = get_alias(aliases, where, DEFAULT_ALIAS) except UnknownAliasError as e: e.display(stderr) return 1 url = nodeurl + "uri/%s" % urllib.quote(rootcap) if path: # move where.endswith check here? url += "/" + escape_path(path) assert not url.endswith("/") url += "?t=json" resp = do_http("GET", url) if resp.status == 404: print("No such file or directory", file=stderr) return 2 if resp.status != 200: print(format_http_error("Error during GET", resp), file=stderr) if resp.status == 0: return 3 else: return resp.status data = resp.read() if options['json']: # The webapi server should always output printable ASCII. if is_printable_ascii(data): print(data, file=stdout) return 0 else: print("The JSON response contained unprintable characters:", file=stderr) print(quote_output(data, quotemarks=False), file=stderr) return 1 try: parsed = json.loads(data) except Exception as e: print("error: %s" % quote_output(e.args[0], quotemarks=False), file=stderr) print("Could not parse JSON response:", file=stderr) print(quote_output(data, quotemarks=False), file=stderr) return 1 nodetype, d = parsed children = {} if nodetype == "dirnode": children = d['children'] else: # paths returned from get_alias are always valid UTF-8 childname = path.split("/")[-1].decode('utf-8') children = {childname: (nodetype, d)} if "metadata" not in d: d["metadata"] = {} childnames = sorted(children.keys()) now = time.time() # we build up a series of rows, then we loop through them to compute a # maxwidth so we can format them tightly. Size, filename, and URI are the # variable-width ones. rows = [] has_unknowns = False for name in childnames: child = children[name] name = unicode(name) childtype = child[0] # See webapi.txt for a discussion of the meanings of unix local # filesystem mtime and ctime, Tahoe mtime and ctime, and Tahoe # linkmotime and linkcrtime. ctime = child[1].get("metadata", {}).get('tahoe', {}).get("linkcrtime") if not ctime: ctime = child[1]["metadata"].get("ctime") mtime = child[1].get("metadata", {}).get('tahoe', {}).get("linkmotime") if not mtime: mtime = child[1]["metadata"].get("mtime") rw_uri = to_str(child[1].get("rw_uri")) ro_uri = to_str(child[1].get("ro_uri")) if ctime: # match for formatting that GNU 'ls' does if (now - ctime) > 6*30*24*60*60: # old files fmt = "%b %d %Y" else: fmt = "%b %d %H:%M" ctime_s = time.strftime(fmt, time.localtime(ctime)) else: ctime_s = "-" if childtype == "dirnode": t0 = "d" size = "-" classify = "/" elif childtype == "filenode": t0 = "-" size = str(child[1].get("size", "?")) classify = "" if rw_uri: classify = "*" else: has_unknowns = True t0 = "?" size = "?" classify = "?" t1 = "-" if ro_uri: t1 = "r" t2 = "-" if rw_uri: t2 = "w" t3 = "-" if childtype == "dirnode": t3 = "x" uri = rw_uri or ro_uri line = [] if options["long"]: line.append(t0+t1+t2+t3) line.append(size) line.append(ctime_s) if not options["classify"]: classify = "" encoding_error = False try: line.append(unicode_to_output(name) + classify) except UnicodeEncodeError: encoding_error = True line.append(quote_output(name) + classify) if options["uri"]: line.append(uri) if options["readonly-uri"]: line.append(quote_output(ro_uri or "-", quotemarks=False)) rows.append((encoding_error, line)) max_widths = [] left_justifys = [] for (encoding_error, row) in rows: for i,cell in enumerate(row): while len(max_widths) <= i: max_widths.append(0) while len(left_justifys) <= i: left_justifys.append(False) max_widths[i] = max(max_widths[i], len(cell)) if cell.startswith("URI"): left_justifys[i] = True if len(left_justifys) == 1: left_justifys[0] = True fmt_pieces = [] for i in range(len(max_widths)): piece = "%" if left_justifys[i]: piece += "-" piece += str(max_widths[i]) piece += "s" fmt_pieces.append(piece) fmt = " ".join(fmt_pieces) rc = 0 for (encoding_error, row) in rows: if encoding_error: print((fmt % tuple(row)).rstrip(), file=stderr) rc = 1 else: print((fmt % tuple(row)).rstrip(), file=stdout) if rc == 1: print("\nThis listing included files whose names could not be converted to the terminal" \ "\noutput encoding. Their names are shown using backslash escapes and in quotes.", file=stderr) if has_unknowns: print("\nThis listing included unknown objects. Using a webapi server that supports" \ "\na later version of Tahoe may help.", file=stderr) return rc
def test_cp_replaces_mutable_file_contents(self): self.basedir = "cli/Cp/cp_replaces_mutable_file_contents" self.set_up_grid(oneshare=True) # Write a test file, which we'll copy to the grid. test_txt_path = os.path.join(self.basedir, "test.txt") test_txt_contents = "foo bar baz" f = open(test_txt_path, "w") f.write(test_txt_contents) f.close() d = self.do_cli("create-alias", "tahoe") d.addCallback(lambda ignored: self.do_cli("mkdir", "tahoe:test")) # We have to use 'tahoe put' here because 'tahoe cp' doesn't # know how to make mutable files at the destination. d.addCallback(lambda ignored: self.do_cli("put", "--mutable", test_txt_path, "tahoe:test/test.txt")) d.addCallback(lambda ignored: self.do_cli("get", "tahoe:test/test.txt")) def _check((rc, out, err)): self.failUnlessEqual(rc, 0) self.failUnlessEqual(out, test_txt_contents) d.addCallback(_check) # We'll do ls --json to get the read uri and write uri for the # file we've just uploaded. d.addCallback(lambda ignored: self.do_cli("ls", "--json", "tahoe:test/test.txt")) def _get_test_txt_uris((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = json.loads(out) self.failUnlessEqual(filetype, "filenode") self.failUnless(data['mutable']) self.failUnlessIn("rw_uri", data) self.rw_uri = to_str(data["rw_uri"]) self.failUnlessIn("ro_uri", data) self.ro_uri = to_str(data["ro_uri"]) d.addCallback(_get_test_txt_uris) # Now make a new file to copy in place of test.txt. new_txt_path = os.path.join(self.basedir, "new.txt") new_txt_contents = "baz bar foo" * 100000 f = open(new_txt_path, "w") f.write(new_txt_contents) f.close() # Copy the new file on top of the old file. d.addCallback(lambda ignored: self.do_cli("cp", new_txt_path, "tahoe:test/test.txt")) # If we get test.txt now, we should see the new data. d.addCallback(lambda ignored: self.do_cli("get", "tahoe:test/test.txt")) d.addCallback(lambda (rc, out, err): self.failUnlessEqual(out, new_txt_contents)) # If we get the json of the new file, we should see that the old # uri is there d.addCallback(lambda ignored: self.do_cli("ls", "--json", "tahoe:test/test.txt")) def _check_json((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = json.loads(out) self.failUnlessEqual(filetype, "filenode") self.failUnless(data['mutable']) self.failUnlessIn("ro_uri", data) self.failUnlessEqual(to_str(data["ro_uri"]), self.ro_uri) self.failUnlessIn("rw_uri", data) self.failUnlessEqual(to_str(data["rw_uri"]), self.rw_uri) d.addCallback(_check_json) # and, finally, doing a GET directly on one of the old uris # should give us the new contents. d.addCallback(lambda ignored: self.do_cli("get", self.rw_uri)) d.addCallback(lambda (rc, out, err): self.failUnlessEqual(out, new_txt_contents)) # Now copy the old test.txt without an explicit destination # file. tahoe cp will match it to the existing file and # overwrite it appropriately. d.addCallback(lambda ignored: self.do_cli("cp", test_txt_path, "tahoe:test")) d.addCallback(lambda ignored: self.do_cli("get", "tahoe:test/test.txt")) d.addCallback(lambda (rc, out, err): self.failUnlessEqual(out, test_txt_contents)) d.addCallback(lambda ignored: self.do_cli("ls", "--json", "tahoe:test/test.txt")) d.addCallback(_check_json) d.addCallback(lambda ignored: self.do_cli("get", self.rw_uri)) d.addCallback(lambda (rc, out, err): self.failUnlessEqual(out, test_txt_contents)) # Now we'll make a more complicated directory structure. # test2/ # test2/mutable1 # test2/mutable2 # test2/imm1 # test2/imm2 imm_test_txt_path = os.path.join(self.basedir, "imm_test.txt") imm_test_txt_contents = test_txt_contents * 10000 fileutil.write(imm_test_txt_path, imm_test_txt_contents) d.addCallback(lambda ignored: self.do_cli("mkdir", "tahoe:test2")) d.addCallback(lambda ignored: self.do_cli("put", "--mutable", new_txt_path, "tahoe:test2/mutable1")) d.addCallback(lambda ignored: self.do_cli("put", "--mutable", new_txt_path, "tahoe:test2/mutable2")) d.addCallback(lambda ignored: self.do_cli('put', new_txt_path, "tahoe:test2/imm1")) d.addCallback(lambda ignored: self.do_cli("put", imm_test_txt_path, "tahoe:test2/imm2")) d.addCallback(lambda ignored: self.do_cli("ls", "--json", "tahoe:test2")) def _process_directory_json((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = json.loads(out) self.failUnlessEqual(filetype, "dirnode") self.failUnless(data['mutable']) self.failUnlessIn("children", data) children = data['children'] # Store the URIs for later use. self.childuris = {} for k in ["mutable1", "mutable2", "imm1", "imm2"]: self.failUnlessIn(k, children) childtype, childdata = children[k] self.failUnlessEqual(childtype, "filenode") if "mutable" in k: self.failUnless(childdata['mutable']) self.failUnlessIn("rw_uri", childdata) uri_key = "rw_uri" else: self.failIf(childdata['mutable']) self.failUnlessIn("ro_uri", childdata) uri_key = "ro_uri" self.childuris[k] = to_str(childdata[uri_key]) d.addCallback(_process_directory_json) # Now build a local directory to copy into place, like the following: # test2/ # test2/mutable1 # test2/mutable2 # test2/imm1 # test2/imm3 def _build_local_directory(ignored): test2_path = os.path.join(self.basedir, "test2") fileutil.make_dirs(test2_path) for fn in ("mutable1", "mutable2", "imm1", "imm3"): fileutil.write(os.path.join(test2_path, fn), fn * 1000) self.test2_path = test2_path d.addCallback(_build_local_directory) d.addCallback(lambda ignored: self.do_cli("cp", "-r", self.test2_path, "tahoe:")) # We expect that mutable1 and mutable2 are overwritten in-place, # so they'll retain their URIs but have different content. def _process_file_json((rc, out, err), fn): self.failUnlessEqual(rc, 0) filetype, data = json.loads(out) self.failUnlessEqual(filetype, "filenode") if "mutable" in fn: self.failUnless(data['mutable']) self.failUnlessIn("rw_uri", data) self.failUnlessEqual(to_str(data["rw_uri"]), self.childuris[fn]) else: self.failIf(data['mutable']) self.failUnlessIn("ro_uri", data) self.failIfEqual(to_str(data["ro_uri"]), self.childuris[fn])
def dump_immutable_chk_share(f, out, options): from allmydata import uri from allmydata.util import base32 from allmydata.immutable.layout import ReadBucketProxy from allmydata.util.encodingutil import quote_output, to_str # use a ReadBucketProxy to parse the bucket and find the uri extension bp = ReadBucketProxy(None, None, '') offsets = bp._parse_offsets(f.read_share_data(0, 0x44)) print("%20s: %d" % ("version", bp._version), file=out) seek = offsets['uri_extension'] length = struct.unpack(bp._fieldstruct, f.read_share_data(seek, bp._fieldsize))[0] seek += bp._fieldsize UEB_data = f.read_share_data(seek, length) unpacked = uri.unpack_extension_readable(UEB_data) keys1 = ("size", "num_segments", "segment_size", "needed_shares", "total_shares") keys2 = ("codec_name", "codec_params", "tail_codec_params") keys3 = ("plaintext_hash", "plaintext_root_hash", "crypttext_hash", "crypttext_root_hash", "share_root_hash", "UEB_hash") display_keys = {"size": "file_size"} for k in keys1: if k in unpacked: dk = display_keys.get(k, k) print("%20s: %s" % (dk, unpacked[k]), file=out) print(file=out) for k in keys2: if k in unpacked: dk = display_keys.get(k, k) print("%20s: %s" % (dk, unpacked[k]), file=out) print(file=out) for k in keys3: if k in unpacked: dk = display_keys.get(k, k) print("%20s: %s" % (dk, unpacked[k]), file=out) leftover = set(unpacked.keys()) - set(keys1 + keys2 + keys3) if leftover: print(file=out) print("LEFTOVER:", file=out) for k in sorted(leftover): print("%20s: %s" % (k, unpacked[k]), file=out) # the storage index isn't stored in the share itself, so we depend upon # knowing the parent directory name to get it pieces = options['filename'].split(os.sep) if len(pieces) >= 2: piece = to_str(pieces[-2]) if base32.could_be_base32_encoded(piece): storage_index = base32.a2b(piece) uri_extension_hash = base32.a2b(unpacked["UEB_hash"]) u = uri.CHKFileVerifierURI(storage_index, uri_extension_hash, unpacked["needed_shares"], unpacked["total_shares"], unpacked["size"]) verify_cap = u.to_string() print("%20s: %s" % ("verify-cap", quote_output(verify_cap, quotemarks=False)), file=out) sizes = {} sizes['data'] = (offsets['plaintext_hash_tree'] - offsets['data']) sizes['validation'] = (offsets['uri_extension'] - offsets['plaintext_hash_tree']) sizes['uri-extension'] = len(UEB_data) print(file=out) print(" Size of data within the share:", file=out) for k in sorted(sizes): print("%20s: %s" % (k, sizes[k]), file=out) if options['offsets']: print(file=out) print(" Section Offsets:", file=out) print("%20s: %s" % ("share data", f._data_offset), file=out) for k in [ "data", "plaintext_hash_tree", "crypttext_hash_tree", "block_hashes", "share_hashes", "uri_extension" ]: name = {"data": "block data"}.get(k, k) offset = f._data_offset + offsets[k] print(" %20s: %s (0x%x)" % (name, offset, offset), file=out) print("%20s: %s" % ("leases", f._lease_offset), file=out)
print >>out, " total_shares: %d" % N print >>out, " segsize: %d" % segsize print >>out, " datalen: %d" % datalen print >>out, " enc_privkey: %d bytes" % len(enc_privkey) print >>out, " pubkey: %d bytes" % len(pubkey) print >>out, " signature: %d bytes" % len(signature) share_hash_ids = ",".join(sorted([str(hid) for hid in share_hash_chain.keys()])) print >>out, " share_hash_chain: %s" % share_hash_ids print >>out, " block_hash_tree: %d nodes" % len(block_hash_tree) # the storage index isn't stored in the share itself, so we depend upon # knowing the parent directory name to get it pieces = options['filename'].split(os.sep) if len(pieces) >= 2: piece = to_str(pieces[-2]) if base32.could_be_base32_encoded(piece): storage_index = base32.a2b(piece) fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey) u = SSKVerifierURI(storage_index, fingerprint) verify_cap = u.to_string() print >>out, " verify-cap:", quote_output(verify_cap, quotemarks=False) if options['offsets']: # NOTE: this offset-calculation code is fragile, and needs to be # merged with MutableShareFile's internals. print >>out print >>out, " Section Offsets:" def printoffset(name, value, shift=0): print >>out, "%s%20s: %s (0x%x)" % (" "*shift, name, value, value) printoffset("first lease", m.HEADER_SIZE)
def dump_immutable_chk_share(f, out, options): from allmydata import uri from allmydata.util import base32 from allmydata.immutable.layout import ReadBucketProxy from allmydata.util.encodingutil import quote_output, to_str # use a ReadBucketProxy to parse the bucket and find the uri extension bp = ReadBucketProxy(None, '', '') offsets = bp._parse_offsets(f.read_share_data(0, 0x44)) print >>out, "%20s: %d" % ("version", bp._version) seek = offsets['uri_extension'] length = struct.unpack(bp._fieldstruct, f.read_share_data(seek, bp._fieldsize))[0] seek += bp._fieldsize UEB_data = f.read_share_data(seek, length) unpacked = uri.unpack_extension_readable(UEB_data) keys1 = ("size", "num_segments", "segment_size", "needed_shares", "total_shares") keys2 = ("codec_name", "codec_params", "tail_codec_params") keys3 = ("plaintext_hash", "plaintext_root_hash", "crypttext_hash", "crypttext_root_hash", "share_root_hash", "UEB_hash") display_keys = {"size": "file_size"} for k in keys1: if k in unpacked: dk = display_keys.get(k, k) print >>out, "%20s: %s" % (dk, unpacked[k]) print >>out for k in keys2: if k in unpacked: dk = display_keys.get(k, k) print >>out, "%20s: %s" % (dk, unpacked[k]) print >>out for k in keys3: if k in unpacked: dk = display_keys.get(k, k) print >>out, "%20s: %s" % (dk, unpacked[k]) leftover = set(unpacked.keys()) - set(keys1 + keys2 + keys3) if leftover: print >>out print >>out, "LEFTOVER:" for k in sorted(leftover): print >>out, "%20s: %s" % (k, unpacked[k]) # the storage index isn't stored in the share itself, so we depend upon # knowing the parent directory name to get it pieces = options['filename'].split(os.sep) if len(pieces) >= 2: piece = to_str(pieces[-2]) if base32.could_be_base32_encoded(piece): storage_index = base32.a2b(piece) uri_extension_hash = base32.a2b(unpacked["UEB_hash"]) u = uri.CHKFileVerifierURI(storage_index, uri_extension_hash, unpacked["needed_shares"], unpacked["total_shares"], unpacked["size"]) verify_cap = u.to_string() print >>out, "%20s: %s" % ("verify-cap", quote_output(verify_cap, quotemarks=False)) sizes = {} sizes['data'] = (offsets['plaintext_hash_tree'] - offsets['data']) sizes['validation'] = (offsets['uri_extension'] - offsets['plaintext_hash_tree']) sizes['uri-extension'] = len(UEB_data) print >>out print >>out, " Size of data within the share:" for k in sorted(sizes): print >>out, "%20s: %s" % (k, sizes[k]) if options['offsets']: print >>out print >>out, " Section Offsets:" print >>out, "%20s: %s" % ("share data", f._data_offset) for k in ["data", "plaintext_hash_tree", "crypttext_hash_tree", "block_hashes", "share_hashes", "uri_extension"]: name = {"data": "block data"}.get(k,k) offset = f._data_offset + offsets[k] print >>out, " %20s: %s (0x%x)" % (name, offset, offset) print >>out, "%20s: %s" % ("leases", f._lease_offset)
def dump_MDMF_share(m, length, options): from allmydata.mutable.layout import MDMFSlotReadProxy from allmydata.util import base32, hashutil from allmydata.uri import MDMFVerifierURI from allmydata.util.encodingutil import quote_output, to_str offset = m.DATA_OFFSET out = options.stdout f = open(options['filename'], "rb") storage_index = None; shnum = 0 class ShareDumper(MDMFSlotReadProxy): def _read(self, readvs, force_remote=False, queue=False): data = [] for (where,length) in readvs: f.seek(offset+where) data.append(f.read(length)) return defer.succeed({shnum: data}) p = ShareDumper(None, storage_index, shnum) def extract(func): stash = [] # these methods return Deferreds, but we happen to know that they run # synchronously when not actually talking to a remote server d = func() d.addCallback(stash.append) return stash[0] verinfo = extract(p.get_verinfo) encprivkey = extract(p.get_encprivkey) signature = extract(p.get_signature) pubkey = extract(p.get_verification_key) block_hash_tree = extract(p.get_blockhashes) share_hash_chain = extract(p.get_sharehashes) f.close() (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix, offsets) = verinfo print >>out, " MDMF contents:" print >>out, " seqnum: %d" % seqnum print >>out, " root_hash: %s" % base32.b2a(root_hash) #print >>out, " IV: %s" % base32.b2a(IV) print >>out, " required_shares: %d" % k print >>out, " total_shares: %d" % N print >>out, " segsize: %d" % segsize print >>out, " datalen: %d" % datalen print >>out, " enc_privkey: %d bytes" % len(encprivkey) print >>out, " pubkey: %d bytes" % len(pubkey) print >>out, " signature: %d bytes" % len(signature) share_hash_ids = ",".join([str(hid) for hid in sorted(share_hash_chain.keys())]) print >>out, " share_hash_chain: %s" % share_hash_ids print >>out, " block_hash_tree: %d nodes" % len(block_hash_tree) # the storage index isn't stored in the share itself, so we depend upon # knowing the parent directory name to get it pieces = options['filename'].split(os.sep) if len(pieces) >= 2: piece = to_str(pieces[-2]) if base32.could_be_base32_encoded(piece): storage_index = base32.a2b(piece) fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey) u = MDMFVerifierURI(storage_index, fingerprint) verify_cap = u.to_string() print >>out, " verify-cap:", quote_output(verify_cap, quotemarks=False) if options['offsets']: # NOTE: this offset-calculation code is fragile, and needs to be # merged with MutableShareFile's internals. print >>out print >>out, " Section Offsets:" def printoffset(name, value, shift=0): print >>out, "%s%.20s: %s (0x%x)" % (" "*shift, name, value, value) printoffset("first lease", m.HEADER_SIZE, 2) printoffset("share data", m.DATA_OFFSET, 2) o_seqnum = m.DATA_OFFSET + struct.calcsize(">B") printoffset("seqnum", o_seqnum, 4) o_root_hash = m.DATA_OFFSET + struct.calcsize(">BQ") printoffset("root_hash", o_root_hash, 4) for k in ["enc_privkey", "share_hash_chain", "signature", "verification_key", "verification_key_end", "share_data", "block_hash_tree", "EOF"]: name = {"share_data": "block data", "verification_key": "pubkey", "verification_key_end": "end of pubkey", "EOF": "end of share data"}.get(k,k) offset = m.DATA_OFFSET + offsets[k] printoffset(name, offset, 4) f = open(options['filename'], "rb") printoffset("extra leases", m._read_extra_lease_offset(f) + 4, 2) f.close() print >>out
def parseArgs(self, invite_code, local_dir): BasedirOptions.parseArgs(self) # Expand the path relative to the current directory of the CLI command, not the node. self.local_dir = None if local_dir is None else argv_to_abspath(local_dir, long_path=False) self.invite_code = to_str(argv_to_unicode(invite_code))
stderr = options.stderr if nodeurl[-1] != "/": nodeurl += "/" try: rootcap, from_path = get_alias(aliases, from_file, DEFAULT_ALIAS) except UnknownAliasError, e: e.display(stderr) return 1 from_url = nodeurl + "uri/%s" % urllib.quote(rootcap) if from_path: from_url += "/" + escape_path(from_path) # figure out the source cap data = urllib.urlopen(from_url + "?t=json").read() nodetype, attrs = simplejson.loads(data) cap = to_str(attrs.get("rw_uri") or attrs["ro_uri"]) # now get the target try: rootcap, path = get_alias(aliases, to_file, DEFAULT_ALIAS) except UnknownAliasError, e: e.display(stderr) return 1 to_url = nodeurl + "uri/%s" % urllib.quote(rootcap) if path: to_url += "/" + escape_path(path) if to_url.endswith("/"): # "mv foo.txt bar/" == "mv foo.txt bar/foo.txt" to_url += escape_path(from_path[from_path.rfind("/")+1:])
try: rootcap, from_path = get_alias(aliases, from_file, DEFAULT_ALIAS) except UnknownAliasError, e: e.display(stderr) return 1 from_url = nodeurl + "uri/%s" % urllib.quote(rootcap) if from_path: from_url += "/" + escape_path(from_path) # figure out the source cap resp = do_http("GET", from_url + "?t=json") if not re.search(r'^2\d\d$', str(resp.status)): print >> stderr, format_http_error("Error", resp) return 1 data = resp.read() nodetype, attrs = simplejson.loads(data) cap = to_str(attrs.get("rw_uri") or attrs["ro_uri"]) # now get the target try: rootcap, path = get_alias(aliases, to_file, DEFAULT_ALIAS) except UnknownAliasError, e: e.display(stderr) return 1 to_url = nodeurl + "uri/%s" % urllib.quote(rootcap) if path: to_url += "/" + escape_path(path) if to_url.endswith("/"): # "mv foo.txt bar/" == "mv foo.txt bar/foo.txt" to_url += escape_path(from_path[from_path.rfind("/") + 1:])
def check_file(self, path, use_timestamps=True): """I will tell you if a given local file needs to be uploaded or not, by looking in a database and seeing if I have a record of this file having been uploaded earlier. I return a FileResults object, synchronously. If r.was_uploaded() returns False, you should upload the file. When you are finished uploading it, call r.did_upload(filecap), so I can update my database. If was_uploaded() returns a filecap, you might be able to avoid an upload. Call r.should_check(), and if it says False, you can skip the upload and use the filecap returned by was_uploaded(). If should_check() returns True, you should perform a filecheck on the filecap returned by was_uploaded(). If the check indicates the file is healthy, please call r.did_check_healthy(checker_results) so I can update the database, using the de-JSONized response from the webapi t=check call for 'checker_results'. If the check indicates the file is not healthy, please upload the file and call r.did_upload(filecap) when you're done. I use_timestamps=True (the default), I will compare ctime and mtime of the local file against an entry in my database, and consider the file to be unchanged if ctime, mtime, and filesize are all the same as the earlier version. If use_timestamps=False, I will not trust the timestamps, so more files (perhaps all) will be marked as needing upload. A future version of this database may hash the file to make equality decisions, in which case use_timestamps=False will not always imply r.must_upload()==True. 'path' points to a local file on disk, possibly relative to the current working directory. The database stores absolute pathnames. """ path = abspath_expanduser_unicode(path) s = os.stat(path) size = s[stat.ST_SIZE] ctime = s[stat.ST_CTIME] mtime = s[stat.ST_MTIME] now = time.time() c = self.cursor c.execute("SELECT size,mtime,ctime,fileid" " FROM local_files" " WHERE path=?", (path,)) row = self.cursor.fetchone() if not row: return FileResult(self, None, False, path, mtime, ctime, size) (last_size,last_mtime,last_ctime,last_fileid) = row c.execute("SELECT caps.filecap, last_upload.last_checked" " FROM caps,last_upload" " WHERE caps.fileid=? AND last_upload.fileid=?", (last_fileid, last_fileid)) row2 = c.fetchone() if ((last_size != size or not use_timestamps or last_mtime != mtime or last_ctime != ctime) # the file has been changed or (not row2) # we somehow forgot where we put the file last time ): c.execute("DELETE FROM local_files WHERE path=?", (path,)) self.connection.commit() return FileResult(self, None, False, path, mtime, ctime, size) # at this point, we're allowed to assume the file hasn't been changed (filecap, last_checked) = row2 age = now - last_checked probability = ((age - self.NO_CHECK_BEFORE) / (self.ALWAYS_CHECK_AFTER - self.NO_CHECK_BEFORE)) probability = min(max(probability, 0.0), 1.0) should_check = bool(random.random() < probability) return FileResult(self, to_str(filecap), should_check, path, mtime, ctime, size)
for name in childnames: child = children[name] name = unicode(name) childtype = child[0] # See webapi.txt for a discussion of the meanings of unix local # filesystem mtime and ctime, Tahoe mtime and ctime, and Tahoe # linkmotime and linkcrtime. ctime = child[1].get("metadata", {}).get('tahoe', {}).get("linkcrtime") if not ctime: ctime = child[1]["metadata"].get("ctime") mtime = child[1].get("metadata", {}).get('tahoe', {}).get("linkmotime") if not mtime: mtime = child[1]["metadata"].get("mtime") rw_uri = to_str(child[1].get("rw_uri")) ro_uri = to_str(child[1].get("ro_uri")) if ctime: # match for formatting that GNU 'ls' does if (now - ctime) > 6*30*24*60*60: # old files fmt = "%b %d %Y" else: fmt = "%b %d %H:%M" ctime_s = time.strftime(fmt, time.localtime(ctime)) else: ctime_s = "-" if childtype == "dirnode": t0 = "d" size = "-" classify = "/"
def check_file(self, path, use_timestamps=True): """I will tell you if a given local file needs to be uploaded or not, by looking in a database and seeing if I have a record of this file having been uploaded earlier. I return a FileResults object, synchronously. If r.was_uploaded() returns False, you should upload the file. When you are finished uploading it, call r.did_upload(filecap), so I can update my database. If was_uploaded() returns a filecap, you might be able to avoid an upload. Call r.should_check(), and if it says False, you can skip the upload and use the filecap returned by was_uploaded(). If should_check() returns True, you should perform a filecheck on the filecap returned by was_uploaded(). If the check indicates the file is healthy, please call r.did_check_healthy(checker_results) so I can update the database, using the de-JSONized response from the webapi t=check call for 'checker_results'. If the check indicates the file is not healthy, please upload the file and call r.did_upload(filecap) when you're done. I use_timestamps=True (the default), I will compare ctime and mtime of the local file against an entry in my database, and consider the file to be unchanged if ctime, mtime, and filesize are all the same as the earlier version. If use_timestamps=False, I will not trust the timestamps, so more files (perhaps all) will be marked as needing upload. A future version of this database may hash the file to make equality decisions, in which case use_timestamps=False will not always imply r.must_upload()==True. 'path' points to a local file on disk, possibly relative to the current working directory. The database stores absolute pathnames. """ path = abspath_expanduser_unicode(path) s = os.stat(path) size = s[stat.ST_SIZE] ctime = s[stat.ST_CTIME] mtime = s[stat.ST_MTIME] now = time.time() c = self.cursor c.execute( "SELECT size,mtime,ctime,fileid" " FROM local_files" " WHERE path=?", (path, )) row = self.cursor.fetchone() if not row: return FileResult(self, None, False, path, mtime, ctime, size) (last_size, last_mtime, last_ctime, last_fileid) = row c.execute( "SELECT caps.filecap, last_upload.last_checked" " FROM caps,last_upload" " WHERE caps.fileid=? AND last_upload.fileid=?", (last_fileid, last_fileid)) row2 = c.fetchone() if ((last_size != size or not use_timestamps or last_mtime != mtime or last_ctime != ctime) # the file has been changed or (not row2) # we somehow forgot where we put the file last time ): c.execute("DELETE FROM local_files WHERE path=?", (path, )) self.connection.commit() return FileResult(self, None, False, path, mtime, ctime, size) # at this point, we're allowed to assume the file hasn't been changed (filecap, last_checked) = row2 age = now - last_checked probability = ((age - self.NO_CHECK_BEFORE) / (self.ALWAYS_CHECK_AFTER - self.NO_CHECK_BEFORE)) probability = min(max(probability, 0.0), 1.0) should_check = bool(random.random() < probability) return FileResult(self, to_str(filecap), should_check, path, mtime, ctime, size)
def tahoe_mv(nodeurl, aliases, from_file, to_file, treq): """ :param DecodedURL nodeurl: The web end point of the Tahoe-LAFS node associated with the magic-folder client. :param [unicode] aliases: XXX :param unicode from_file: cap of the source. :param unicode to_file: cap of the destination. :param HTTPClient treq: An ``HTTPClient`` or similar object to use to make the queries. :return integer: Returns 0 for successful execution. In the case of a failure, an exception is raised. """ try: rootcap, from_path = get_alias(aliases, from_file, DEFAULT_ALIAS) except Exception as e: raise e from_url = nodeurl.child( u"uri", u"{}".format(rootcap), ) if from_path: from_url = from_url.child(u"{}".format(escape_path(from_path))) from_url = from_url.add( u"t", u"json", ) get_url = from_url.to_uri().to_text().encode("ascii") response = yield treq.get(get_url) if response.code != OK: returnValue((yield bad_response(get_url, response))) result = yield readBody(response) nodetype, attrs = json.loads(result) cap = to_str(attrs.get("rw_uri") or attrs["ro_uri"]) # now get the target try: rootcap, path = get_alias(aliases, to_file, DEFAULT_ALIAS) except Exception as e: raise e to_url = nodeurl.child(u"uri", u"{}".format(rootcap)) if path: to_url = to_url.child(u"{}".format(escape_path(path))) if path.endswith("/"): # "mv foo.txt bar/" == "mv foo.txt bar/foo.txt" to_url = to_url.child(u"{}".format( escape_path(from_path[from_path.rfind("/") + 1:]))) put_url = to_url.add( u"t", u"uri", ).add( u"replace", u"only-files", ) response = yield treq.put(put_url.to_text(), cap) if not re.search(r'^2\d\d$', str(response.code)): if response.code == CONFLICT: raise Exception("You cannot overwrite a directory with a file") else: raise Exception(format_http_error("Error", response)) returnValue(0)
def test_cp_replaces_mutable_file_contents(self): self.basedir = "cli/Cp/cp_replaces_mutable_file_contents" self.set_up_grid() # Write a test file, which we'll copy to the grid. test_txt_path = os.path.join(self.basedir, "test.txt") test_txt_contents = "foo bar baz" f = open(test_txt_path, "w") f.write(test_txt_contents) f.close() d = self.do_cli("create-alias", "tahoe") d.addCallback(lambda ignored: self.do_cli("mkdir", "tahoe:test")) # We have to use 'tahoe put' here because 'tahoe cp' doesn't # know how to make mutable files at the destination. d.addCallback(lambda ignored: self.do_cli("put", "--mutable", test_txt_path, "tahoe:test/test.txt")) d.addCallback(lambda ignored: self.do_cli("get", "tahoe:test/test.txt")) def _check((rc, out, err)): self.failUnlessEqual(rc, 0) self.failUnlessEqual(out, test_txt_contents) d.addCallback(_check) # We'll do ls --json to get the read uri and write uri for the # file we've just uploaded. d.addCallback(lambda ignored: self.do_cli("ls", "--json", "tahoe:test/test.txt")) def _get_test_txt_uris((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = simplejson.loads(out) self.failUnlessEqual(filetype, "filenode") self.failUnless(data['mutable']) self.failUnlessIn("rw_uri", data) self.rw_uri = to_str(data["rw_uri"]) self.failUnlessIn("ro_uri", data) self.ro_uri = to_str(data["ro_uri"]) d.addCallback(_get_test_txt_uris) # Now make a new file to copy in place of test.txt. new_txt_path = os.path.join(self.basedir, "new.txt") new_txt_contents = "baz bar foo" * 100000 f = open(new_txt_path, "w") f.write(new_txt_contents) f.close() # Copy the new file on top of the old file. d.addCallback(lambda ignored: self.do_cli("cp", new_txt_path, "tahoe:test/test.txt")) # If we get test.txt now, we should see the new data. d.addCallback(lambda ignored: self.do_cli("get", "tahoe:test/test.txt")) d.addCallback(lambda (rc, out, err): self.failUnlessEqual(out, new_txt_contents)) # If we get the json of the new file, we should see that the old # uri is there d.addCallback(lambda ignored: self.do_cli("ls", "--json", "tahoe:test/test.txt")) def _check_json((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = simplejson.loads(out) self.failUnlessEqual(filetype, "filenode") self.failUnless(data['mutable']) self.failUnlessIn("ro_uri", data) self.failUnlessEqual(to_str(data["ro_uri"]), self.ro_uri) self.failUnlessIn("rw_uri", data) self.failUnlessEqual(to_str(data["rw_uri"]), self.rw_uri) d.addCallback(_check_json) # and, finally, doing a GET directly on one of the old uris # should give us the new contents. d.addCallback(lambda ignored: self.do_cli("get", self.rw_uri)) d.addCallback(lambda (rc, out, err): self.failUnlessEqual(out, new_txt_contents)) # Now copy the old test.txt without an explicit destination # file. tahoe cp will match it to the existing file and # overwrite it appropriately. d.addCallback(lambda ignored: self.do_cli("cp", test_txt_path, "tahoe:test")) d.addCallback(lambda ignored: self.do_cli("get", "tahoe:test/test.txt")) d.addCallback(lambda (rc, out, err): self.failUnlessEqual(out, test_txt_contents)) d.addCallback(lambda ignored: self.do_cli("ls", "--json", "tahoe:test/test.txt")) d.addCallback(_check_json) d.addCallback(lambda ignored: self.do_cli("get", self.rw_uri)) d.addCallback(lambda (rc, out, err): self.failUnlessEqual(out, test_txt_contents)) # Now we'll make a more complicated directory structure. # test2/ # test2/mutable1 # test2/mutable2 # test2/imm1 # test2/imm2 imm_test_txt_path = os.path.join(self.basedir, "imm_test.txt") imm_test_txt_contents = test_txt_contents * 10000 fileutil.write(imm_test_txt_path, imm_test_txt_contents) d.addCallback(lambda ignored: self.do_cli("mkdir", "tahoe:test2")) d.addCallback(lambda ignored: self.do_cli("put", "--mutable", new_txt_path, "tahoe:test2/mutable1")) d.addCallback(lambda ignored: self.do_cli("put", "--mutable", new_txt_path, "tahoe:test2/mutable2")) d.addCallback(lambda ignored: self.do_cli('put', new_txt_path, "tahoe:test2/imm1")) d.addCallback(lambda ignored: self.do_cli("put", imm_test_txt_path, "tahoe:test2/imm2")) d.addCallback(lambda ignored: self.do_cli("ls", "--json", "tahoe:test2")) def _process_directory_json((rc, out, err)): self.failUnlessEqual(rc, 0) filetype, data = simplejson.loads(out) self.failUnlessEqual(filetype, "dirnode") self.failUnless(data['mutable']) self.failUnlessIn("children", data) children = data['children'] # Store the URIs for later use. self.childuris = {} for k in ["mutable1", "mutable2", "imm1", "imm2"]: self.failUnlessIn(k, children) childtype, childdata = children[k] self.failUnlessEqual(childtype, "filenode") if "mutable" in k: self.failUnless(childdata['mutable']) self.failUnlessIn("rw_uri", childdata) uri_key = "rw_uri" else: self.failIf(childdata['mutable']) self.failUnlessIn("ro_uri", childdata) uri_key = "ro_uri" self.childuris[k] = to_str(childdata[uri_key]) d.addCallback(_process_directory_json) # Now build a local directory to copy into place, like the following: # test2/ # test2/mutable1 # test2/mutable2 # test2/imm1 # test2/imm3 def _build_local_directory(ignored): test2_path = os.path.join(self.basedir, "test2") fileutil.make_dirs(test2_path) for fn in ("mutable1", "mutable2", "imm1", "imm3"): fileutil.write(os.path.join(test2_path, fn), fn * 1000) self.test2_path = test2_path d.addCallback(_build_local_directory) d.addCallback(lambda ignored: self.do_cli("cp", "-r", self.test2_path, "tahoe:")) # We expect that mutable1 and mutable2 are overwritten in-place, # so they'll retain their URIs but have different content. def _process_file_json((rc, out, err), fn): self.failUnlessEqual(rc, 0) filetype, data = simplejson.loads(out) self.failUnlessEqual(filetype, "filenode") if "mutable" in fn: self.failUnless(data['mutable']) self.failUnlessIn("rw_uri", data) self.failUnlessEqual(to_str(data["rw_uri"]), self.childuris[fn]) else: self.failIf(data['mutable']) self.failUnlessIn("ro_uri", data) self.failIfEqual(to_str(data["ro_uri"]), self.childuris[fn])