def _got(data):
if self._aborted:
raise UploadAborted()
encrypted_pieces = []
length = 0
while data:
encrypted_piece = data.pop(0)
length += len(encrypted_piece)
crypttext_segment_hasher.update(encrypted_piece)
self._crypttext_hasher.update(encrypted_piece)
encrypted_pieces.append(encrypted_piece)
precondition(length <= input_chunk_size,
"length=%d > input_chunk_size=%d" %
(length, input_chunk_size))
if allow_short:
if length < input_chunk_size:
# padding
pad_size = input_chunk_size - length
encrypted_pieces.append('\x00' * pad_size)
else:
# non-tail segments should be the full segment size
if length != input_chunk_size:
log.msg("non-tail segment should be full segment size: %d!=%d"
% (length, input_chunk_size),
level=log.BAD, umid="jNk5Yw")
precondition(length == input_chunk_size,
"length=%d != input_chunk_size=%d" %
(length, input_chunk_size))
encrypted_piece = "".join(encrypted_pieces)
return previous_chunks + [encrypted_piece]
def _decode(s):
precondition(isinstance(s, basestring), s=s)
if isinstance(s, bytes):
return s.decode(filesystem_encoding)
else:
return s
def _handle_bad_share(self, f, readers):
"""
I am called when a block or a salt fails to correctly validate, or when
the decryption or decoding operation fails for some reason. I react to
this failure by notifying the remote server of corruption, and then
removing the remote server from further activity.
"""
# these are the errors we can tolerate: by giving up on this share
# and finding others to replace it. Any other errors (i.e. coding
# bugs) are re-raised, causing the download to fail.
f.trap(DeadReferenceError, RemoteException, BadShareError)
# DeadReferenceError happens when we try to fetch data from a server
# that has gone away. RemoteException happens if the server had an
# internal error. BadShareError encompasses: (UnknownVersionError,
# LayoutInvalid, struct.error) which happen when we get obviously
# wrong data, and CorruptShareError which happens later, when we
# perform integrity checks on the data.
precondition(isinstance(readers, list), readers)
bad_shnums = [reader.shnum for reader in readers]
self.log("validation or decoding failed on share(s) %s, server(s) %s "
", segment %d: %s" % \
(bad_shnums, readers, self._current_segment, str(f)))
for reader in readers:
self._mark_bad_share(reader.server, reader.shnum, reader, f)
return None
def _write_share_data(self, f, offset, data):
length = len(data)
precondition(offset >= 0)
data_length = self._read_data_length(f)
extra_lease_offset = self._read_extra_lease_offset(f)
if offset+length >= data_length:
# They are expanding their data size.
if self.DATA_OFFSET+offset+length > extra_lease_offset:
# Their new data won't fit in the current container, so we
# have to move the leases. With luck, they're expanding it
# more than the size of the extra lease block, which will
# minimize the corrupt-the-share window
self._change_container_size(f, offset+length)
extra_lease_offset = self._read_extra_lease_offset(f)
# an interrupt here is ok.. the container has been enlarged
# but the data remains untouched
assert self.DATA_OFFSET+offset+length <= extra_lease_offset
# Their data now fits in the current container. We must write
# their new data and modify the recorded data size.
new_data_length = offset+length
self._write_data_length(f, new_data_length)
# an interrupt here will result in a corrupted share
# now all that's left to do is write out their data
f.seek(self.DATA_OFFSET+offset)
f.write(data)
return
def addDeferred(self, d):
precondition(not self.closed, "don't call addDeferred() on a closed ExpandableDeferredList")
index = len(self.resultList)
self.resultList.append(None)
d.addCallbacks(self._cbDeferred, self._ebDeferred,
callbackArgs=(index,))
return d
def create_node(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_local_unicode_path(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
defer.returnValue(-1)
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "client")
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
yield write_node_config(c, config)
write_client_config(c, config)
print >>out, "Node created in %s" % quote_local_unicode_path(basedir)
if not config.get("introducer", ""):
print >>out, " Please set [client]introducer.furl= in tahoe.cfg!"
print >>out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >>out, " Please set [node]nickname= in tahoe.cfg"
defer.returnValue(0)
def do_join(self, client_num, local_dir, invite_code):
action = start_action(
action_type=u"join-magic-folder",
client_num=client_num,
local_dir=local_dir,
invite_code=invite_code,
)
with action.context():
precondition(isinstance(local_dir, unicode), local_dir=local_dir)
precondition(isinstance(invite_code, str), invite_code=invite_code)
local_dir_arg = unicode_to_argv(local_dir)
d = DeferredContext(
self.do_cli(
"magic-folder",
"join",
invite_code,
local_dir_arg,
client_num=client_num,
)
)
def _done(args):
(rc, stdout, stderr) = args
self.failUnlessEqual(rc, 0)
self.failUnlessEqual(stdout, "")
self.failUnlessEqual(stderr, "")
return (rc, stdout, stderr)
d.addCallback(_done)
return d.addActionFinish()
def modify(self, old_contents, servermap, first_time):
children = self.node._unpack_contents(old_contents)
now = time.time()
for (namex, (child, new_metadata)) in self.entries.iteritems():
name = normalize(namex)
precondition(IFilesystemNode.providedBy(child), child)
# Strictly speaking this is redundant because we would raise the
# error again in _pack_normalized_children.
child.raise_error()
metadata = None
if name in children:
if not self.overwrite:
raise ExistingChildError("child %s already exists" % quote_output(name, encoding='utf-8'))
if self.overwrite == "only-files" and IDirectoryNode.providedBy(children[name][0]):
raise ExistingChildError("child %s already exists as a directory" % quote_output(name, encoding='utf-8'))
metadata = children[name][1].copy()
metadata = update_metadata(metadata, new_metadata, now)
if self.create_readonly_node and metadata.get('no-write', False):
child = self.create_readonly_node(child, name)
children[name] = (child, metadata)
new_contents = self.node._pack_contents(children)
return new_contents
def quote_output(s, quotemarks=True, encoding=None):
"""
Encode either a Unicode string or a UTF-8-encoded bytestring for representation
on stdout or stderr, tolerating errors. If 'quotemarks' is True, the string is
always quoted; otherwise, it is quoted only if necessary to avoid ambiguity or
control bytes in the output.
Quoting may use either single or double quotes. Within single quotes, all
characters stand for themselves, and ' will not appear. Within double quotes,
Python-compatible backslash escaping is used.
"""
precondition(isinstance(s, (str, unicode)), s)
if isinstance(s, str):
try:
s = s.decode('utf-8')
except UnicodeDecodeError:
return 'b"%s"' % (ESCAPABLE_8BIT.sub(_str_escape, s),)
if MUST_DOUBLE_QUOTE.search(s) is None:
try:
out = s.encode(encoding or io_encoding)
if quotemarks or out.startswith('"'):
return "'%s'" % (out,)
else:
return out
except (UnicodeDecodeError, UnicodeEncodeError):
pass
escaped = ESCAPABLE_UNICODE.sub(_unicode_escape, s)
return '"%s"' % (escaped.encode(encoding or io_encoding, 'backslashreplace'),)
def add_or_renew_lease(self, lease_info):
precondition(lease_info.owner_num != 0) # 0 means "no lease here"
try:
self.renew_lease(lease_info.renew_secret,
lease_info.expiration_time)
except IndexError:
self.add_lease(lease_info)
def _finished(self, ur):
assert interfaces.IUploadResults.providedBy(ur), ur
vcapstr = ur.get_verifycapstr()
precondition(isinstance(vcapstr, str), vcapstr)
v = uri.from_string(vcapstr)
f_times = self._fetcher.get_times()
hur = upload.HelperUploadResults()
hur.timings = {"cumulative_fetch": f_times["cumulative_fetch"],
"total_fetch": f_times["total"],
}
for key,val in ur.get_timings().items():
hur.timings[key] = val
hur.uri_extension_hash = v.uri_extension_hash
hur.ciphertext_fetched = self._fetcher.get_ciphertext_fetched()
hur.preexisting_shares = ur.get_preexisting_shares()
# hur.sharemap needs to be {shnum: set(serverid)}
hur.sharemap = {}
for shnum, servers in ur.get_sharemap().items():
hur.sharemap[shnum] = set([s.get_serverid() for s in servers])
# and hur.servermap needs to be {serverid: set(shnum)}
hur.servermap = {}
for server, shnums in ur.get_servermap().items():
hur.servermap[server.get_serverid()] = set(shnums)
hur.pushed_shares = ur.get_pushed_shares()
hur.file_size = ur.get_file_size()
hur.uri_extension_data = ur.get_uri_extension_data()
hur.verifycapstr = vcapstr
self._reader.close()
os.unlink(self._encoding_file)
self._finished_observers.fire(hur)
self._helper.upload_finished(self._storage_index, v.size)
del self._reader
def _got_announcement(self, key_s, ann):
if key_s is not None:
precondition(isinstance(key_s, str), key_s)
precondition(key_s.startswith("v0-"), key_s)
assert ann["service-name"] == "storage"
s = NativeStorageServer(key_s, ann, self._tub_options)
s.on_status_changed(lambda _: self._got_connection())
serverid = s.get_serverid()
old = self.servers.get(serverid)
if old:
if old.get_announcement() == ann:
return # duplicate
# replacement
del self.servers[serverid]
old.stop_connecting()
old.disownServiceParent()
# NOTE: this disownServiceParent() returns a Deferred that
# doesn't fire until Tub.stopService fires, which will wait for
# any existing connections to be shut down. This doesn't
# generally matter for normal runtime, but unit tests can run
# into DirtyReactorErrors if they don't block on these. If a test
# replaces one server with a newer version, then terminates
# before the old one has been shut down, it might get
# DirtyReactorErrors. The fix would be to gather these Deferreds
# into a structure that will block StorageFarmBroker.stopService
# until they have fired (but hopefully don't keep reference
# cycles around when they fire earlier than that, which will
# almost always be the case for normal runtime).
# now we forget about them and start using the new one
self.servers[serverid] = s
s.setServiceParent(self)
s.start_connecting(self._trigger_connections)
def create_stats_gatherer(config):
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print("The base directory %s is not empty." % quote_output(basedir), file=err)
print("To avoid clobbering anything, I am going to quit now.", file=err)
print("Please use a different directory, or empty this one.", file=err)
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "stats-gatherer")
if config["hostname"]:
portnum = iputil.allocate_tcp_port()
location = "tcp:%s:%d" % (config["hostname"], portnum)
port = "tcp:%d" % portnum
else:
location = config["location"]
port = config["port"]
fileutil.write(os.path.join(basedir, "location"), location+"\n")
fileutil.write(os.path.join(basedir, "port"), port+"\n")
return 0
def could_be_base32_encoded_l(s, lengthinbits, s5=s5, tr=string.translate, identitytranstable=identitytranstable, chars=chars):
precondition(isinstance(s, str), s)
if s == '':
return True
assert lengthinbits%5 < len(s5), lengthinbits
assert ord(s[-1]) < s5[lengthinbits%5]
return (((lengthinbits+4)/5) == len(s)) and s5[lengthinbits%5][ord(s[-1])] and not string.translate(s, identitytranstable, chars)
def add_alias(options):
nodedir = options['node-directory']
alias = options.alias
precondition(isinstance(alias, unicode), alias=alias)
cap = options.cap
stdout = options.stdout
stderr = options.stderr
if u":" in alias:
# a single trailing colon will already have been stripped if present
print >>stderr, "Alias names cannot contain colons."
return 1
if u" " in alias:
print >>stderr, "Alias names cannot contain spaces."
return 1
old_aliases = get_aliases(nodedir)
if alias in old_aliases:
print >>stderr, "Alias %s already exists!" % quote_output(alias)
return 1
aliasfile = os.path.join(nodedir, "private", "aliases")
cap = uri.from_string_dirnode(cap).to_string()
add_line_to_aliasfile(aliasfile, alias, cap)
print >>stdout, "Alias %s added" % quote_output(alias)
return 0
def put_uri_extension(self, data):
offset = self._offsets['uri_extension']
assert isinstance(data, str)
precondition(len(data) <= self._uri_extension_size_max,
len(data), self._uri_extension_size_max)
length = struct.pack(self.fieldstruct, len(data))
return self._write(offset, length+data)
def upload(self, childpath):
precondition(isinstance(childpath, unicode), childpath)
#self.verboseprint("uploading %s.." % quote_output(childpath))
metadata = get_local_metadata(childpath)
# we can use the backupdb here
must_upload, bdb_results = self.check_backupdb_file(childpath)
if must_upload:
self.verboseprint("uploading %s.." % quote_output(childpath))
infileobj = open(childpath, "rb")
url = self.options['node-url'] + "uri"
resp = do_http("PUT", url, infileobj)
if resp.status not in (200, 201):
raise HTTPError("Error during file PUT", resp)
filecap = resp.read().strip()
self.verboseprint(" %s -> %s" % (quote_output(childpath, quotemarks=False),
quote_output(filecap, quotemarks=False)))
#self.verboseprint(" metadata: %s" % (quote_output(metadata, quotemarks=False),))
if bdb_results:
bdb_results.did_upload(filecap)
self.files_uploaded += 1
return filecap, metadata
else:
self.verboseprint("skipping %s.." % quote_output(childpath))
self.files_reused += 1
return bdb_results.was_uploaded(), metadata
def process(self, localpath):
precondition(isinstance(localpath, unicode), localpath)
# returns newdircap
self.verboseprint("processing %s" % quote_output(localpath))
create_contents = {} # childname -> (type, rocap, metadata)
compare_contents = {} # childname -> rocap
try:
children = listdir_unicode(localpath)
except EnvironmentError:
self.directories_skipped += 1
self.warn("WARNING: permission denied on directory %s" % quote_output(localpath))
children = []
except FilenameEncodingError:
self.directories_skipped += 1
self.warn("WARNING: could not list directory %s due to a filename encoding error" % quote_output(localpath))
children = []
for child in self.options.filter_listdir(children):
assert isinstance(child, unicode), child
childpath = os.path.join(localpath, child)
# note: symlinks to directories are both islink() and isdir()
if os.path.isdir(childpath) and not os.path.islink(childpath):
metadata = get_local_metadata(childpath)
# recurse on the child directory
childcap = self.process(childpath)
assert isinstance(childcap, str)
create_contents[child] = ("dirnode", childcap, metadata)
compare_contents[child] = childcap
elif os.path.isfile(childpath) and not os.path.islink(childpath):
try:
childcap, metadata = self.upload(childpath)
assert isinstance(childcap, str)
create_contents[child] = ("filenode", childcap, metadata)
compare_contents[child] = childcap
except EnvironmentError:
self.files_skipped += 1
self.warn("WARNING: permission denied on file %s" % quote_output(childpath))
else:
self.files_skipped += 1
if os.path.islink(childpath):
self.warn("WARNING: cannot backup symlink %s" % quote_output(childpath))
else:
self.warn("WARNING: cannot backup special file %s" % quote_output(childpath))
must_create, r = self.check_backupdb_directory(compare_contents)
if must_create:
self.verboseprint(" creating directory for %s" % quote_output(localpath))
newdircap = mkdir(create_contents, self.options)
assert isinstance(newdircap, str)
if r:
r.did_create(newdircap)
self.directories_created += 1
return newdircap
else:
self.verboseprint(" re-using old directory for %s" % quote_output(localpath))
self.directories_reused += 1
return r.was_created()
def _pack_normalized_children(children, writekey, deep_immutable=False):
"""Take a dict that maps:
children[unicode_nfc_name] = (IFileSystemNode, metadata_dict)
and pack it into a single string, for use as the contents of the backing
file. This is the same format as is returned by _unpack_contents. I also
accept an AuxValueDict, in which case I'll use the auxilliary cached data
as the pre-packed entry, which is faster than re-packing everything each
time.
If writekey is provided then I will superencrypt the child's writecap with
writekey.
If deep_immutable is True, I will require that all my children are deeply
immutable, and will raise a MustBeDeepImmutableError if not.
"""
precondition((writekey is None) or isinstance(writekey, str), writekey)
has_aux = isinstance(children, AuxValueDict)
entries = []
for name in sorted(children.keys()):
assert isinstance(name, unicode)
entry = None
(child, metadata) = children[name]
child.raise_error()
if deep_immutable and not child.is_allowed_in_immutable_directory():
raise MustBeDeepImmutableError(
"child %s is not allowed in an immutable directory" % quote_output(name, encoding="utf-8"), name
)
if has_aux:
entry = children.get_aux(name)
if not entry:
assert IFilesystemNode.providedBy(child), (name, child)
assert isinstance(metadata, dict)
rw_uri = child.get_write_uri()
if rw_uri is None:
rw_uri = ""
assert isinstance(rw_uri, str), rw_uri
# should be prevented by MustBeDeepImmutableError check above
assert not (rw_uri and deep_immutable)
ro_uri = child.get_readonly_uri()
if ro_uri is None:
ro_uri = ""
assert isinstance(ro_uri, str), ro_uri
if writekey is not None:
writecap = netstring(_encrypt_rw_uri(writekey, rw_uri))
else:
writecap = ZERO_LEN_NETSTR
entry = "".join(
[
netstring(name.encode("utf-8")),
netstring(strip_prefix_for_ro(ro_uri, deep_immutable)),
writecap,
netstring(simplejson.dumps(metadata)),
]
)
entries.append(netstring(entry))
return "".join(entries)
def remote_write(self, offset, data):
start = time.time()
precondition(not self.closed)
if self.throw_out_all_data:
return
self._sharefile.write_share_data(offset, data)
self.ss.add_latency("write", time.time() - start)
self.ss.count("write")
def _got_data(self, results, blocknum):
precondition(blocknum < self.num_blocks, self, blocknum, self.num_blocks)
sharehashes, blockhashes, blockdata = results
try:
sharehashes = dict(sharehashes)
except ValueError, le:
le.args = tuple(le.args + (sharehashes,))
raise
def decode(self, some_shares, their_shareids):
precondition(len(some_shares) == len(their_shareids),
len(some_shares), len(their_shareids))
precondition(len(some_shares) == self.required_shares,
len(some_shares), self.required_shares)
data = self.decoder.decode(some_shares,
[int(s) for s in their_shareids])
return defer.succeed(data)
def set_nodes(self, entries, overwrite=True):
precondition(isinstance(entries, dict), entries)
if self.is_readonly():
return defer.fail(NotWriteableError())
a = Adder(self, entries, overwrite=overwrite, create_readonly_node=self._create_readonly_node)
d = self._node.modify(a.modify)
d.addCallback(lambda res: self)
return d
def next(self):
precondition(self.i <= len(self.c.l), "The iterated ValueOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the ValueOrderedDict while the iteration was in progress.", self.i, self.c)
precondition((self.i == len(self.c.l)) or self.c.d.has_key(self.c.l[self.i][1]), "The iterated ValueOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the ValueOrderedDict while the iteration was in progress.", self.i, (self.i < len(self.c.l)) and self.c.l[self.i], self.c)
if self.i == len(self.c.l):
raise StopIteration
le = self.c.l[self.i]
self.i += 1
return le[0]
def a2b(cs):
"""
@param cs the base-32 encoded data (a string)
"""
precondition(could_be_base32_encoded(cs), "cs is required to be possibly base32 encoded data.", cs=cs)
precondition(isinstance(cs, str), cs)
return a2b_l(cs, num_octets_that_encode_to_this_many_quintets(len(cs))*8)
def get_source_info(self, source_spec):
"""
This turns an argv string into a (Local|Tahoe)(File|Directory)Source.
"""
precondition(isinstance(source_spec, unicode), source_spec)
rootcap, path_utf8 = get_alias(self.aliases, source_spec, None)
path = path_utf8.decode("utf-8")
# any trailing slash is removed in abspath_expanduser_unicode(), so
# make a note of it here, to throw an error later
had_trailing_slash = path.endswith("/")
if rootcap == DefaultAliasMarker:
# no alias, so this is a local file
pathname = abspath_expanduser_unicode(path)
name = os.path.basename(pathname)
if not os.path.exists(pathname):
raise MissingSourceError(source_spec, quotefn=quote_local_unicode_path)
if os.path.isdir(pathname):
t = LocalDirectorySource(self.progress, pathname, name)
else:
if had_trailing_slash:
raise FilenameWithTrailingSlashError(source_spec,
quotefn=quote_local_unicode_path)
if not os.path.isfile(pathname):
raise WeirdSourceError(pathname)
t = LocalFileSource(pathname, name) # non-empty
else:
# this is a tahoe object
url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
name = None
if path:
if path.endswith("/"):
path = path[:-1]
url += "/" + escape_path(path)
last_slash = path.rfind(u"/")
name = path
if last_slash != -1:
name = path[last_slash+1:]
resp = do_http("GET", url + "?t=json")
if resp.status == 404:
raise MissingSourceError(source_spec)
elif resp.status != 200:
raise HTTPError("Error examining source %s" % quote_output(source_spec),
resp)
parsed = json.loads(resp.read())
nodetype, d = parsed
if nodetype == "dirnode":
t = TahoeDirectorySource(self.nodeurl, self.cache,
self.progress, name)
t.init_from_parsed(parsed)
else:
if had_trailing_slash:
raise FilenameWithTrailingSlashError(source_spec)
writecap = to_str(d.get("rw_uri"))
readcap = to_str(d.get("ro_uri"))
mutable = d.get("mutable", False) # older nodes don't provide it
t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap, name)
return t
def pack_children(childrenx, writekey, deep_immutable=False):
# initial_children must have metadata (i.e. {} instead of None)
children = {}
for (namex, (node, metadata)) in childrenx.iteritems():
precondition(isinstance(metadata, dict),
"directory creation requires metadata to be a dict, not None", metadata)
children[normalize(namex)] = (node, metadata)
return _pack_normalized_children(children, writekey=writekey, deep_immutable=deep_immutable)
def get_alias(aliases, path_unicode, default):
"""
Transform u"work:path/filename" into (aliases[u"work"], u"path/filename".encode('utf-8')).
If default=None, then an empty alias is indicated by returning
DefaultAliasMarker. We special-case strings with a recognized cap URI
prefix, to make it easy to access specific files/directories by their
caps.
If the transformed alias is either not found in aliases, or is blank
and default is not found in aliases, an UnknownAliasError is
raised.
"""
precondition(isinstance(path_unicode, unicode), path_unicode)
from allmydata import uri
path = path_unicode.encode('utf-8').strip(" ")
if uri.has_uri_prefix(path):
# We used to require "URI:blah:./foo" in order to get a subpath,
# stripping out the ":./" sequence. We still allow that for compatibility,
# but now also allow just "URI:blah/foo".
sep = path.find(":./")
if sep != -1:
return path[:sep], path[sep+3:]
sep = path.find("/")
if sep != -1:
return path[:sep], path[sep+1:]
return path, ""
colon = path.find(":")
if colon == -1:
# no alias
if default == None:
return DefaultAliasMarker, path
if default not in aliases:
raise UnknownAliasError("No alias specified, and the default %s alias doesn't exist. "
"To create it, use 'tahoe create-alias %s'."
% (quote_output(default), quote_output(default, quotemarks=False)))
return uri.from_string_dirnode(aliases[default]).to_string(), path
if colon == 1 and default is None and platform_uses_lettercolon_drivename():
# treat C:\why\must\windows\be\so\weird as a local path, not a tahoe
# file in the "C:" alias
return DefaultAliasMarker, path
# decoding must succeed because path is valid UTF-8 and colon & space are ASCII
alias = path[:colon].decode('utf-8')
if u"/" in alias:
# no alias, but there's a colon in a dirname/filename, like
# "foo/bar:7"
if default == None:
return DefaultAliasMarker, path
if default not in aliases:
raise UnknownAliasError("No alias specified, and the default %s alias doesn't exist. "
"To create it, use 'tahoe create-alias %s'."
% (quote_output(default), quote_output(default, quotemarks=False)))
return uri.from_string_dirnode(aliases[default]).to_string(), path
if alias not in aliases:
raise UnknownAliasError("Unknown alias %s, please create it with 'tahoe add-alias' or 'tahoe create-alias'." %
quote_output(alias))
return uri.from_string_dirnode(aliases[alias]).to_string(), path[colon+1:]
def subscribe_to(self, service_name, cb, *args, **kwargs):
self._local_subscribers.append( (service_name,cb,args,kwargs) )
self._subscribed_service_names.add(service_name)
self._maybe_subscribe()
for index,(ann,key_s,when) in self._inbound_announcements.items():
precondition(isinstance(key_s, str), key_s)
servicename = index[0]
if servicename == service_name:
eventually(cb, key_s, ann, *args, **kwargs)
def __init__(self, node, entries=None, overwrite=True, create_readonly_node=None):
self.node = node
if entries is None:
entries = {}
precondition(isinstance(entries, dict), entries)
# keys of 'entries' may not be normalized.
self.entries = entries
self.overwrite = overwrite
self.create_readonly_node = create_readonly_node
def _write_share_data(self, f, offset, data):
length = len(data)
precondition(offset >= 0)
data_length = self._read_data_length(f)
extra_lease_offset = self._read_extra_lease_offset(f)
if offset + length >= data_length:
# They are expanding their data size.
if self.DATA_OFFSET + offset + length > extra_lease_offset:
# TODO: allow containers to shrink. For now, they remain
# large.
# Their new data won't fit in the current container, so we
# have to move the leases. With luck, they're expanding it
# more than the size of the extra lease block, which will
# minimize the corrupt-the-share window
self._change_container_size(f, offset + length)
extra_lease_offset = self._read_extra_lease_offset(f)
# an interrupt here is ok.. the container has been enlarged
# but the data remains untouched
assert self.DATA_OFFSET + offset + length <= extra_lease_offset
# Their data now fits in the current container. We must write
# their new data and modify the recorded data size.
# Fill any newly exposed empty space with 0's.
if offset > data_length:
f.seek(self.DATA_OFFSET + data_length)
f.write('\x00' * (offset - data_length))
f.flush()
new_data_length = offset + length
self._write_data_length(f, new_data_length)
# an interrupt here will result in a corrupted share
# now all that's left to do is write out their data
f.seek(self.DATA_OFFSET + offset)
f.write(data)
return
def run(config, stdout, stderr):
from twisted.internet import reactor
from twisted.python import log, logfile
from allmydata import client
basedir = config['basedir']
precondition(isinstance(basedir, unicode), basedir)
if not os.path.isdir(basedir):
print >> stderr, "%s does not look like a directory at all" % quote_output(
basedir)
return 1
for fn in listdir_unicode(basedir):
if fn.endswith(u".tac"):
tac = str(fn)
break
else:
print >> stderr, "%s does not look like a node directory (no .tac file)" % quote_output(
basedir)
return 1
if "client" not in tac:
print >> stderr, ("%s looks like it contains a non-client node (%s).\n"
"Use 'tahoe start' instead of 'tahoe run'." %
(quote_output(basedir), tac))
return 1
os.chdir(basedir)
# set up twisted logging. this will become part of the node rsn.
logdir = os.path.join(basedir, 'logs')
if not os.path.exists(logdir):
os.makedirs(logdir)
lf = logfile.LogFile('tahoesvc.log', logdir)
log.startLogging(lf)
# run the node itself
c = client.Client(basedir)
reactor.callLater(0, c.startService) # after reactor startup
reactor.run()
return 0
def invite(options):
precondition(isinstance(options.alias, unicode), alias=options.alias)
precondition(isinstance(options.nickname, unicode),
nickname=options.nickname)
from allmydata.scripts import tahoe_mkdir
mkdir_options = _delegate_options(options, MakeDirectoryOptions())
mkdir_options.where = None
rc = tahoe_mkdir.mkdir(mkdir_options)
if rc != 0:
print >> options.stderr, "magic-folder: failed to mkdir\n"
return rc
# FIXME this assumes caps are ASCII.
dmd_write_cap = mkdir_options.stdout.getvalue().strip()
dmd_readonly_cap = uri.from_string(
dmd_write_cap).get_readonly().to_string()
if dmd_readonly_cap is None:
print >> options.stderr, "magic-folder: failed to diminish dmd write cap\n"
return 1
magic_write_cap = get_aliases(options["node-directory"])[options.alias]
magic_readonly_cap = uri.from_string(
magic_write_cap).get_readonly().to_string()
# tahoe ln CLIENT_READCAP COLLECTIVE_WRITECAP/NICKNAME
ln_options = _delegate_options(options, LnOptions())
ln_options.from_file = unicode(dmd_readonly_cap, 'utf-8')
ln_options.to_file = u"%s/%s" % (unicode(magic_write_cap,
'utf-8'), options.nickname)
rc = tahoe_mv.mv(ln_options, mode="link")
if rc != 0:
print >> options.stderr, "magic-folder: failed to create link\n"
print >> options.stderr, ln_options.stderr.getvalue()
return rc
# FIXME: this assumes caps are ASCII.
print >> options.stdout, "%s%s%s" % (magic_readonly_cap, INVITE_SEPARATOR,
dmd_write_cap)
return 0
def __init__(self, readbucketproxy, verifycap, fetch_failures=None):
# fetch_failures is for debugging -- see test_encode.py
self._fetch_failures = fetch_failures
self._readbucketproxy = readbucketproxy
precondition(IVerifierURI.providedBy(verifycap), verifycap)
self._verifycap = verifycap
# required
self.segment_size = None
self.crypttext_root_hash = None
self.share_root_hash = None
# computed
self.block_size = None
self.share_size = None
self.num_segments = None
self.tail_data_size = None
self.tail_segment_size = None
# optional
self.crypttext_hash = None
def set_node(self, namex, child, metadata=None, overwrite=True):
"""I add a child at the specific name. I return a Deferred that fires
when the operation finishes. This Deferred will fire with the child
node that was just added. I will replace any existing child of the
same name.
If this directory node is read-only, the Deferred will errback with a
NotWriteableError."""
precondition(IFilesystemNode.providedBy(child), child)
if self.is_readonly():
return defer.fail(NotWriteableError())
assert IFilesystemNode.providedBy(child), child
a = Adder(self,
overwrite=overwrite,
create_readonly_node=self._create_readonly_node)
a.set_node(namex, child, metadata)
d = self._node.modify(a.modify)
d.addCallback(lambda res: child)
return d
def _parse_offsets(self, data):
precondition(len(data) >= 0x4)
self._offsets = {}
(version, ) = struct.unpack(">L", data[0:4])
if version != 1 and version != 2:
raise ShareVersionIncompatible(version)
if version == 1:
precondition(len(data) >= 0x24)
x = 0x0c
fieldsize = 0x4
fieldstruct = ">L"
else:
precondition(len(data) >= 0x44)
x = 0x14
fieldsize = 0x8
fieldstruct = ">Q"
self._version = version
self._fieldsize = fieldsize
self._fieldstruct = fieldstruct
for field in (
'data',
'plaintext_hash_tree', # UNUSED
'crypttext_hash_tree',
'block_hashes',
'share_hashes',
'uri_extension',
):
offset = struct.unpack(fieldstruct, data[x:x + fieldsize])[0]
x += fieldsize
self._offsets[field] = offset
return self._offsets
def _got_announcement(self, key_s, ann, handlers=None):
precondition(isinstance(key_s, str), key_s)
precondition(key_s.startswith("v0-"), key_s)
precondition(ann["service-name"] == "storage", ann["service-name"])
if handlers is not None:
s = NativeStorageServer(key_s, ann, self._tub_options, handlers)
else:
s = NativeStorageServer(key_s, ann, self._tub_options,
self._tub_handlers)
s.on_status_changed(lambda _: self._got_connection())
server_id = s.get_serverid()
old = self.servers.get(server_id)
if old and server_id not in self.static_servers:
if old.get_announcement() == ann:
return # duplicate
# replacement
del self.servers[server_id]
old.stop_connecting()
old.disownServiceParent()
# NOTE: this disownServiceParent() returns a Deferred that
# doesn't fire until Tub.stopService fires, which will wait for
# any existing connections to be shut down. This doesn't
# generally matter for normal runtime, but unit tests can run
# into DirtyReactorErrors if they don't block on these. If a test
# replaces one server with a newer version, then terminates
# before the old one has been shut down, it might get
# DirtyReactorErrors. The fix would be to gather these Deferreds
# into a structure that will block StorageFarmBroker.stopService
# until they have fired (but hopefully don't keep reference
# cycles around when they fire earlier than that, which will
# almost always be the case for normal runtime).
# now we forget about them and start using the new one
s.setServiceParent(self)
self.servers[server_id] = s
s.start_connecting(self._trigger_connections)
def create(options):
precondition(isinstance(options.alias, unicode), alias=options.alias)
precondition(isinstance(options.nickname, (unicode, NoneType)), nickname=options.nickname)
precondition(isinstance(options.local_dir, (unicode, NoneType)), local_dir=options.local_dir)
from allmydata.scripts import tahoe_add_alias
create_alias_options = _delegate_options(options, CreateAliasOptions())
create_alias_options.alias = options.alias
rc = tahoe_add_alias.create_alias(create_alias_options)
if rc != 0:
print >>options.stderr, create_alias_options.stderr.getvalue()
return rc
print >>options.stdout, create_alias_options.stdout.getvalue()
if options.nickname is not None:
invite_options = _delegate_options(options, InviteOptions())
invite_options.alias = options.alias
invite_options.nickname = options.nickname
rc = invite(invite_options)
if rc != 0:
print >>options.stderr, "magic-folder: failed to invite after create\n"
print >>options.stderr, invite_options.stderr.getvalue()
return rc
invite_code = invite_options.stdout.getvalue().strip()
join_options = _delegate_options(options, JoinOptions())
join_options.local_dir = options.local_dir
join_options.invite_code = invite_code
rc = join(join_options)
if rc != 0:
print >>options.stderr, "magic-folder: failed to join after create\n"
print >>options.stderr, join_options.stderr.getvalue()
return rc
return 0
def create(options):
precondition(isinstance(options.alias, unicode), alias=options.alias)
precondition(isinstance(options.nickname, (unicode, NoneType)),
nickname=options.nickname)
precondition(isinstance(options.local_dir, (unicode, NoneType)),
local_dir=options.local_dir)
# make sure we don't already have a magic-folder with this name before we create the alias
maybe_upgrade_magic_folders(options["node-directory"])
folders = load_magic_folders(options["node-directory"])
if options['name'] in folders:
print("Already have a magic-folder named '{}'".format(options['name']),
file=options.stderr)
return 1
# create an alias; this basically just remembers the cap for the
# master directory
create_alias_options = _delegate_options(options, CreateAliasOptions())
create_alias_options.alias = options.alias
rc = tahoe_add_alias.create_alias(create_alias_options)
if rc != 0:
print(create_alias_options.stderr.getvalue(), file=options.stderr)
return rc
print(create_alias_options.stdout.getvalue(), file=options.stdout)
if options.nickname is not None:
print(u"Inviting myself as client '{}':".format(options.nickname),
file=options.stdout)
invite_options = _delegate_options(options, InviteOptions())
invite_options.alias = options.alias
invite_options.nickname = options.nickname
invite_options['name'] = options['name']
rc = invite(invite_options)
if rc != 0:
print(u"magic-folder: failed to invite after create\n",
file=options.stderr)
print(invite_options.stderr.getvalue(), file=options.stderr)
return rc
invite_code = invite_options.stdout.getvalue().strip()
print(u" created invite code", file=options.stdout)
join_options = _delegate_options(options, JoinOptions())
join_options['poll-interval'] = options['poll-interval']
join_options.nickname = options.nickname
join_options.local_dir = options.local_dir
join_options.invite_code = invite_code
rc = join(join_options)
if rc != 0:
print(u"magic-folder: failed to join after create\n",
file=options.stderr)
print(join_options.stderr.getvalue(), file=options.stderr)
return rc
print(u" joined new magic-folder", file=options.stdout)
print(u"Successfully created magic-folder '{}' with alias '{}:' "
u"and client '{}'\nYou must re-start your node before the "
u"magic-folder will be active.".format(options['name'],
options.alias,
options.nickname),
file=options.stdout)
return 0
def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None):
"""
Encode either a Unicode string or a UTF-8-encoded bytestring for representation
on stdout or stderr, tolerating errors. If 'quotemarks' is True, the string is
always quoted; otherwise, it is quoted only if necessary to avoid ambiguity or
control bytes in the output. (Newlines are counted as control bytes iff
quote_newlines is True.)
Quoting may use either single or double quotes. Within single quotes, all
characters stand for themselves, and ' will not appear. Within double quotes,
Python-compatible backslash escaping is used.
If not explicitly given, quote_newlines is True when quotemarks is True.
"""
precondition(isinstance(s, (str, unicode)), s)
if quote_newlines is None:
quote_newlines = quotemarks
if isinstance(s, str):
try:
s = s.decode('utf-8')
except UnicodeDecodeError:
return 'b"%s"' % (ESCAPABLE_8BIT.sub(
lambda m: _str_escape(m, quote_newlines), s), )
must_double_quote = quote_newlines and MUST_DOUBLE_QUOTE_NL or MUST_DOUBLE_QUOTE
if must_double_quote.search(s) is None:
try:
out = s.encode(encoding or io_encoding)
if quotemarks or out.startswith('"'):
return "'%s'" % (out, )
else:
return out
except (UnicodeDecodeError, UnicodeEncodeError):
pass
escaped = ESCAPABLE_UNICODE.sub(
lambda m: _unicode_escape(m, quote_newlines), s)
return '"%s"' % (escaped.encode(encoding or io_encoding,
'backslashreplace'), )
def unicode_to_argv(s, mangle=False):
"""
Encode the given Unicode argument as a bytestring.
If the argument is to be passed to a different process, then the 'mangle' argument
should be true; on Windows, this uses a mangled encoding that will be reversed by
code in runner.py.
On Python 3, just return the string unchanged, since argv is unicode.
"""
precondition(isinstance(s, unicode), s)
if PY3:
warnings.warn("This will be unnecessary once Python 2 is dropped.",
DeprecationWarning)
return s
if mangle and sys.platform == "win32":
# This must be the same as 'mangle' in bin/tahoe-script.template.
return bytes(
re.sub(u'[^\\x20-\\x7F]', lambda m: u'\x7F%x;' %
(ord(m.group(0)), ), s), io_encoding)
else:
return s.encode(io_encoding)
def set_children(self, entries, overwrite=True):
# this takes URIs
a = Adder(self, overwrite=overwrite,
create_readonly_node=self._create_readonly_node)
for (namex, e) in entries.iteritems():
assert isinstance(namex, unicode), namex
if len(e) == 2:
writecap, readcap = e
metadata = None
else:
assert len(e) == 3
writecap, readcap, metadata = e
precondition(isinstance(writecap, (str,type(None))), writecap)
precondition(isinstance(readcap, (str,type(None))), readcap)
# We now allow packing unknown nodes, provided they are valid
# for this type of directory.
child_node = self._create_and_validate_node(writecap, readcap, namex)
a.set_node(namex, child_node, metadata)
d = self._node.modify(a.modify)
d.addCallback(lambda ign: self)
return d
def _apply(old_contents):
new_contents = modifier(old_contents, self._servermap, first_time)
precondition((isinstance(new_contents, str) or
new_contents is None),
"Modifier function must return a string "
"or None")
if new_contents is None or new_contents == old_contents:
log.msg("no changes")
# no changes need to be made
if first_time:
return
# However, since Publish is not automatically doing a
# recovery when it observes UCWE, we need to do a second
# publish. See #551 for details. We'll basically loop until
# we managed an uncontested publish.
old_uploadable = MutableData(old_contents)
new_contents = old_uploadable
else:
new_contents = MutableData(new_contents)
return self._upload(new_contents)
def create_introducer(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_local_unicode_path(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "introducer")
c = open(os.path.join(basedir, "tahoe.cfg"), "w")
write_node_config(c, config)
c.close()
print >>out, "Introducer created in %s" % quote_local_unicode_path(basedir)
return 0
def __init__(self, client, accountfile, accounturl, ftp_portstr):
precondition(isinstance(accountfile, (unicode, NoneType)), accountfile)
service.MultiService.__init__(self)
r = Dispatcher(client)
p = portal.Portal(r)
if accountfile:
c = AccountFileChecker(self, accountfile)
p.registerChecker(c)
if accounturl:
c = AccountURLChecker(self, accounturl)
p.registerChecker(c)
if not accountfile and not accounturl:
# we could leave this anonymous, with just the /uri/CAP form
raise NeedRootcapLookupScheme("must provide some translation")
f = ftp.FTPFactory(p)
# strports requires a native string.
ftp_portstr = ensure_str(ftp_portstr)
s = strports.service(ftp_portstr, f)
s.setServiceParent(self)
def create_alias(options):
# mkdir+add_alias
nodedir = options['node-directory']
alias = options.alias
precondition(isinstance(alias, unicode), alias=alias)
stdout = options.stdout
stderr = options.stderr
if u":" in alias:
# a single trailing colon will already have been stripped if present
print >>stderr, "Alias names cannot contain colons."
return 1
if u" " in alias:
print >>stderr, "Alias names cannot contain spaces."
return 1
old_aliases = get_aliases(nodedir)
if alias in old_aliases:
print >>stderr, "Alias %s already exists!" % quote_output(alias)
return 1
aliasfile = os.path.join(nodedir, "private", "aliases")
nodeurl = options['node-url']
if not nodeurl.endswith("/"):
nodeurl += "/"
url = nodeurl + "uri?t=mkdir"
resp = do_http("POST", url)
rc = check_http_error(resp, stderr)
if rc:
return rc
new_uri = resp.read().strip()
# probably check for others..
add_line_to_aliasfile(aliasfile, alias, new_uri)
print >>stdout, "Alias %s created" % (quote_output(alias),)
return 0
def remote_close(self):
precondition(not self.closed)
start = time.time()
fileutil.make_dirs(os.path.dirname(self.finalhome))
fileutil.rename(self.incominghome, self.finalhome)
try:
# self.incominghome is like storage/shares/incoming/ab/abcde/4 .
# We try to delete the parent (.../ab/abcde) to avoid leaving
# these directories lying around forever, but the delete might
# fail if we're working on another share for the same storage
# index (like ab/abcde/5). The alternative approach would be to
# use a hierarchy of objects (PrefixHolder, BucketHolder,
# ShareWriter), each of which is responsible for a single
# directory on disk, and have them use reference counting of
# their children to know when they should do the rmdir. This
# approach is simpler, but relies on os.rmdir refusing to delete
# a non-empty directory. Do *not* use fileutil.rm_dir() here!
os.rmdir(os.path.dirname(self.incominghome))
# we also delete the grandparent (prefix) directory, .../ab ,
# again to avoid leaving directories lying around. This might
# fail if there is another bucket open that shares a prefix (like
# ab/abfff).
os.rmdir(os.path.dirname(os.path.dirname(self.incominghome)))
# we leave the great-grandparent (incoming/) directory in place.
except EnvironmentError:
# ignore the "can't rmdir because the directory is not empty"
# exceptions, those are normal consequences of the
# above-mentioned conditions.
pass
self._sharefile = None
self.closed = True
self._canary.dontNotifyOnDisconnect(self._disconnect_marker)
filelen = os.stat(self.finalhome)[stat.ST_SIZE]
self.ss.bucket_writer_closed(self, filelen)
self.ss.add_latency("close", time.time() - start)
self.ss.count("close")
def watch(self,
path,
mask=IN_WATCH_MASK,
autoAdd=False,
callbacks=None,
recursive=False):
precondition(isinstance(autoAdd, bool), autoAdd=autoAdd)
precondition(isinstance(recursive, bool), recursive=recursive)
assert autoAdd == False
path_u = path.path
if not isinstance(path_u, unicode):
path_u = path_u.decode('utf-8')
_assert(isinstance(path_u, unicode), path_u=path_u)
if path_u not in self._callbacks.keys():
self._callbacks[path_u] = callbacks or []
self._watches[path_u] = self._observer.schedule(
INotifyEventHandler(path_u, mask, self._callbacks[path_u],
self._pending_delay),
path=path_u,
recursive=False,
)
def add_lease(self, available_space, lease_info):
"""
Add a new lease to this share.
:param int available_space: The maximum number of bytes of storage to
commit in this operation. If more than this number of bytes is
required, raise ``NoSpace`` instead.
:raise NoSpace: If more than ``available_space`` bytes is required to
complete the operation. In this case, no lease is added.
:return: ``None``
"""
precondition(lease_info.owner_num != 0) # 0 means "no lease here"
with open(self.home, 'rb+') as f:
num_lease_slots = self._get_num_lease_slots(f)
empty_slot = self._get_first_empty_lease_slot(f)
if empty_slot is not None:
self._write_lease_record(f, empty_slot, lease_info)
else:
if lease_info.mutable_size() > available_space:
raise NoSpace()
self._write_lease_record(f, num_lease_slots, lease_info)
def create_introducer(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_local_unicode_path(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
defer.returnValue(-1)
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "introducer")
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
yield write_node_config(c, config)
print >>out, "Introducer created in %s" % quote_local_unicode_path(basedir)
defer.returnValue(0)
def write(self, offset, data): # type: (int, bytes) -> bool
"""
Write data at given offset, return whether the upload is complete.
"""
# Delay the timeout, since we received data:
self._timeout.reset(30 * 60)
start = self._clock.seconds()
precondition(not self.closed)
if self.throw_out_all_data:
return False
# Make sure we're not conflicting with existing data:
end = offset + len(data)
for (chunk_start, chunk_stop,
_) in self._already_written.ranges(offset, end):
chunk_len = chunk_stop - chunk_start
actual_chunk = self._sharefile.read_share_data(
chunk_start, chunk_len)
writing_chunk = data[chunk_start - offset:chunk_stop - offset]
if actual_chunk != writing_chunk:
raise ConflictingWriteError(
"Chunk {}-{} doesn't match already written data.".format(
chunk_start, chunk_stop))
self._sharefile.write_share_data(offset, data)
self._already_written.set(True, offset, end)
self.ss.add_latency("write", self._clock.seconds() - start)
self.ss.count("write")
# Return whether the whole thing has been written. See
# https://github.com/mlenzen/collections-extended/issues/169 and
# https://github.com/mlenzen/collections-extended/issues/172 for why
# it's done this way.
return sum([
mr.stop - mr.start for mr in self._already_written.ranges()
]) == self._max_size
def __init__(self, filename, max_size=None, create=False):
""" If max_size is not None then I won't allow more than max_size to be written to me. If create=True and max_size must not be None. """
precondition((max_size is not None) or (not create), max_size, create)
self.home = filename
self._max_size = max_size
if create:
# touch the file, so later callers will see that we're working on
# it. Also construct the metadata.
assert not os.path.exists(self.home)
fileutil.make_dirs(os.path.dirname(self.home))
f = open(self.home, 'wb')
# The second field -- the four-byte share data length -- is no
# longer used as of Tahoe v1.3.0, but we continue to write it in
# there in case someone downgrades a storage server from >=
# Tahoe-1.3.0 to < Tahoe-1.3.0, or moves a share file from one
# server to another, etc. We do saturation -- a share data length
# larger than 2**32-1 (what can fit into the field) is marked as
# the largest length that can fit into the field. That way, even
# if this does happen, the old < v1.3.0 server will still allow
# clients to read the first part of the share.
f.write(struct.pack(">LLL", 1, min(2**32 - 1, max_size), 0))
f.close()
self._lease_offset = max_size + 0x0c
self._num_leases = 0
else:
f = open(self.home, 'rb')
filesize = os.path.getsize(self.home)
(version, unused, num_leases) = struct.unpack(">LLL", f.read(0xc))
f.close()
if version != 1:
msg = "sharefile %s had version %d but we wanted 1" % \
(filename, version)
raise UnknownImmutableContainerVersionError(msg)
self._num_leases = num_leases
self._lease_offset = filesize - (num_leases * self.LEASE_SIZE)
self._data_offset = 0xc
def copy_file_into_dir(self, source, name, target):
precondition(isinstance(source, FileSources), source)
precondition(isinstance(target, DirectoryTargets), target)
precondition(isinstance(name, str), name)
if self.need_to_copy_bytes(source, target):
# if the target is a local directory, this will just write the
# bytes to disk. If it is a tahoe directory, it will upload the
# data, and stash the new filecap for a later set_children call.
f = source.open(self.caps_only)
target.put_file(name, f)
return
# otherwise we're copying tahoe to tahoe, and using immutable files,
# so we can just make a link
target.put_uri(name, source.bestcap())
def _read_share_data(self, f, offset, length):
precondition(offset >= 0)
precondition(length >= 0)
data_length = self._read_data_length(f)
if offset+length > data_length:
# reads beyond the end of the data are truncated. Reads that
# start beyond the end of the data return an empty string.
length = max(0, data_length-offset)
if length == 0:
return b""
precondition(offset+length <= data_length)
f.seek(self.DATA_OFFSET+offset)
data = f.read(length)
return data
def __init__(self, verifycap, servers, verify, add_lease, secret_holder,
monitor):
assert precondition(isinstance(verifycap, CHKFileVerifierURI), verifycap, type(verifycap))
prefix = str(base32.b2a(verifycap.get_storage_index()[:8])[:12], "utf-8")
log.PrefixingLogMixin.__init__(self, facility="tahoe.immutable.checker", prefix=prefix)
self._verifycap = verifycap
self._monitor = monitor
self._servers = servers
self._verify = verify # bool: verify what the servers claim, or not?
self._add_lease = add_lease
frs = file_renewal_secret_hash(secret_holder.get_renewal_secret(),
self._verifycap.get_storage_index())
self.file_renewal_secret = frs
fcs = file_cancel_secret_hash(secret_holder.get_cancel_secret(),
self._verifycap.get_storage_index())
self.file_cancel_secret = fcs
def _got_announcement(self, key_s, ann):
"""
This callback is given to the introducer and called any time an
announcement is received which has a valid signature and does not have
a sequence number less than or equal to a previous sequence number
seen for that server by that introducer.
Note sequence numbers are not considered between different introducers
so if we use more than one introducer it is possible for them to
deliver us stale announcements in some cases.
"""
precondition(isinstance(key_s, bytes), key_s)
precondition(key_s.startswith(b"v0-"), key_s)
precondition(ann["service-name"] == "storage", ann["service-name"])
server_id = key_s
if self._should_ignore_announcement(server_id, ann):
return
s = self._make_storage_server(
server_id,
{u"ann": ann},
)
try:
old = self.servers.pop(server_id)
except KeyError:
pass
else:
# It's a replacement, get rid of the old one.
old.stop_connecting()
old.disownServiceParent()
# NOTE: this disownServiceParent() returns a Deferred that
# doesn't fire until Tub.stopService fires, which will wait for
# any existing connections to be shut down. This doesn't
# generally matter for normal runtime, but unit tests can run
# into DirtyReactorErrors if they don't block on these. If a test
# replaces one server with a newer version, then terminates
# before the old one has been shut down, it might get
# DirtyReactorErrors. The fix would be to gather these Deferreds
# into a structure that will block StorageFarmBroker.stopService
# until they have fired (but hopefully don't keep reference
# cycles around when they fire earlier than that, which will
# almost always be the case for normal runtime).
# now we forget about them and start using the new one
s.setServiceParent(self)
self.servers[server_id] = s
s.start_connecting(self._trigger_connections)
def watch(self, path, mask=IN_WATCH_MASK, autoAdd=False, callbacks=None, recursive=False):
precondition(self._state == NOT_STARTED, "watch() can only be called before startReading()", state=self._state)
precondition(self._filter is None, "only one watch is supported")
precondition(isinstance(autoAdd, bool), autoAdd=autoAdd)
precondition(isinstance(recursive, bool), recursive=recursive)
#precondition(autoAdd == recursive, "need autoAdd and recursive to be the same", autoAdd=autoAdd, recursive=recursive)
self._path = path
path_u = path.path
if not isinstance(path_u, unicode):
path_u = path_u.decode(sys.getfilesystemencoding())
_assert(isinstance(path_u, unicode), path_u=path_u)
self._filter = FILE_NOTIFY_CHANGE_FILE_NAME | FILE_NOTIFY_CHANGE_DIR_NAME | FILE_NOTIFY_CHANGE_LAST_WRITE
if mask & (IN_ACCESS | IN_CLOSE_NOWRITE | IN_OPEN):
self._filter = self._filter | FILE_NOTIFY_CHANGE_LAST_ACCESS
if mask & IN_ATTRIB:
self._filter = self._filter | FILE_NOTIFY_CHANGE_ATTRIBUTES | FILE_NOTIFY_CHANGE_SECURITY
self._recursive = TRUE if recursive else FALSE
self._callbacks = callbacks or []
self._hDirectory = _open_directory(path_u)
def _got_announcement(self, key_s, ann):
precondition(isinstance(key_s, str), key_s)
precondition(key_s.startswith("v0-"), key_s)
precondition(ann["service-name"] == "storage", ann["service-name"])
server_id = key_s
if server_id in self._static_server_ids:
log.msg(format="ignoring announcement for static server '%(id)s'",
id=server_id,
facility="tahoe.storage_broker",
umid="AlxzqA",
level=log.UNUSUAL)
return
s = self._make_storage_server(
server_id.decode("utf-8"),
{u"ann": ann},
)
server_id = s.get_serverid()
old = self.servers.get(server_id)
if old:
if old.get_announcement() == ann:
return # duplicate
# replacement
del self.servers[server_id]
old.stop_connecting()
old.disownServiceParent()
# NOTE: this disownServiceParent() returns a Deferred that
# doesn't fire until Tub.stopService fires, which will wait for
# any existing connections to be shut down. This doesn't
# generally matter for normal runtime, but unit tests can run
# into DirtyReactorErrors if they don't block on these. If a test
# replaces one server with a newer version, then terminates
# before the old one has been shut down, it might get
# DirtyReactorErrors. The fix would be to gather these Deferreds
# into a structure that will block StorageFarmBroker.stopService
# until they have fired (but hopefully don't keep reference
# cycles around when they fire earlier than that, which will
# almost always be the case for normal runtime).
# now we forget about them and start using the new one
s.setServiceParent(self)
self.servers[server_id] = s
s.start_connecting(self._trigger_connections)
def create(options):
precondition(isinstance(options.alias, unicode), alias=options.alias)
precondition(isinstance(options.nickname, (unicode, NoneType)),
nickname=options.nickname)
precondition(isinstance(options.local_dir, (unicode, NoneType)),
local_dir=options.local_dir)
try:
from twisted.internet import reactor
treq = HTTPClient(Agent(reactor))
name = options['name']
nodedir = options.parent.node_directory
localdir = options.local_dir
rc = yield _create(options.alias, options.nickname, name, nodedir,
localdir, options["poll-interval"], treq)
print("Alias %s created" % (quote_output(options.alias), ),
file=options.stdout)
except Exception as e:
print("%s" % str(e), file=options.stderr)
returnValue(1)
returnValue(rc)
def create_node(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >> err, "The base directory %s is not empty." % quote_output(
basedir)
print >> err, "To avoid clobbering anything, I am going to quit now."
print >> err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
f = open(os.path.join(basedir, "tahoe-client.tac"), "w")
f.write(client_tac)
f.close()
c = open(os.path.join(basedir, "tahoe.cfg"), "w")
write_node_config(c, config)
c.write("[client]\n")
c.write("# Which services should this client connect to?\n")
c.write("introducer.furl = %s\n" % config.get("introducer", ""))
c.write("helper.furl =\n")
c.write("#key_generator.furl =\n")
c.write("#stats_gatherer.furl =\n")
c.write("\n")
c.write("# What encoding parameters should this client use for uploads?\n")
c.write("#shares.needed = 3\n")
c.write("#shares.happy = 7\n")
c.write("#shares.total = 10\n")
c.write("\n")
boolstr = {True: "true", False: "false"}
c.write("[storage]\n")
c.write("# Shall this node provide storage service?\n")
storage_enabled = not config.get("no-storage", None)
c.write("enabled = %s\n" % boolstr[storage_enabled])
c.write("#readonly =\n")
c.write("reserved_space = 1G\n")
c.write("#expire.enabled =\n")
c.write("#expire.mode =\n")
c.write("\n")
c.write("[helper]\n")
c.write("# Shall this node run a helper service that clients can use?\n")
c.write("enabled = false\n")
c.write("\n")
c.write("[drop_upload]\n")
c.write(
"# Shall this node automatically upload files created or modified in a local directory?\n"
)
c.write("enabled = false\n")
c.write(
"# To specify the target of uploads, a mutable directory writecap URI must be placed\n"
"# in 'private/drop_upload_dircap'.\n")
c.write("local.directory = ~/drop_upload\n")
c.write("\n")
c.close()
from allmydata.util import fileutil
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
print >> out, "Node created in %s" % quote_output(basedir)
if not config.get("introducer", ""):
print >> out, " Please set [client]introducer.furl= in tahoe.cfg!"
print >> out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >> out, " Please set [node]nickname= in tahoe.cfg"
return 0
