def _got_key( (pubkey, privkey) ):
nm.key_generator = SameKeyGenerator(pubkey, privkey)
pubkey_s = pubkey.serialize()
privkey_s = privkey.serialize()
u = uri.WriteableSSKFileURI(ssk_writekey_hash(privkey_s),
ssk_pubkey_fingerprint_hash(pubkey_s))
self._storage_index = u.get_storage_index()
def create_with_keys(self, keypair, contents,
version=SDMF_VERSION):
"""Call this to create a brand-new mutable file. It will create the
shares, find homes for them, and upload the initial contents (created
with the same rules as IClient.create_mutable_file() ). Returns a
Deferred that fires (with the MutableFileNode instance you should
use) when it completes.
"""
(pubkey, privkey) = keypair
self._pubkey, self._privkey = pubkey, privkey
pubkey_s = rsa.der_string_from_verifying_key(self._pubkey)
privkey_s = rsa.der_string_from_signing_key(self._privkey)
self._writekey = hashutil.ssk_writekey_hash(privkey_s)
self._encprivkey = self._encrypt_privkey(self._writekey, privkey_s)
self._fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s)
if version == MDMF_VERSION:
self._uri = WriteableMDMFFileURI(self._writekey, self._fingerprint)
self._protocol_version = version
elif version == SDMF_VERSION:
self._uri = WriteableSSKFileURI(self._writekey, self._fingerprint)
self._protocol_version = version
self._readkey = self._uri.readkey
self._storage_index = self._uri.storage_index
initial_contents = self._get_initial_contents(contents)
return self._upload(initial_contents, None)
def _got_key((pubkey, privkey)):
nm.key_generator = SameKeyGenerator(pubkey, privkey)
pubkey_s = pubkey.serialize()
privkey_s = privkey.serialize()
u = uri.WriteableSSKFileURI(ssk_writekey_hash(privkey_s),
ssk_pubkey_fingerprint_hash(pubkey_s))
self._storage_index = u.get_storage_index()
def create_with_keys(self, keypair, contents,
version=SDMF_VERSION):
"""Call this to create a brand-new mutable file. It will create the
shares, find homes for them, and upload the initial contents (created
with the same rules as IClient.create_mutable_file() ). Returns a
Deferred that fires (with the MutableFileNode instance you should
use) when it completes.
"""
(pubkey, privkey) = keypair
self._pubkey, self._privkey = pubkey, privkey
pubkey_s = self._pubkey.serialize()
privkey_s = self._privkey.serialize()
self._writekey = hashutil.ssk_writekey_hash(privkey_s)
self._encprivkey = self._encrypt_privkey(self._writekey, privkey_s)
self._fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s)
if version == MDMF_VERSION:
self._uri = WriteableMDMFFileURI(self._writekey, self._fingerprint)
self._protocol_version = version
elif version == SDMF_VERSION:
self._uri = WriteableSSKFileURI(self._writekey, self._fingerprint)
self._protocol_version = version
self._readkey = self._uri.readkey
self._storage_index = self._uri.storage_index
initial_contents = self._get_initial_contents(contents)
return self._upload(initial_contents, None)
def _got_key(keypair):
(pubkey, privkey) = keypair
nm.key_generator = SameKeyGenerator(pubkey, privkey)
pubkey_s = rsa.der_string_from_verifying_key(pubkey)
privkey_s = rsa.der_string_from_signing_key(privkey)
u = uri.WriteableSSKFileURI(ssk_writekey_hash(privkey_s),
ssk_pubkey_fingerprint_hash(pubkey_s))
self._storage_index = u.get_storage_index()
def _try_to_validate_privkey(self, enc_privkey, reader, server):
alleged_privkey_s = self._node._decrypt_privkey(enc_privkey)
alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
if alleged_writekey != self._node.get_writekey():
self.log("invalid privkey from %s shnum %d" % (reader, reader.shnum), level=log.WEIRD, umid="YIw4tA")
if self._verify:
self.servermap.mark_bad_share(server, reader.shnum, self.verinfo[-2])
e = CorruptShareError(server, reader.shnum, "invalid privkey")
f = failure.Failure(e)
self._bad_shares.add((server, reader.shnum, f))
return
# it's good
self.log("got valid privkey from shnum %d on reader %s" % (reader.shnum, reader))
privkey = rsa.create_signing_key_from_string(alleged_privkey_s)
self._node._populate_encprivkey(enc_privkey)
self._node._populate_privkey(privkey)
self._need_privkey = False
def _try_to_validate_privkey(self, enc_privkey, peerid, shnum, lp):
alleged_privkey_s = self._node._decrypt_privkey(enc_privkey)
alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
if alleged_writekey != self._node.get_writekey():
self.log("invalid privkey from %s shnum %d" %
(idlib.nodeid_b2a(peerid)[:8], shnum),
parent=lp, level=log.WEIRD, umid="YIw4tA")
return
# it's good
self.log("got valid privkey from shnum %d on peerid %s" %
(shnum, idlib.shortnodeid_b2a(peerid)),
parent=lp)
privkey = rsa.create_signing_key_from_string(alleged_privkey_s)
self._node._populate_encprivkey(enc_privkey)
self._node._populate_privkey(privkey)
self._need_privkey = False
def _try_to_validate_privkey(self, enc_privkey, peerid, shnum, lp):
alleged_privkey_s = self._node._decrypt_privkey(enc_privkey)
alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
if alleged_writekey != self._node.get_writekey():
self.log("invalid privkey from %s shnum %d" %
(idlib.nodeid_b2a(peerid)[:8], shnum),
parent=lp,
level=log.WEIRD,
umid="YIw4tA")
return
# it's good
self.log("got valid privkey from shnum %d on peerid %s" %
(shnum, idlib.shortnodeid_b2a(peerid)),
parent=lp)
privkey = rsa.create_signing_key_from_string(alleged_privkey_s)
self._node._populate_encprivkey(enc_privkey)
self._node._populate_privkey(privkey)
self._need_privkey = False
def _try_to_validate_privkey(self, enc_privkey, reader, server):
alleged_privkey_s = self._node._decrypt_privkey(enc_privkey)
alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
if alleged_writekey != self._node.get_writekey():
self.log("invalid privkey from %s shnum %d" %
(reader, reader.shnum),
level=log.WEIRD,
umid="YIw4tA")
if self._verify:
self.servermap.mark_bad_share(server, reader.shnum,
self.verinfo[-2])
e = CorruptShareError(server, reader.shnum, "invalid privkey")
f = failure.Failure(e)
self._bad_shares.add((server, reader.shnum, f))
return
# it's good
self.log("got valid privkey from shnum %d on reader %s" %
(reader.shnum, reader))
privkey = rsa.create_signing_key_from_string(alleged_privkey_s)
self._node._populate_encprivkey(enc_privkey)
self._node._populate_privkey(privkey)
self._need_privkey = False
class MutableChecker:
def __init__(self, node, storage_broker, history, monitor):
self._node = node
self._storage_broker = storage_broker
self._history = history
self._monitor = monitor
self.bad_shares = [] # list of (nodeid,shnum,failure)
self._storage_index = self._node.get_storage_index()
self.results = CheckResults(from_string(node.get_uri()),
self._storage_index)
self.need_repair = False
self.responded = set() # set of (binary) nodeids
def check(self, verify=False, add_lease=False):
servermap = ServerMap()
u = ServermapUpdater(self._node,
self._storage_broker,
self._monitor,
servermap,
MODE_CHECK,
add_lease=add_lease)
if self._history:
self._history.notify_mapupdate(u.get_status())
d = u.update()
d.addCallback(self._got_mapupdate_results)
if verify:
d.addCallback(self._verify_all_shares)
d.addCallback(lambda res: servermap)
d.addCallback(self._fill_checker_results, self.results)
d.addCallback(lambda res: self.results)
return d
def _got_mapupdate_results(self, servermap):
# the file is healthy if there is exactly one recoverable version, it
# has at least N distinct shares, and there are no unrecoverable
# versions: all existing shares will be for the same version.
self._monitor.raise_if_cancelled()
self.best_version = None
num_recoverable = len(servermap.recoverable_versions())
if num_recoverable:
self.best_version = servermap.best_recoverable_version()
if servermap.unrecoverable_versions():
self.need_repair = True
if num_recoverable != 1:
self.need_repair = True
if self.best_version:
available_shares = servermap.shares_available()
(num_distinct_shares, k, N) = available_shares[self.best_version]
if num_distinct_shares < N:
self.need_repair = True
return servermap
def _verify_all_shares(self, servermap):
# read every byte of each share
if not self.best_version:
return
versionmap = servermap.make_versionmap()
shares = versionmap[self.best_version]
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
offsets_tuple) = self.best_version
offsets = dict(offsets_tuple)
readv = [(0, offsets["EOF"])]
dl = []
for (shnum, peerid, timestamp) in shares:
ss = servermap.connections[peerid]
d = self._do_read(ss, peerid, self._storage_index, [shnum], readv)
d.addCallback(self._got_answer, peerid, servermap)
dl.append(d)
return defer.DeferredList(dl,
fireOnOneErrback=True,
consumeErrors=True)
def _do_read(self, ss, peerid, storage_index, shnums, readv):
# isolate the callRemote to a separate method, so tests can subclass
# Publish and override it
d = ss.callRemote("slot_readv", storage_index, shnums, readv)
return d
def _got_answer(self, datavs, peerid, servermap):
for shnum, datav in datavs.items():
data = datav[0]
try:
self._got_results_one_share(shnum, peerid, data)
except CorruptShareError:
f = failure.Failure()
self.need_repair = True
self.bad_shares.append((peerid, shnum, f))
prefix = data[:SIGNED_PREFIX_LENGTH]
servermap.mark_bad_share(peerid, shnum, prefix)
ss = servermap.connections[peerid]
self.notify_server_corruption(ss, shnum, str(f.value))
def check_prefix(self, peerid, shnum, data):
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
offsets_tuple) = self.best_version
got_prefix = data[:SIGNED_PREFIX_LENGTH]
if got_prefix != prefix:
raise CorruptShareError(
peerid, shnum,
"prefix mismatch: share changed while we were reading it")
def _got_results_one_share(self, shnum, peerid, data):
self.check_prefix(peerid, shnum, data)
# the [seqnum:signature] pieces are validated by _compare_prefix,
# which checks their signature against the pubkey known to be
# associated with this file.
(seqnum, root_hash, IV, k, N, segsize, datalen, pubkey, signature,
share_hash_chain, block_hash_tree, share_data,
enc_privkey) = unpack_share(data)
# validate [share_hash_chain,block_hash_tree,share_data]
leaves = [hashutil.block_hash(share_data)]
t = hashtree.HashTree(leaves)
if list(t) != block_hash_tree:
raise CorruptShareError(peerid, shnum, "block hash tree failure")
share_hash_leaf = t[0]
t2 = hashtree.IncompleteHashTree(N)
# root_hash was checked by the signature
t2.set_hashes({0: root_hash})
try:
t2.set_hashes(hashes=share_hash_chain,
leaves={shnum: share_hash_leaf})
except (hashtree.BadHashError, hashtree.NotEnoughHashesError,
IndexError), e:
msg = "corrupt hashes: %s" % (e, )
raise CorruptShareError(peerid, shnum, msg)
# validate enc_privkey: only possible if we have a write-cap
if not self._node.is_readonly():
alleged_privkey_s = self._node._decrypt_privkey(enc_privkey)
alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
if alleged_writekey != self._node.get_writekey():
raise CorruptShareError(peerid, shnum, "invalid privkey")
class MutableFileNode:
implements(IMutableFileNode, ICheckable)
def __init__(self, storage_broker, secret_holder,
default_encoding_parameters, history):
self._storage_broker = storage_broker
self._secret_holder = secret_holder
self._default_encoding_parameters = default_encoding_parameters
self._history = history
self._pubkey = None # filled in upon first read
self._privkey = None # filled in if we're mutable
# we keep track of the last encoding parameters that we use. These
# are updated upon retrieve, and used by publish. If we publish
# without ever reading (i.e. overwrite()), then we use these values.
self._required_shares = default_encoding_parameters["k"]
self._total_shares = default_encoding_parameters["n"]
self._sharemap = {} # known shares, shnum-to-[nodeids]
self._cache = ResponseCache()
self._most_recent_size = None
# all users of this MutableFileNode go through the serializer. This
# takes advantage of the fact that Deferreds discard the callbacks
# that they're done with, so we can keep using the same Deferred
# forever without consuming more and more memory.
self._serializer = defer.succeed(None)
def __repr__(self):
if hasattr(self, '_uri'):
return "<%s %x %s %s>" % (self.__class__.__name__, id(self),
self.is_readonly() and 'RO'
or 'RW', self._uri.abbrev())
else:
return "<%s %x %s %s>" % (self.__class__.__name__, id(self), None,
None)
def init_from_cap(self, filecap):
# we have the URI, but we have not yet retrieved the public
# verification key, nor things like 'k' or 'N'. If and when someone
# wants to get our contents, we'll pull from shares and fill those
# in.
assert isinstance(filecap, (ReadonlySSKFileURI, WriteableSSKFileURI))
self._uri = filecap
self._writekey = None
if isinstance(filecap, WriteableSSKFileURI):
self._writekey = self._uri.writekey
self._readkey = self._uri.readkey
self._storage_index = self._uri.storage_index
self._fingerprint = self._uri.fingerprint
# the following values are learned during Retrieval
# self._pubkey
# self._required_shares
# self._total_shares
# and these are needed for Publish. They are filled in by Retrieval
# if possible, otherwise by the first peer that Publish talks to.
self._privkey = None
self._encprivkey = None
return self
def create_with_keys(self, (pubkey, privkey), contents):
"""Call this to create a brand-new mutable file. It will create the
shares, find homes for them, and upload the initial contents (created
with the same rules as IClient.create_mutable_file() ). Returns a
Deferred that fires (with the MutableFileNode instance you should
use) when it completes.
"""
self._pubkey, self._privkey = pubkey, privkey
pubkey_s = self._pubkey.serialize()
privkey_s = self._privkey.serialize()
self._writekey = hashutil.ssk_writekey_hash(privkey_s)
self._encprivkey = self._encrypt_privkey(self._writekey, privkey_s)
self._fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s)
self._uri = WriteableSSKFileURI(self._writekey, self._fingerprint)
self._readkey = self._uri.readkey
self._storage_index = self._uri.storage_index
initial_contents = self._get_initial_contents(contents)
return self._upload(initial_contents, None)
