def remote_cancel_lease(self, storage_index, cancel_secret): start = time.time() self.count("cancel") total_space_freed = 0 found_buckets = False for sf in self._iter_share_files(storage_index): # note: if we can't find a lease on one share, we won't bother # looking in the others. Unless something broke internally # (perhaps we ran out of disk space while adding a lease), the # leases on all shares will be identical. found_buckets = True # this raises IndexError if the lease wasn't present XXXX total_space_freed += sf.cancel_lease(cancel_secret) if found_buckets: storagedir = os.path.join(self.sharedir, storage_index_to_dir(storage_index)) if not os.listdir(storagedir): os.rmdir(storagedir) if self.stats_provider: self.stats_provider.count('storage_server.bytes_freed', total_space_freed) self.add_latency("cancel", time.time() - start) if not found_buckets: raise IndexError("no such storage index")
def remote_cancel_lease(self, storage_index, cancel_secret): start = time.time() self.count("cancel") total_space_freed = 0 found_buckets = False for sf in self._iter_share_files(storage_index): # note: if we can't find a lease on one share, we won't bother # looking in the others. Unless something broke internally # (perhaps we ran out of disk space while adding a lease), the # leases on all shares will be identical. found_buckets = True # this raises IndexError if the lease wasn't present XXXX total_space_freed += sf.cancel_lease(cancel_secret) if found_buckets: storagedir = os.path.join(self.sharedir, storage_index_to_dir(storage_index)) if not os.listdir(storagedir): os.rmdir(storagedir) if self.stats_provider: self.stats_provider.count('storage_server.bytes_freed', total_space_freed) self.add_latency("cancel", time.time() - start) if not found_buckets: raise IndexError("no such storage index")
def remote_slot_readv(self, storage_index, shares, readv): start = time.time() self.count("readv") si_s = si_b2a(storage_index) lp = log.msg("storage: slot_readv %s %s" % (si_s, shares), facility="tahoe.storage", level=log.OPERATIONAL) si_dir = storage_index_to_dir(storage_index) # shares exist if there is a file for them bucketdir = os.path.join(self.sharedir, si_dir) if not os.path.isdir(bucketdir): self.add_latency("readv", time.time() - start) return {} datavs = {} for sharenum_s in os.listdir(bucketdir): try: sharenum = int(sharenum_s) except ValueError: continue if sharenum in shares or not shares: filename = os.path.join(bucketdir, sharenum_s) msf = MutableShareFile(filename, self) datavs[sharenum] = msf.readv(readv) log.msg("returning shares %s" % (datavs.keys(), ), facility="tahoe.storage", level=log.NOISY, parent=lp) self.add_latency("readv", time.time() - start) return datavs
def remote_slot_readv(self, storage_index, shares, readv): start = time.time() self.count("readv") si_s = si_b2a(storage_index) lp = log.msg("storage: slot_readv %s %s" % (si_s, shares), facility="tahoe.storage", level=log.OPERATIONAL) si_dir = storage_index_to_dir(storage_index) # shares exist if there is a file for them bucketdir = os.path.join(self.sharedir, si_dir) if not os.path.isdir(bucketdir): self.add_latency("readv", time.time() - start) return {} datavs = {} for sharenum_s in os.listdir(bucketdir): try: sharenum = int(sharenum_s) except ValueError: continue if sharenum in shares or not shares: filename = os.path.join(bucketdir, sharenum_s) msf = MutableShareFile(filename, self) datavs[sharenum] = msf.readv(readv) log.msg("returning shares %s" % (datavs.keys(),), facility="tahoe.storage", level=log.NOISY, parent=lp) self.add_latency("readv", time.time() - start) return datavs
def get_all_share_paths(storage_server, storage_index): """ Get the paths of all shares in the given storage index (or slot). :param allmydata.storage.server.StorageServer storage_server: The storage server which owns the storage index. :param bytes storage_index: The storage index (or slot) in which to look up shares. :return: A generator of tuples of (int, bytes) giving a share number and the path to storage for that share number. """ bucket = join(storage_server.sharedir, storage_index_to_dir(storage_index)) try: contents = listdir(bucket) except OSError as e: if e.errno == ENOENT: return raise for candidate in contents: try: sharenum = int(candidate) except ValueError: pass else: yield sharenum, join(bucket, candidate)
def remote_slot_testv_and_readv_and_writev( self, passes, storage_index, secrets, tw_vectors, r_vector, ): """ Pass-through after a pass check to ensure clients can only allocate storage for mutable shares if they present valid passes. :note: This method can be used both to allocate storage and to rewrite data in already-allocated storage. These cases may not be the same from the perspective of pass validation. """ with start_action( action_type= u"zkapauthorizer:storage-server:remote:slot-testv-and-readv-and-writev", storage_index=b2a(storage_index), path=storage_index_to_dir(storage_index), ): result = self._slot_testv_and_readv_and_writev( passes, storage_index, secrets, tw_vectors, r_vector, ) if isinstance(result, Deferred): raise TypeError( "_slot_testv_and_readv_and_writev returned Deferred") return result
def _get_bucket_shares(self, storage_index): """Return a list of (shnum, pathname) tuples for files that hold shares for this storage_index. In each tuple, 'shnum' will always be the integer form of the last component of 'pathname'.""" storagedir = os.path.join(self.sharedir, storage_index_to_dir(storage_index)) try: for f in os.listdir(storagedir): if NUM_RE.match(f): filename = os.path.join(storagedir, f) yield (int(f), filename) except OSError: # Commonly caused by there being no buckets at all. pass
def _get_bucket_shares(self, storage_index): """Return a list of (shnum, pathname) tuples for files that hold shares for this storage_index. In each tuple, 'shnum' will always be the integer form of the last component of 'pathname'.""" storagedir = os.path.join(self.sharedir, storage_index_to_dir(storage_index)) try: for f in os.listdir(storagedir): if NUM_RE.match(f): filename = os.path.join(storagedir, f) yield (int(f), filename) except OSError: # Commonly caused by there being no buckets at all. pass
def _copy_share(self, share, to_server): (sharenum, sharefile) = share (id, ss) = to_server shares_dir = os.path.join(ss.original.storedir, "shares") si = uri.from_string(self.uri).get_storage_index() si_dir = os.path.join(shares_dir, storage_index_to_dir(si)) if not os.path.exists(si_dir): os.makedirs(si_dir) new_sharefile = os.path.join(si_dir, str(sharenum)) shutil.copy(sharefile, new_sharefile) self.shares = self.find_uri_shares(self.uri) # Make sure that the storage server has the share. self.failUnless((sharenum, ss.original.my_nodeid, new_sharefile) in self.shares)
def enumerate_mutable_shares(self, storage_index: bytes) -> set[int]: """Return all share numbers for the given mutable.""" si_dir = storage_index_to_dir(storage_index) # shares exist if there is a file for them bucketdir = os.path.join(self.sharedir, si_dir) if not os.path.isdir(bucketdir): return set() result = set() for sharenum_s in os.listdir(bucketdir): try: result.add(int(sharenum_s)) except ValueError: continue return result
def _copy_share(self, share, to_server): (sharenum, sharefile) = share (id, ss) = to_server shares_dir = os.path.join(ss.original._server.storedir, "shares") si = uri.from_string(self.uri).get_storage_index() si_dir = os.path.join(shares_dir, storage_index_to_dir(si)) if not os.path.exists(si_dir): os.makedirs(si_dir) new_sharefile = os.path.join(si_dir, str(sharenum)) shutil.copy(sharefile, new_sharefile) self.shares = self.find_uri_shares(self.uri) # Make sure that the storage server has the share. self.failUnless((sharenum, ss.original._server.my_nodeid, new_sharefile) in self.shares)
def write_shares(storage_server, storage_index, sharenums, size, canary): sharedir = FilePath(storage_server.sharedir).preauthChild( # storage_index_to_dir likes to return multiple segments # joined by pathsep storage_index_to_dir(storage_index), ) for sharenum in sharenums: sharepath = sharedir.child(u"{}".format(sharenum)) sharepath.parent().makedirs() whitebox_write_sparse_share( sharepath, version=1, size=size, leases=leases, now=clock.seconds(), )
def copy_sdmf_shares(self): # We'll basically be short-circuiting the upload process. servernums = list(self.g.servers_by_number.keys()) assert len(servernums) == 10 assignments = list(zip(self.sdmf_old_shares.keys(), servernums)) # Get the storage index. cap = uri.from_string(self.sdmf_old_cap) si = cap.get_storage_index() # Now execute each assignment by writing the storage. for (share, servernum) in assignments: sharedata = base64.b64decode(self.sdmf_old_shares[share]) storedir = self.get_serverdir(servernum) storage_path = os.path.join(storedir, "shares", storage_index_to_dir(si)) fileutil.make_dirs(storage_path) fileutil.write(os.path.join(storage_path, "%d" % share), sharedata) # ...and verify that the shares are there. shares = self.find_uri_shares(self.sdmf_old_cap) assert len(shares) == 10
def copy_sdmf_shares(self): # We'll basically be short-circuiting the upload process. servernums = self.g.servers_by_number.keys() assert len(servernums) == 10 assignments = zip(self.sdmf_old_shares.keys(), servernums) # Get the storage index. cap = uri.from_string(self.sdmf_old_cap) si = cap.get_storage_index() # Now execute each assignment by writing the storage. for (share, servernum) in assignments: sharedata = base64.b64decode(self.sdmf_old_shares[share]) storedir = self.get_serverdir(servernum) storage_path = os.path.join(storedir, "shares", storage_index_to_dir(si)) fileutil.make_dirs(storage_path) fileutil.write(os.path.join(storage_path, "%d" % share), sharedata) # ...and verify that the shares are there. shares = self.find_uri_shares(self.sdmf_old_cap) assert len(shares) == 10
def test_stat_shares_truncated_file(self, storage_index, sharenum, size, clock, version, position): """ If a share file is truncated in the middle of the header, ``stat_shares`` declines to offer a result (by raising ``ValueError``). """ # Hypothesis causes our storage server to be used many times. Clean # up between iterations. cleanup_storage_server(self.anonymous_storage_server) sharedir = FilePath( self.anonymous_storage_server.sharedir ).preauthChild( # storage_index_to_dir likes to return multiple segments # joined by pathsep storage_index_to_dir(storage_index), ) sharepath = sharedir.child(u"{}".format(sharenum)) sharepath.parent().makedirs() whitebox_write_sparse_share( sharepath, version=version, size=size, # We know leases are at the end, where they'll get chopped off, so # we don't bother to write any. leases=[], now=clock.seconds(), ) with sharepath.open("wb") as fobj: fobj.truncate(position) self.assertThat( self.client.stat_shares([storage_index]), failed( AfterPreprocessing( lambda f: f.value, IsInstance(ValueError), ), ), )
def test_1654(self): # test that the Retrieve object unconditionally verifies the block # hash tree root for mutable shares. The failure mode is that # carefully crafted shares can cause undetected corruption (the # retrieve appears to finish successfully, but the result is # corrupted). When fixed, these shares always cause a # CorruptShareError, which results in NotEnoughSharesError in this # 2-of-2 file. self.basedir = "mutable/Problems/test_1654" self.set_up_grid(num_servers=2) cap = uri.from_string(TEST_1654_CAP) si = cap.get_storage_index() for share, shnum in [(TEST_1654_SH0, 0), (TEST_1654_SH1, 1)]: sharedata = base64.b64decode(share) storedir = self.get_serverdir(shnum) storage_path = os.path.join(storedir, "shares", storage_index_to_dir(si)) fileutil.make_dirs(storage_path) fileutil.write(os.path.join(storage_path, "%d" % shnum), sharedata) nm = self.g.clients[0].nodemaker n = nm.create_from_cap(TEST_1654_CAP) # to exercise the problem correctly, we must ensure that sh0 is # processed first, and sh1 second. NoNetworkGrid has facilities to # stall the first request from a single server, but it's not # currently easy to extend that to stall the second request (mutable # retrievals will see two: first the mapupdate, then the fetch). # However, repeated executions of this run without the #1654 fix # suggests that we're failing reliably even without explicit stalls, # probably because the servers are queried in a fixed order. So I'm # ok with relying upon that. d = self.shouldFail(NotEnoughSharesError, "test #1654 share corruption", "ran out of servers", n.download_best_version) return d
def test_1654(self): # test that the Retrieve object unconditionally verifies the block # hash tree root for mutable shares. The failure mode is that # carefully crafted shares can cause undetected corruption (the # retrieve appears to finish successfully, but the result is # corrupted). When fixed, these shares always cause a # CorruptShareError, which results in NotEnoughSharesError in this # 2-of-2 file. self.basedir = "mutable/Problems/test_1654" self.set_up_grid(num_servers=2) cap = uri.from_string(TEST_1654_CAP) si = cap.get_storage_index() for share, shnum in [(TEST_1654_SH0, 0), (TEST_1654_SH1, 1)]: sharedata = base64.b64decode(share) storedir = self.get_serverdir(shnum) storage_path = os.path.join(storedir, "shares", storage_index_to_dir(si)) fileutil.make_dirs(storage_path) fileutil.write(os.path.join(storage_path, "%d" % shnum), sharedata) nm = self.g.clients[0].nodemaker n = nm.create_from_cap(TEST_1654_CAP) # to exercise the problem correctly, we must ensure that sh0 is # processed first, and sh1 second. NoNetworkGrid has facilities to # stall the first request from a single server, but it's not # currently easy to extend that to stall the second request (mutable # retrievals will see two: first the mapupdate, then the fetch). # However, repeated executions of this run without the #1654 fix # suggests that we're failing reliably even without explicit stalls, # probably because the servers are queried in a fixed order. So I'm # ok with relying upon that. d = self.shouldFail(NotEnoughSharesError, "test #1654 share corruption", "ran out of servers", n.download_best_version) return d
def test_stat_shares_immutable_wrong_version(self, storage_index, sharenum, size, clock, leases, version): """ If a share file with an unexpected version is found, ``stat_shares`` declines to offer a result (by raising ``ValueError``). """ assume(version != 1) # Hypothesis causes our storage server to be used many times. Clean # up between iterations. cleanup_storage_server(self.anonymous_storage_server) sharedir = FilePath( self.anonymous_storage_server.sharedir ).preauthChild( # storage_index_to_dir likes to return multiple segments # joined by pathsep storage_index_to_dir(storage_index), ) sharepath = sharedir.child(u"{}".format(sharenum)) sharepath.parent().makedirs() whitebox_write_sparse_share( sharepath, version=version, size=size, leases=leases, now=clock.seconds(), ) self.assertThat( self.client.stat_shares([storage_index]), failed( AfterPreprocessing( lambda f: f.value, IsInstance(ValueError), ), ), )
def remote_slot_testv_and_readv_and_writev(self, storage_index, secrets, test_and_write_vectors, read_vector): start = time.time() self.count("writev") si_s = si_b2a(storage_index) log.msg("storage: slot_writev %s" % si_s) si_dir = storage_index_to_dir(storage_index) (write_enabler, renew_secret, cancel_secret) = secrets # shares exist if there is a file for them bucketdir = os.path.join(self.sharedir, si_dir) shares = {} if os.path.isdir(bucketdir): for sharenum_s in os.listdir(bucketdir): try: sharenum = int(sharenum_s) except ValueError: continue filename = os.path.join(bucketdir, sharenum_s) msf = MutableShareFile(filename, self) msf.check_write_enabler(write_enabler, si_s) shares[sharenum] = msf # write_enabler is good for all existing shares. # Now evaluate test vectors. testv_is_good = True for sharenum in test_and_write_vectors: (testv, datav, new_length) = test_and_write_vectors[sharenum] if sharenum in shares: if not shares[sharenum].check_testv(testv): self.log("testv failed: [%d]: %r" % (sharenum, testv)) testv_is_good = False break else: # compare the vectors against an empty share, in which all # reads return empty strings. if not EmptyShare().check_testv(testv): self.log("testv failed (empty): [%d] %r" % (sharenum, testv)) testv_is_good = False break # now gather the read vectors, before we do any writes read_data = {} for sharenum, share in shares.items(): read_data[sharenum] = share.readv(read_vector) ownerid = 1 # TODO expire_time = time.time() + 31*24*60*60 # one month lease_info = LeaseInfo(ownerid, renew_secret, cancel_secret, expire_time, self.my_nodeid) if testv_is_good: # now apply the write vectors for sharenum in test_and_write_vectors: (testv, datav, new_length) = test_and_write_vectors[sharenum] if new_length == 0: if sharenum in shares: shares[sharenum].unlink() else: if sharenum not in shares: # allocate a new share allocated_size = 2000 # arbitrary, really share = self._allocate_slot_share(bucketdir, secrets, sharenum, allocated_size, owner_num=0) shares[sharenum] = share shares[sharenum].writev(datav, new_length) # and update the lease shares[sharenum].add_or_renew_lease(lease_info) if new_length == 0: # delete empty bucket directories if not os.listdir(bucketdir): os.rmdir(bucketdir) # all done self.add_latency("writev", time.time() - start) return (testv_is_good, read_data)
def slot_testv_and_readv_and_writev( # type: ignore # warner/foolscap#78 self, storage_index, secrets, test_and_write_vectors, read_vector, renew_leases, ): """ Read data from shares and conditionally write some data to them. :param bool renew_leases: If and only if this is ``True`` and the test vectors pass then shares in this slot will also have an updated lease applied to them. See ``allmydata.interfaces.RIStorageServer`` for details about other parameters and return value. """ start = time.time() self.count("writev") si_s = si_b2a(storage_index) log.msg("storage: slot_writev %s" % si_s) si_dir = storage_index_to_dir(storage_index) (write_enabler, renew_secret, cancel_secret) = secrets bucketdir = os.path.join(self.sharedir, si_dir) # If collection succeeds we know the write_enabler is good for all # existing shares. shares = self._collect_mutable_shares_for_storage_index( bucketdir, write_enabler, si_s, ) # Now evaluate test vectors. testv_is_good = self._evaluate_test_vectors( test_and_write_vectors, shares, ) # now gather the read vectors, before we do any writes read_data = self._evaluate_read_vectors( read_vector, shares, ) if testv_is_good: # now apply the write vectors remaining_shares = self._evaluate_write_vectors( bucketdir, secrets, test_and_write_vectors, shares, ) if renew_leases: lease_info = self._make_lease_info(renew_secret, cancel_secret) self._add_or_renew_leases(remaining_shares, lease_info) # all done self.add_latency("writev", time.time() - start) return (testv_is_good, read_data)
def remote_allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, canary, owner_num=0): # owner_num is not for clients to set, but rather it should be # curried into the PersonalStorageServer instance that is dedicated # to a particular owner. start = time.time() self.count("allocate") alreadygot = set() bucketwriters = {} # k: shnum, v: BucketWriter si_dir = storage_index_to_dir(storage_index) si_s = si_b2a(storage_index) log.msg("storage: allocate_buckets %s" % si_s) # in this implementation, the lease information (including secrets) # goes into the share files themselves. It could also be put into a # separate database. Note that the lease should not be added until # the BucketWriter has been closed. expire_time = time.time() + 31*24*60*60 lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, expire_time, self.my_nodeid) max_space_per_bucket = allocated_size remaining_space = self.get_available_space() limited = remaining_space is not None if limited: # this is a bit conservative, since some of this allocated_size() # has already been written to disk, where it will show up in # get_available_space. remaining_space -= self.allocated_size() # self.readonly_storage causes remaining_space <= 0 # fill alreadygot with all shares that we have, not just the ones # they asked about: this will save them a lot of work. Add or update # leases for all of them: if they want us to hold shares for this # file, they'll want us to hold leases for this file. for (shnum, fn) in self._get_bucket_shares(storage_index): alreadygot.add(shnum) sf = ShareFile(fn) sf.add_or_renew_lease(lease_info) for shnum in sharenums: incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum) finalhome = os.path.join(self.sharedir, si_dir, "%d" % shnum) if os.path.exists(finalhome): # great! we already have it. easy. pass elif os.path.exists(incominghome): # Note that we don't create BucketWriters for shnums that # have a partial share (in incoming/), so if a second upload # occurs while the first is still in progress, the second # uploader will use different storage servers. pass elif (not limited) or (remaining_space >= max_space_per_bucket): # ok! we need to create the new share file. bw = BucketWriter(self, incominghome, finalhome, max_space_per_bucket, lease_info, canary) if self.no_storage: bw.throw_out_all_data = True bucketwriters[shnum] = bw self._active_writers[bw] = 1 if limited: remaining_space -= max_space_per_bucket else: # bummer! not enough space to accept this bucket pass if bucketwriters: fileutil.make_dirs(os.path.join(self.sharedir, si_dir)) self.add_latency("allocate", time.time() - start) return alreadygot, bucketwriters
def remote_slot_testv_and_readv_and_writev(self, storage_index, secrets, test_and_write_vectors, read_vector): start = time.time() self.count("writev") si_s = si_b2a(storage_index) log.msg("storage: slot_writev %s" % si_s) si_dir = storage_index_to_dir(storage_index) (write_enabler, renew_secret, cancel_secret) = secrets # shares exist if there is a file for them bucketdir = os.path.join(self.sharedir, si_dir) shares = {} if os.path.isdir(bucketdir): for sharenum_s in os.listdir(bucketdir): try: sharenum = int(sharenum_s) except ValueError: continue filename = os.path.join(bucketdir, sharenum_s) msf = MutableShareFile(filename, self) msf.check_write_enabler(write_enabler, si_s) shares[sharenum] = msf # write_enabler is good for all existing shares. # Now evaluate test vectors. testv_is_good = True for sharenum in test_and_write_vectors: (testv, datav, new_length) = test_and_write_vectors[sharenum] if sharenum in shares: if not shares[sharenum].check_testv(testv): self.log("testv failed: [%d]: %r" % (sharenum, testv)) testv_is_good = False break else: # compare the vectors against an empty share, in which all # reads return empty strings. if not EmptyShare().check_testv(testv): self.log("testv failed (empty): [%d] %r" % (sharenum, testv)) testv_is_good = False break # now gather the read vectors, before we do any writes read_data = {} for sharenum, share in shares.items(): read_data[sharenum] = share.readv(read_vector) ownerid = 1 # TODO expire_time = time.time() + 31 * 24 * 60 * 60 # one month lease_info = LeaseInfo(ownerid, renew_secret, cancel_secret, expire_time, self.my_nodeid) if testv_is_good: # now apply the write vectors for sharenum in test_and_write_vectors: (testv, datav, new_length) = test_and_write_vectors[sharenum] if new_length == 0: if sharenum in shares: shares[sharenum].unlink() else: if sharenum not in shares: # allocate a new share allocated_size = 2000 # arbitrary, really share = self._allocate_slot_share(bucketdir, secrets, sharenum, allocated_size, owner_num=0) shares[sharenum] = share shares[sharenum].writev(datav, new_length) # and update the lease shares[sharenum].add_or_renew_lease(lease_info) if new_length == 0: # delete empty bucket directories if not os.listdir(bucketdir): os.rmdir(bucketdir) # all done self.add_latency("writev", time.time() - start) return (testv_is_good, read_data)
def remote_allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, canary, owner_num=0): # owner_num is not for clients to set, but rather it should be # curried into the PersonalStorageServer instance that is dedicated # to a particular owner. start = time.time() self.count("allocate") alreadygot = set() bucketwriters = {} # k: shnum, v: BucketWriter si_dir = storage_index_to_dir(storage_index) si_s = si_b2a(storage_index) log.msg("storage: allocate_buckets %s" % si_s) # in this implementation, the lease information (including secrets) # goes into the share files themselves. It could also be put into a # separate database. Note that the lease should not be added until # the BucketWriter has been closed. expire_time = time.time() + 31 * 24 * 60 * 60 lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, expire_time, self.my_nodeid) max_space_per_bucket = allocated_size remaining_space = self.get_available_space() limited = remaining_space is not None if limited: # this is a bit conservative, since some of this allocated_size() # has already been written to disk, where it will show up in # get_available_space. remaining_space -= self.allocated_size() # self.readonly_storage causes remaining_space <= 0 # fill alreadygot with all shares that we have, not just the ones # they asked about: this will save them a lot of work. Add or update # leases for all of them: if they want us to hold shares for this # file, they'll want us to hold leases for this file. for (shnum, fn) in self._get_bucket_shares(storage_index): alreadygot.add(shnum) sf = ShareFile(fn) sf.add_or_renew_lease(lease_info) for shnum in sharenums: incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum) finalhome = os.path.join(self.sharedir, si_dir, "%d" % shnum) if os.path.exists(finalhome): # great! we already have it. easy. pass elif os.path.exists(incominghome): # Note that we don't create BucketWriters for shnums that # have a partial share (in incoming/), so if a second upload # occurs while the first is still in progress, the second # uploader will use different storage servers. pass elif (not limited) or (remaining_space >= max_space_per_bucket): # ok! we need to create the new share file. bw = BucketWriter(self, incominghome, finalhome, max_space_per_bucket, lease_info, canary) if self.no_storage: bw.throw_out_all_data = True bucketwriters[shnum] = bw self._active_writers[bw] = 1 if limited: remaining_space -= max_space_per_bucket else: # bummer! not enough space to accept this bucket pass if bucketwriters: fileutil.make_dirs(os.path.join(self.sharedir, si_dir)) self.add_latency("allocate", time.time() - start) return alreadygot, bucketwriters
def allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, owner_num=0, renew_leases=True): """ Generic bucket allocation API. :param bool renew_leases: If and only if this is ``True`` then renew a secret-matching lease on (or, if none match, add a new lease to) existing shares in this bucket. Any *new* shares are given a new lease regardless. """ # owner_num is not for clients to set, but rather it should be # curried into the PersonalStorageServer instance that is dedicated # to a particular owner. start = self._clock.seconds() self.count("allocate") alreadygot = {} bucketwriters = {} # k: shnum, v: BucketWriter si_dir = storage_index_to_dir(storage_index) si_s = si_b2a(storage_index) log.msg("storage: allocate_buckets %r" % si_s) # in this implementation, the lease information (including secrets) # goes into the share files themselves. It could also be put into a # separate database. Note that the lease should not be added until # the BucketWriter has been closed. expire_time = self._clock.seconds() + DEFAULT_RENEWAL_TIME lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, expire_time, self.my_nodeid) max_space_per_bucket = allocated_size remaining_space = self.get_available_space() limited = remaining_space is not None if limited: # this is a bit conservative, since some of this allocated_size() # has already been written to disk, where it will show up in # get_available_space. remaining_space -= self.allocated_size() # self.readonly_storage causes remaining_space <= 0 # fill alreadygot with all shares that we have, not just the ones # they asked about: this will save them a lot of work. Add or update # leases for all of them: if they want us to hold shares for this # file, they'll want us to hold leases for this file. for (shnum, fn) in self.get_shares(storage_index): alreadygot[shnum] = ShareFile(fn) if renew_leases: self._add_or_renew_leases(alreadygot.values(), lease_info) for shnum in sharenums: incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum) finalhome = os.path.join(self.sharedir, si_dir, "%d" % shnum) if os.path.exists(finalhome): # great! we already have it. easy. pass elif os.path.exists(incominghome): # For Foolscap we don't create BucketWriters for shnums that # have a partial share (in incoming/), so if a second upload # occurs while the first is still in progress, the second # uploader will use different storage servers. pass elif (not limited) or (remaining_space >= max_space_per_bucket): # ok! we need to create the new share file. bw = BucketWriter(self, incominghome, finalhome, max_space_per_bucket, lease_info, clock=self._clock) if self.no_storage: # Really this should be done by having a separate class for # this situation; see # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3862 bw.throw_out_all_data = True bucketwriters[shnum] = bw self._bucket_writers[incominghome] = bw if limited: remaining_space -= max_space_per_bucket else: # bummer! not enough space to accept this bucket pass if bucketwriters: fileutil.make_dirs(os.path.join(self.sharedir, si_dir)) self.add_latency("allocate", self._clock.seconds() - start) return set(alreadygot), bucketwriters