def remote_cancel_lease(self, storage_index, cancel_secret):
start = time.time()
self.count("cancel")
total_space_freed = 0
found_buckets = False
for sf in self._iter_share_files(storage_index):
# note: if we can't find a lease on one share, we won't bother
# looking in the others. Unless something broke internally
# (perhaps we ran out of disk space while adding a lease), the
# leases on all shares will be identical.
found_buckets = True
# this raises IndexError if the lease wasn't present XXXX
total_space_freed += sf.cancel_lease(cancel_secret)
if found_buckets:
storagedir = os.path.join(self.sharedir,
storage_index_to_dir(storage_index))
if not os.listdir(storagedir):
os.rmdir(storagedir)
if self.stats_provider:
self.stats_provider.count('storage_server.bytes_freed',
total_space_freed)
self.add_latency("cancel", time.time() - start)
if not found_buckets:
raise IndexError("no such storage index")
def remote_cancel_lease(self, storage_index, cancel_secret):
start = time.time()
self.count("cancel")
total_space_freed = 0
found_buckets = False
for sf in self._iter_share_files(storage_index):
# note: if we can't find a lease on one share, we won't bother
# looking in the others. Unless something broke internally
# (perhaps we ran out of disk space while adding a lease), the
# leases on all shares will be identical.
found_buckets = True
# this raises IndexError if the lease wasn't present XXXX
total_space_freed += sf.cancel_lease(cancel_secret)
if found_buckets:
storagedir = os.path.join(self.sharedir,
storage_index_to_dir(storage_index))
if not os.listdir(storagedir):
os.rmdir(storagedir)
if self.stats_provider:
self.stats_provider.count('storage_server.bytes_freed',
total_space_freed)
self.add_latency("cancel", time.time() - start)
if not found_buckets:
raise IndexError("no such storage index")
def remote_slot_readv(self, storage_index, shares, readv):
start = time.time()
self.count("readv")
si_s = si_b2a(storage_index)
lp = log.msg("storage: slot_readv %s %s" % (si_s, shares),
facility="tahoe.storage",
level=log.OPERATIONAL)
si_dir = storage_index_to_dir(storage_index)
# shares exist if there is a file for them
bucketdir = os.path.join(self.sharedir, si_dir)
if not os.path.isdir(bucketdir):
self.add_latency("readv", time.time() - start)
return {}
datavs = {}
for sharenum_s in os.listdir(bucketdir):
try:
sharenum = int(sharenum_s)
except ValueError:
continue
if sharenum in shares or not shares:
filename = os.path.join(bucketdir, sharenum_s)
msf = MutableShareFile(filename, self)
datavs[sharenum] = msf.readv(readv)
log.msg("returning shares %s" % (datavs.keys(), ),
facility="tahoe.storage",
level=log.NOISY,
parent=lp)
self.add_latency("readv", time.time() - start)
return datavs
def remote_slot_readv(self, storage_index, shares, readv):
start = time.time()
self.count("readv")
si_s = si_b2a(storage_index)
lp = log.msg("storage: slot_readv %s %s" % (si_s, shares),
facility="tahoe.storage", level=log.OPERATIONAL)
si_dir = storage_index_to_dir(storage_index)
# shares exist if there is a file for them
bucketdir = os.path.join(self.sharedir, si_dir)
if not os.path.isdir(bucketdir):
self.add_latency("readv", time.time() - start)
return {}
datavs = {}
for sharenum_s in os.listdir(bucketdir):
try:
sharenum = int(sharenum_s)
except ValueError:
continue
if sharenum in shares or not shares:
filename = os.path.join(bucketdir, sharenum_s)
msf = MutableShareFile(filename, self)
datavs[sharenum] = msf.readv(readv)
log.msg("returning shares %s" % (datavs.keys(),),
facility="tahoe.storage", level=log.NOISY, parent=lp)
self.add_latency("readv", time.time() - start)
return datavs
def get_all_share_paths(storage_server, storage_index):
"""
Get the paths of all shares in the given storage index (or slot).
:param allmydata.storage.server.StorageServer storage_server: The storage
server which owns the storage index.
:param bytes storage_index: The storage index (or slot) in which to look
up shares.
:return: A generator of tuples of (int, bytes) giving a share number and
the path to storage for that share number.
"""
bucket = join(storage_server.sharedir, storage_index_to_dir(storage_index))
try:
contents = listdir(bucket)
except OSError as e:
if e.errno == ENOENT:
return
raise
for candidate in contents:
try:
sharenum = int(candidate)
except ValueError:
pass
else:
yield sharenum, join(bucket, candidate)
def remote_slot_testv_and_readv_and_writev(
self,
passes,
storage_index,
secrets,
tw_vectors,
r_vector,
):
"""
Pass-through after a pass check to ensure clients can only allocate
storage for mutable shares if they present valid passes.
:note: This method can be used both to allocate storage and to rewrite
data in already-allocated storage. These cases may not be the
same from the perspective of pass validation.
"""
with start_action(
action_type=
u"zkapauthorizer:storage-server:remote:slot-testv-and-readv-and-writev",
storage_index=b2a(storage_index),
path=storage_index_to_dir(storage_index),
):
result = self._slot_testv_and_readv_and_writev(
passes,
storage_index,
secrets,
tw_vectors,
r_vector,
)
if isinstance(result, Deferred):
raise TypeError(
"_slot_testv_and_readv_and_writev returned Deferred")
return result
def _get_bucket_shares(self, storage_index):
"""Return a list of (shnum, pathname) tuples for files that hold
shares for this storage_index. In each tuple, 'shnum' will always be
the integer form of the last component of 'pathname'."""
storagedir = os.path.join(self.sharedir, storage_index_to_dir(storage_index))
try:
for f in os.listdir(storagedir):
if NUM_RE.match(f):
filename = os.path.join(storagedir, f)
yield (int(f), filename)
except OSError:
# Commonly caused by there being no buckets at all.
pass
def _get_bucket_shares(self, storage_index):
"""Return a list of (shnum, pathname) tuples for files that hold
shares for this storage_index. In each tuple, 'shnum' will always be
the integer form of the last component of 'pathname'."""
storagedir = os.path.join(self.sharedir, storage_index_to_dir(storage_index))
try:
for f in os.listdir(storagedir):
if NUM_RE.match(f):
filename = os.path.join(storagedir, f)
yield (int(f), filename)
except OSError:
# Commonly caused by there being no buckets at all.
pass
def _copy_share(self, share, to_server):
(sharenum, sharefile) = share
(id, ss) = to_server
shares_dir = os.path.join(ss.original.storedir, "shares")
si = uri.from_string(self.uri).get_storage_index()
si_dir = os.path.join(shares_dir, storage_index_to_dir(si))
if not os.path.exists(si_dir):
os.makedirs(si_dir)
new_sharefile = os.path.join(si_dir, str(sharenum))
shutil.copy(sharefile, new_sharefile)
self.shares = self.find_uri_shares(self.uri)
# Make sure that the storage server has the share.
self.failUnless((sharenum, ss.original.my_nodeid, new_sharefile)
in self.shares)
def enumerate_mutable_shares(self, storage_index: bytes) -> set[int]:
"""Return all share numbers for the given mutable."""
si_dir = storage_index_to_dir(storage_index)
# shares exist if there is a file for them
bucketdir = os.path.join(self.sharedir, si_dir)
if not os.path.isdir(bucketdir):
return set()
result = set()
for sharenum_s in os.listdir(bucketdir):
try:
result.add(int(sharenum_s))
except ValueError:
continue
return result
def _copy_share(self, share, to_server):
(sharenum, sharefile) = share
(id, ss) = to_server
shares_dir = os.path.join(ss.original._server.storedir, "shares")
si = uri.from_string(self.uri).get_storage_index()
si_dir = os.path.join(shares_dir, storage_index_to_dir(si))
if not os.path.exists(si_dir):
os.makedirs(si_dir)
new_sharefile = os.path.join(si_dir, str(sharenum))
shutil.copy(sharefile, new_sharefile)
self.shares = self.find_uri_shares(self.uri)
# Make sure that the storage server has the share.
self.failUnless((sharenum, ss.original._server.my_nodeid,
new_sharefile) in self.shares)
def write_shares(storage_server, storage_index, sharenums, size,
canary):
sharedir = FilePath(storage_server.sharedir).preauthChild(
# storage_index_to_dir likes to return multiple segments
# joined by pathsep
storage_index_to_dir(storage_index), )
for sharenum in sharenums:
sharepath = sharedir.child(u"{}".format(sharenum))
sharepath.parent().makedirs()
whitebox_write_sparse_share(
sharepath,
version=1,
size=size,
leases=leases,
now=clock.seconds(),
)
def copy_sdmf_shares(self):
# We'll basically be short-circuiting the upload process.
servernums = list(self.g.servers_by_number.keys())
assert len(servernums) == 10
assignments = list(zip(self.sdmf_old_shares.keys(), servernums))
# Get the storage index.
cap = uri.from_string(self.sdmf_old_cap)
si = cap.get_storage_index()
# Now execute each assignment by writing the storage.
for (share, servernum) in assignments:
sharedata = base64.b64decode(self.sdmf_old_shares[share])
storedir = self.get_serverdir(servernum)
storage_path = os.path.join(storedir, "shares",
storage_index_to_dir(si))
fileutil.make_dirs(storage_path)
fileutil.write(os.path.join(storage_path, "%d" % share), sharedata)
# ...and verify that the shares are there.
shares = self.find_uri_shares(self.sdmf_old_cap)
assert len(shares) == 10
def copy_sdmf_shares(self):
# We'll basically be short-circuiting the upload process.
servernums = self.g.servers_by_number.keys()
assert len(servernums) == 10
assignments = zip(self.sdmf_old_shares.keys(), servernums)
# Get the storage index.
cap = uri.from_string(self.sdmf_old_cap)
si = cap.get_storage_index()
# Now execute each assignment by writing the storage.
for (share, servernum) in assignments:
sharedata = base64.b64decode(self.sdmf_old_shares[share])
storedir = self.get_serverdir(servernum)
storage_path = os.path.join(storedir, "shares",
storage_index_to_dir(si))
fileutil.make_dirs(storage_path)
fileutil.write(os.path.join(storage_path, "%d" % share),
sharedata)
# ...and verify that the shares are there.
shares = self.find_uri_shares(self.sdmf_old_cap)
assert len(shares) == 10
def test_stat_shares_truncated_file(self, storage_index, sharenum, size,
clock, version, position):
"""
If a share file is truncated in the middle of the header,
``stat_shares`` declines to offer a result (by raising
``ValueError``).
"""
# Hypothesis causes our storage server to be used many times. Clean
# up between iterations.
cleanup_storage_server(self.anonymous_storage_server)
sharedir = FilePath(
self.anonymous_storage_server.sharedir
).preauthChild(
# storage_index_to_dir likes to return multiple segments
# joined by pathsep
storage_index_to_dir(storage_index), )
sharepath = sharedir.child(u"{}".format(sharenum))
sharepath.parent().makedirs()
whitebox_write_sparse_share(
sharepath,
version=version,
size=size,
# We know leases are at the end, where they'll get chopped off, so
# we don't bother to write any.
leases=[],
now=clock.seconds(),
)
with sharepath.open("wb") as fobj:
fobj.truncate(position)
self.assertThat(
self.client.stat_shares([storage_index]),
failed(
AfterPreprocessing(
lambda f: f.value,
IsInstance(ValueError),
), ),
)
def test_1654(self):
# test that the Retrieve object unconditionally verifies the block
# hash tree root for mutable shares. The failure mode is that
# carefully crafted shares can cause undetected corruption (the
# retrieve appears to finish successfully, but the result is
# corrupted). When fixed, these shares always cause a
# CorruptShareError, which results in NotEnoughSharesError in this
# 2-of-2 file.
self.basedir = "mutable/Problems/test_1654"
self.set_up_grid(num_servers=2)
cap = uri.from_string(TEST_1654_CAP)
si = cap.get_storage_index()
for share, shnum in [(TEST_1654_SH0, 0), (TEST_1654_SH1, 1)]:
sharedata = base64.b64decode(share)
storedir = self.get_serverdir(shnum)
storage_path = os.path.join(storedir, "shares",
storage_index_to_dir(si))
fileutil.make_dirs(storage_path)
fileutil.write(os.path.join(storage_path, "%d" % shnum),
sharedata)
nm = self.g.clients[0].nodemaker
n = nm.create_from_cap(TEST_1654_CAP)
# to exercise the problem correctly, we must ensure that sh0 is
# processed first, and sh1 second. NoNetworkGrid has facilities to
# stall the first request from a single server, but it's not
# currently easy to extend that to stall the second request (mutable
# retrievals will see two: first the mapupdate, then the fetch).
# However, repeated executions of this run without the #1654 fix
# suggests that we're failing reliably even without explicit stalls,
# probably because the servers are queried in a fixed order. So I'm
# ok with relying upon that.
d = self.shouldFail(NotEnoughSharesError, "test #1654 share corruption",
"ran out of servers",
n.download_best_version)
return d
def test_1654(self):
# test that the Retrieve object unconditionally verifies the block
# hash tree root for mutable shares. The failure mode is that
# carefully crafted shares can cause undetected corruption (the
# retrieve appears to finish successfully, but the result is
# corrupted). When fixed, these shares always cause a
# CorruptShareError, which results in NotEnoughSharesError in this
# 2-of-2 file.
self.basedir = "mutable/Problems/test_1654"
self.set_up_grid(num_servers=2)
cap = uri.from_string(TEST_1654_CAP)
si = cap.get_storage_index()
for share, shnum in [(TEST_1654_SH0, 0), (TEST_1654_SH1, 1)]:
sharedata = base64.b64decode(share)
storedir = self.get_serverdir(shnum)
storage_path = os.path.join(storedir, "shares",
storage_index_to_dir(si))
fileutil.make_dirs(storage_path)
fileutil.write(os.path.join(storage_path, "%d" % shnum),
sharedata)
nm = self.g.clients[0].nodemaker
n = nm.create_from_cap(TEST_1654_CAP)
# to exercise the problem correctly, we must ensure that sh0 is
# processed first, and sh1 second. NoNetworkGrid has facilities to
# stall the first request from a single server, but it's not
# currently easy to extend that to stall the second request (mutable
# retrievals will see two: first the mapupdate, then the fetch).
# However, repeated executions of this run without the #1654 fix
# suggests that we're failing reliably even without explicit stalls,
# probably because the servers are queried in a fixed order. So I'm
# ok with relying upon that.
d = self.shouldFail(NotEnoughSharesError, "test #1654 share corruption",
"ran out of servers",
n.download_best_version)
return d
def test_stat_shares_immutable_wrong_version(self, storage_index, sharenum,
size, clock, leases, version):
"""
If a share file with an unexpected version is found, ``stat_shares``
declines to offer a result (by raising ``ValueError``).
"""
assume(version != 1)
# Hypothesis causes our storage server to be used many times. Clean
# up between iterations.
cleanup_storage_server(self.anonymous_storage_server)
sharedir = FilePath(
self.anonymous_storage_server.sharedir
).preauthChild(
# storage_index_to_dir likes to return multiple segments
# joined by pathsep
storage_index_to_dir(storage_index), )
sharepath = sharedir.child(u"{}".format(sharenum))
sharepath.parent().makedirs()
whitebox_write_sparse_share(
sharepath,
version=version,
size=size,
leases=leases,
now=clock.seconds(),
)
self.assertThat(
self.client.stat_shares([storage_index]),
failed(
AfterPreprocessing(
lambda f: f.value,
IsInstance(ValueError),
), ),
)
def remote_slot_testv_and_readv_and_writev(self, storage_index,
secrets,
test_and_write_vectors,
read_vector):
start = time.time()
self.count("writev")
si_s = si_b2a(storage_index)
log.msg("storage: slot_writev %s" % si_s)
si_dir = storage_index_to_dir(storage_index)
(write_enabler, renew_secret, cancel_secret) = secrets
# shares exist if there is a file for them
bucketdir = os.path.join(self.sharedir, si_dir)
shares = {}
if os.path.isdir(bucketdir):
for sharenum_s in os.listdir(bucketdir):
try:
sharenum = int(sharenum_s)
except ValueError:
continue
filename = os.path.join(bucketdir, sharenum_s)
msf = MutableShareFile(filename, self)
msf.check_write_enabler(write_enabler, si_s)
shares[sharenum] = msf
# write_enabler is good for all existing shares.
# Now evaluate test vectors.
testv_is_good = True
for sharenum in test_and_write_vectors:
(testv, datav, new_length) = test_and_write_vectors[sharenum]
if sharenum in shares:
if not shares[sharenum].check_testv(testv):
self.log("testv failed: [%d]: %r" % (sharenum, testv))
testv_is_good = False
break
else:
# compare the vectors against an empty share, in which all
# reads return empty strings.
if not EmptyShare().check_testv(testv):
self.log("testv failed (empty): [%d] %r" % (sharenum,
testv))
testv_is_good = False
break
# now gather the read vectors, before we do any writes
read_data = {}
for sharenum, share in shares.items():
read_data[sharenum] = share.readv(read_vector)
ownerid = 1 # TODO
expire_time = time.time() + 31*24*60*60 # one month
lease_info = LeaseInfo(ownerid,
renew_secret, cancel_secret,
expire_time, self.my_nodeid)
if testv_is_good:
# now apply the write vectors
for sharenum in test_and_write_vectors:
(testv, datav, new_length) = test_and_write_vectors[sharenum]
if new_length == 0:
if sharenum in shares:
shares[sharenum].unlink()
else:
if sharenum not in shares:
# allocate a new share
allocated_size = 2000 # arbitrary, really
share = self._allocate_slot_share(bucketdir, secrets,
sharenum,
allocated_size,
owner_num=0)
shares[sharenum] = share
shares[sharenum].writev(datav, new_length)
# and update the lease
shares[sharenum].add_or_renew_lease(lease_info)
if new_length == 0:
# delete empty bucket directories
if not os.listdir(bucketdir):
os.rmdir(bucketdir)
# all done
self.add_latency("writev", time.time() - start)
return (testv_is_good, read_data)
def slot_testv_and_readv_and_writev( # type: ignore # warner/foolscap#78
self,
storage_index,
secrets,
test_and_write_vectors,
read_vector,
renew_leases,
):
"""
Read data from shares and conditionally write some data to them.
:param bool renew_leases: If and only if this is ``True`` and the test
vectors pass then shares in this slot will also have an updated
lease applied to them.
See ``allmydata.interfaces.RIStorageServer`` for details about other
parameters and return value.
"""
start = time.time()
self.count("writev")
si_s = si_b2a(storage_index)
log.msg("storage: slot_writev %s" % si_s)
si_dir = storage_index_to_dir(storage_index)
(write_enabler, renew_secret, cancel_secret) = secrets
bucketdir = os.path.join(self.sharedir, si_dir)
# If collection succeeds we know the write_enabler is good for all
# existing shares.
shares = self._collect_mutable_shares_for_storage_index(
bucketdir,
write_enabler,
si_s,
)
# Now evaluate test vectors.
testv_is_good = self._evaluate_test_vectors(
test_and_write_vectors,
shares,
)
# now gather the read vectors, before we do any writes
read_data = self._evaluate_read_vectors(
read_vector,
shares,
)
if testv_is_good:
# now apply the write vectors
remaining_shares = self._evaluate_write_vectors(
bucketdir,
secrets,
test_and_write_vectors,
shares,
)
if renew_leases:
lease_info = self._make_lease_info(renew_secret, cancel_secret)
self._add_or_renew_leases(remaining_shares, lease_info)
# all done
self.add_latency("writev", time.time() - start)
return (testv_is_good, read_data)
def remote_allocate_buckets(self, storage_index,
renew_secret, cancel_secret,
sharenums, allocated_size,
canary, owner_num=0):
# owner_num is not for clients to set, but rather it should be
# curried into the PersonalStorageServer instance that is dedicated
# to a particular owner.
start = time.time()
self.count("allocate")
alreadygot = set()
bucketwriters = {} # k: shnum, v: BucketWriter
si_dir = storage_index_to_dir(storage_index)
si_s = si_b2a(storage_index)
log.msg("storage: allocate_buckets %s" % si_s)
# in this implementation, the lease information (including secrets)
# goes into the share files themselves. It could also be put into a
# separate database. Note that the lease should not be added until
# the BucketWriter has been closed.
expire_time = time.time() + 31*24*60*60
lease_info = LeaseInfo(owner_num,
renew_secret, cancel_secret,
expire_time, self.my_nodeid)
max_space_per_bucket = allocated_size
remaining_space = self.get_available_space()
limited = remaining_space is not None
if limited:
# this is a bit conservative, since some of this allocated_size()
# has already been written to disk, where it will show up in
# get_available_space.
remaining_space -= self.allocated_size()
# self.readonly_storage causes remaining_space <= 0
# fill alreadygot with all shares that we have, not just the ones
# they asked about: this will save them a lot of work. Add or update
# leases for all of them: if they want us to hold shares for this
# file, they'll want us to hold leases for this file.
for (shnum, fn) in self._get_bucket_shares(storage_index):
alreadygot.add(shnum)
sf = ShareFile(fn)
sf.add_or_renew_lease(lease_info)
for shnum in sharenums:
incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum)
finalhome = os.path.join(self.sharedir, si_dir, "%d" % shnum)
if os.path.exists(finalhome):
# great! we already have it. easy.
pass
elif os.path.exists(incominghome):
# Note that we don't create BucketWriters for shnums that
# have a partial share (in incoming/), so if a second upload
# occurs while the first is still in progress, the second
# uploader will use different storage servers.
pass
elif (not limited) or (remaining_space >= max_space_per_bucket):
# ok! we need to create the new share file.
bw = BucketWriter(self, incominghome, finalhome,
max_space_per_bucket, lease_info, canary)
if self.no_storage:
bw.throw_out_all_data = True
bucketwriters[shnum] = bw
self._active_writers[bw] = 1
if limited:
remaining_space -= max_space_per_bucket
else:
# bummer! not enough space to accept this bucket
pass
if bucketwriters:
fileutil.make_dirs(os.path.join(self.sharedir, si_dir))
self.add_latency("allocate", time.time() - start)
return alreadygot, bucketwriters
def remote_slot_testv_and_readv_and_writev(self, storage_index, secrets,
test_and_write_vectors,
read_vector):
start = time.time()
self.count("writev")
si_s = si_b2a(storage_index)
log.msg("storage: slot_writev %s" % si_s)
si_dir = storage_index_to_dir(storage_index)
(write_enabler, renew_secret, cancel_secret) = secrets
# shares exist if there is a file for them
bucketdir = os.path.join(self.sharedir, si_dir)
shares = {}
if os.path.isdir(bucketdir):
for sharenum_s in os.listdir(bucketdir):
try:
sharenum = int(sharenum_s)
except ValueError:
continue
filename = os.path.join(bucketdir, sharenum_s)
msf = MutableShareFile(filename, self)
msf.check_write_enabler(write_enabler, si_s)
shares[sharenum] = msf
# write_enabler is good for all existing shares.
# Now evaluate test vectors.
testv_is_good = True
for sharenum in test_and_write_vectors:
(testv, datav, new_length) = test_and_write_vectors[sharenum]
if sharenum in shares:
if not shares[sharenum].check_testv(testv):
self.log("testv failed: [%d]: %r" % (sharenum, testv))
testv_is_good = False
break
else:
# compare the vectors against an empty share, in which all
# reads return empty strings.
if not EmptyShare().check_testv(testv):
self.log("testv failed (empty): [%d] %r" %
(sharenum, testv))
testv_is_good = False
break
# now gather the read vectors, before we do any writes
read_data = {}
for sharenum, share in shares.items():
read_data[sharenum] = share.readv(read_vector)
ownerid = 1 # TODO
expire_time = time.time() + 31 * 24 * 60 * 60 # one month
lease_info = LeaseInfo(ownerid, renew_secret, cancel_secret,
expire_time, self.my_nodeid)
if testv_is_good:
# now apply the write vectors
for sharenum in test_and_write_vectors:
(testv, datav, new_length) = test_and_write_vectors[sharenum]
if new_length == 0:
if sharenum in shares:
shares[sharenum].unlink()
else:
if sharenum not in shares:
# allocate a new share
allocated_size = 2000 # arbitrary, really
share = self._allocate_slot_share(bucketdir,
secrets,
sharenum,
allocated_size,
owner_num=0)
shares[sharenum] = share
shares[sharenum].writev(datav, new_length)
# and update the lease
shares[sharenum].add_or_renew_lease(lease_info)
if new_length == 0:
# delete empty bucket directories
if not os.listdir(bucketdir):
os.rmdir(bucketdir)
# all done
self.add_latency("writev", time.time() - start)
return (testv_is_good, read_data)
def remote_allocate_buckets(self,
storage_index,
renew_secret,
cancel_secret,
sharenums,
allocated_size,
canary,
owner_num=0):
# owner_num is not for clients to set, but rather it should be
# curried into the PersonalStorageServer instance that is dedicated
# to a particular owner.
start = time.time()
self.count("allocate")
alreadygot = set()
bucketwriters = {} # k: shnum, v: BucketWriter
si_dir = storage_index_to_dir(storage_index)
si_s = si_b2a(storage_index)
log.msg("storage: allocate_buckets %s" % si_s)
# in this implementation, the lease information (including secrets)
# goes into the share files themselves. It could also be put into a
# separate database. Note that the lease should not be added until
# the BucketWriter has been closed.
expire_time = time.time() + 31 * 24 * 60 * 60
lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret,
expire_time, self.my_nodeid)
max_space_per_bucket = allocated_size
remaining_space = self.get_available_space()
limited = remaining_space is not None
if limited:
# this is a bit conservative, since some of this allocated_size()
# has already been written to disk, where it will show up in
# get_available_space.
remaining_space -= self.allocated_size()
# self.readonly_storage causes remaining_space <= 0
# fill alreadygot with all shares that we have, not just the ones
# they asked about: this will save them a lot of work. Add or update
# leases for all of them: if they want us to hold shares for this
# file, they'll want us to hold leases for this file.
for (shnum, fn) in self._get_bucket_shares(storage_index):
alreadygot.add(shnum)
sf = ShareFile(fn)
sf.add_or_renew_lease(lease_info)
for shnum in sharenums:
incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum)
finalhome = os.path.join(self.sharedir, si_dir, "%d" % shnum)
if os.path.exists(finalhome):
# great! we already have it. easy.
pass
elif os.path.exists(incominghome):
# Note that we don't create BucketWriters for shnums that
# have a partial share (in incoming/), so if a second upload
# occurs while the first is still in progress, the second
# uploader will use different storage servers.
pass
elif (not limited) or (remaining_space >= max_space_per_bucket):
# ok! we need to create the new share file.
bw = BucketWriter(self, incominghome, finalhome,
max_space_per_bucket, lease_info, canary)
if self.no_storage:
bw.throw_out_all_data = True
bucketwriters[shnum] = bw
self._active_writers[bw] = 1
if limited:
remaining_space -= max_space_per_bucket
else:
# bummer! not enough space to accept this bucket
pass
if bucketwriters:
fileutil.make_dirs(os.path.join(self.sharedir, si_dir))
self.add_latency("allocate", time.time() - start)
return alreadygot, bucketwriters
def allocate_buckets(self,
storage_index,
renew_secret,
cancel_secret,
sharenums,
allocated_size,
owner_num=0,
renew_leases=True):
"""
Generic bucket allocation API.
:param bool renew_leases: If and only if this is ``True`` then renew a
secret-matching lease on (or, if none match, add a new lease to)
existing shares in this bucket. Any *new* shares are given a new
lease regardless.
"""
# owner_num is not for clients to set, but rather it should be
# curried into the PersonalStorageServer instance that is dedicated
# to a particular owner.
start = self._clock.seconds()
self.count("allocate")
alreadygot = {}
bucketwriters = {} # k: shnum, v: BucketWriter
si_dir = storage_index_to_dir(storage_index)
si_s = si_b2a(storage_index)
log.msg("storage: allocate_buckets %r" % si_s)
# in this implementation, the lease information (including secrets)
# goes into the share files themselves. It could also be put into a
# separate database. Note that the lease should not be added until
# the BucketWriter has been closed.
expire_time = self._clock.seconds() + DEFAULT_RENEWAL_TIME
lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret,
expire_time, self.my_nodeid)
max_space_per_bucket = allocated_size
remaining_space = self.get_available_space()
limited = remaining_space is not None
if limited:
# this is a bit conservative, since some of this allocated_size()
# has already been written to disk, where it will show up in
# get_available_space.
remaining_space -= self.allocated_size()
# self.readonly_storage causes remaining_space <= 0
# fill alreadygot with all shares that we have, not just the ones
# they asked about: this will save them a lot of work. Add or update
# leases for all of them: if they want us to hold shares for this
# file, they'll want us to hold leases for this file.
for (shnum, fn) in self.get_shares(storage_index):
alreadygot[shnum] = ShareFile(fn)
if renew_leases:
self._add_or_renew_leases(alreadygot.values(), lease_info)
for shnum in sharenums:
incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum)
finalhome = os.path.join(self.sharedir, si_dir, "%d" % shnum)
if os.path.exists(finalhome):
# great! we already have it. easy.
pass
elif os.path.exists(incominghome):
# For Foolscap we don't create BucketWriters for shnums that
# have a partial share (in incoming/), so if a second upload
# occurs while the first is still in progress, the second
# uploader will use different storage servers.
pass
elif (not limited) or (remaining_space >= max_space_per_bucket):
# ok! we need to create the new share file.
bw = BucketWriter(self,
incominghome,
finalhome,
max_space_per_bucket,
lease_info,
clock=self._clock)
if self.no_storage:
# Really this should be done by having a separate class for
# this situation; see
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3862
bw.throw_out_all_data = True
bucketwriters[shnum] = bw
self._bucket_writers[incominghome] = bw
if limited:
remaining_space -= max_space_per_bucket
else:
# bummer! not enough space to accept this bucket
pass
if bucketwriters:
fileutil.make_dirs(os.path.join(self.sharedir, si_dir))
self.add_latency("allocate", self._clock.seconds() - start)
return set(alreadygot), bucketwriters
