class Storage(object):
def __init__(self, config, filepool, save_path, files, add_task,
external_add_task, doneflag):
self.filepool = filepool
self.config = config
self.doneflag = doneflag
self.add_task = add_task
self.external_add_task = external_add_task
self.initialize(save_path, files)
def initialize(self, save_path, files):
# a list of bytes ranges and filenames for window-based IO
self.ranges = []
# a dict of filename-to-ranges for piece priorities and filename lookup
self.range_by_name = {}
# a sparse set for smart allocation detection
self.allocated_regions = SparseSet()
# dict of filename-to-length on disk (for % complete in the file view)
self.undownloaded = {}
self.save_path = save_path
# Rather implement this as an ugly hack here than change all the
# individual calls. Affects all torrent instances using this module.
if self.config['bad_libc_workaround']:
bad_libc_workaround()
self.initialized = False
self.startup_df = ThreadedDeferred(wrap_task(self.external_add_task),
self._build_file_structs,
self.filepool, files)
return self.startup_df
def _build_file_structs(self, filepool, files):
total = 0
for filename, length in files:
# we're shutting down, abort.
if self.doneflag.isSet():
return False
self.undownloaded[filename] = length
if length > 0:
self.ranges.append((total, total + length, filename))
self.range_by_name[filename] = (total, total + length)
if os.path.exists(filename):
if not os.path.isfile(filename):
raise BTFailure(_("File %s already exists, but is not a "
"regular file") % filename)
l = os.path.getsize(filename)
if l > length:
# This is the truncation Bram was talking about that no one
# else thinks is a good idea.
#h = file(filename, 'rb+')
#make_file_sparse(filename, h, length)
#h.truncate(length)
#h.close()
l = length
a = get_allocated_regions(filename, begin=0, length=l)
if a is not None:
a.offset(total)
else:
a = SparseSet()
if l > 0:
a.add(total, total + l)
self.allocated_regions += a
total += length
self.total_length = total
self.initialized = True
return True
def get_byte_range_for_filename(self, filename):
if filename not in self.range_by_name:
filename = os.path.normpath(filename)
filename = os.path.join(self.save_path, filename)
return self.range_by_name[filename]
def was_preallocated(self, pos, length):
return self.allocated_regions.is_range_in(pos, pos+length)
def get_total_length(self):
return self.total_length
def _intervals(self, pos, amount):
r = []
stop = pos + amount
p = max(bisect_right(self.ranges, (pos, 2 ** 500)) - 1, 0)
for begin, end, filename in self.ranges[p:]:
if begin >= stop:
break
r.append((filename,
max(pos, begin) - begin, min(end, stop) - begin))
return r
def _file_op(self, filename, pos, param, write):
begin, end = self.get_byte_range_for_filename(filename)
length = end - begin
hdf = self.filepool.acquire_handle(filename, for_write=write,
length=length)
def op(h):
h.seek(pos)
if write:
odf = h.write(param)
else:
odf = h.read(param)
def like_finally(r):
self.filepool.release_handle(filename, h)
return r
odf.addBoth(like_finally)
return odf
hdf.addCallback(op)
return hdf
def _batch_read(self, pos, amount):
dfs = []
r = []
# queue all the reads
for filename, pos, end in self._intervals(pos, amount):
df = self._file_op(filename, pos, end - pos, write=False)
dfs.append(df)
# yield on all the reads in order - they complete in any order
exc = None
for df in dfs:
yield df
try:
r.append(df.getResult())
except:
exc = exc or sys.exc_info()
if exc:
raise exc[0], exc[1], exc[2]
r = ''.join(r)
if len(r) != amount:
raise BTFailure(_("Short read (%d of %d) - "
"something truncated files?") %
(len(r), amount))
yield r
def read(self, pos, amount):
df = launch_coroutine(wrap_task(self.add_task),
self._batch_read, pos, amount)
return df
def _batch_write(self, pos, s):
dfs = []
total = 0
amount = len(s)
# queue all the writes
for filename, begin, end in self._intervals(pos, amount):
length = end - begin
assert length > 0, '%s %s' % (pos, amount)
d = buffer(s, total, length)
total += length
df = self._file_op(filename, begin, d, write=True)
dfs.append(df)
assert total == amount, '%s and %s' % (total, amount)
written = 0
# yield on all the writes - they complete in any order
exc = None
for df in dfs:
yield df
try:
written += df.getResult()
except:
exc = exc or sys.exc_info()
if exc:
raise exc[0], exc[1], exc[2]
assert total == written, '%s and %s' % (total, written)
yield total
def write(self, pos, s):
df = launch_coroutine(wrap_task(self.add_task),
self._batch_write, pos, s)
return df
def close(self):
if not self.initialized:
def post_init(r):
return self.filepool.close_files(self.range_by_name)
self.startup_df.addCallback(post_init)
return self.startup_df
df = self.filepool.close_files(self.range_by_name)
return df
def downloaded(self, pos, length):
for filename, begin, end in self._intervals(pos, length):
self.undownloaded[filename] -= end - begin
def main():
_t = time.clock()
s = SparseSet()
def blank():
#print "-" * 79
s._begins = []
s._ends = []
def reset():
#print "-" * 79
s._begins = [ 1,
10,
25,
300,
]
s._ends = [ 3,
15,
45,
1000,
]
def test(l):
a = zip(s._begins, s._ends)
assert a == l, str(a) + " is not " + str(l)
reset()
s.add(2, 24)
test([(1, 24), (25, 45), (300, 1000)])
reset()
s.add(4, 27)
test([(1, 3), (4, 45), (300, 1000)])
reset()
s.add(4, 24)
test([(1, 3), (4, 24), (25, 45), (300, 1000)])
reset()
s.add(4, 23)
test([(1, 3), (4, 23), (25, 45), (300, 1000)])
reset()
s.add(4, 7)
test([(1, 3), (4, 7), (10, 15), (25, 45), (300, 1000)])
reset()
s.add(4, 46)
test([(1, 3), (4, 46), (300, 1000)])
blank()
s.add_range(range(1, 3))
s.add_range(range(10, 15))
s.add_range(range(25, 45))
s.add_range(range(300, 1000))
s.add(4, 46)
test([(1, 3), (4, 46), (300, 1000)])
blank()
s.add_range(range(1, 3))
s.add_range(range(10, 15))
s.add_range(range(25, 45))
s.add_range(range(0))
s.add_range(range(300, 1000))
s.add(4, 46)
test([(1, 3), (4, 46), (300, 1000)])
blank()
s.add_range(range(1, 3))
s.add_range(range(10, 15))
s.add_range(range(25, 45))
s.add_range(range(1))
s.add_range(range(300, 1000))
s.add(4, 46)
test([(0, 3), (4, 46), (300, 1000)])
blank()
s.add_range(range(1, 3))
s.add_range(range(10, 15))
s.add_range(range(25, 45))
s.add_range(range(300, 1000))
for i in xrange(1, 3):
assert i in s, str(i) + " is in " + str(s)
assert not (i+1) in s
for i in xrange(10, 15):
assert i in s, str(i) + " is in " + str(s)
assert not (i+1) in s
for i in xrange(300, 1000):
assert i in s, str(i) + " is in " + str(s)
assert not (i+1) in s
assert s.is_range_in(1, 3)
assert not s.is_range_in(1, 3 + 1)
assert s.is_range_in(300, 1000)
assert not s.is_range_in(300 - 1, 1000)
assert not s.is_range_in(300, 2000)
assert s.is_range_in(300, 700)
reset()
s.add(2, 700)
test([(1, 1000)])
blank()
s.add(0, 10)
test([(0, 10)])
s.add(-2, 1)
test([(-2, 10)])
def reset2():
blank()
s.add(0, 10)
s.add(20, 30)
s.add(40, 50)
s.add(60, 70)
reset2()
s.add(0, 70)
test([(0, 70)])
reset2()
s.add(1, 70)
test([(0, 70)])
reset2()
s.add(0, 69)
test([(0, 70)])
reset2()
s.add(-1, 70)
test([(-1, 70)])
reset2()
s.add(0, 71)
test([(0, 71)])
reset2()
s.add(15, 55)
test([(0, 10), (15, 55), (60, 70)])
blank()
try:
s.add(1, 1)
except ValueError:
pass
else:
assert False
test([])
blank()
try:
s.add(1, 0)
except ValueError:
pass
else:
assert False
test([])
blank()
try:
s.add(1, 2)
except ValueError:
assert False
test([(1, 2)])
blank()
s.add(1.5, 3.7)
s.add(2.5, 4.7)
blank()
s.add(1, 3)
s.add(2, 4)
test([(1, 4)])
blank()
s.add(2, 4)
s.add(1, 3)
test([(1, 4)])
blank()
s.add(0, 2)
s.add(2, 4)
test([(0, 4)])
blank()
s.add(2, 4)
s.add(0, 2)
test([(0, 4)])
blank()
s.add(2, 3)
s.add(0, 1)
test([(0, 1), (2, 3)])
blank()
s.add(0, 1)
s.add(2, 4)
test([(0, 1), (2, 4)])
blank()
from random import shuffle
l = range(0, 11)
shuffle(l)
for i in l:
s.add(i, i+1)
del l
def testy_thing(d):
blank()
for i in d:
s.add(i)
test([(0, 5)])
def testy_thing2(d):
blank()
for i in d:
s.add(i*2)
test([(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)])
all = []
def xcombinations(items, n):
if n == 0:
yield []
else:
for i in xrange(len(items)):
for cc in xcombinations(items[:i] + items[i+1:], n - 1):
yield [items[i]] + cc
for uc in xcombinations(range(5), 5):
all.append(uc)
for d in all:
testy_thing(d)
for d in all:
testy_thing2(d)
blank()
s.add(0, 1000)
s.subtract(200, 500)
test([(0, 200), (500, 1000)])
#blank()
s.subtract(200, 500)
test([(0, 200), (500, 1000)])
s.subtract(100, 201)
test([(0, 100), (500, 1000)])
s.subtract(300, 500)
test([(0, 100), (500, 1000)])
s.subtract(30, 50)
test([(0, 30), (50, 100), (500, 1000)])
s.subtract(29, 1001)
test([(0, 29)])
blank()
s.add(0, 30)
s.add(51, 100)
s.add(501, 1000)
s.subtract(-1, 900)
test([(900, 1000)])
blank()
s.add(0, 30)
s.add(51, 100)
s.add(501, 1000)
s.subtract(29, 502)
test([(0, 29), (502, 1000)])
blank()
s.add(0, 30)
s.add(51, 100)
s.add(501, 1000)
s.subtract(35, 200)
test([(0, 30), (501, 1000)])
blank()
s.add(0, 30)
s.add(51, 100)
s.add(501, 1000)
s.subtract(55, 601)
test([(0, 30), (51, 55), (601, 1000)])
blank()
s.add(0, 30)
s.add(51, 100)
s.add(501, 1000)
s.subtract(25, 70)
test([(0, 25), (70, 100), (501, 1000)])
blank()
s.add(0, 30)
s.add(51, 100)
s.add(501, 1000)
s.add(2000, 10000)
s.subtract(25, 502)
test([(0, 25), (502, 1000), (2000, 10000)])
blank()
s.add(0, 30)
s.add(51, 100)
s.add(102, 189)
s.add(501, 1000)
s.add(2000, 10000)
s.subtract(25, 502)
test([(0, 25), (502, 1000), (2000, 10000)])
blank()
s.add(0, 30)
s.add(51, 100)
s.add(102, 189)
s.add(501, 1000)
s.subtract(25, 1000)
test([(0, 25)])
blank()
s.add(0, 30)
s.add(51, 100)
s.add(102, 189)
s.add(501, 1000)
s.subtract_range(range(25, 1000))
test([(0, 25)])
blank()
s.add(0, 30)
s.add(51, 100)
s.add(102, 189)
s.add(501, 1000)
s.subtract(52, 999)
test([(0, 30), (51, 52), (999, 1000)])
blank()
import random
all = []
assert len(s._begins) == 0
for i in range(0, 10):
b = random.randint(0, 10000)
l = random.randint(1, 1000)
all.append((b, b+l))
s.add_range(range(b, b+l))
for b, e in all:
s.subtract(b, e)
assert len(s._begins) == 0
blank()
s.add(0, 100)
s.add(1000, 2000)
assert s[-1] == 1999
assert s[0] == 0
assert s[99] == 99
assert s[100] == 1000
assert s[101] == 1001
blank()
s.add(-10, -5)
s.add(0, 100)
assert s[0] == -10
assert s[10] == 5
assert s[-1] == 99
blank()
s.add(0, 100)
s.add(1000, 1100)
f = range(0, 100) + range(1000, 1100)
for i in s:
assert i == f.pop(0)
blank()
s.add(0, 100)
s.add(1000, 1100)
f = range(0, 100) + range(1000, 1100)
for i in s.iterneg(0, 1100):
assert i not in f
blank()
s.add(0, 100)
y = range(0, 100)
n = range(100, 200)
for i in s.iterneg(0, 200):
assert i not in y
assert i in n
blank()
s.add(100, 200)
y = range(100, 200)
n = range(0, 100)
for i in s.iterneg(0, 200):
assert i not in y
assert i in n
s = SparseSet()
s.add(2, 50)
s.add(100, 1000)
t = SparseSet(s)
assert t == s
assert not (t != s)
assert id(t) != id(s)
s = SparseSet()
s.add(2, 50)
s.add(100, 1000)
t = SparseSet()
t.add(20, 500)
t.add(1000, 10000)
o = SparseSet(t)
n = t - s
assert t == o
assert t._begins == o._begins
assert t._ends == o._ends
assert n != t
s = SparseSet()
s.add(2, 50)
s.add(100, 1000)
t = SparseSet()
t.add(20, 500)
t.add(1000, 10000)
o = SparseSet(t)
n = t + s
assert t == o
assert t._begins == o._begins
assert t._ends == o._ends
assert n != t
s = SparseSet()
s.add(2, 50)
s.add(100, 1000)
t = SparseSet()
t.add(2, 50)
t.add(100, 10000)
assert t != s
s = SparseSet()
s.add(2, 50)
s.add(100, 1000)
t = SparseSet()
t.add(20, 500)
t.add(1000, 10000)
n = t - s
t -= s
assert n == t, '%s %s' % (n, t)
print "passed all tests in", time.clock() - _t
class BtStorage(object):
def __init__(self, filepool, save_path, files):
self.filepool = filepool
self.initialize(save_path, files)
def initialize(self, save_path, files):
# a list of bytes ranges and filenames for window-based IO
self.ranges = []
# a dict of filename-to-ranges for piece priorities and filename lookup
self.range_by_name = {}
# a sparse set for smart allocation detection
self.allocated_regions = SparseSet()
# dict of filename-to-length on disk (for % complete in the file view)
self.undownloaded = {}
self.save_path = save_path
# Rather implement this as an ugly hack here than change all the
# individual calls. Affects all torrent instances using this module.
#bad_libc_workaround()
self.initialized = False
return self._build_file_structs(self.filepool, files)
def _build_file_structs(self, filepool, files):
total = 0
for filename, length in files:
self.undownloaded[filename] = length
if length > 0:
self.ranges.append((total, total + length, filename))
self.range_by_name[filename] = (total, total + length)
if os.path.exists(filename):
if not os.path.isfile(filename):
raise BTFailure(_("File %s already exists, but is not a "
"regular file") % filename)
l = os.path.getsize(filename)
if l > length:
# This is the truncation Bram was talking about that no one
# else thinks is a good idea.
#h = file(filename, 'rb+')
#make_file_sparse(filename, h, length)
#h.truncate(length)
#h.close()
l = length
a = get_allocated_regions(filename, begin=0, length=l)
if a is not None:
a.offset(total)
else:
a = SparseSet()
if l > 0:
a.add(total, total + l)
self.allocated_regions += a
total += length
self.total_length = total
self.initialized = True
return True
def get_byte_range_for_filename(self, filename):
if filename not in self.range_by_name:
filename = os.path.normpath(filename)
filename = os.path.join(self.save_path, filename)
return self.range_by_name[filename]
def was_preallocated(self, pos, length):
return self.allocated_regions.is_range_in(pos, pos+length)
def get_total_length(self):
return self.total_length
def _intervals(self, pos, amount):
r = []
stop = pos + amount
p = max(bisect_right(self.ranges, (pos, 2 ** 500)) - 1, 0)
for begin, end, filename in self.ranges[p:]:
if begin >= stop:
break
r.append((filename, max(pos, begin) - begin, min(end, stop) - begin))
return r
def _read(self, filename, pos, amount):
begin, end = self.get_byte_range_for_filename(filename)
length = end - begin
h = self.filepool.acquire_handle(filename, for_write=False, length=length)
if h is None:
return
try:
h.seek(pos)
r = h.read(amount)
finally:
self.filepool.release_handle(filename, h)
return r
def _batch_read(self, pos, amount):
dfs = []
r = []
# queue all the reads
for filename, pos, end in self._intervals(pos, amount):
r.append(self._read(filename, pos, end - pos))
r = ''.join(r)
if len(r) != amount:
raise BTFailure(_("Short read (%d of %d) - something truncated files?") %
(len(r), amount))
return r
def read(self, pos, amount):
return self._batch_read(pos, amount)
def _write(self, filename, pos, s):
begin, end = self.get_byte_range_for_filename(filename)
length = end - begin
h = self.filepool.acquire_handle(filename, for_write=True, length=length)
if h is None:
return
try:
h.seek(pos)
h.write(s)
finally:
self.filepool.release_handle(filename, h)
return len(s)
def _batch_write(self, pos, s):
dfs = []
total = 0
amount = len(s)
# queue all the writes
for filename, begin, end in self._intervals(pos, amount):
length = end - begin
d = buffer(s, total, length)
total += length
self._write(filename, begin, d)
return total
def write(self, pos, s):
return self._batch_write(pos, s)
def close(self):
self.filepool.close_files(self.range_by_name)
def downloaded(self, pos, length):
for filename, begin, end in self._intervals(pos, length):
self.undownloaded[filename] -= end - begin
