class Storage(object): def __init__(self, config, filepool, save_path, files, add_task, external_add_task, doneflag): self.filepool = filepool self.config = config self.doneflag = doneflag self.add_task = add_task self.external_add_task = external_add_task self.initialize(save_path, files) def initialize(self, save_path, files): # a list of bytes ranges and filenames for window-based IO self.ranges = [] # a dict of filename-to-ranges for piece priorities and filename lookup self.range_by_name = {} # a sparse set for smart allocation detection self.allocated_regions = SparseSet() # dict of filename-to-length on disk (for % complete in the file view) self.undownloaded = {} self.save_path = save_path # Rather implement this as an ugly hack here than change all the # individual calls. Affects all torrent instances using this module. if self.config['bad_libc_workaround']: bad_libc_workaround() self.initialized = False self.startup_df = ThreadedDeferred(wrap_task(self.external_add_task), self._build_file_structs, self.filepool, files) return self.startup_df def _build_file_structs(self, filepool, files): total = 0 for filename, length in files: # we're shutting down, abort. if self.doneflag.isSet(): return False self.undownloaded[filename] = length if length > 0: self.ranges.append((total, total + length, filename)) self.range_by_name[filename] = (total, total + length) if os.path.exists(filename): if not os.path.isfile(filename): raise BTFailure(_("File %s already exists, but is not a " "regular file") % filename) l = os.path.getsize(filename) if l > length: # This is the truncation Bram was talking about that no one # else thinks is a good idea. #h = file(filename, 'rb+') #make_file_sparse(filename, h, length) #h.truncate(length) #h.close() l = length a = get_allocated_regions(filename, begin=0, length=l) if a is not None: a.offset(total) else: a = SparseSet() if l > 0: a.add(total, total + l) self.allocated_regions += a total += length self.total_length = total self.initialized = True return True def get_byte_range_for_filename(self, filename): if filename not in self.range_by_name: filename = os.path.normpath(filename) filename = os.path.join(self.save_path, filename) return self.range_by_name[filename] def was_preallocated(self, pos, length): return self.allocated_regions.is_range_in(pos, pos+length) def get_total_length(self): return self.total_length def _intervals(self, pos, amount): r = [] stop = pos + amount p = max(bisect_right(self.ranges, (pos, 2 ** 500)) - 1, 0) for begin, end, filename in self.ranges[p:]: if begin >= stop: break r.append((filename, max(pos, begin) - begin, min(end, stop) - begin)) return r def _file_op(self, filename, pos, param, write): begin, end = self.get_byte_range_for_filename(filename) length = end - begin hdf = self.filepool.acquire_handle(filename, for_write=write, length=length) def op(h): h.seek(pos) if write: odf = h.write(param) else: odf = h.read(param) def like_finally(r): self.filepool.release_handle(filename, h) return r odf.addBoth(like_finally) return odf hdf.addCallback(op) return hdf def _batch_read(self, pos, amount): dfs = [] r = [] # queue all the reads for filename, pos, end in self._intervals(pos, amount): df = self._file_op(filename, pos, end - pos, write=False) dfs.append(df) # yield on all the reads in order - they complete in any order exc = None for df in dfs: yield df try: r.append(df.getResult()) except: exc = exc or sys.exc_info() if exc: raise exc[0], exc[1], exc[2] r = ''.join(r) if len(r) != amount: raise BTFailure(_("Short read (%d of %d) - " "something truncated files?") % (len(r), amount)) yield r def read(self, pos, amount): df = launch_coroutine(wrap_task(self.add_task), self._batch_read, pos, amount) return df def _batch_write(self, pos, s): dfs = [] total = 0 amount = len(s) # queue all the writes for filename, begin, end in self._intervals(pos, amount): length = end - begin assert length > 0, '%s %s' % (pos, amount) d = buffer(s, total, length) total += length df = self._file_op(filename, begin, d, write=True) dfs.append(df) assert total == amount, '%s and %s' % (total, amount) written = 0 # yield on all the writes - they complete in any order exc = None for df in dfs: yield df try: written += df.getResult() except: exc = exc or sys.exc_info() if exc: raise exc[0], exc[1], exc[2] assert total == written, '%s and %s' % (total, written) yield total def write(self, pos, s): df = launch_coroutine(wrap_task(self.add_task), self._batch_write, pos, s) return df def close(self): if not self.initialized: def post_init(r): return self.filepool.close_files(self.range_by_name) self.startup_df.addCallback(post_init) return self.startup_df df = self.filepool.close_files(self.range_by_name) return df def downloaded(self, pos, length): for filename, begin, end in self._intervals(pos, length): self.undownloaded[filename] -= end - begin
def main(): _t = time.clock() s = SparseSet() def blank(): #print "-" * 79 s._begins = [] s._ends = [] def reset(): #print "-" * 79 s._begins = [ 1, 10, 25, 300, ] s._ends = [ 3, 15, 45, 1000, ] def test(l): a = zip(s._begins, s._ends) assert a == l, str(a) + " is not " + str(l) reset() s.add(2, 24) test([(1, 24), (25, 45), (300, 1000)]) reset() s.add(4, 27) test([(1, 3), (4, 45), (300, 1000)]) reset() s.add(4, 24) test([(1, 3), (4, 24), (25, 45), (300, 1000)]) reset() s.add(4, 23) test([(1, 3), (4, 23), (25, 45), (300, 1000)]) reset() s.add(4, 7) test([(1, 3), (4, 7), (10, 15), (25, 45), (300, 1000)]) reset() s.add(4, 46) test([(1, 3), (4, 46), (300, 1000)]) blank() s.add_range(range(1, 3)) s.add_range(range(10, 15)) s.add_range(range(25, 45)) s.add_range(range(300, 1000)) s.add(4, 46) test([(1, 3), (4, 46), (300, 1000)]) blank() s.add_range(range(1, 3)) s.add_range(range(10, 15)) s.add_range(range(25, 45)) s.add_range(range(0)) s.add_range(range(300, 1000)) s.add(4, 46) test([(1, 3), (4, 46), (300, 1000)]) blank() s.add_range(range(1, 3)) s.add_range(range(10, 15)) s.add_range(range(25, 45)) s.add_range(range(1)) s.add_range(range(300, 1000)) s.add(4, 46) test([(0, 3), (4, 46), (300, 1000)]) blank() s.add_range(range(1, 3)) s.add_range(range(10, 15)) s.add_range(range(25, 45)) s.add_range(range(300, 1000)) for i in xrange(1, 3): assert i in s, str(i) + " is in " + str(s) assert not (i+1) in s for i in xrange(10, 15): assert i in s, str(i) + " is in " + str(s) assert not (i+1) in s for i in xrange(300, 1000): assert i in s, str(i) + " is in " + str(s) assert not (i+1) in s assert s.is_range_in(1, 3) assert not s.is_range_in(1, 3 + 1) assert s.is_range_in(300, 1000) assert not s.is_range_in(300 - 1, 1000) assert not s.is_range_in(300, 2000) assert s.is_range_in(300, 700) reset() s.add(2, 700) test([(1, 1000)]) blank() s.add(0, 10) test([(0, 10)]) s.add(-2, 1) test([(-2, 10)]) def reset2(): blank() s.add(0, 10) s.add(20, 30) s.add(40, 50) s.add(60, 70) reset2() s.add(0, 70) test([(0, 70)]) reset2() s.add(1, 70) test([(0, 70)]) reset2() s.add(0, 69) test([(0, 70)]) reset2() s.add(-1, 70) test([(-1, 70)]) reset2() s.add(0, 71) test([(0, 71)]) reset2() s.add(15, 55) test([(0, 10), (15, 55), (60, 70)]) blank() try: s.add(1, 1) except ValueError: pass else: assert False test([]) blank() try: s.add(1, 0) except ValueError: pass else: assert False test([]) blank() try: s.add(1, 2) except ValueError: assert False test([(1, 2)]) blank() s.add(1.5, 3.7) s.add(2.5, 4.7) blank() s.add(1, 3) s.add(2, 4) test([(1, 4)]) blank() s.add(2, 4) s.add(1, 3) test([(1, 4)]) blank() s.add(0, 2) s.add(2, 4) test([(0, 4)]) blank() s.add(2, 4) s.add(0, 2) test([(0, 4)]) blank() s.add(2, 3) s.add(0, 1) test([(0, 1), (2, 3)]) blank() s.add(0, 1) s.add(2, 4) test([(0, 1), (2, 4)]) blank() from random import shuffle l = range(0, 11) shuffle(l) for i in l: s.add(i, i+1) del l def testy_thing(d): blank() for i in d: s.add(i) test([(0, 5)]) def testy_thing2(d): blank() for i in d: s.add(i*2) test([(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)]) all = [] def xcombinations(items, n): if n == 0: yield [] else: for i in xrange(len(items)): for cc in xcombinations(items[:i] + items[i+1:], n - 1): yield [items[i]] + cc for uc in xcombinations(range(5), 5): all.append(uc) for d in all: testy_thing(d) for d in all: testy_thing2(d) blank() s.add(0, 1000) s.subtract(200, 500) test([(0, 200), (500, 1000)]) #blank() s.subtract(200, 500) test([(0, 200), (500, 1000)]) s.subtract(100, 201) test([(0, 100), (500, 1000)]) s.subtract(300, 500) test([(0, 100), (500, 1000)]) s.subtract(30, 50) test([(0, 30), (50, 100), (500, 1000)]) s.subtract(29, 1001) test([(0, 29)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(-1, 900) test([(900, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(29, 502) test([(0, 29), (502, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(35, 200) test([(0, 30), (501, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(55, 601) test([(0, 30), (51, 55), (601, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(25, 70) test([(0, 25), (70, 100), (501, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.add(2000, 10000) s.subtract(25, 502) test([(0, 25), (502, 1000), (2000, 10000)]) blank() s.add(0, 30) s.add(51, 100) s.add(102, 189) s.add(501, 1000) s.add(2000, 10000) s.subtract(25, 502) test([(0, 25), (502, 1000), (2000, 10000)]) blank() s.add(0, 30) s.add(51, 100) s.add(102, 189) s.add(501, 1000) s.subtract(25, 1000) test([(0, 25)]) blank() s.add(0, 30) s.add(51, 100) s.add(102, 189) s.add(501, 1000) s.subtract_range(range(25, 1000)) test([(0, 25)]) blank() s.add(0, 30) s.add(51, 100) s.add(102, 189) s.add(501, 1000) s.subtract(52, 999) test([(0, 30), (51, 52), (999, 1000)]) blank() import random all = [] assert len(s._begins) == 0 for i in range(0, 10): b = random.randint(0, 10000) l = random.randint(1, 1000) all.append((b, b+l)) s.add_range(range(b, b+l)) for b, e in all: s.subtract(b, e) assert len(s._begins) == 0 blank() s.add(0, 100) s.add(1000, 2000) assert s[-1] == 1999 assert s[0] == 0 assert s[99] == 99 assert s[100] == 1000 assert s[101] == 1001 blank() s.add(-10, -5) s.add(0, 100) assert s[0] == -10 assert s[10] == 5 assert s[-1] == 99 blank() s.add(0, 100) s.add(1000, 1100) f = range(0, 100) + range(1000, 1100) for i in s: assert i == f.pop(0) blank() s.add(0, 100) s.add(1000, 1100) f = range(0, 100) + range(1000, 1100) for i in s.iterneg(0, 1100): assert i not in f blank() s.add(0, 100) y = range(0, 100) n = range(100, 200) for i in s.iterneg(0, 200): assert i not in y assert i in n blank() s.add(100, 200) y = range(100, 200) n = range(0, 100) for i in s.iterneg(0, 200): assert i not in y assert i in n s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet(s) assert t == s assert not (t != s) assert id(t) != id(s) s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet() t.add(20, 500) t.add(1000, 10000) o = SparseSet(t) n = t - s assert t == o assert t._begins == o._begins assert t._ends == o._ends assert n != t s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet() t.add(20, 500) t.add(1000, 10000) o = SparseSet(t) n = t + s assert t == o assert t._begins == o._begins assert t._ends == o._ends assert n != t s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet() t.add(2, 50) t.add(100, 10000) assert t != s s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet() t.add(20, 500) t.add(1000, 10000) n = t - s t -= s assert n == t, '%s %s' % (n, t) print "passed all tests in", time.clock() - _t
class BtStorage(object): def __init__(self, filepool, save_path, files): self.filepool = filepool self.initialize(save_path, files) def initialize(self, save_path, files): # a list of bytes ranges and filenames for window-based IO self.ranges = [] # a dict of filename-to-ranges for piece priorities and filename lookup self.range_by_name = {} # a sparse set for smart allocation detection self.allocated_regions = SparseSet() # dict of filename-to-length on disk (for % complete in the file view) self.undownloaded = {} self.save_path = save_path # Rather implement this as an ugly hack here than change all the # individual calls. Affects all torrent instances using this module. #bad_libc_workaround() self.initialized = False return self._build_file_structs(self.filepool, files) def _build_file_structs(self, filepool, files): total = 0 for filename, length in files: self.undownloaded[filename] = length if length > 0: self.ranges.append((total, total + length, filename)) self.range_by_name[filename] = (total, total + length) if os.path.exists(filename): if not os.path.isfile(filename): raise BTFailure(_("File %s already exists, but is not a " "regular file") % filename) l = os.path.getsize(filename) if l > length: # This is the truncation Bram was talking about that no one # else thinks is a good idea. #h = file(filename, 'rb+') #make_file_sparse(filename, h, length) #h.truncate(length) #h.close() l = length a = get_allocated_regions(filename, begin=0, length=l) if a is not None: a.offset(total) else: a = SparseSet() if l > 0: a.add(total, total + l) self.allocated_regions += a total += length self.total_length = total self.initialized = True return True def get_byte_range_for_filename(self, filename): if filename not in self.range_by_name: filename = os.path.normpath(filename) filename = os.path.join(self.save_path, filename) return self.range_by_name[filename] def was_preallocated(self, pos, length): return self.allocated_regions.is_range_in(pos, pos+length) def get_total_length(self): return self.total_length def _intervals(self, pos, amount): r = [] stop = pos + amount p = max(bisect_right(self.ranges, (pos, 2 ** 500)) - 1, 0) for begin, end, filename in self.ranges[p:]: if begin >= stop: break r.append((filename, max(pos, begin) - begin, min(end, stop) - begin)) return r def _read(self, filename, pos, amount): begin, end = self.get_byte_range_for_filename(filename) length = end - begin h = self.filepool.acquire_handle(filename, for_write=False, length=length) if h is None: return try: h.seek(pos) r = h.read(amount) finally: self.filepool.release_handle(filename, h) return r def _batch_read(self, pos, amount): dfs = [] r = [] # queue all the reads for filename, pos, end in self._intervals(pos, amount): r.append(self._read(filename, pos, end - pos)) r = ''.join(r) if len(r) != amount: raise BTFailure(_("Short read (%d of %d) - something truncated files?") % (len(r), amount)) return r def read(self, pos, amount): return self._batch_read(pos, amount) def _write(self, filename, pos, s): begin, end = self.get_byte_range_for_filename(filename) length = end - begin h = self.filepool.acquire_handle(filename, for_write=True, length=length) if h is None: return try: h.seek(pos) h.write(s) finally: self.filepool.release_handle(filename, h) return len(s) def _batch_write(self, pos, s): dfs = [] total = 0 amount = len(s) # queue all the writes for filename, begin, end in self._intervals(pos, amount): length = end - begin d = buffer(s, total, length) total += length self._write(filename, begin, d) return total def write(self, pos, s): return self._batch_write(pos, s) def close(self): self.filepool.close_files(self.range_by_name) def downloaded(self, pos, length): for filename, begin, end in self._intervals(pos, length): self.undownloaded[filename] -= end - begin