def safe_register_port(self, new_mapping): # check for the host now, while we're in the thread and before # we need to read it. new_mapping.populate_host() nat_logger.info("You asked for: " + str(new_mapping)) new_mapping.original_external_port = new_mapping.external_port mappings = self._list_ports() used_ports = [] for mapping in mappings: # only consider ports which match the same protocol if mapping.protocol == new_mapping.protocol: # look for exact matches if (mapping.host == new_mapping.host and mapping.internal_port == new_mapping.internal_port): # the service name could not match, that's ok. new_mapping.d.callback(mapping.external_port) nat_logger.info("Already effectively mapped: " + str(mapping)) return # otherwise, add it to the list of used external ports used_ports.append(mapping.external_port) used_ports.sort() used_ports = SparseSet(used_ports) all_ports = SparseSet() all_ports.add(1024, 65535) free_ports = all_ports - used_ports new_mapping.external_port = random.choice(free_ports) nat_logger.info("I'll give you: " + str(new_mapping)) self.register_port(new_mapping)
def SetValue(self, value, state=None, data=None, redraw=True): if data is not None: sorted_data = {} length, update, piece_states = data self.resolution = length keys = piece_states.keys() keys.sort(self.sort) pos = 0 h = piece_states.get('h', SparseSet()) t = piece_states.get('t', set()) t = list(t) t.sort() have_trans_sparse_set = h + t for k in keys: p = piece_states[k] if k in ('h', 't'): count = len(p) else: count = 0 # OW for i in p: if i not in have_trans_sparse_set: count += 1 if not count: continue newpos = pos + count s = SparseSet() s.add(pos, newpos) sorted_data[k] = s pos = newpos data = (length, update, sorted_data) FancyDownloadGauge.SetValue(self, value, state, data, redraw)
def _build_file_structs(self, filepool, files): total = 0 for filename, length in files: self.undownloaded[filename] = length if length > 0: self.ranges.append((total, total + length, filename)) self.range_by_name[filename] = (total, total + length) if os.path.exists(filename): if not os.path.isfile(filename): raise BTFailure(_("File %s already exists, but is not a " "regular file") % filename) l = os.path.getsize(filename) if l > length: # This is the truncation Bram was talking about that no one # else thinks is a good idea. #h = file(filename, 'rb+') #make_file_sparse(filename, h, length) #h.truncate(length) #h.close() l = length a = get_allocated_regions(filename, begin=0, length=l) if a is not None: a.offset(total) else: a = SparseSet() if l > 0: a.add(total, total + l) self.allocated_regions += a total += length self.total_length = total self.initialized = True return True
def get_allocated_regions(path, f=None, begin=0, length=None): supported = get_sparse_files_support(path) if not supported: return if os.name == 'nt': if not os.path.exists(path): return False if f is None: f = file(path, 'r') handle = win32file._get_osfhandle(f.fileno()) if length is None: length = os.path.getsize(path) - begin a = SparseSet() interval = 10000000 i = begin end = begin + length while i < end: d = struct.pack("<QQ", i, interval) try: r = win32file.DeviceIoControl(handle, FSCTL_QUERY_ALLOCATED_RANGES, d, interval, None) except pywintypes.error, e: # I've seen: # error: (1784, 'DeviceIoControl', 'The supplied user buffer is not valid for the requested operation.') return for c in xrange(0, len(r), 16): qq = struct.unpack("<QQ", r[c:c+16]) b = qq[0] e = b + qq[1] a.add(b, e) i += interval return a
def get_allocated_regions(path, f=None, begin=0, length=None): supported = get_sparse_files_support(path) if not supported: return if os.name == 'nt': if not os.path.exists(path): return False if f is None: f = file(path, 'r') handle = win32file._get_osfhandle(f.fileno()) if length is None: length = os.path.getsize(path) - begin a = SparseSet() interval = 10000000 i = begin end = begin + length while i < end: d = struct.pack("<QQ", i, interval) try: r = win32file.DeviceIoControl(handle, FSCTL_QUERY_ALLOCATED_RANGES, d, interval, None) except pywintypes.error, e: # I've seen: # error: (1784, 'DeviceIoControl', 'The supplied user buffer is not valid for the requested operation.') return for c in xrange(0, len(r), 16): qq = struct.unpack("<QQ", r[c:c + 16]) b = qq[0] e = b + qq[1] a.add(b, e) i += interval return a
def SetValue(self, value, state=None, data=None, redraw=True): if data is not None: sorted_data = {} length, update, piece_states = data self.resolution = length keys = piece_states.keys() keys.sort(self.sort) pos = 0 h = piece_states.get('h', SparseSet()) t = piece_states.get('t', set()) t = list(t) t.sort() have_trans_sparse_set = h + t for k in keys: p = piece_states[k] if k in ('h', 't'): count = len(p) else: count = 0 # OW for i in p: if i not in have_trans_sparse_set: count += 1 if not count: continue newpos = pos+count s = SparseSet() s.add(pos, newpos) sorted_data[k] = s pos = newpos data = (length, update, sorted_data) FancyDownloadGauge.SetValue(self, value, state, data, redraw)
def new_request(self, index, full=False): # returns (begin, length) if index not in self.inactive_requests: self._make_inactive(index) rs = self.inactive_requests[index] if full: s = SparseSet() while rs: r = rs.pop() s.add(r[0], r[0] + r[1]) b, e = s.largest_range() s.remove(b, e) reqs = self._break_up(b, e - b) for r in reqs: assert r[1] <= self.request_size self.active_requests[index].append(r) # I don't like this. the function should return reqs r = (b, e - b) for b, e in s.iterrange(): rs.extend(self._break_up(b, e - b)) else: # why min? do we want all the blocks in order? r = min(rs) rs.remove(r) assert r[1] <= self.request_size self.active_requests[index].append(r) self.amount_inactive -= r[1] assert self.amount_inactive >= 0, ('Amount inactive: %d' % self.amount_inactive) if len(rs) == 0: self.fully_active.add(index) if self.amount_inactive == 0: self.endgame = True assert (r[0] + r[1]) <= self._piecelen(index) return r
def add(self, piece, bucketindex): assert not self.place_in_buckets.has_key(piece) while len(self.buckets) <= bucketindex: self.buckets.append(SparseSet()) bucket = self.buckets[bucketindex] bucket.add(piece) self.place_in_buckets[piece] = bucketindex
def prepend_bucket(self): # it' possible we had everything to begin with if len(self.buckets) == 0: return self.buckets.insert(0, SparseSet()) # bleh. for piece in self.place_in_buckets: self.place_in_buckets[piece] += 1
def __init__(self, config, storage, rm, urlage, picker, numpieces, finished, errorfunc, kickfunc, banfunc, get_downrate): self.config = config self.storage = storage self.rm = rm self.urlage = urlage self.picker = picker self.errorfunc = errorfunc self.rerequester = None self.entered_endgame = False self.connection_manager = None self.chunksize = config['download_chunk_size'] self.numpieces = numpieces self.finished = finished self.snub_time = config['snub_time'] self.kickfunc = kickfunc self.banfunc = banfunc self.get_downrate = get_downrate self.downloads = [] self.perip = {} self.bad_peers = {} self.discarded_bytes = 0 self.useful_received_listeners = set() self.raw_received_listeners = set() if SPARSE_SET: self.piece_states = PieceSetBuckets() nothing = SparseSet() nothing.add(0, self.numpieces) self.piece_states.buckets.append(nothing) # I hate this nowhere = [(i, 0) for i in xrange(self.numpieces)] self.piece_states.place_in_buckets = dict(nowhere) else: typecode = resolve_typecode(self.numpieces) self.piece_states = SortedPieceBuckets(typecode) nothing = array.array(typecode, range(self.numpieces)) self.piece_states.buckets.append(nothing) # I hate this nowhere = [(i, (0, i)) for i in xrange(self.numpieces)] self.piece_states.place_in_buckets = dict(nowhere) self.last_update = 0 self.all_requests = set()
def _build_file_structs(self, filepool, files): total = 0 for filename, length in files: print "filename %s in stor tpool length %d " % (filename, length) # we're shutting down, abort. if self.doneflag.isSet(): return False self.undownloaded[filename] = length if length > 0: self.ranges.append((total, total + length, filename)) self.range_by_name[filename] = (total, total + length) if os.path.exists(filename): if not os.path.isfile(filename): raise BTFailure( _("File %s already exists, but is not a " "regular file") % filename) l = os.path.getsize(filename) print "size on disk %d length = %d" % (l, length) if l > length: # This is the truncation Bram was talking about that no one # else thinks is a good idea. #h = file(filename, 'rb+') #make_file_sparse(filename, h, length) #h.truncate(length) #h.close() l = length a = get_allocated_regions(filename, begin=0, length=l) if a is not None: a.offset(total) else: a = SparseSet() if l > 0: a.add(total, total + l) self.allocated_regions += a total += length self.total_length = total self.initialized = True return True
def get_allocated_regions(path, f=None, begin=0, length=None): supported = get_sparse_files_support(path) if not supported: return if os.name == 'nt': if not os.path.exists(path): return False if f is None: f = file(path, 'r') handle = win32file._get_osfhandle(f.fileno()) if length is None: length = os.path.getsize(path) - begin a = SparseSet() run = 128 i = begin end = begin + length while i < end: d = struct.pack("<QQ", i, length) try: r = win32file.DeviceIoControl(handle, FSCTL_QUERY_ALLOCATED_RANGES, d, struct.calcsize("<QQ") * run, None) except pywintypes.error, e: if e.args[0] == winerror.ERROR_MORE_DATA: run *= 2 continue # I've also seen: # error: (1784, 'DeviceIoControl', 'The supplied user buffer is not valid for the requested operation.') return if not r: break for c in xrange(0, len(r), 16): qq = struct.unpack("<QQ", r[c:c + 16]) b = qq[0] e = b + qq[1] a.add(b, e) i = max(i, e) return a
def initialize(self, save_path, files): # a list of bytes ranges and filenames for window-based IO self.ranges = [] # a dict of filename-to-ranges for piece priorities and filename lookup self.range_by_name = {} # a sparse set for smart allocation detection self.allocated_regions = SparseSet() # dict of filename-to-length on disk (for % complete in the file view) self.undownloaded = {} self.save_path = save_path # Rather implement this as an ugly hack here than change all the # individual calls. Affects all torrent instances using this module. if self.config['bad_libc_workaround']: bad_libc_workaround() self.initialized = False self.startup_df = ThreadedDeferred(wrap_task(self.external_add_task), self._build_file_structs, self.filepool, files) return self.startup_df
def get_allocated_regions(path, f=None, begin=0, length=None): supported = get_sparse_files_support(path) if not supported: return if os.name == 'nt': if not os.path.exists(path): return False if f is None: f = file(path, 'r') handle = win32file._get_osfhandle(f.fileno()) if length is None: length = os.path.getsize(path) - begin a = SparseSet() run = 128 i = begin end = begin + length while i < end: d = struct.pack("<QQ", i, length) try: r = win32file.DeviceIoControl(handle, FSCTL_QUERY_ALLOCATED_RANGES, d, struct.calcsize("<QQ")*run, None) except pywintypes.error, e: if e.args[0] == winerror.ERROR_MORE_DATA: run *= 2 continue # I've also seen: # error: (1784, 'DeviceIoControl', 'The supplied user buffer is not valid for the requested operation.') return if not r: break for c in xrange(0, len(r), 16): qq = struct.unpack("<QQ", r[c:c+16]) b = qq[0] e = b + qq[1] a.add(b, e) i = max(i, e) return a
def initialize(self, save_path, files): # a list of bytes ranges and filenames for window-based IO self.ranges = [] # a dict of filename-to-ranges for piece priorities and filename lookup self.range_by_name = {} # a sparse set for smart allocation detection self.allocated_regions = SparseSet() # dict of filename-to-length on disk (for % complete in the file view) self.undownloaded = {} self.save_path = save_path # Rather implement this as an ugly hack here than change all the # individual calls. Affects all torrent instances using this module. #bad_libc_workaround() self.initialized = False return self._build_file_structs(self.filepool, files)
def __init__(self, config, storage, rm, urlage, picker, numpieces, finished, errorfunc, kickfunc, banfunc, get_downrate, micropayments=False): self.config = config self.storage = storage self.rm = rm self.urlage = urlage self.picker = picker self.errorfunc = errorfunc self.rerequester = None self.entered_endgame = False self.connection_manager = None self.chunksize = config['download_chunk_size'] self.numpieces = numpieces self.finished = finished self.snub_time = config['snub_time'] self.kickfunc = kickfunc self.banfunc = banfunc self.get_downrate = get_downrate self.downloads = [] self.perip = {} self.bad_peers = {} self.discarded_bytes = 0 self.useful_received_listeners = set() self.raw_received_listeners = set() self.micropayments = micropayments #boolean on/off self.key_rewards = {} #key rewards we use to "pay" uploaders self.waiting_for_reward = { } #hash with first key peerid then [(idx1,offset1,len1),(idx2,offset2,len2) ] self.payment_key_hash_cache = { } #stores payment key hashes received from tracker self.private_key = None self.public_key = None self.certificate = None self.pk_tools = None self.public_key_tracker = None if micropayments: """load keys and check certificates""" log("micropayments=" + str(micropayments)) #check if certifcates are OK cert = self.config["micropayment_certificate"] self.ca_dir = self.config["micropayment_trusted_ca_dir"] self.pk_tools = PKTools(self.ca_dir, config['save_incomplete_in']) if not self.pk_tools.validate_certificate(cert): log("invalid certificates") return else: log("valid certificates") self.certificate = parse_PEM_certificate( open(self.config["micropayment_certificate"])) self.public_key = self.certificate.publicKey self.private_key = parse_PEM_private_key( open(self.config["micropayment_private_key"])) log("cert tracker filename=%s:" % (self.config["micropayment_tracker_certificate"])) cert_tracker = parse_PEM_certificate( open(self.config["micropayment_tracker_certificate"])) self.public_key_tracker = cert_tracker.publicKey if SPARSE_SET: self.piece_states = PieceSetBuckets() nothing = SparseSet() nothing.add(0, self.numpieces) self.piece_states.buckets.append(nothing) # I hate this nowhere = [(i, 0) for i in xrange(self.numpieces)] self.piece_states.place_in_buckets = dict(nowhere) else: typecode = resolve_typecode(self.numpieces) self.piece_states = SortedPieceBuckets(typecode) nothing = array.array(typecode, range(self.numpieces)) self.piece_states.buckets.append(nothing) # I hate this nowhere = [(i, (0, i)) for i in xrange(self.numpieces)] self.piece_states.place_in_buckets = dict(nowhere) self.last_update = 0 self.all_requests = set()
def main(): _t = time.clock() s = SparseSet() def blank(): #print "-" * 79 s._begins = [] s._ends = [] def reset(): #print "-" * 79 s._begins = [ 1, 10, 25, 300, ] s._ends = [ 3, 15, 45, 1000, ] def test(l): a = zip(s._begins, s._ends) assert a == l, str(a) + " is not " + str(l) reset() s.add(2, 24) test([(1, 24), (25, 45), (300, 1000)]) reset() s.add(4, 27) test([(1, 3), (4, 45), (300, 1000)]) reset() s.add(4, 24) test([(1, 3), (4, 24), (25, 45), (300, 1000)]) reset() s.add(4, 23) test([(1, 3), (4, 23), (25, 45), (300, 1000)]) reset() s.add(4, 7) test([(1, 3), (4, 7), (10, 15), (25, 45), (300, 1000)]) reset() s.add(4, 46) test([(1, 3), (4, 46), (300, 1000)]) blank() s.add_range(range(1, 3)) s.add_range(range(10, 15)) s.add_range(range(25, 45)) s.add_range(range(300, 1000)) s.add(4, 46) test([(1, 3), (4, 46), (300, 1000)]) blank() s.add_range(range(1, 3)) s.add_range(range(10, 15)) s.add_range(range(25, 45)) s.add_range(range(0)) s.add_range(range(300, 1000)) s.add(4, 46) test([(1, 3), (4, 46), (300, 1000)]) blank() s.add_range(range(1, 3)) s.add_range(range(10, 15)) s.add_range(range(25, 45)) s.add_range(range(1)) s.add_range(range(300, 1000)) s.add(4, 46) test([(0, 3), (4, 46), (300, 1000)]) blank() s.add_range(range(1, 3)) s.add_range(range(10, 15)) s.add_range(range(25, 45)) s.add_range(range(300, 1000)) for i in xrange(1, 3): assert i in s, str(i) + " is in " + str(s) assert not (i+1) in s for i in xrange(10, 15): assert i in s, str(i) + " is in " + str(s) assert not (i+1) in s for i in xrange(300, 1000): assert i in s, str(i) + " is in " + str(s) assert not (i+1) in s assert s.is_range_in(1, 3) assert not s.is_range_in(1, 3 + 1) assert s.is_range_in(300, 1000) assert not s.is_range_in(300 - 1, 1000) assert not s.is_range_in(300, 2000) assert s.is_range_in(300, 700) reset() s.add(2, 700) test([(1, 1000)]) blank() s.add(0, 10) test([(0, 10)]) s.add(-2, 1) test([(-2, 10)]) def reset2(): blank() s.add(0, 10) s.add(20, 30) s.add(40, 50) s.add(60, 70) reset2() s.add(0, 70) test([(0, 70)]) reset2() s.add(1, 70) test([(0, 70)]) reset2() s.add(0, 69) test([(0, 70)]) reset2() s.add(-1, 70) test([(-1, 70)]) reset2() s.add(0, 71) test([(0, 71)]) reset2() s.add(15, 55) test([(0, 10), (15, 55), (60, 70)]) blank() try: s.add(1, 1) except ValueError: pass else: assert False test([]) blank() try: s.add(1, 0) except ValueError: pass else: assert False test([]) blank() try: s.add(1, 2) except ValueError: assert False test([(1, 2)]) blank() s.add(1.5, 3.7) s.add(2.5, 4.7) blank() s.add(1, 3) s.add(2, 4) test([(1, 4)]) blank() s.add(2, 4) s.add(1, 3) test([(1, 4)]) blank() s.add(0, 2) s.add(2, 4) test([(0, 4)]) blank() s.add(2, 4) s.add(0, 2) test([(0, 4)]) blank() s.add(2, 3) s.add(0, 1) test([(0, 1), (2, 3)]) blank() s.add(0, 1) s.add(2, 4) test([(0, 1), (2, 4)]) blank() from random import shuffle l = range(0, 11) shuffle(l) for i in l: s.add(i, i+1) del l def testy_thing(d): blank() for i in d: s.add(i) test([(0, 5)]) def testy_thing2(d): blank() for i in d: s.add(i*2) test([(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)]) all = [] def xcombinations(items, n): if n == 0: yield [] else: for i in xrange(len(items)): for cc in xcombinations(items[:i] + items[i+1:], n - 1): yield [items[i]] + cc for uc in xcombinations(range(5), 5): all.append(uc) for d in all: testy_thing(d) for d in all: testy_thing2(d) blank() s.add(0, 1000) s.subtract(200, 500) test([(0, 200), (500, 1000)]) #blank() s.subtract(200, 500) test([(0, 200), (500, 1000)]) s.subtract(100, 201) test([(0, 100), (500, 1000)]) s.subtract(300, 500) test([(0, 100), (500, 1000)]) s.subtract(30, 50) test([(0, 30), (50, 100), (500, 1000)]) s.subtract(29, 1001) test([(0, 29)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(-1, 900) test([(900, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(29, 502) test([(0, 29), (502, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(35, 200) test([(0, 30), (501, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(55, 601) test([(0, 30), (51, 55), (601, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(25, 70) test([(0, 25), (70, 100), (501, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.add(2000, 10000) s.subtract(25, 502) test([(0, 25), (502, 1000), (2000, 10000)]) blank() s.add(0, 30) s.add(51, 100) s.add(102, 189) s.add(501, 1000) s.add(2000, 10000) s.subtract(25, 502) test([(0, 25), (502, 1000), (2000, 10000)]) blank() s.add(0, 30) s.add(51, 100) s.add(102, 189) s.add(501, 1000) s.subtract(25, 1000) test([(0, 25)]) blank() s.add(0, 30) s.add(51, 100) s.add(102, 189) s.add(501, 1000) s.subtract_range(range(25, 1000)) test([(0, 25)]) blank() s.add(0, 30) s.add(51, 100) s.add(102, 189) s.add(501, 1000) s.subtract(52, 999) test([(0, 30), (51, 52), (999, 1000)]) blank() import random all = [] assert len(s._begins) == 0 for i in range(0, 10): b = random.randint(0, 10000) l = random.randint(1, 1000) all.append((b, b+l)) s.add_range(range(b, b+l)) for b, e in all: s.subtract(b, e) assert len(s._begins) == 0 blank() s.add(0, 100) s.add(1000, 2000) assert s[-1] == 1999 assert s[0] == 0 assert s[99] == 99 assert s[100] == 1000 assert s[101] == 1001 blank() s.add(-10, -5) s.add(0, 100) assert s[0] == -10 assert s[10] == 5 assert s[-1] == 99 blank() s.add(0, 100) s.add(1000, 1100) f = range(0, 100) + range(1000, 1100) for i in s: assert i == f.pop(0) blank() s.add(0, 100) s.add(1000, 1100) f = range(0, 100) + range(1000, 1100) for i in s.iterneg(0, 1100): assert i not in f blank() s.add(0, 100) y = range(0, 100) n = range(100, 200) for i in s.iterneg(0, 200): assert i not in y assert i in n blank() s.add(100, 200) y = range(100, 200) n = range(0, 100) for i in s.iterneg(0, 200): assert i not in y assert i in n s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet(s) assert t == s assert not (t != s) assert id(t) != id(s) s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet() t.add(20, 500) t.add(1000, 10000) o = SparseSet(t) n = t - s assert t == o assert t._begins == o._begins assert t._ends == o._ends assert n != t s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet() t.add(20, 500) t.add(1000, 10000) o = SparseSet(t) n = t + s assert t == o assert t._begins == o._begins assert t._ends == o._ends assert n != t s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet() t.add(2, 50) t.add(100, 10000) assert t != s s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet() t.add(20, 500) t.add(1000, 10000) n = t - s t -= s assert n == t, '%s %s' % (n, t) print "passed all tests in", time.clock() - _t
class BtStorage(object): def __init__(self, filepool, save_path, files): self.filepool = filepool self.initialize(save_path, files) def initialize(self, save_path, files): # a list of bytes ranges and filenames for window-based IO self.ranges = [] # a dict of filename-to-ranges for piece priorities and filename lookup self.range_by_name = {} # a sparse set for smart allocation detection self.allocated_regions = SparseSet() # dict of filename-to-length on disk (for % complete in the file view) self.undownloaded = {} self.save_path = save_path # Rather implement this as an ugly hack here than change all the # individual calls. Affects all torrent instances using this module. #bad_libc_workaround() self.initialized = False return self._build_file_structs(self.filepool, files) def _build_file_structs(self, filepool, files): total = 0 for filename, length in files: self.undownloaded[filename] = length if length > 0: self.ranges.append((total, total + length, filename)) self.range_by_name[filename] = (total, total + length) if os.path.exists(filename): if not os.path.isfile(filename): raise BTFailure(_("File %s already exists, but is not a " "regular file") % filename) l = os.path.getsize(filename) if l > length: # This is the truncation Bram was talking about that no one # else thinks is a good idea. #h = file(filename, 'rb+') #make_file_sparse(filename, h, length) #h.truncate(length) #h.close() l = length a = get_allocated_regions(filename, begin=0, length=l) if a is not None: a.offset(total) else: a = SparseSet() if l > 0: a.add(total, total + l) self.allocated_regions += a total += length self.total_length = total self.initialized = True return True def get_byte_range_for_filename(self, filename): if filename not in self.range_by_name: filename = os.path.normpath(filename) filename = os.path.join(self.save_path, filename) return self.range_by_name[filename] def was_preallocated(self, pos, length): return self.allocated_regions.is_range_in(pos, pos+length) def get_total_length(self): return self.total_length def _intervals(self, pos, amount): r = [] stop = pos + amount p = max(bisect_right(self.ranges, (pos, 2 ** 500)) - 1, 0) for begin, end, filename in self.ranges[p:]: if begin >= stop: break r.append((filename, max(pos, begin) - begin, min(end, stop) - begin)) return r def _read(self, filename, pos, amount): begin, end = self.get_byte_range_for_filename(filename) length = end - begin h = self.filepool.acquire_handle(filename, for_write=False, length=length) if h is None: return try: h.seek(pos) r = h.read(amount) finally: self.filepool.release_handle(filename, h) return r def _batch_read(self, pos, amount): dfs = [] r = [] # queue all the reads for filename, pos, end in self._intervals(pos, amount): r.append(self._read(filename, pos, end - pos)) r = ''.join(r) if len(r) != amount: raise BTFailure(_("Short read (%d of %d) - something truncated files?") % (len(r), amount)) return r def read(self, pos, amount): return self._batch_read(pos, amount) def _write(self, filename, pos, s): begin, end = self.get_byte_range_for_filename(filename) length = end - begin h = self.filepool.acquire_handle(filename, for_write=True, length=length) if h is None: return try: h.seek(pos) h.write(s) finally: self.filepool.release_handle(filename, h) return len(s) def _batch_write(self, pos, s): dfs = [] total = 0 amount = len(s) # queue all the writes for filename, begin, end in self._intervals(pos, amount): length = end - begin d = buffer(s, total, length) total += length self._write(filename, begin, d) return total def write(self, pos, s): return self._batch_write(pos, s) def close(self): self.filepool.close_files(self.range_by_name) def downloaded(self, pos, length): for filename, begin, end in self._intervals(pos, length): self.undownloaded[filename] -= end - begin
class Storage(object): def __init__(self, config, filepool, save_path, files, add_task, external_add_task, doneflag): self.filepool = filepool self.config = config self.doneflag = doneflag self.add_task = add_task self.external_add_task = external_add_task self.initialize(save_path, files) def initialize(self, save_path, files): # a list of bytes ranges and filenames for window-based IO self.ranges = [] # a dict of filename-to-ranges for piece priorities and filename lookup self.range_by_name = {} # a sparse set for smart allocation detection self.allocated_regions = SparseSet() # dict of filename-to-length on disk (for % complete in the file view) self.undownloaded = {} self.save_path = save_path # Rather implement this as an ugly hack here than change all the # individual calls. Affects all torrent instances using this module. if self.config['bad_libc_workaround']: bad_libc_workaround() self.initialized = False self.startup_df = ThreadedDeferred(wrap_task(self.external_add_task), self._build_file_structs, self.filepool, files) return self.startup_df def _build_file_structs(self, filepool, files): total = 0 for filename, length in files: # we're shutting down, abort. if self.doneflag.isSet(): return False self.undownloaded[filename] = length if length > 0: self.ranges.append((total, total + length, filename)) self.range_by_name[filename] = (total, total + length) if os.path.exists(filename): if not os.path.isfile(filename): raise BTFailure(_("File %s already exists, but is not a " "regular file") % filename) l = os.path.getsize(filename) if l > length: # This is the truncation Bram was talking about that no one # else thinks is a good idea. #h = file(filename, 'rb+') #make_file_sparse(filename, h, length) #h.truncate(length) #h.close() l = length a = get_allocated_regions(filename, begin=0, length=l) if a is not None: a.offset(total) else: a = SparseSet() if l > 0: a.add(total, total + l) self.allocated_regions += a total += length self.total_length = total self.initialized = True return True def get_byte_range_for_filename(self, filename): if filename not in self.range_by_name: filename = os.path.normpath(filename) filename = os.path.join(self.save_path, filename) return self.range_by_name[filename] def was_preallocated(self, pos, length): return self.allocated_regions.is_range_in(pos, pos+length) def get_total_length(self): return self.total_length def _intervals(self, pos, amount): r = [] stop = pos + amount p = max(bisect_right(self.ranges, (pos, 2 ** 500)) - 1, 0) for begin, end, filename in self.ranges[p:]: if begin >= stop: break r.append((filename, max(pos, begin) - begin, min(end, stop) - begin)) return r def _file_op(self, filename, pos, param, write): begin, end = self.get_byte_range_for_filename(filename) length = end - begin hdf = self.filepool.acquire_handle(filename, for_write=write, length=length) def op(h): h.seek(pos) if write: odf = h.write(param) else: odf = h.read(param) def like_finally(r): self.filepool.release_handle(filename, h) return r odf.addBoth(like_finally) return odf hdf.addCallback(op) return hdf def _batch_read(self, pos, amount): dfs = [] r = [] # queue all the reads for filename, pos, end in self._intervals(pos, amount): df = self._file_op(filename, pos, end - pos, write=False) dfs.append(df) # yield on all the reads in order - they complete in any order exc = None for df in dfs: yield df try: r.append(df.getResult()) except: exc = exc or sys.exc_info() if exc: raise exc[0], exc[1], exc[2] r = ''.join(r) if len(r) != amount: raise BTFailure(_("Short read (%d of %d) - " "something truncated files?") % (len(r), amount)) yield r def read(self, pos, amount): df = launch_coroutine(wrap_task(self.add_task), self._batch_read, pos, amount) return df def _batch_write(self, pos, s): dfs = [] total = 0 amount = len(s) # queue all the writes for filename, begin, end in self._intervals(pos, amount): length = end - begin assert length > 0, '%s %s' % (pos, amount) d = buffer(s, total, length) total += length df = self._file_op(filename, begin, d, write=True) dfs.append(df) assert total == amount, '%s and %s' % (total, amount) written = 0 # yield on all the writes - they complete in any order exc = None for df in dfs: yield df try: written += df.getResult() except: exc = exc or sys.exc_info() if exc: raise exc[0], exc[1], exc[2] assert total == written, '%s and %s' % (total, written) yield total def write(self, pos, s): df = launch_coroutine(wrap_task(self.add_task), self._batch_write, pos, s) return df def close(self): if not self.initialized: def post_init(r): return self.filepool.close_files(self.range_by_name) self.startup_df.addCallback(post_init) return self.startup_df df = self.filepool.close_files(self.range_by_name) return df def downloaded(self, pos, length): for filename, begin, end in self._intervals(pos, length): self.undownloaded[filename] -= end - begin
class AppearanceSettingsPanel(SettingsPanel): label = _("Appearance") pb_config_key = 'progressbar_style' # sample data sample_value = 0.4 sample_data = { 'h': SparseSet(), 't': SparseSet(), } sample_data['h'].add(0, 80) sample_data['t'].add(80, 100) for i in range(20, 0, -1): s = SparseSet() s.add(200 - i * 5, 200 - (i - 1) * 5) sample_data[i - 1] = s del i, s def __init__(self, parent, *a, **k): SettingsPanel.__init__(self, parent, *a, **k) # widgets self.gauge_box = wx.StaticBox(self, label=_("Progress bar style:")) self.gauge_sizer = wx.StaticBoxSizer(self.gauge_box, wx.VERTICAL) self.null_radio = wx.RadioButton( self, label=_("&None (just show percent complete)"), style=wx.RB_GROUP) self.null_radio.value = 0 self.simple_radio = wx.RadioButton(self, label=_("&Ordinary progress bar")) self.simple_radio.value = 1 self.simple_sample = self.new_sample(SimpleDownloadGauge, 1) self.moderate_radio = wx.RadioButton(self, label=_("&Detailed progress bar")) self.moderate_radio.value = 2 msg = _( "(shows the percentage of complete, transferring, available and missing pieces in the torrent)" ) if not text_wrappable: half = len(msg) // 2 for i in xrange(half): if msg[half + i] == ' ': msg = msg[:half + i + 1] + '\n' + msg[half + i + 1:] break elif msg[half - i] == ' ': msg = msg[:half - i + 1] + '\n' + msg[half - i + 1:] break self.moderate_text = ElectroStaticText(self, wx.ID_ANY, msg) if text_wrappable: self.moderate_text.Wrap(250) self.moderate_sample = self.new_sample(ModerateDownloadGauge, 2) self.fancy_radio = wx.RadioButton(self, label=_("&Piece bar")) self.fancy_radio.value = 3 self.fancy_text = ElectroStaticText( self, wx.ID_ANY, _("(shows the status of each piece in the torrent)")) if text_wrappable: self.fancy_text.Wrap(250) # generate random sample data r = set(xrange(200)) self.sample_data = {} for key, count in (('h', 80), ('t', 20)) + tuple([(i, 5) for i in range(19)]): self.sample_data[key] = SparseSet() for d in random.sample(r, count): self.sample_data[key].add(d) r.remove(d) for d in r: self.sample_data[0].add(d) self.fancy_sample = self.new_sample(FancyDownloadGauge, 3) # sizers gauge = wx.TOP | wx.LEFT | wx.RIGHT extra = wx.TOP | wx.LEFT | wx.RIGHT | wx.GROW self.gauge_sizer.Add(self.null_radio, flag=gauge, border=SPACING) self.gauge_sizer.AddSpacer((SPACING, SPACING)) self.gauge_sizer.Add(self.simple_radio, flag=gauge, border=SPACING) self.gauge_sizer.Add(self.simple_sample, flag=extra, border=SPACING) self.gauge_sizer.AddSpacer((SPACING, SPACING)) self.gauge_sizer.Add(self.moderate_radio, flag=gauge, border=SPACING) self.gauge_sizer.Add(self.moderate_sample, flag=extra, border=SPACING) self.gauge_sizer.Add(self.moderate_text, flag=extra, border=SPACING) self.gauge_sizer.AddSpacer((SPACING, SPACING)) self.gauge_sizer.Add(self.fancy_radio, flag=gauge, border=SPACING) self.gauge_sizer.Add(self.fancy_sample, flag=extra, border=SPACING) self.gauge_sizer.Add(self.fancy_text, flag=extra, border=SPACING) self.sizer.AddFirst(self.gauge_sizer, flag=wx.GROW) # setup self.pb_group = (self.null_radio, self.simple_radio, self.moderate_radio, self.fancy_radio) for r in self.pb_group: r.Bind(wx.EVT_RADIOBUTTON, self.radio) if r.value == wx.the_app.config[self.pb_config_key]: r.SetValue(True) else: r.SetValue(False) # toolbar widgets self.toolbar_box = wx.StaticBox(self, label=_("Toolbar style:")) self.toolbar_text = CheckButton( self, _("Show text"), self.settings_window, 'toolbar_text', self.settings_window.config['toolbar_text'], wx.the_app.reset_toolbar_style) self.toolbar_size_text = ElectroStaticText(self, id=wx.ID_ANY, label=_("Icon size:")) self.toolbar_size_choice = wx.Choice(self, choices=(_("Small"), _("Normal"), _("Large"))) self.toolbar_config_to_choice(wx.the_app.config['toolbar_size']) self.toolbar_size_choice.Bind(wx.EVT_CHOICE, self.toolbar_choice_to_config) # toolbar sizers self.toolbar_sizer = HSizer() self.toolbar_sizer.AddFirst(self.toolbar_text, flag=wx.ALIGN_CENTER_VERTICAL) line = wx.StaticLine(self, id=wx.ID_ANY, style=wx.VERTICAL) self.toolbar_sizer.Add(line, flag=wx.ALIGN_CENTER_VERTICAL | wx.GROW) self.toolbar_sizer.Add(self.toolbar_size_text, flag=wx.ALIGN_CENTER_VERTICAL) self.toolbar_sizer.Add(self.toolbar_size_choice, flag=wx.GROW | wx.ALIGN_TOP, proportion=1) self.toolbar_box_sizer = wx.StaticBoxSizer(self.toolbar_box, wx.VERTICAL) self.toolbar_box_sizer.Add(self.toolbar_sizer, flag=wx.GROW) self.sizer.Add(self.toolbar_box_sizer, flag=wx.GROW) if wx.the_app.config['debug']: # the T-Word widgets self.themes = [] self.theme_choice = wx.Choice(self, choices=[]) self.theme_choice.Enable(False) self.theme_choice.Bind(wx.EVT_CHOICE, self.set_theme) self.restart_hint = ElectroStaticText( self, id=wx.ID_ANY, label=_("(Changing themes requires restart.)")) self.theme_static_box = wx.StaticBox(self, label=_("Theme:")) # the T-Word sizers self.theme_sizer = VSizer() self.theme_sizer.AddFirst(self.theme_choice, flag=wx.GROW | wx.ALIGN_RIGHT) self.theme_sizer.Add(self.restart_hint, flag=wx.GROW | wx.ALIGN_RIGHT) self.theme_static_box_sizer = wx.StaticBoxSizer( self.theme_static_box, wx.VERTICAL) self.theme_static_box_sizer.Add(self.theme_sizer, flag=wx.GROW) self.sizer.Add(self.theme_static_box_sizer, flag=wx.GROW) self.get_themes() def get_themes(self): def _callback(themes): self.themes.extend(themes) self.theme_choice.AppendItems(strings=themes) curr_theme = wx.the_app.config['theme'] if curr_theme not in self.themes: self.settings_window.setfunc('theme', 'default') curr_theme = wx.the_app.config['theme'] curr_idx = self.themes.index(curr_theme) self.theme_choice.SetSelection(curr_idx) self.theme_choice.Enable(True) def callback(themes): gui_wrap(_callback, themes) df = list_themes() df.addCallback(callback) df.getResult() def set_theme(self, e): i = self.theme_choice.GetSelection() t = self.themes[i] self.settings_window.setfunc('theme', t) def toolbar_choice_to_config(self, *a): i = self.toolbar_size_choice.GetSelection(), size = 8 * (i[0] + 2) self.settings_window.setfunc('toolbar_size', size) wx.the_app.reset_toolbar_style() def toolbar_config_to_choice(self, value): i = (value // 8) - 2 self.toolbar_size_choice.SetSelection(i) def new_sample(self, sample_class, value): sample = sample_class(self, size=wx.Size(-1, 20), style=wx.SUNKEN_BORDER) # I happen to know 200 is the right number because I looked. sample.SetValue(self.sample_value, 'running', (200, 0, self.sample_data)) sample.Bind(wx.EVT_LEFT_DOWN, self.sample) sample.Bind(wx.EVT_CONTEXT_MENU, None) sample.value = value return sample def radio(self, event): widget = event.GetEventObject() value = widget.value self.settings_window.setfunc(self.pb_config_key, value) gui_wrap(wx.the_app.main_window.torrentlist.change_gauge_type, value) def sample(self, event): self.radio(event) pb = event.GetEventObject() value = pb.value for p in self.pb_group: if p.value == value: p.SetValue(True) break
def __init__(self, parent, *a, **k): SettingsPanel.__init__(self, parent, *a, **k) # widgets self.gauge_box = wx.StaticBox(self, label=_("Progress bar style:")) self.gauge_sizer = wx.StaticBoxSizer(self.gauge_box, wx.VERTICAL) self.null_radio = wx.RadioButton( self, label=_("&None (just show percent complete)"), style=wx.RB_GROUP) self.null_radio.value = 0 self.simple_radio = wx.RadioButton(self, label=_("&Ordinary progress bar")) self.simple_radio.value = 1 self.simple_sample = self.new_sample(SimpleDownloadGauge, 1) self.moderate_radio = wx.RadioButton(self, label=_("&Detailed progress bar")) self.moderate_radio.value = 2 msg = _( "(shows the percentage of complete, transferring, available and missing pieces in the torrent)" ) if not text_wrappable: half = len(msg) // 2 for i in xrange(half): if msg[half + i] == ' ': msg = msg[:half + i + 1] + '\n' + msg[half + i + 1:] break elif msg[half - i] == ' ': msg = msg[:half - i + 1] + '\n' + msg[half - i + 1:] break self.moderate_text = ElectroStaticText(self, wx.ID_ANY, msg) if text_wrappable: self.moderate_text.Wrap(250) self.moderate_sample = self.new_sample(ModerateDownloadGauge, 2) self.fancy_radio = wx.RadioButton(self, label=_("&Piece bar")) self.fancy_radio.value = 3 self.fancy_text = ElectroStaticText( self, wx.ID_ANY, _("(shows the status of each piece in the torrent)")) if text_wrappable: self.fancy_text.Wrap(250) # generate random sample data r = set(xrange(200)) self.sample_data = {} for key, count in (('h', 80), ('t', 20)) + tuple([(i, 5) for i in range(19)]): self.sample_data[key] = SparseSet() for d in random.sample(r, count): self.sample_data[key].add(d) r.remove(d) for d in r: self.sample_data[0].add(d) self.fancy_sample = self.new_sample(FancyDownloadGauge, 3) # sizers gauge = wx.TOP | wx.LEFT | wx.RIGHT extra = wx.TOP | wx.LEFT | wx.RIGHT | wx.GROW self.gauge_sizer.Add(self.null_radio, flag=gauge, border=SPACING) self.gauge_sizer.AddSpacer((SPACING, SPACING)) self.gauge_sizer.Add(self.simple_radio, flag=gauge, border=SPACING) self.gauge_sizer.Add(self.simple_sample, flag=extra, border=SPACING) self.gauge_sizer.AddSpacer((SPACING, SPACING)) self.gauge_sizer.Add(self.moderate_radio, flag=gauge, border=SPACING) self.gauge_sizer.Add(self.moderate_sample, flag=extra, border=SPACING) self.gauge_sizer.Add(self.moderate_text, flag=extra, border=SPACING) self.gauge_sizer.AddSpacer((SPACING, SPACING)) self.gauge_sizer.Add(self.fancy_radio, flag=gauge, border=SPACING) self.gauge_sizer.Add(self.fancy_sample, flag=extra, border=SPACING) self.gauge_sizer.Add(self.fancy_text, flag=extra, border=SPACING) self.sizer.AddFirst(self.gauge_sizer, flag=wx.GROW) # setup self.pb_group = (self.null_radio, self.simple_radio, self.moderate_radio, self.fancy_radio) for r in self.pb_group: r.Bind(wx.EVT_RADIOBUTTON, self.radio) if r.value == wx.the_app.config[self.pb_config_key]: r.SetValue(True) else: r.SetValue(False) # toolbar widgets self.toolbar_box = wx.StaticBox(self, label=_("Toolbar style:")) self.toolbar_text = CheckButton( self, _("Show text"), self.settings_window, 'toolbar_text', self.settings_window.config['toolbar_text'], wx.the_app.reset_toolbar_style) self.toolbar_size_text = ElectroStaticText(self, id=wx.ID_ANY, label=_("Icon size:")) self.toolbar_size_choice = wx.Choice(self, choices=(_("Small"), _("Normal"), _("Large"))) self.toolbar_config_to_choice(wx.the_app.config['toolbar_size']) self.toolbar_size_choice.Bind(wx.EVT_CHOICE, self.toolbar_choice_to_config) # toolbar sizers self.toolbar_sizer = HSizer() self.toolbar_sizer.AddFirst(self.toolbar_text, flag=wx.ALIGN_CENTER_VERTICAL) line = wx.StaticLine(self, id=wx.ID_ANY, style=wx.VERTICAL) self.toolbar_sizer.Add(line, flag=wx.ALIGN_CENTER_VERTICAL | wx.GROW) self.toolbar_sizer.Add(self.toolbar_size_text, flag=wx.ALIGN_CENTER_VERTICAL) self.toolbar_sizer.Add(self.toolbar_size_choice, flag=wx.GROW | wx.ALIGN_TOP, proportion=1) self.toolbar_box_sizer = wx.StaticBoxSizer(self.toolbar_box, wx.VERTICAL) self.toolbar_box_sizer.Add(self.toolbar_sizer, flag=wx.GROW) self.sizer.Add(self.toolbar_box_sizer, flag=wx.GROW) if wx.the_app.config['debug']: # the T-Word widgets self.themes = [] self.theme_choice = wx.Choice(self, choices=[]) self.theme_choice.Enable(False) self.theme_choice.Bind(wx.EVT_CHOICE, self.set_theme) self.restart_hint = ElectroStaticText( self, id=wx.ID_ANY, label=_("(Changing themes requires restart.)")) self.theme_static_box = wx.StaticBox(self, label=_("Theme:")) # the T-Word sizers self.theme_sizer = VSizer() self.theme_sizer.AddFirst(self.theme_choice, flag=wx.GROW | wx.ALIGN_RIGHT) self.theme_sizer.Add(self.restart_hint, flag=wx.GROW | wx.ALIGN_RIGHT) self.theme_static_box_sizer = wx.StaticBoxSizer( self.theme_static_box, wx.VERTICAL) self.theme_static_box_sizer.Add(self.theme_sizer, flag=wx.GROW) self.sizer.Add(self.theme_static_box_sizer, flag=wx.GROW) self.get_themes()