def new_request(self, index, full=False): # returns (begin, length) if index not in self.inactive_requests: self._make_inactive(index) rs = self.inactive_requests[index] if full: s = SparseSet() while rs: r = rs.pop() s.add(r[0], r[0] + r[1]) b, e = s.largest_range() s.remove(b, e) reqs = self._break_up(b, e - b) for r in reqs: assert r[1] <= self.request_size self.active_requests[index].append(r) # I don't like this. the function should return reqs r = (b, e - b) for b, e in s.iterrange(): rs.extend(self._break_up(b, e - b)) else: # why min? do we want all the blocks in order? r = min(rs) rs.remove(r) assert r[1] <= self.request_size self.active_requests[index].append(r) self.amount_inactive -= r[1] assert self.amount_inactive >= 0, ('Amount inactive: %d' % self.amount_inactive) if len(rs) == 0: self.fully_active.add(index) if self.amount_inactive == 0: self.endgame = True assert (r[0] + r[1]) <= self._piecelen(index) return r
def safe_register_port(self, new_mapping): # check for the host now, while we're in the thread and before # we need to read it. new_mapping.populate_host() nat_logger.info("You asked for: " + str(new_mapping)) new_mapping.original_external_port = new_mapping.external_port mappings = self._list_ports() used_ports = [] for mapping in mappings: # only consider ports which match the same protocol if mapping.protocol == new_mapping.protocol: # look for exact matches if (mapping.host == new_mapping.host and mapping.internal_port == new_mapping.internal_port): # the service name could not match, that's ok. new_mapping.d.callback(mapping.external_port) nat_logger.info("Already effectively mapped: " + str(mapping)) return # otherwise, add it to the list of used external ports used_ports.append(mapping.external_port) used_ports.sort() used_ports = SparseSet(used_ports) all_ports = SparseSet() all_ports.add(1024, 65535) free_ports = all_ports - used_ports new_mapping.external_port = random.choice(free_ports) nat_logger.info("I'll give you: " + str(new_mapping)) self.register_port(new_mapping)
def safe_register_port(self, new_mapping): # check for the host now, while we're in the thread and before # we need to read it. new_mapping.populate_host() nat_logger.info("You asked for: " + str(new_mapping)) new_mapping.original_external_port = new_mapping.external_port mappings = self._list_ports() used_ports = [] for mapping in mappings: # only consider ports which match the same protocol if mapping.protocol == new_mapping.protocol: # look for exact matches if (mapping.host == new_mapping.host and mapping.internal_port == new_mapping.internal_port): # the service name could not match, that's ok. new_mapping.d.callback(mapping.external_port) nat_logger.info("Already effectively mapped: " + str(mapping)) return # otherwise, add it to the list of used external ports used_ports.append(mapping.external_port) used_ports.sort() used_ports = SparseSet(used_ports) all_ports = SparseSet() all_ports.add(1024, 65535) free_ports = all_ports - used_ports new_mapping.external_port = random.choice(free_ports) nat_logger.info("I'll give you: " + str(new_mapping)) self.register_port(new_mapping)
def SetValue(self, value, state=None, data=None, redraw=True): if data is not None: sorted_data = {} length, update, piece_states = data self.resolution = length keys = piece_states.keys() keys.sort(self.sort) pos = 0 h = piece_states.get('h', SparseSet()) t = piece_states.get('t', set()) t = list(t) t.sort() have_trans_sparse_set = h + t for k in keys: p = piece_states[k] if k in ('h', 't'): count = len(p) else: count = 0 # OW for i in p: if i not in have_trans_sparse_set: count += 1 if not count: continue newpos = pos + count s = SparseSet() s.add(pos, newpos) sorted_data[k] = s pos = newpos data = (length, update, sorted_data) FancyDownloadGauge.SetValue(self, value, state, data, redraw)
def _build_file_structs(self, filepool, files): total = 0 for filename, length in files: self.undownloaded[filename] = length if length > 0: self.ranges.append((total, total + length, filename)) self.range_by_name[filename] = (total, total + length) if os.path.exists(filename): if not os.path.isfile(filename): raise BTFailure(_("File %s already exists, but is not a " "regular file") % filename) l = os.path.getsize(filename) if l > length: # This is the truncation Bram was talking about that no one # else thinks is a good idea. #h = file(filename, 'rb+') #make_file_sparse(filename, h, length) #h.truncate(length) #h.close() l = length a = get_allocated_regions(filename, begin=0, length=l) if a is not None: a.offset(total) else: a = SparseSet() if l > 0: a.add(total, total + l) self.allocated_regions += a total += length self.total_length = total self.initialized = True return True
def get_allocated_regions(path, f=None, begin=0, length=None): supported = get_sparse_files_support(path) if not supported: return if os.name == 'nt': if not os.path.exists(path): return False if f is None: f = file(path, 'r') handle = win32file._get_osfhandle(f.fileno()) if length is None: length = os.path.getsize(path) - begin a = SparseSet() interval = 10000000 i = begin end = begin + length while i < end: d = struct.pack("<QQ", i, interval) try: r = win32file.DeviceIoControl(handle, FSCTL_QUERY_ALLOCATED_RANGES, d, interval, None) except pywintypes.error, e: # I've seen: # error: (1784, 'DeviceIoControl', 'The supplied user buffer is not valid for the requested operation.') return for c in xrange(0, len(r), 16): qq = struct.unpack("<QQ", r[c:c+16]) b = qq[0] e = b + qq[1] a.add(b, e) i += interval return a
def get_allocated_regions(path, f=None, begin=0, length=None): supported = get_sparse_files_support(path) if not supported: return if os.name == 'nt': if not os.path.exists(path): return False if f is None: f = file(path, 'r') handle = win32file._get_osfhandle(f.fileno()) if length is None: length = os.path.getsize(path) - begin a = SparseSet() interval = 10000000 i = begin end = begin + length while i < end: d = struct.pack("<QQ", i, interval) try: r = win32file.DeviceIoControl(handle, FSCTL_QUERY_ALLOCATED_RANGES, d, interval, None) except pywintypes.error, e: # I've seen: # error: (1784, 'DeviceIoControl', 'The supplied user buffer is not valid for the requested operation.') return for c in xrange(0, len(r), 16): qq = struct.unpack("<QQ", r[c:c + 16]) b = qq[0] e = b + qq[1] a.add(b, e) i += interval return a
def SetValue(self, value, state=None, data=None, redraw=True): if data is not None: sorted_data = {} length, update, piece_states = data self.resolution = length keys = piece_states.keys() keys.sort(self.sort) pos = 0 h = piece_states.get('h', SparseSet()) t = piece_states.get('t', set()) t = list(t) t.sort() have_trans_sparse_set = h + t for k in keys: p = piece_states[k] if k in ('h', 't'): count = len(p) else: count = 0 # OW for i in p: if i not in have_trans_sparse_set: count += 1 if not count: continue newpos = pos+count s = SparseSet() s.add(pos, newpos) sorted_data[k] = s pos = newpos data = (length, update, sorted_data) FancyDownloadGauge.SetValue(self, value, state, data, redraw)
def __init__(self, config, storage, rm, urlage, picker, numpieces, finished, errorfunc, kickfunc, banfunc, get_downrate): self.config = config self.storage = storage self.rm = rm self.urlage = urlage self.picker = picker self.errorfunc = errorfunc self.rerequester = None self.entered_endgame = False self.connection_manager = None self.chunksize = config['download_chunk_size'] self.numpieces = numpieces self.finished = finished self.snub_time = config['snub_time'] self.kickfunc = kickfunc self.banfunc = banfunc self.get_downrate = get_downrate self.downloads = [] self.perip = {} self.bad_peers = {} self.discarded_bytes = 0 self.useful_received_listeners = set() self.raw_received_listeners = set() if SPARSE_SET: self.piece_states = PieceSetBuckets() nothing = SparseSet() nothing.add(0, self.numpieces) self.piece_states.buckets.append(nothing) # I hate this nowhere = [(i, 0) for i in xrange(self.numpieces)] self.piece_states.place_in_buckets = dict(nowhere) else: typecode = resolve_typecode(self.numpieces) self.piece_states = SortedPieceBuckets(typecode) nothing = array.array(typecode, range(self.numpieces)) self.piece_states.buckets.append(nothing) # I hate this nowhere = [(i, (0, i)) for i in xrange(self.numpieces)] self.piece_states.place_in_buckets = dict(nowhere) self.last_update = 0 self.all_requests = set()
def __init__(self, config, storage, rm, urlage, picker, numpieces, finished, errorfunc, kickfunc, banfunc, get_downrate): self.config = config self.storage = storage self.rm = rm self.urlage = urlage self.picker = picker self.errorfunc = errorfunc self.rerequester = None self.entered_endgame = False self.connection_manager = None self.chunksize = config['download_chunk_size'] self.numpieces = numpieces self.finished = finished self.snub_time = config['snub_time'] self.kickfunc = kickfunc self.banfunc = banfunc self.get_downrate = get_downrate self.downloads = [] self.perip = {} self.bad_peers = {} self.discarded_bytes = 0 self.useful_received_listeners = set() self.raw_received_listeners = set() if SPARSE_SET: self.piece_states = PieceSetBuckets() nothing = SparseSet() nothing.add(0, self.numpieces) self.piece_states.buckets.append(nothing) # I hate this nowhere = [(i, 0) for i in xrange(self.numpieces)] self.piece_states.place_in_buckets = dict(nowhere) else: typecode = resolve_typecode(self.numpieces) self.piece_states = SortedPieceBuckets(typecode) nothing = array.array(typecode, range(self.numpieces)) self.piece_states.buckets.append(nothing) # I hate this nowhere = [(i, (0, i)) for i in xrange(self.numpieces)] self.piece_states.place_in_buckets = dict(nowhere) self.last_update = 0 self.all_requests = set()
def _build_file_structs(self, filepool, files): total = 0 for filename, length in files: print "filename %s in stor tpool length %d " % (filename, length) # we're shutting down, abort. if self.doneflag.isSet(): return False self.undownloaded[filename] = length if length > 0: self.ranges.append((total, total + length, filename)) self.range_by_name[filename] = (total, total + length) if os.path.exists(filename): if not os.path.isfile(filename): raise BTFailure( _("File %s already exists, but is not a " "regular file") % filename) l = os.path.getsize(filename) print "size on disk %d length = %d" % (l, length) if l > length: # This is the truncation Bram was talking about that no one # else thinks is a good idea. #h = file(filename, 'rb+') #make_file_sparse(filename, h, length) #h.truncate(length) #h.close() l = length a = get_allocated_regions(filename, begin=0, length=l) if a is not None: a.offset(total) else: a = SparseSet() if l > 0: a.add(total, total + l) self.allocated_regions += a total += length self.total_length = total self.initialized = True return True
def get_allocated_regions(path, f=None, begin=0, length=None): supported = get_sparse_files_support(path) if not supported: return if os.name == 'nt': if not os.path.exists(path): return False if f is None: f = file(path, 'r') handle = win32file._get_osfhandle(f.fileno()) if length is None: length = os.path.getsize(path) - begin a = SparseSet() run = 128 i = begin end = begin + length while i < end: d = struct.pack("<QQ", i, length) try: r = win32file.DeviceIoControl(handle, FSCTL_QUERY_ALLOCATED_RANGES, d, struct.calcsize("<QQ") * run, None) except pywintypes.error, e: if e.args[0] == winerror.ERROR_MORE_DATA: run *= 2 continue # I've also seen: # error: (1784, 'DeviceIoControl', 'The supplied user buffer is not valid for the requested operation.') return if not r: break for c in xrange(0, len(r), 16): qq = struct.unpack("<QQ", r[c:c + 16]) b = qq[0] e = b + qq[1] a.add(b, e) i = max(i, e) return a
def get_allocated_regions(path, f=None, begin=0, length=None): supported = get_sparse_files_support(path) if not supported: return if os.name == 'nt': if not os.path.exists(path): return False if f is None: f = file(path, 'r') handle = win32file._get_osfhandle(f.fileno()) if length is None: length = os.path.getsize(path) - begin a = SparseSet() run = 128 i = begin end = begin + length while i < end: d = struct.pack("<QQ", i, length) try: r = win32file.DeviceIoControl(handle, FSCTL_QUERY_ALLOCATED_RANGES, d, struct.calcsize("<QQ")*run, None) except pywintypes.error, e: if e.args[0] == winerror.ERROR_MORE_DATA: run *= 2 continue # I've also seen: # error: (1784, 'DeviceIoControl', 'The supplied user buffer is not valid for the requested operation.') return if not r: break for c in xrange(0, len(r), 16): qq = struct.unpack("<QQ", r[c:c+16]) b = qq[0] e = b + qq[1] a.add(b, e) i = max(i, e) return a
def main(): _t = time.clock() s = SparseSet() def blank(): #print "-" * 79 s._begins = [] s._ends = [] def reset(): #print "-" * 79 s._begins = [ 1, 10, 25, 300, ] s._ends = [ 3, 15, 45, 1000, ] def test(l): a = zip(s._begins, s._ends) assert a == l, str(a) + " is not " + str(l) reset() s.add(2, 24) test([(1, 24), (25, 45), (300, 1000)]) reset() s.add(4, 27) test([(1, 3), (4, 45), (300, 1000)]) reset() s.add(4, 24) test([(1, 3), (4, 24), (25, 45), (300, 1000)]) reset() s.add(4, 23) test([(1, 3), (4, 23), (25, 45), (300, 1000)]) reset() s.add(4, 7) test([(1, 3), (4, 7), (10, 15), (25, 45), (300, 1000)]) reset() s.add(4, 46) test([(1, 3), (4, 46), (300, 1000)]) blank() s.add_range(range(1, 3)) s.add_range(range(10, 15)) s.add_range(range(25, 45)) s.add_range(range(300, 1000)) s.add(4, 46) test([(1, 3), (4, 46), (300, 1000)]) blank() s.add_range(range(1, 3)) s.add_range(range(10, 15)) s.add_range(range(25, 45)) s.add_range(range(0)) s.add_range(range(300, 1000)) s.add(4, 46) test([(1, 3), (4, 46), (300, 1000)]) blank() s.add_range(range(1, 3)) s.add_range(range(10, 15)) s.add_range(range(25, 45)) s.add_range(range(1)) s.add_range(range(300, 1000)) s.add(4, 46) test([(0, 3), (4, 46), (300, 1000)]) blank() s.add_range(range(1, 3)) s.add_range(range(10, 15)) s.add_range(range(25, 45)) s.add_range(range(300, 1000)) for i in xrange(1, 3): assert i in s, str(i) + " is in " + str(s) assert not (i+1) in s for i in xrange(10, 15): assert i in s, str(i) + " is in " + str(s) assert not (i+1) in s for i in xrange(300, 1000): assert i in s, str(i) + " is in " + str(s) assert not (i+1) in s assert s.is_range_in(1, 3) assert not s.is_range_in(1, 3 + 1) assert s.is_range_in(300, 1000) assert not s.is_range_in(300 - 1, 1000) assert not s.is_range_in(300, 2000) assert s.is_range_in(300, 700) reset() s.add(2, 700) test([(1, 1000)]) blank() s.add(0, 10) test([(0, 10)]) s.add(-2, 1) test([(-2, 10)]) def reset2(): blank() s.add(0, 10) s.add(20, 30) s.add(40, 50) s.add(60, 70) reset2() s.add(0, 70) test([(0, 70)]) reset2() s.add(1, 70) test([(0, 70)]) reset2() s.add(0, 69) test([(0, 70)]) reset2() s.add(-1, 70) test([(-1, 70)]) reset2() s.add(0, 71) test([(0, 71)]) reset2() s.add(15, 55) test([(0, 10), (15, 55), (60, 70)]) blank() try: s.add(1, 1) except ValueError: pass else: assert False test([]) blank() try: s.add(1, 0) except ValueError: pass else: assert False test([]) blank() try: s.add(1, 2) except ValueError: assert False test([(1, 2)]) blank() s.add(1.5, 3.7) s.add(2.5, 4.7) blank() s.add(1, 3) s.add(2, 4) test([(1, 4)]) blank() s.add(2, 4) s.add(1, 3) test([(1, 4)]) blank() s.add(0, 2) s.add(2, 4) test([(0, 4)]) blank() s.add(2, 4) s.add(0, 2) test([(0, 4)]) blank() s.add(2, 3) s.add(0, 1) test([(0, 1), (2, 3)]) blank() s.add(0, 1) s.add(2, 4) test([(0, 1), (2, 4)]) blank() from random import shuffle l = range(0, 11) shuffle(l) for i in l: s.add(i, i+1) del l def testy_thing(d): blank() for i in d: s.add(i) test([(0, 5)]) def testy_thing2(d): blank() for i in d: s.add(i*2) test([(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)]) all = [] def xcombinations(items, n): if n == 0: yield [] else: for i in xrange(len(items)): for cc in xcombinations(items[:i] + items[i+1:], n - 1): yield [items[i]] + cc for uc in xcombinations(range(5), 5): all.append(uc) for d in all: testy_thing(d) for d in all: testy_thing2(d) blank() s.add(0, 1000) s.subtract(200, 500) test([(0, 200), (500, 1000)]) #blank() s.subtract(200, 500) test([(0, 200), (500, 1000)]) s.subtract(100, 201) test([(0, 100), (500, 1000)]) s.subtract(300, 500) test([(0, 100), (500, 1000)]) s.subtract(30, 50) test([(0, 30), (50, 100), (500, 1000)]) s.subtract(29, 1001) test([(0, 29)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(-1, 900) test([(900, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(29, 502) test([(0, 29), (502, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(35, 200) test([(0, 30), (501, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(55, 601) test([(0, 30), (51, 55), (601, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.subtract(25, 70) test([(0, 25), (70, 100), (501, 1000)]) blank() s.add(0, 30) s.add(51, 100) s.add(501, 1000) s.add(2000, 10000) s.subtract(25, 502) test([(0, 25), (502, 1000), (2000, 10000)]) blank() s.add(0, 30) s.add(51, 100) s.add(102, 189) s.add(501, 1000) s.add(2000, 10000) s.subtract(25, 502) test([(0, 25), (502, 1000), (2000, 10000)]) blank() s.add(0, 30) s.add(51, 100) s.add(102, 189) s.add(501, 1000) s.subtract(25, 1000) test([(0, 25)]) blank() s.add(0, 30) s.add(51, 100) s.add(102, 189) s.add(501, 1000) s.subtract_range(range(25, 1000)) test([(0, 25)]) blank() s.add(0, 30) s.add(51, 100) s.add(102, 189) s.add(501, 1000) s.subtract(52, 999) test([(0, 30), (51, 52), (999, 1000)]) blank() import random all = [] assert len(s._begins) == 0 for i in range(0, 10): b = random.randint(0, 10000) l = random.randint(1, 1000) all.append((b, b+l)) s.add_range(range(b, b+l)) for b, e in all: s.subtract(b, e) assert len(s._begins) == 0 blank() s.add(0, 100) s.add(1000, 2000) assert s[-1] == 1999 assert s[0] == 0 assert s[99] == 99 assert s[100] == 1000 assert s[101] == 1001 blank() s.add(-10, -5) s.add(0, 100) assert s[0] == -10 assert s[10] == 5 assert s[-1] == 99 blank() s.add(0, 100) s.add(1000, 1100) f = range(0, 100) + range(1000, 1100) for i in s: assert i == f.pop(0) blank() s.add(0, 100) s.add(1000, 1100) f = range(0, 100) + range(1000, 1100) for i in s.iterneg(0, 1100): assert i not in f blank() s.add(0, 100) y = range(0, 100) n = range(100, 200) for i in s.iterneg(0, 200): assert i not in y assert i in n blank() s.add(100, 200) y = range(100, 200) n = range(0, 100) for i in s.iterneg(0, 200): assert i not in y assert i in n s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet(s) assert t == s assert not (t != s) assert id(t) != id(s) s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet() t.add(20, 500) t.add(1000, 10000) o = SparseSet(t) n = t - s assert t == o assert t._begins == o._begins assert t._ends == o._ends assert n != t s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet() t.add(20, 500) t.add(1000, 10000) o = SparseSet(t) n = t + s assert t == o assert t._begins == o._begins assert t._ends == o._ends assert n != t s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet() t.add(2, 50) t.add(100, 10000) assert t != s s = SparseSet() s.add(2, 50) s.add(100, 1000) t = SparseSet() t.add(20, 500) t.add(1000, 10000) n = t - s t -= s assert n == t, '%s %s' % (n, t) print "passed all tests in", time.clock() - _t
def __init__(self, config, storage, rm, urlage, picker, numpieces, finished, errorfunc, kickfunc, banfunc, get_downrate, micropayments=False): self.config = config self.storage = storage self.rm = rm self.urlage = urlage self.picker = picker self.errorfunc = errorfunc self.rerequester = None self.entered_endgame = False self.connection_manager = None self.chunksize = config['download_chunk_size'] self.numpieces = numpieces self.finished = finished self.snub_time = config['snub_time'] self.kickfunc = kickfunc self.banfunc = banfunc self.get_downrate = get_downrate self.downloads = [] self.perip = {} self.bad_peers = {} self.discarded_bytes = 0 self.useful_received_listeners = set() self.raw_received_listeners = set() self.micropayments = micropayments #boolean on/off self.key_rewards = {} #key rewards we use to "pay" uploaders self.waiting_for_reward = { } #hash with first key peerid then [(idx1,offset1,len1),(idx2,offset2,len2) ] self.payment_key_hash_cache = { } #stores payment key hashes received from tracker self.private_key = None self.public_key = None self.certificate = None self.pk_tools = None self.public_key_tracker = None if micropayments: """load keys and check certificates""" log("micropayments=" + str(micropayments)) #check if certifcates are OK cert = self.config["micropayment_certificate"] self.ca_dir = self.config["micropayment_trusted_ca_dir"] self.pk_tools = PKTools(self.ca_dir, config['save_incomplete_in']) if not self.pk_tools.validate_certificate(cert): log("invalid certificates") return else: log("valid certificates") self.certificate = parse_PEM_certificate( open(self.config["micropayment_certificate"])) self.public_key = self.certificate.publicKey self.private_key = parse_PEM_private_key( open(self.config["micropayment_private_key"])) log("cert tracker filename=%s:" % (self.config["micropayment_tracker_certificate"])) cert_tracker = parse_PEM_certificate( open(self.config["micropayment_tracker_certificate"])) self.public_key_tracker = cert_tracker.publicKey if SPARSE_SET: self.piece_states = PieceSetBuckets() nothing = SparseSet() nothing.add(0, self.numpieces) self.piece_states.buckets.append(nothing) # I hate this nowhere = [(i, 0) for i in xrange(self.numpieces)] self.piece_states.place_in_buckets = dict(nowhere) else: typecode = resolve_typecode(self.numpieces) self.piece_states = SortedPieceBuckets(typecode) nothing = array.array(typecode, range(self.numpieces)) self.piece_states.buckets.append(nothing) # I hate this nowhere = [(i, (0, i)) for i in xrange(self.numpieces)] self.piece_states.place_in_buckets = dict(nowhere) self.last_update = 0 self.all_requests = set()