def get_allocated_regions(path, f=None, begin=0, length=None): supported = get_sparse_files_support(path) if not supported: return if os.name == 'nt': if not os.path.exists(path): return False if f is None: f = file(path, 'r') handle = win32file._get_osfhandle(f.fileno()) if length is None: length = os.path.getsize(path) - begin a = SparseSet() interval = 10000000 i = begin end = begin + length while i < end: d = struct.pack("<QQ", i, interval) try: r = win32file.DeviceIoControl(handle, FSCTL_QUERY_ALLOCATED_RANGES, d, interval, None) except pywintypes.error, e: # I've seen: # error: (1784, 'DeviceIoControl', 'The supplied user buffer is not valid for the requested operation.') return for c in xrange(0, len(r), 16): qq = struct.unpack("<QQ", buffer(r, c, 16)) b = qq[0] e = b + qq[1] a.add(b, e) i += interval return a
def _build_file_structs(self, filepool, files): total = 0 for filename, length in files: # we're shutting down, abort. if self.doneflag.isSet(): return False self.undownloaded[filename] = length if length > 0: self.ranges.append((total, total + length, filename)) self.range_by_name[filename] = (total, total + length) if os.path.exists(filename): if not os.path.isfile(filename): raise BTFailure(_("File %s already exists, but is not a " "regular file") % filename) l = os.path.getsize(filename) if l > length: # This is the truncation Bram was talking about that no one # else thinks is a good idea. #h = file(filename, 'rb+') #make_file_sparse(filename, h, length) #h.truncate(length) #h.close() l = length a = get_allocated_regions(filename, begin=0, length=l) if a is not None: a.offset(total) else: a = SparseSet() if l > 0: a.add(total, total + l) self.allocated_regions += a total += length self.total_length = total self.initialized = True return True
def initialize(self, save_path, files): # a list of bytes ranges and filenames for window-based IO self.ranges = [] # a dict of filename-to-ranges for piece priorities and filename lookup self.range_by_name = {} # a sparse set for smart allocation detection self.allocated_regions = SparseSet() # dict of filename-to-length on disk (for % complete in the file view) self.undownloaded = {} self.save_path = save_path # Rather implement this as an ugly hack here than change all the # individual calls. Affects all torrent instances using this module. if self.config['bad_libc_workaround']: bad_libc_workaround() self.initialized = False self.startup_df = ThreadedDeferred(_wrap_task(self.external_add_task), self._build_file_structs, self.filepool, files) return self.startup_df
def __init__(self, config, storage, urlage, picker, numpieces, finished, total_downmeasure, downmeasure, measurefunc, kickfunc, banfunc): self.config = config self.storage = storage self.urlage = urlage self.picker = picker self.rerequester = None self.connection_manager = None self.chunksize = config['download_slice_size'] self.total_downmeasure = total_downmeasure self.downmeasure = downmeasure self.numpieces = numpieces self.finished = finished self.snub_time = config['snub_time'] self.measurefunc = measurefunc self.kickfunc = kickfunc self.banfunc = banfunc self.downloads = [] self.perip = {} self.bad_peers = {} self.discarded_bytes = 0 self.burst_avg = 0 self.piece_states = PieceSetBuckets() nothing = SparseSet(xrange(self.numpieces)) self.piece_states.buckets.append(nothing) # I hate this nowhere = [(i, 0) for i in xrange(self.numpieces)] self.piece_states.place_in_buckets = dict(nowhere) self.last_update = 0 indexes = self.storage.inactive_requests.keys() indexes.sort() self.active_requests = SparseSet(indexes) self.all_requests = set()
class Storage(object): def __init__(self, config, filepool, save_path, files, add_task, external_add_task, doneflag): self.filepool = filepool self.config = config self.doneflag = doneflag self.add_task = add_task self.external_add_task = external_add_task self.initialize(save_path, files) def initialize(self, save_path, files): # a list of bytes ranges and filenames for window-based IO self.ranges = [] # a dict of filename-to-ranges for piece priorities and filename lookup self.range_by_name = {} # a sparse set for smart allocation detection self.allocated_regions = SparseSet() # dict of filename-to-length on disk (for % complete in the file view) self.undownloaded = {} self.save_path = save_path # Rather implement this as an ugly hack here than change all the # individual calls. Affects all torrent instances using this module. if self.config['bad_libc_workaround']: bad_libc_workaround() self.initialized = False self.startup_df = ThreadedDeferred(_wrap_task(self.external_add_task), self._build_file_structs, self.filepool, files) return self.startup_df def _build_file_structs(self, filepool, files): total = 0 for filename, length in files: # we're shutting down, abort. if self.doneflag.isSet(): return False self.undownloaded[filename] = length if length > 0: self.ranges.append((total, total + length, filename)) self.range_by_name[filename] = (total, total + length) if os.path.exists(filename): if not os.path.isfile(filename): raise BTFailure(_("File %s already exists, but is not a " "regular file") % filename) l = os.path.getsize(filename) if l > length: # This is the truncation Bram was talking about that no one # else thinks is a good idea. #h = file(filename, 'rb+') #make_file_sparse(filename, h, length) #h.truncate(length) #h.close() l = length a = get_allocated_regions(filename, begin=0, length=l) if a is not None: a.offset(total) else: a = SparseSet() if l > 0: a.add(total, total + l) self.allocated_regions += a total += length self.total_length = total self.initialized = True return True def get_byte_range_for_filename(self, filename): if filename not in self.range_by_name: filename = os.path.normpath(filename) filename = os.path.join(self.save_path, filename) return self.range_by_name[filename] def was_preallocated(self, pos, length): return self.allocated_regions.is_range_in(pos, pos+length) def get_total_length(self): return self.total_length def _intervals(self, pos, amount): r = [] stop = pos + amount p = max(bisect_right(self.ranges, (pos, 2 ** 500)) - 1, 0) for begin, end, filename in self.ranges[p:]: if begin >= stop: break r.append((filename, max(pos, begin) - begin, min(end, stop) - begin)) return r def _file_op(self, filename, pos, param, write): begin, end = self.get_byte_range_for_filename(filename) length = end - begin final = Deferred() hdf = self.filepool.acquire_handle(filename, for_write=write, length=length) def handle_error(f=None): final.callback(0) # error acquiring handle if hdf is None: handle_error() return final def op(h): h.seek(pos) if write: odf = h.write(param) else: odf = h.read(param) def complete(r): self.filepool.release_handle(filename, h) final.callback(r) odf.addCallback(complete) odf.addErrback(final.errback) hdf.addCallback(op) hdf.addErrback(handle_error) return final def _batch_read(self, pos, amount): dfs = [] r = [] # queue all the reads for filename, pos, end in self._intervals(pos, amount): df = self._file_op(filename, pos, end - pos, write=False) dfs.append(df) # yield on all the reads in order - they complete in any order for df in dfs: yield df r.append(df.getResult()) r = ''.join(r) if len(r) != amount: raise BTFailure(_("Short read (%d of %d) - something truncated files?") % (len(r), amount)) yield r def read(self, pos, amount): df = launch_coroutine(_wrap_task(self.add_task), self._batch_read, pos, amount) return df def _batch_write(self, pos, s): dfs = [] total = 0 amount = len(s) # queue all the writes for filename, begin, end in self._intervals(pos, amount): length = end - begin assert length > 0, '%s %s' % (pos, amount) d = buffer(s, total, length) total += length df = self._file_op(filename, begin, d, write=True) dfs.append(df) assert total == amount, '%s and %s' % (total, amount) written = 0 # yield on all the writes - they complete in any order for df in dfs: yield df written += df.getResult() assert total == written, '%s and %s' % (total, written) yield total def write(self, pos, s): df = launch_coroutine(_wrap_task(self.add_task), self._batch_write, pos, s) return df def close(self): if not self.initialized: end = Deferred() def post_init(r): df = self.filepool.close_files(self.range_by_name) df.addCallback(end.callback) self.startup_df.addCallback(post_init) return end df = self.filepool.close_files(self.range_by_name) return df def downloaded(self, pos, length): for filename, begin, end in self._intervals(pos, length): self.undownloaded[filename] -= end - begin
class MultiDownload(object): def __init__(self, config, storage, urlage, picker, numpieces, finished, total_downmeasure, downmeasure, measurefunc, kickfunc, banfunc): self.config = config self.storage = storage self.urlage = urlage self.picker = picker self.rerequester = None self.connection_manager = None self.chunksize = config['download_slice_size'] self.total_downmeasure = total_downmeasure self.downmeasure = downmeasure self.numpieces = numpieces self.finished = finished self.snub_time = config['snub_time'] self.measurefunc = measurefunc self.kickfunc = kickfunc self.banfunc = banfunc self.downloads = [] self.perip = {} self.bad_peers = {} self.discarded_bytes = 0 self.burst_avg = 0 self.piece_states = PieceSetBuckets() nothing = SparseSet(xrange(self.numpieces)) self.piece_states.buckets.append(nothing) # I hate this nowhere = [(i, 0) for i in xrange(self.numpieces)] self.piece_states.place_in_buckets = dict(nowhere) self.last_update = 0 indexes = self.storage.inactive_requests.keys() indexes.sort() self.active_requests = SparseSet(indexes) self.all_requests = set() def attach_connection_manager(self, connection_manager): self.connection_manager = connection_manager def aggregate_piece_states(self): d = {} d['h'] = self.storage.have_set d['t'] = self.active_requests for i, bucket in enumerate(self.piece_states.buckets): d[i] = bucket r = (self.numpieces, self.last_update, d) return r def get_adjusted_distributed_copies(self): # compensate for the fact that piece picker does not contain all the pieces num = self.picker.get_distributed_copies() percent_have = (float(len(self.storage.have_set)) / float(self.numpieces)) num += percent_have if self.rerequester and self.rerequester.tracker_num_seeds: num += self.rerequester.tracker_num_seeds return num def active_requests_add(self, r): self.last_update += 1 self.active_requests.add(r) def active_requests_remove(self, r): self.last_update += 1 self.active_requests.remove(r) def got_have(self, piece): self.picker.got_have(piece) self.last_update += 1 p = self.piece_states p.add(piece, p.remove(piece) + 1) def got_have_all(self): self.picker.got_have_all() self.last_update += 1 self.piece_states.prepend_bucket() def lost_have(self, piece): self.picker.lost_have(piece) self.last_update += 1 p = self.piece_states p.add(piece, p.remove(piece) - 1) def lost_have_all(self): self.picker.lost_have_all() self.last_update += 1 self.piece_states.popleft_bucket() def hashchecked(self, index): if not self.storage.do_I_have(index): if self.storage.endgame: while self.storage.want_requests(index): nb, nl = self.storage.new_request(index) self.all_requests.add((index, nb, nl)) for d in self.downloads: d.fix_download_endgame() else: ds = [d for d in self.downloads if not d.choked] random.shuffle(ds) for d in ds: d._request_more([index]) return self.picker.complete(index) self.active_requests_remove(index) self.connection_manager.hashcheck_succeeded(index) if self.storage.have.numfalse == 0: for d in self.downloads: if d.have.numfalse == 0: d.connection.close() self.finished() def time_until_rate(self): rate = self.config['max_download_rate'] # obey "no max" setting if rate == -1: return 0 possible_burst = self.burst_avg # substract the possible burst so that the download rate is a maximum adjusted_rate = rate - possible_burst # don't let it go to 0, just because the burst is greater than the # desired rate. this could be improved by no throttling and unthrottling # all connections at once. adjusted_rate = max(adjusted_rate, possible_burst) # don't let it go above the explicit max either. adjusted_rate = min(adjusted_rate, rate) # we need to call this because the rate measure is dumb self.total_downmeasure.get_rate() t = self.total_downmeasure.time_until_rate(adjusted_rate) #print "DOWNLOAD: burst_size:", possible_burst, "rate:", self.total_downmeasure.get_rate(), "rate_limit:", rate, "new_rate:", adjusted_rate, t return t def check_rate(self): t = self.time_until_rate() if t > 0: self.postpone_func() else: self.resume_func() def update_rate(self, amount): self.burst_avg = (self.burst_avg/2.0 + float(amount)/2.0) self.measurefunc(amount) self.total_downmeasure.update_rate(amount) self.downmeasure.update_rate(amount) self.check_rate() def get_rate(self): return self.downmeasure.get_rate() def make_download(self, connection): ip = connection.ip perip = self.perip.setdefault(ip, PerIPStats()) perip.numconnections += 1 d = Download(self, connection) perip.lastdownload = d perip.peerid = connection.id self.downloads.append(d) return d def lost_peer(self, download): self.downloads.remove(download) ip = download.connection.ip self.perip[ip].numconnections -= 1 if self.perip[ip].lastdownload == download: self.perip[ip].lastdownload = None def kick(self, download): if not self.config['retaliate_to_garbled_data']: return ip = download.connection.ip peerid = download.connection.id # kickfunc will schedule connection.close() to be executed later; we # might now be inside RawServer event loop with events from that # connection already queued, and trying to handle them after doing # close() now could cause problems. self.kickfunc(download.connection) def ban(self, ip): if not self.config['retaliate_to_garbled_data']: return self.banfunc(ip) self.bad_peers[ip] = (True, self.perip[ip])