def get_allocated_regions(path, f=None, begin=0, length=None):
supported = get_sparse_files_support(path)
if not supported:
return
if os.name == 'nt':
if not os.path.exists(path):
return False
if f is None:
f = file(path, 'r')
handle = win32file._get_osfhandle(f.fileno())
if length is None:
length = os.path.getsize(path) - begin
a = SparseSet()
interval = 10000000
i = begin
end = begin + length
while i < end:
d = struct.pack("<QQ", i, interval)
try:
r = win32file.DeviceIoControl(handle, FSCTL_QUERY_ALLOCATED_RANGES,
d, interval, None)
except pywintypes.error, e:
# I've seen:
# error: (1784, 'DeviceIoControl', 'The supplied user buffer is not valid for the requested operation.')
return
for c in xrange(0, len(r), 16):
qq = struct.unpack("<QQ", buffer(r, c, 16))
b = qq[0]
e = b + qq[1]
a.add(b, e)
i += interval
return a
def _build_file_structs(self, filepool, files):
total = 0
for filename, length in files:
# we're shutting down, abort.
if self.doneflag.isSet():
return False
self.undownloaded[filename] = length
if length > 0:
self.ranges.append((total, total + length, filename))
self.range_by_name[filename] = (total, total + length)
if os.path.exists(filename):
if not os.path.isfile(filename):
raise BTFailure(_("File %s already exists, but is not a "
"regular file") % filename)
l = os.path.getsize(filename)
if l > length:
# This is the truncation Bram was talking about that no one
# else thinks is a good idea.
#h = file(filename, 'rb+')
#make_file_sparse(filename, h, length)
#h.truncate(length)
#h.close()
l = length
a = get_allocated_regions(filename, begin=0, length=l)
if a is not None:
a.offset(total)
else:
a = SparseSet()
if l > 0:
a.add(total, total + l)
self.allocated_regions += a
total += length
self.total_length = total
self.initialized = True
return True
def initialize(self, save_path, files):
# a list of bytes ranges and filenames for window-based IO
self.ranges = []
# a dict of filename-to-ranges for piece priorities and filename lookup
self.range_by_name = {}
# a sparse set for smart allocation detection
self.allocated_regions = SparseSet()
# dict of filename-to-length on disk (for % complete in the file view)
self.undownloaded = {}
self.save_path = save_path
# Rather implement this as an ugly hack here than change all the
# individual calls. Affects all torrent instances using this module.
if self.config['bad_libc_workaround']:
bad_libc_workaround()
self.initialized = False
self.startup_df = ThreadedDeferred(_wrap_task(self.external_add_task),
self._build_file_structs,
self.filepool, files)
return self.startup_df
def __init__(self, config, storage, urlage, picker, numpieces,
finished, total_downmeasure, downmeasure, measurefunc,
kickfunc, banfunc):
self.config = config
self.storage = storage
self.urlage = urlage
self.picker = picker
self.rerequester = None
self.connection_manager = None
self.chunksize = config['download_slice_size']
self.total_downmeasure = total_downmeasure
self.downmeasure = downmeasure
self.numpieces = numpieces
self.finished = finished
self.snub_time = config['snub_time']
self.measurefunc = measurefunc
self.kickfunc = kickfunc
self.banfunc = banfunc
self.downloads = []
self.perip = {}
self.bad_peers = {}
self.discarded_bytes = 0
self.burst_avg = 0
self.piece_states = PieceSetBuckets()
nothing = SparseSet(xrange(self.numpieces))
self.piece_states.buckets.append(nothing)
# I hate this
nowhere = [(i, 0) for i in xrange(self.numpieces)]
self.piece_states.place_in_buckets = dict(nowhere)
self.last_update = 0
indexes = self.storage.inactive_requests.keys()
indexes.sort()
self.active_requests = SparseSet(indexes)
self.all_requests = set()
class Storage(object):
def __init__(self, config, filepool, save_path, files, add_task,
external_add_task, doneflag):
self.filepool = filepool
self.config = config
self.doneflag = doneflag
self.add_task = add_task
self.external_add_task = external_add_task
self.initialize(save_path, files)
def initialize(self, save_path, files):
# a list of bytes ranges and filenames for window-based IO
self.ranges = []
# a dict of filename-to-ranges for piece priorities and filename lookup
self.range_by_name = {}
# a sparse set for smart allocation detection
self.allocated_regions = SparseSet()
# dict of filename-to-length on disk (for % complete in the file view)
self.undownloaded = {}
self.save_path = save_path
# Rather implement this as an ugly hack here than change all the
# individual calls. Affects all torrent instances using this module.
if self.config['bad_libc_workaround']:
bad_libc_workaround()
self.initialized = False
self.startup_df = ThreadedDeferred(_wrap_task(self.external_add_task),
self._build_file_structs,
self.filepool, files)
return self.startup_df
def _build_file_structs(self, filepool, files):
total = 0
for filename, length in files:
# we're shutting down, abort.
if self.doneflag.isSet():
return False
self.undownloaded[filename] = length
if length > 0:
self.ranges.append((total, total + length, filename))
self.range_by_name[filename] = (total, total + length)
if os.path.exists(filename):
if not os.path.isfile(filename):
raise BTFailure(_("File %s already exists, but is not a "
"regular file") % filename)
l = os.path.getsize(filename)
if l > length:
# This is the truncation Bram was talking about that no one
# else thinks is a good idea.
#h = file(filename, 'rb+')
#make_file_sparse(filename, h, length)
#h.truncate(length)
#h.close()
l = length
a = get_allocated_regions(filename, begin=0, length=l)
if a is not None:
a.offset(total)
else:
a = SparseSet()
if l > 0:
a.add(total, total + l)
self.allocated_regions += a
total += length
self.total_length = total
self.initialized = True
return True
def get_byte_range_for_filename(self, filename):
if filename not in self.range_by_name:
filename = os.path.normpath(filename)
filename = os.path.join(self.save_path, filename)
return self.range_by_name[filename]
def was_preallocated(self, pos, length):
return self.allocated_regions.is_range_in(pos, pos+length)
def get_total_length(self):
return self.total_length
def _intervals(self, pos, amount):
r = []
stop = pos + amount
p = max(bisect_right(self.ranges, (pos, 2 ** 500)) - 1, 0)
for begin, end, filename in self.ranges[p:]:
if begin >= stop:
break
r.append((filename, max(pos, begin) - begin, min(end, stop) - begin))
return r
def _file_op(self, filename, pos, param, write):
begin, end = self.get_byte_range_for_filename(filename)
length = end - begin
final = Deferred()
hdf = self.filepool.acquire_handle(filename, for_write=write, length=length)
def handle_error(f=None):
final.callback(0)
# error acquiring handle
if hdf is None:
handle_error()
return final
def op(h):
h.seek(pos)
if write:
odf = h.write(param)
else:
odf = h.read(param)
def complete(r):
self.filepool.release_handle(filename, h)
final.callback(r)
odf.addCallback(complete)
odf.addErrback(final.errback)
hdf.addCallback(op)
hdf.addErrback(handle_error)
return final
def _batch_read(self, pos, amount):
dfs = []
r = []
# queue all the reads
for filename, pos, end in self._intervals(pos, amount):
df = self._file_op(filename, pos, end - pos, write=False)
dfs.append(df)
# yield on all the reads in order - they complete in any order
for df in dfs:
yield df
r.append(df.getResult())
r = ''.join(r)
if len(r) != amount:
raise BTFailure(_("Short read (%d of %d) - something truncated files?") %
(len(r), amount))
yield r
def read(self, pos, amount):
df = launch_coroutine(_wrap_task(self.add_task),
self._batch_read, pos, amount)
return df
def _batch_write(self, pos, s):
dfs = []
total = 0
amount = len(s)
# queue all the writes
for filename, begin, end in self._intervals(pos, amount):
length = end - begin
assert length > 0, '%s %s' % (pos, amount)
d = buffer(s, total, length)
total += length
df = self._file_op(filename, begin, d, write=True)
dfs.append(df)
assert total == amount, '%s and %s' % (total, amount)
written = 0
# yield on all the writes - they complete in any order
for df in dfs:
yield df
written += df.getResult()
assert total == written, '%s and %s' % (total, written)
yield total
def write(self, pos, s):
df = launch_coroutine(_wrap_task(self.add_task),
self._batch_write, pos, s)
return df
def close(self):
if not self.initialized:
end = Deferred()
def post_init(r):
df = self.filepool.close_files(self.range_by_name)
df.addCallback(end.callback)
self.startup_df.addCallback(post_init)
return end
df = self.filepool.close_files(self.range_by_name)
return df
def downloaded(self, pos, length):
for filename, begin, end in self._intervals(pos, length):
self.undownloaded[filename] -= end - begin
class MultiDownload(object):
def __init__(self, config, storage, urlage, picker, numpieces,
finished, total_downmeasure, downmeasure, measurefunc,
kickfunc, banfunc):
self.config = config
self.storage = storage
self.urlage = urlage
self.picker = picker
self.rerequester = None
self.connection_manager = None
self.chunksize = config['download_slice_size']
self.total_downmeasure = total_downmeasure
self.downmeasure = downmeasure
self.numpieces = numpieces
self.finished = finished
self.snub_time = config['snub_time']
self.measurefunc = measurefunc
self.kickfunc = kickfunc
self.banfunc = banfunc
self.downloads = []
self.perip = {}
self.bad_peers = {}
self.discarded_bytes = 0
self.burst_avg = 0
self.piece_states = PieceSetBuckets()
nothing = SparseSet(xrange(self.numpieces))
self.piece_states.buckets.append(nothing)
# I hate this
nowhere = [(i, 0) for i in xrange(self.numpieces)]
self.piece_states.place_in_buckets = dict(nowhere)
self.last_update = 0
indexes = self.storage.inactive_requests.keys()
indexes.sort()
self.active_requests = SparseSet(indexes)
self.all_requests = set()
def attach_connection_manager(self, connection_manager):
self.connection_manager = connection_manager
def aggregate_piece_states(self):
d = {}
d['h'] = self.storage.have_set
d['t'] = self.active_requests
for i, bucket in enumerate(self.piece_states.buckets):
d[i] = bucket
r = (self.numpieces, self.last_update, d)
return r
def get_adjusted_distributed_copies(self):
# compensate for the fact that piece picker does not contain all the pieces
num = self.picker.get_distributed_copies()
percent_have = (float(len(self.storage.have_set)) /
float(self.numpieces))
num += percent_have
if self.rerequester and self.rerequester.tracker_num_seeds:
num += self.rerequester.tracker_num_seeds
return num
def active_requests_add(self, r):
self.last_update += 1
self.active_requests.add(r)
def active_requests_remove(self, r):
self.last_update += 1
self.active_requests.remove(r)
def got_have(self, piece):
self.picker.got_have(piece)
self.last_update += 1
p = self.piece_states
p.add(piece, p.remove(piece) + 1)
def got_have_all(self):
self.picker.got_have_all()
self.last_update += 1
self.piece_states.prepend_bucket()
def lost_have(self, piece):
self.picker.lost_have(piece)
self.last_update += 1
p = self.piece_states
p.add(piece, p.remove(piece) - 1)
def lost_have_all(self):
self.picker.lost_have_all()
self.last_update += 1
self.piece_states.popleft_bucket()
def hashchecked(self, index):
if not self.storage.do_I_have(index):
if self.storage.endgame:
while self.storage.want_requests(index):
nb, nl = self.storage.new_request(index)
self.all_requests.add((index, nb, nl))
for d in self.downloads:
d.fix_download_endgame()
else:
ds = [d for d in self.downloads if not d.choked]
random.shuffle(ds)
for d in ds:
d._request_more([index])
return
self.picker.complete(index)
self.active_requests_remove(index)
self.connection_manager.hashcheck_succeeded(index)
if self.storage.have.numfalse == 0:
for d in self.downloads:
if d.have.numfalse == 0:
d.connection.close()
self.finished()
def time_until_rate(self):
rate = self.config['max_download_rate']
# obey "no max" setting
if rate == -1:
return 0
possible_burst = self.burst_avg
# substract the possible burst so that the download rate is a maximum
adjusted_rate = rate - possible_burst
# don't let it go to 0, just because the burst is greater than the
# desired rate. this could be improved by no throttling and unthrottling
# all connections at once.
adjusted_rate = max(adjusted_rate, possible_burst)
# don't let it go above the explicit max either.
adjusted_rate = min(adjusted_rate, rate)
# we need to call this because the rate measure is dumb
self.total_downmeasure.get_rate()
t = self.total_downmeasure.time_until_rate(adjusted_rate)
#print "DOWNLOAD: burst_size:", possible_burst, "rate:", self.total_downmeasure.get_rate(), "rate_limit:", rate, "new_rate:", adjusted_rate, t
return t
def check_rate(self):
t = self.time_until_rate()
if t > 0:
self.postpone_func()
else:
self.resume_func()
def update_rate(self, amount):
self.burst_avg = (self.burst_avg/2.0 + float(amount)/2.0)
self.measurefunc(amount)
self.total_downmeasure.update_rate(amount)
self.downmeasure.update_rate(amount)
self.check_rate()
def get_rate(self):
return self.downmeasure.get_rate()
def make_download(self, connection):
ip = connection.ip
perip = self.perip.setdefault(ip, PerIPStats())
perip.numconnections += 1
d = Download(self, connection)
perip.lastdownload = d
perip.peerid = connection.id
self.downloads.append(d)
return d
def lost_peer(self, download):
self.downloads.remove(download)
ip = download.connection.ip
self.perip[ip].numconnections -= 1
if self.perip[ip].lastdownload == download:
self.perip[ip].lastdownload = None
def kick(self, download):
if not self.config['retaliate_to_garbled_data']:
return
ip = download.connection.ip
peerid = download.connection.id
# kickfunc will schedule connection.close() to be executed later; we
# might now be inside RawServer event loop with events from that
# connection already queued, and trying to handle them after doing
# close() now could cause problems.
self.kickfunc(download.connection)
def ban(self, ip):
if not self.config['retaliate_to_garbled_data']:
return
self.banfunc(ip)
self.bad_peers[ip] = (True, self.perip[ip])
