def checkPiece(self, piece): metainfo = self.metainfo if metainfo.is_batch: files = [os.path.join(self.pathName, f) for f in metainfo.files_fs] else: files = [self.pathName] filePool = FilePool(self.config['max_files_open']) filePool.add_files(files, None) storage = Storage(self.config, filePool, zip(files, metainfo.sizes)) doneFlag = threading.Event() config = Preferences(self.config) config['check_hashes'] = True storageWrapper = StorageWrapper(storage, config, metainfo.hashes, metainfo.piece_length, None, self.statusFunc, doneFlag, None, metainfo.infohash, None, None) result = storageWrapper.do_I_have(piece) storage.close() filePool.close_all() return result
def _start_download(self, metainfo, feedback, save_path): self.feedback = feedback config = self.config self.infohash = metainfo.infohash self.total_bytes = metainfo.total_bytes if not metainfo.reported_errors: metainfo.show_encoding_errors(self._error) myid = self._make_id() seed(myid) def schedfunc(func, delay): self._rawserver.add_task(func, delay, context=self) def externalsched(func, delay): self._rawserver.external_add_task(func, delay, context=self) if metainfo.is_batch: myfiles = [os.path.join(save_path, f) for f in metainfo.files_fs] else: myfiles = [save_path] self._filepool.add_files(myfiles, self) self._myfiles = myfiles self._storage = Storage(config, self._filepool, zip(myfiles, metainfo.sizes)) resumefile = None if config['data_dir']: filename = os.path.join(config['data_dir'], 'resume', self.infohash.encode('hex')) if os.path.exists(filename): try: resumefile = file(filename, 'rb') if self._storage.check_fastresume(resumefile) == 0: resumefile.close() resumefile = None except Exception, e: self._error(WARNING, _("Could not load fastresume data: %s") % str(e) + ' ' + _("Will perform full hash check.")) if resumefile is not None: resumefile.close() resumefile = None
def get_completion(self, config, metainfo, save_path, filelist=False): if not config['data_dir']: return None infohash = metainfo.infohash if metainfo.is_batch: myfiles = [os.path.join(save_path, f) for f in metainfo.files_fs] else: myfiles = [save_path] if metainfo.total_bytes == 0: if filelist: return None return 1 try: s = Storage(None, None, zip(myfiles, metainfo.sizes), check_only=True) except: return None filename = os.path.join(config['data_dir'], 'resume', infohash.encode('hex')) try: f = file(filename, 'rb') except: f = None try: r = s.check_fastresume(f, filelist, metainfo.piece_length, len(metainfo.hashes), myfiles) except: r = None if f is not None: f.close() if r is None: return None if filelist: return r[0] / metainfo.total_bytes, r[1], r[2] return r / metainfo.total_bytes
def _start_download(self, metainfo, feedback, save_path): self.feedback = feedback config = self.config #constrains the upload rate according to the config default or command line arguments self._set_auto_uploads() self.infohash = metainfo.infohash self.total_bytes = metainfo.total_bytes if not metainfo.reported_errors: metainfo.show_encoding_errors(self._error) #create my peer ID myid = self._make_id() self.logcollector.log(None , 'P ID ' + myid) #initialize the seed of the ramdom number generator based on my peer ID seed(myid) def schedfunc(func, delay): self._rawserver.add_task(func, delay, self) def externalsched(func, delay): self._rawserver.external_add_task(func, delay, self) if metainfo.is_batch: myfiles = [os.path.join(save_path, f) for f in metainfo.files_fs] else: myfiles = [save_path] self._filepool.add_files(myfiles, self) self._myfiles = myfiles self._storage = Storage(config, self._filepool, zip(myfiles, metainfo.sizes)) resumefile = None if config['data_dir']: filename = os.path.join(config['data_dir'], 'resume', self.infohash.encode('hex')) if os.path.exists(filename): try: resumefile = file(filename, 'rb') if self._storage.check_fastresume(resumefile) == 0: resumefile.close() resumefile = None except Exception, e: self._error(WARNING, 'Could not load fastresume data: '+ str(e) + '. Will perform full hash check.') if resumefile is not None: resumefile.close() resumefile = None
def _start_download(self, metainfo, feedback, save_path): self.feedback = feedback config = self.config self.infohash = metainfo.infohash self.total_bytes = metainfo.total_bytes if not metainfo.reported_errors: metainfo.show_encoding_errors(self._error) myid = self._make_id() seed(myid) def schedfunc(func, delay): self._rawserver.add_task(func, delay, context=self) def externalsched(func, delay): self._rawserver.external_add_task(func, delay, context=self) if metainfo.is_batch: myfiles = [os.path.join(save_path, f) for f in metainfo.files_fs] else: myfiles = [save_path] self._filepool.add_files(myfiles, self) self._myfiles = myfiles if config['upload_only']: self.cachehandler = Cache_handler(myfiles,True) else: self.cachehandler = Cache_handler(myfiles) #self.cachehandler.update_cache() self._storage = Storage(self.cachehandler,config, self._filepool, zip(myfiles, metainfo.sizes)) resumefile = None if config['data_dir']: filename = os.path.join(config['data_dir'], 'resume', self.infohash.encode('hex')) if os.path.exists(filename): try: resumefile = file(filename, 'rb') if self._storage.check_fastresume(resumefile) == 0: resumefile.close() resumefile = None except Exception, e: self._error(WARNING, _("Could not load fastresume data: %s") % str(e) + ' ' + _("Will perform full hash check.")) if resumefile is not None: resumefile.close() resumefile = None
class _SingleTorrent(object): def __init__(self, rawserver, singleport_listener, ratelimiter, filepool, config, logcollector): self._rawserver = rawserver self._singleport_listener = singleport_listener self._ratelimiter = ratelimiter self._filepool = filepool self.config = dict(config) self._storage = None self._storagewrapper = None self._ratemeasure = None self._upmeasure = None self._downmeasure = None self._encoder = None self._rerequest = None self._statuscollecter = None self._announced = False self._listening = False self.reserved_ports = [] self.reported_port = None self._myfiles = None self.started = False self.is_seed = False self.closed = False self.infohash = None self.total_bytes = None self._doneflag = threading.Event() self.finflag = threading.Event() self._hashcheck_thread = None self._contfunc = None self._activity = ('Initial startup', 0) self.feedback = None self.errors = [] self.logcollector=logcollector def start_download(self, *args, **kwargs): it = self._start_download(*args, **kwargs) def cont(): try: it.next() except StopIteration: self._contfunc = None def contfunc(): self._rawserver.external_add_task(cont, 0, self) self._contfunc = contfunc contfunc() def _start_download(self, metainfo, feedback, save_path): self.feedback = feedback config = self.config #constrains the upload rate according to the config default or command line arguments self._set_auto_uploads() self.infohash = metainfo.infohash self.total_bytes = metainfo.total_bytes if not metainfo.reported_errors: metainfo.show_encoding_errors(self._error) #create my peer ID myid = self._make_id() self.logcollector.log(None , 'P ID ' + myid) #initialize the seed of the ramdom number generator based on my peer ID seed(myid) def schedfunc(func, delay): self._rawserver.add_task(func, delay, self) def externalsched(func, delay): self._rawserver.external_add_task(func, delay, self) if metainfo.is_batch: myfiles = [os.path.join(save_path, f) for f in metainfo.files_fs] else: myfiles = [save_path] self._filepool.add_files(myfiles, self) self._myfiles = myfiles self._storage = Storage(config, self._filepool, zip(myfiles, metainfo.sizes)) resumefile = None if config['data_dir']: filename = os.path.join(config['data_dir'], 'resume', self.infohash.encode('hex')) if os.path.exists(filename): try: resumefile = file(filename, 'rb') if self._storage.check_fastresume(resumefile) == 0: resumefile.close() resumefile = None except Exception, e: self._error(WARNING, 'Could not load fastresume data: '+ str(e) + '. Will perform full hash check.') if resumefile is not None: resumefile.close() resumefile = None def data_flunked(amount, index): self._ratemeasure.data_rejected(amount) self._error(INFO, 'piece %d failed hash check, ' 're-downloading it' % index) backthread_exception = [] def errorfunc(level, text): def e(): self._error(level, text) externalsched(e, 0) def hashcheck(): def statusfunc(activity = None, fractionDone = 0): if activity is None: activity = self._activity[0] self._activity = (activity, fractionDone) try: self._storagewrapper = StorageWrapper(self._storage, config, metainfo.hashes, metainfo.piece_length, self._finished, statusfunc, self._doneflag, data_flunked, self.infohash, errorfunc, resumefile, self.logcollector) except: backthread_exception.append(sys.exc_info()) self._contfunc() thread = threading.Thread(target = hashcheck) thread.setDaemon(False) self._hashcheck_thread = thread thread.start() yield None self._hashcheck_thread = None if resumefile is not None: resumefile.close() if backthread_exception: a, b, c = backthread_exception[0] raise a, b, c if self._storagewrapper.amount_left == 0: self._finished() choker = Choker(config, schedfunc, self.logcollector, self.finflag.isSet) upmeasure = Measure(config['max_rate_period']) upmeasure_seedtime = Measure(config['max_rate_period_seedtime']) downmeasure = Measure(config['max_rate_period']) self._upmeasure = upmeasure self._downmeasure = downmeasure self._ratemeasure = RateMeasure(self._storagewrapper. amount_left_with_partials) picker = PiecePicker(len(metainfo.hashes), config) #initialize the PiecePicker object for i in xrange(len(metainfo.hashes)): if self._storagewrapper.do_I_have(i): picker.complete(i) for i in self._storagewrapper.stat_dirty: picker.requested(i) def kickpeer(connection): def kick(): connection.close() schedfunc(kick, 0) def banpeer(ip): self._encoder.ban(ip) downloader = Downloader(config, self._storagewrapper, picker, len(metainfo.hashes), downmeasure, self._ratemeasure.data_came_in, kickpeer, banpeer, self.logcollector) def make_upload(connection): return Upload(connection, self._ratelimiter, upmeasure, upmeasure_seedtime, choker, self._storagewrapper, config['max_slice_length'], config['max_rate_period'], self.logcollector) self._encoder = Encoder(make_upload, downloader, choker, len(metainfo.hashes), self._ratelimiter, self._rawserver, config, myid, schedfunc, self.infohash, self, self.logcollector) self.reported_port = self.config['forwarded_port'] if not self.reported_port: self.reported_port = self._singleport_listener.get_port() self.reserved_ports.append(self.reported_port) self._singleport_listener.add_torrent(self.infohash, self._encoder) self._listening = True #retreives peer set from the tracker self._rerequest = Rerequester(metainfo.announce, config, schedfunc, self._encoder.how_many_connections, self._encoder.start_connection, externalsched, self._storagewrapper.get_amount_left, upmeasure.get_total, downmeasure.get_total, self.reported_port, myid, self.infohash, self._error, self.finflag, upmeasure.get_rate, downmeasure.get_rate, self._encoder.ever_got_incoming, self.internal_shutdown, self._announce_done, self.logcollector) self._statuscollecter = DownloaderFeedback(choker, upmeasure.get_rate, upmeasure_seedtime.get_rate, downmeasure.get_rate, upmeasure.get_total, downmeasure.get_total, self._ratemeasure.get_time_left, self._ratemeasure.get_size_left, self.total_bytes, self.finflag, downloader, self._myfiles) self._announced = True self._rerequest.begin() self.started = True if not self.finflag.isSet(): self._activity = ('downloading', 0) self.feedback.started(self)
class _SingleTorrent(object): def __init__(self, rawserver, singleport_listener, ratelimiter, filepool, config, dht): self._rawserver = rawserver self._singleport_listener = singleport_listener self._ratelimiter = ratelimiter self._filepool = filepool self._dht = dht self._storage = None self._storagewrapper = None self._ratemeasure = None self._upmeasure = None self._downmeasure = None self._encoder = None self._rerequest = None self._statuscollecter = None self._announced = False self._listening = False self.reserved_ports = [] self.reported_port = None self._myfiles = None self.started = False self.is_seed = False self.closed = False self.infohash = None self.total_bytes = None self._doneflag = threading.Event() self.finflag = threading.Event() self._hashcheck_thread = None self._contfunc = None self._activity = (_("Initial startup"), 0) self.feedback = None self.errors = [] self.rlgroup = None self.config = config def start_download(self, *args, **kwargs): it = self._start_download(*args, **kwargs) def cont(): try: it.next() except StopIteration: self._contfunc = None def contfunc(): self._rawserver.external_add_task(cont, 0, context=self) self._contfunc = contfunc contfunc() def _start_download(self, metainfo, feedback, save_path): self.feedback = feedback config = self.config self.infohash = metainfo.infohash self.total_bytes = metainfo.total_bytes if not metainfo.reported_errors: metainfo.show_encoding_errors(self._error) myid = self._make_id() seed(myid) def schedfunc(func, delay): self._rawserver.add_task(func, delay, context=self) def externalsched(func, delay): self._rawserver.external_add_task(func, delay, context=self) if metainfo.is_batch: myfiles = [os.path.join(save_path, f) for f in metainfo.files_fs] else: myfiles = [save_path] self._filepool.add_files(myfiles, self) self._myfiles = myfiles self._storage = Storage(config, self._filepool, zip(myfiles, metainfo.sizes)) resumefile = None if config['data_dir']: filename = os.path.join(config['data_dir'], 'resume', self.infohash.encode('hex')) if os.path.exists(filename): try: resumefile = file(filename, 'rb') if self._storage.check_fastresume(resumefile) == 0: resumefile.close() resumefile = None except Exception, e: self._error(WARNING, _("Could not load fastresume data: %s.") % str(e) + ' ' + _("Will perform full hash check.")) if resumefile is not None: resumefile.close() resumefile = None def data_flunked(amount, index): self._ratemeasure.data_rejected(amount) self._error(INFO, _("piece %d failed hash check, re-downloading it") % index) backthread_exception = [] def errorfunc(level, text): def e(): self._error(level, text) externalsched(e, 0) def hashcheck(): def statusfunc(activity = None, fractionDone = 0): if activity is None: activity = self._activity[0] self._activity = (activity, fractionDone) try: self._storagewrapper = StorageWrapper(self._storage, config, metainfo.hashes, metainfo.piece_length, self._finished, statusfunc, self._doneflag, data_flunked, self.infohash, errorfunc, resumefile) except: backthread_exception.append(sys.exc_info()) self._contfunc() thread = threading.Thread(target = hashcheck) thread.setDaemon(False) self._hashcheck_thread = thread thread.start() yield None self._hashcheck_thread = None if resumefile is not None: resumefile.close() if backthread_exception: a, b, c = backthread_exception[0] raise a, b, c if self._storagewrapper.amount_left == 0: self._finished() choker = Choker(config, schedfunc, self.finflag.isSet) upmeasure = Measure(config['max_rate_period']) upmeasure_seedtime = Measure(config['max_rate_period_seedtime']) downmeasure = Measure(config['max_rate_period']) self._upmeasure = upmeasure self._upmeasure_seedtime = upmeasure_seedtime self._downmeasure = downmeasure self._ratemeasure = RateMeasure(self._storagewrapper. amount_left_with_partials) picker = PiecePicker(len(metainfo.hashes), config) for i in xrange(len(metainfo.hashes)): if self._storagewrapper.do_I_have(i): picker.complete(i) for i in self._storagewrapper.stat_dirty: picker.requested(i) def kickpeer(connection): def kick(): connection.close() schedfunc(kick, 0) def banpeer(ip): self._encoder.ban(ip) downloader = Downloader(config, self._storagewrapper, picker, len(metainfo.hashes), downmeasure, self._ratemeasure.data_came_in, kickpeer, banpeer) def make_upload(connection): return Upload(connection, self._ratelimiter, upmeasure, upmeasure_seedtime, choker, self._storagewrapper, config['max_slice_length'], config['max_rate_period']) self.reported_port = self.config['forwarded_port'] if not self.reported_port: self.reported_port = self._singleport_listener.get_port() self.reserved_ports.append(self.reported_port) if self._dht: addContact = self._dht.addContact else: addContact = None self._encoder = Encoder(make_upload, downloader, choker, len(metainfo.hashes), self._ratelimiter, self._rawserver, config, myid, schedfunc, self.infohash, self, addContact, self.reported_port) self._singleport_listener.add_torrent(self.infohash, self._encoder) self._listening = True if metainfo.is_trackerless: if not self._dht: self._error(self, CRITICAL, _("Attempt to download a trackerless torrent with trackerless client turned off.")) return else: if len(self._dht.table.findNodes(metainfo.infohash, invalid=False)) < const.K: for host, port in metainfo.nodes: self._dht.addContact(host, port) self._rerequest = DHTRerequester(config, schedfunc, self._encoder.how_many_connections, self._encoder.start_connection, externalsched, self._storagewrapper.get_amount_left, upmeasure.get_total, downmeasure.get_total, self.reported_port, myid, self.infohash, self._error, self.finflag, upmeasure.get_rate, downmeasure.get_rate, self._encoder.ever_got_incoming, self.internal_shutdown, self._announce_done, self._dht) else: self._rerequest = Rerequester(metainfo.announce, config, schedfunc, self._encoder.how_many_connections, self._encoder.start_connection, externalsched, self._storagewrapper.get_amount_left, upmeasure.get_total, downmeasure.get_total, self.reported_port, myid, self.infohash, self._error, self.finflag, upmeasure.get_rate, downmeasure.get_rate, self._encoder.ever_got_incoming, self.internal_shutdown, self._announce_done) self._statuscollecter = DownloaderFeedback(choker, upmeasure.get_rate, upmeasure_seedtime.get_rate, downmeasure.get_rate, upmeasure.get_total, downmeasure.get_total, self._ratemeasure.get_time_left, self._ratemeasure.get_size_left, self.total_bytes, self.finflag, downloader, self._myfiles, self._encoder.ever_got_incoming, self._rerequest) self._announced = True if self._dht and len(self._dht.table.findNodes(self.infohash)) == 0: self._rawserver.add_task(self._dht.findCloseNodes, 5) self._rawserver.add_task(self._rerequest.begin, 20) else: self._rerequest.begin() self.started = True if not self.finflag.isSet(): self._activity = (_("downloading"), 0) self.feedback.started(self)
class _SingleTorrent(object): def __init__(self, rawserver, singleport_listener, ratelimiter, filepool, config, dht): self._rawserver = rawserver self._singleport_listener = singleport_listener self._ratelimiter = ratelimiter self._filepool = filepool self._dht = dht self._storage = None self._storagewrapper = None self._ratemeasure = None self._upmeasure = None self._downmeasure = None self._encoder = None self._rerequest = None self._statuscollecter = None self._announced = False self._listening = False self.reserved_ports = [] self.reported_port = None self._myfiles = None self.started = False self.is_seed = False self.closed = False self.infohash = None self.total_bytes = None self._doneflag = threading.Event() self.finflag = threading.Event() self._hashcheck_thread = None self._contfunc = None self._activity = (_("Initial startup"), 0) self.feedback = None self.errors = [] self.rlgroup = None self.config = config def start_download(self, *args, **kwargs): it = self._start_download(*args, **kwargs) def cont(): try: it.next() except StopIteration: self._contfunc = None def contfunc(): self._rawserver.external_add_task(cont, 0, context=self) self._contfunc = contfunc contfunc() def _start_download(self, metainfo, feedback, save_path): self.feedback = feedback config = self.config self.infohash = metainfo.infohash self.total_bytes = metainfo.total_bytes if not metainfo.reported_errors: metainfo.show_encoding_errors(self._error) myid = self._make_id() seed(myid) def schedfunc(func, delay): self._rawserver.add_task(func, delay, context=self) def externalsched(func, delay): self._rawserver.external_add_task(func, delay, context=self) if metainfo.is_batch: myfiles = [os.path.join(save_path, f) for f in metainfo.files_fs] else: myfiles = [save_path] self._filepool.add_files(myfiles, self) self._myfiles = myfiles self._storage = Storage(config, self._filepool, zip(myfiles, metainfo.sizes)) resumefile = None if config['data_dir']: filename = os.path.join(config['data_dir'], 'resume', self.infohash.encode('hex')) if os.path.exists(filename): try: resumefile = file(filename, 'rb') if self._storage.check_fastresume(resumefile) == 0: resumefile.close() resumefile = None except Exception, e: self._error(WARNING, _("Could not load fastresume data: %s") % str(e) + ' ' + _("Will perform full hash check.")) if resumefile is not None: resumefile.close() resumefile = None def data_flunked(amount, index): self._ratemeasure.data_rejected(amount) self._error(INFO, _("piece %d failed hash check, re-downloading it") % index) backthread_exception = [] def errorfunc(level, text): def e(): self._error(level, text) externalsched(e, 0) def hashcheck(): def statusfunc(activity = None, fractionDone = 0): if activity is None: activity = self._activity[0] self._activity = (activity, fractionDone) try: self._storagewrapper = StorageWrapper(self._storage, config, metainfo.hashes, metainfo.piece_length, self._finished, statusfunc, self._doneflag, data_flunked, self.infohash, errorfunc, resumefile) except: backthread_exception.append(sys.exc_info()) self._contfunc() thread = threading.Thread(target = hashcheck) thread.setDaemon(False) self._hashcheck_thread = thread thread.start() yield None self._hashcheck_thread = None if resumefile is not None: resumefile.close() if backthread_exception: a, b, c = backthread_exception[0] raise a, b, c if self._storagewrapper.amount_left == 0: self._finished() choker = Choker(config, schedfunc, self.finflag.isSet) upmeasure = Measure(config['max_rate_period']) upmeasure_seedtime = Measure(config['max_rate_period_seedtime']) downmeasure = Measure(config['max_rate_period']) self._upmeasure = upmeasure self._upmeasure_seedtime = upmeasure_seedtime self._downmeasure = downmeasure self._ratemeasure = RateMeasure(self._storagewrapper. amount_left_with_partials) picker = PiecePicker(len(metainfo.hashes), config) for i in xrange(len(metainfo.hashes)): if self._storagewrapper.do_I_have(i): picker.complete(i) for i in self._storagewrapper.stat_dirty: picker.requested(i) def kickpeer(connection): def kick(): connection.close() schedfunc(kick, 0) def banpeer(ip): self._encoder.ban(ip) downloader = Downloader(config, self._storagewrapper, picker, len(metainfo.hashes), downmeasure, self._ratemeasure.data_came_in, kickpeer, banpeer) def make_upload(connection): return Upload(connection, self._ratelimiter, upmeasure, upmeasure_seedtime, choker, self._storagewrapper, config['max_slice_length'], config['max_rate_period']) self.reported_port = self.config['forwarded_port'] if not self.reported_port: self.reported_port = self._singleport_listener.get_port(self.change_port) self.reserved_ports.append(self.reported_port) if self._dht: addContact = self._dht.addContact else: addContact = None self._encoder = Encoder(make_upload, downloader, choker, len(metainfo.hashes), self._ratelimiter, self._rawserver, config, myid, schedfunc, self.infohash, self, addContact, self.reported_port) self._singleport_listener.add_torrent(self.infohash, self._encoder) self._listening = True if metainfo.is_trackerless: if not self._dht: self._error(self, CRITICAL, _("Attempt to download a trackerless torrent with trackerless client turned off.")) return else: if len(self._dht.table.findNodes(metainfo.infohash, invalid=False)) < const.K: for host, port in metainfo.nodes: self._dht.addContact(host, port) self._rerequest = DHTRerequester(config, schedfunc, self._encoder.how_many_connections, self._encoder.start_connection, externalsched, self._storagewrapper.get_amount_left, upmeasure.get_total, downmeasure.get_total, self.reported_port, myid, self.infohash, self._error, self.finflag, upmeasure.get_rate, downmeasure.get_rate, self._encoder.ever_got_incoming, self.internal_shutdown, self._announce_done, self._dht) else: self._rerequest = Rerequester(metainfo.announce, config, schedfunc, self._encoder.how_many_connections, self._encoder.start_connection, externalsched, self._storagewrapper.get_amount_left, upmeasure.get_total, downmeasure.get_total, self.reported_port, myid, self.infohash, self._error, self.finflag, upmeasure.get_rate, downmeasure.get_rate, self._encoder.ever_got_incoming, self.internal_shutdown, self._announce_done) self._statuscollecter = DownloaderFeedback(choker, upmeasure.get_rate, upmeasure_seedtime.get_rate, downmeasure.get_rate, upmeasure.get_total, downmeasure.get_total, self._ratemeasure.get_time_left, self._ratemeasure.get_size_left, self.total_bytes, self.finflag, downloader, self._myfiles, self._encoder.ever_got_incoming, self._rerequest) self._announced = True if self._dht and len(self._dht.table.findNodes(self.infohash)) == 0: self._rawserver.add_task(self._dht.findCloseNodes, 5) self._rawserver.add_task(self._rerequest.begin, 20) else: self._rerequest.begin() self.started = True if not self.finflag.isSet(): self._activity = (_("downloading"), 0) self.feedback.started(self)