Exemple #1
0
    def initFiles(self, old_style=False, statusfunc=None):
        if self.doneflag.is_set():
            return None
        if not statusfunc:
            statusfunc = self.statusfunc

        disabled_files = None
        if self.selector_enabled:
            self.priority = self.config['priority']
            if self.priority:
                try:
                    self.priority = self.priority.split(',')
                    assert len(self.priority) == len(self.files)
                    self.priority = [int(p) for p in self.priority]
                    for p in self.priority:
                        assert p >= -1
                        assert p <= 2
                except (AssertionError, ValueError):
                    self.errorfunc('bad priority list given, ignored')
                    self.priority = None

            data = self.appdataobj.getTorrentData(self.infohash)
            try:
                d = data['resume data']['priority']
                assert len(d) == len(self.files)
                disabled_files = [x == -1 for x in d]
            except (KeyError, TypeError, AssertionError):
                try:
                    disabled_files = [x == -1 for x in self.priority]
                except TypeError:
                    pass

        piece_length = self.metainfo['info']['piece length']
        try:
            try:
                self.storage = Storage(self.files, piece_length, self.doneflag,
                                       self.config, disabled_files)
            except IOError as e:
                self.errorfunc('trouble accessing files - ' + str(e))
                return None
            if self.doneflag.is_set():
                return None

            self.storagewrapper = StorageWrapper(
                self.storage, self.config['download_slice_size'], self.pieces,
                piece_length, self._finished, self._failed, statusfunc,
                self.doneflag, self.config['check_hashes'], self._data_flunked,
                self.rawserver.add_task, self.config, self.unpauseflag)

        except ValueError as e:
            self._failed('bad data - ' + str(e))
        except IOError as e:
            self._failed('IOError - ' + str(e))
        if self.doneflag.is_set():
            return None

        if self.selector_enabled:
            self.fileselector = FileSelector(
                self.files, piece_length,
                self.appdataobj.getPieceDir(self.infohash), self.storage,
                self.storagewrapper, self.rawserver.add_task, self._failed)
            if data:
                data = data.get('resume data')
                if data:
                    self.fileselector.unpickle(data)

        self.checking = True
        if old_style:
            return self.storagewrapper.old_style_init()
        return self.storagewrapper.initialize
Exemple #2
0
    def initFiles(self, old_style=False, statusfunc=None):
        if self.doneflag.is_set():
            return None
        if not statusfunc:
            statusfunc = self.statusfunc

        disabled_files = None
        if self.selector_enabled:
            self.priority = self.config['priority']
            if self.priority:
                try:
                    self.priority = self.priority.split(',')
                    assert len(self.priority) == len(self.files)
                    self.priority = [int(p) for p in self.priority]
                    for p in self.priority:
                        assert p >= -1
                        assert p <= 2
                except (AssertionError, ValueError):
                    self.errorfunc('bad priority list given, ignored')
                    self.priority = None

            data = self.appdataobj.getTorrentData(self.infohash)
            try:
                d = data['resume data']['priority']
                assert len(d) == len(self.files)
                disabled_files = [x == -1 for x in d]
            except (KeyError, TypeError, AssertionError):
                try:
                    disabled_files = [x == -1 for x in self.priority]
                except TypeError:
                    pass

        piece_length = self.metainfo['info']['piece length']
        try:
            try:
                self.storage = Storage(self.files, piece_length, self.doneflag,
                                       self.config, disabled_files)
            except IOError as e:
                self.errorfunc('trouble accessing files - ' + str(e))
                return None
            if self.doneflag.is_set():
                return None

            self.storagewrapper = StorageWrapper(
                self.storage, self.config['download_slice_size'],
                self.pieces, piece_length, self._finished,
                self._failed, statusfunc, self.doneflag,
                self.config['check_hashes'], self._data_flunked,
                self.rawserver.add_task, self.config, self.unpauseflag)

        except ValueError as e:
            self._failed('bad data - ' + str(e))
        except IOError as e:
            self._failed('IOError - ' + str(e))
        if self.doneflag.is_set():
            return None

        if self.selector_enabled:
            self.fileselector = FileSelector(
                self.files, piece_length,
                self.appdataobj.getPieceDir(self.infohash), self.storage,
                self.storagewrapper, self.rawserver.add_task, self._failed)
            if data:
                data = data.get('resume data')
                if data:
                    self.fileselector.unpickle(data)

        self.checking = True
        if old_style:
            return self.storagewrapper.old_style_init()
        return self.storagewrapper.initialize
Exemple #3
0
class BT1Download:
    def __init__(self,
                 statusfunc,
                 finfunc,
                 errorfunc,
                 excfunc,
                 doneflag,
                 config,
                 metainfo,
                 infohash,
                 peerid,
                 rawserver,
                 port,
                 appdataobj=None):
        self.statusfunc = statusfunc
        self.finfunc = finfunc
        self.errorfunc = errorfunc
        self.excfunc = excfunc
        self.doneflag = doneflag
        self.config = config
        self.metainfo = metainfo  # MetaInfo
        self.infohash = infohash  # bytes[20]
        self.myid = peerid  # bytes
        self.rawserver = rawserver
        self.port = port

        self.pieces = self.metainfo['info'].hasher.pieces  # [bytes[20]]
        self.len_pieces = len(self.pieces)
        self.argslistheader = argslistheader
        self.unpauseflag = threading.Event()
        self.unpauseflag.set()
        self.downloader = None
        self.storagewrapper = None
        self.fileselector = None
        self.super_seeding_active = False
        self.filedatflag = threading.Event()
        self.spewflag = threading.Event()
        self.superseedflag = threading.Event()
        self.whenpaused = None
        self.finflag = threading.Event()
        self.rerequest = None
        self.tcp_ack_fudge = config['tcp_ack_fudge']

        self.selector_enabled = config['selector_enabled']
        if appdataobj:
            self.appdataobj = appdataobj
        elif self.selector_enabled:
            self.appdataobj = ConfigDir()
            self.appdataobj.deleteOldCacheData(config['expire_cache_data'],
                                               [self.infohash])

        self.excflag = self.rawserver.get_exception_flag()
        self.failed = False
        self.checking = False
        self.started = False

        self.picker = PiecePicker(self.len_pieces,
                                  config['rarest_first_cutoff'],
                                  config['rarest_first_priority_cutoff'])
        self.choker = Choker(config, rawserver.add_task, self.picker,
                             self.finflag.is_set)

    def saveAs(self, filefunc, pathfunc=None):
        try:

            def make(f, forcedir=False):
                if not forcedir:
                    f = os.path.split(f)[0]
                if f != '' and not os.path.exists(f):
                    os.makedirs(f)

            info = self.metainfo['info']
            if 'length' in info:
                file_length = info['length']
                fname = filefunc(info['name'], file_length,
                                 self.config['saveas'], False)
                if fname is None:
                    return None
                make(fname)
                files = [(fname, file_length)]
            else:
                file_length = sum(x['length'] for x in info['files'])
                fname = filefunc(info['name'], file_length,
                                 self.config['saveas'], True)
                if fname is None:
                    return None

                # if this path exists, and no files from the info dict exist,
                # we assume it's a new download and the user wants to create a
                # new directory with the default name
                existing = 0
                if os.path.exists(fname):
                    if not os.path.isdir(fname):
                        self.errorfunc(fname + 'is not a dir')
                        return None
                    if len(os.listdir(fname)) > 0:  # if it's not empty
                        existing = any(
                            os.path.exists(os.path.join(fname, x['path'][0]))
                            for x in info['files'])
                        if not existing:
                            fname = os.path.join(fname, info['name'])
                            if os.path.exists(fname) and \
                                    not os.path.isdir(fname):
                                if fname[-8:] == '.torrent':
                                    fname = fname[:-8]
                                if os.path.exists(fname) and \
                                        not os.path.isdir(fname):
                                    self.errorfunc("Can't create dir - " +
                                                   info['name'])
                                    return None
                make(fname, True)

                # alert the UI to any possible change in path
                if pathfunc is not None:
                    pathfunc(fname)

                files = []
                for x in info['files']:
                    n = os.path.join(fname, *x['path'])
                    files.append((n, x['length']))
                    make(n)
        except OSError as e:
            self.errorfunc("Couldn't allocate dir - " + str(e))
            return None

        self.filename = fname
        self.files = files
        self.datalength = file_length

        return fname

    def _finished(self):
        self.finflag.set()
        try:
            self.storage.set_readonly()
        except (IOError, OSError) as e:
            self.errorfunc('trouble setting readonly at end - ' + str(e))
        if self.superseedflag.is_set():
            self._set_super_seed()
        self.choker.set_round_robin_period(
            max(
                self.config['round_robin_period'],
                self.config['round_robin_period'] *
                self.metainfo['info']['piece length'] / 200000))
        self.rerequest_complete()
        self.finfunc()

    def _data_flunked(self, amount, index):
        self.ratemeasure_datarejected(amount)
        if not self.doneflag.is_set():
            self.errorfunc('piece {:d} failed hash check, re-downloading it'
                           ''.format(index))

    def _failed(self, reason):
        self.failed = True
        self.doneflag.set()
        if reason is not None:
            self.errorfunc(reason)

    def initFiles(self, old_style=False, statusfunc=None):
        if self.doneflag.is_set():
            return None
        if not statusfunc:
            statusfunc = self.statusfunc

        disabled_files = None
        if self.selector_enabled:
            self.priority = self.config['priority']
            if self.priority:
                try:
                    self.priority = self.priority.split(',')
                    assert len(self.priority) == len(self.files)
                    self.priority = [int(p) for p in self.priority]
                    for p in self.priority:
                        assert p >= -1
                        assert p <= 2
                except (AssertionError, ValueError):
                    self.errorfunc('bad priority list given, ignored')
                    self.priority = None

            data = self.appdataobj.getTorrentData(self.infohash)
            try:
                d = data['resume data']['priority']
                assert len(d) == len(self.files)
                disabled_files = [x == -1 for x in d]
            except (KeyError, TypeError, AssertionError):
                try:
                    disabled_files = [x == -1 for x in self.priority]
                except TypeError:
                    pass

        piece_length = self.metainfo['info']['piece length']
        try:
            try:
                self.storage = Storage(self.files, piece_length, self.doneflag,
                                       self.config, disabled_files)
            except IOError as e:
                self.errorfunc('trouble accessing files - ' + str(e))
                return None
            if self.doneflag.is_set():
                return None

            self.storagewrapper = StorageWrapper(
                self.storage, self.config['download_slice_size'], self.pieces,
                piece_length, self._finished, self._failed, statusfunc,
                self.doneflag, self.config['check_hashes'], self._data_flunked,
                self.rawserver.add_task, self.config, self.unpauseflag)

        except ValueError as e:
            self._failed('bad data - ' + str(e))
        except IOError as e:
            self._failed('IOError - ' + str(e))
        if self.doneflag.is_set():
            return None

        if self.selector_enabled:
            self.fileselector = FileSelector(
                self.files, piece_length,
                self.appdataobj.getPieceDir(self.infohash), self.storage,
                self.storagewrapper, self.rawserver.add_task, self._failed)
            if data:
                data = data.get('resume data')
                if data:
                    self.fileselector.unpickle(data)

        self.checking = True
        if old_style:
            return self.storagewrapper.old_style_init()
        return self.storagewrapper.initialize

    def _make_upload(self, connection, ratelimiter, totalup):
        return Upload(connection, ratelimiter, totalup, self.choker,
                      self.storagewrapper, self.picker, self.config)

    def _kick_peer(self, connection):
        self.rawserver.add_task(connection.close, 0)

    def _ban_peer(self, ip):
        self.encoder.ban(ip)

    def _received_raw_data(self, x):
        if self.tcp_ack_fudge:
            x = int(x * self.tcp_ack_fudge)
            self.ratelimiter.adjust_sent(x)

    def _received_data(self, x):
        self.downmeasure.update_rate(x)
        self.ratemeasure.data_came_in(x)

    def _received_http_data(self, x):
        self.downmeasure.update_rate(x)
        self.ratemeasure.data_came_in(x)
        self.downloader.external_data_received(x)

    def _cancelfunc(self, pieces):
        self.downloader.cancel_piece_download(pieces)
        self.httpdownloader.cancel_piece_download(pieces)

    def _reqmorefunc(self, pieces):
        self.downloader.requeue_piece_download(pieces)

    def startEngine(self, ratelimiter=None, statusfunc=None):
        if self.doneflag.is_set():
            return False
        if not statusfunc:
            statusfunc = self.statusfunc

        self.checking = False

        if not CRYPTO_OK:
            if self.config['crypto_allowed']:
                self.errorfunc('warning - crypto library not installed')
            self.config['crypto_allowed'] = 0
            self.config['crypto_only'] = 0
            self.config['crypto_stealth'] = 0

        for i in range(self.len_pieces):
            if self.storagewrapper.do_I_have(i):
                self.picker.complete(i)
        self.upmeasure = Measure(self.config['max_rate_period'],
                                 self.config['upload_rate_fudge'])
        self.downmeasure = Measure(self.config['max_rate_period'])

        if ratelimiter:
            self.ratelimiter = ratelimiter
        else:
            self.ratelimiter = RateLimiter(self.rawserver.add_task,
                                           self.config['upload_unit_size'],
                                           self.setConns)
            self.ratelimiter.set_upload_rate(self.config['max_upload_rate'])

        self.ratemeasure = RateMeasure()
        self.ratemeasure_datarejected = self.ratemeasure.data_rejected

        self.downloader = Downloader(
            self.storagewrapper, self.picker, self.config['request_backlog'],
            self.config['max_rate_period'], self.len_pieces,
            self.config['download_slice_size'], self._received_data,
            self.config['snub_time'], self.config['auto_kick'],
            self._kick_peer, self._ban_peer)
        self.downloader.set_download_rate(self.config['max_download_rate'])
        self.connecter = Connecter(self._make_upload, self.downloader,
                                   self.choker, self.len_pieces,
                                   self.upmeasure, self.config,
                                   self.ratelimiter, self.rawserver.add_task)
        self.encoder = Encoder(self.connecter, self.rawserver, self.myid,
                               self.config['max_message_length'],
                               self.rawserver.add_task,
                               self.config['keepalive_interval'],
                               self.infohash, self._received_raw_data,
                               self.config)

        self.httpdownloader = HTTPDownloader(
            self.storagewrapper, self.picker, self.rawserver, self.finflag,
            self.errorfunc, self.downloader, self.config['max_rate_period'],
            self.infohash, self._received_http_data, self.connecter.got_piece)
        if 'httpseeds' in self.metainfo and not self.finflag.is_set():
            for u in self.metainfo['httpseeds']:
                self.httpdownloader.make_download(u)

        if self.selector_enabled:
            self.fileselector.tie_in(self.picker, self._cancelfunc,
                                     self._reqmorefunc,
                                     self.rerequest_ondownloadmore)
            if self.priority:
                self.fileselector.set_priorities_now(self.priority)

            # erase old data once you've started modifying it
            self.appdataobj.deleteTorrentData(self.infohash)

        if self.config['super_seeder']:
            self.set_super_seed()

        self.started = True
        return True

    def rerequest_complete(self):
        if self.rerequest:
            self.rerequest.announce(1)

    def rerequest_stopped(self):
        if self.rerequest:
            self.rerequest.announce(2)

    def rerequest_lastfailed(self):
        if self.rerequest:
            return self.rerequest.last_failed
        return False

    def rerequest_ondownloadmore(self):
        if self.rerequest:
            self.rerequest.hit()

    def startRerequester(self, force_rapid_update=False):
        trackerlist = self.metainfo.get('announce-list',
                                        [[self.metainfo['announce']]])

        self.rerequest = Rerequester(
            self.port, self.myid, self.infohash, trackerlist, self.config,
            self.rawserver.add_task, self.errorfunc, self.excfunc,
            self.encoder.start_connections,
            self.connecter.how_many_connections,
            self.storagewrapper.get_amount_left, self.upmeasure.get_total,
            self.downmeasure.get_total, self.upmeasure.get_rate,
            self.downmeasure.get_rate, self.doneflag, self.unpauseflag,
            force_rapid_update)

        self.rerequest.start()

    def _init_stats(self):
        self.statistics = Statistics(self.upmeasure, self.downmeasure,
                                     self.connecter, self.httpdownloader,
                                     self.ratelimiter,
                                     self.rerequest_lastfailed,
                                     self.filedatflag)
        if 'files' in self.metainfo['info']:
            self.statistics.set_dirstats(self.files,
                                         self.metainfo['info']['piece length'])
        if self.config['spew']:
            self.spewflag.set()

    def autoStats(self, displayfunc=None):
        if not displayfunc:
            displayfunc = self.statusfunc

        self._init_stats()
        DownloaderFeedback(self.choker, self.httpdownloader,
                           self.rawserver.add_task, self.upmeasure.get_rate,
                           self.downmeasure.get_rate, self.ratemeasure,
                           self.storagewrapper.get_stats, self.datalength,
                           self.finflag, self.spewflag, self.statistics,
                           displayfunc, self.config['display_interval'])

    def startStats(self):
        self._init_stats()
        d = DownloaderFeedback(self.choker, self.httpdownloader,
                               self.rawserver.add_task,
                               self.upmeasure.get_rate,
                               self.downmeasure.get_rate, self.ratemeasure,
                               self.storagewrapper.get_stats, self.datalength,
                               self.finflag, self.spewflag, self.statistics)
        return d.gather

    def getPortHandler(self):
        return self.encoder

    def shutdown(self, torrentdata={}):
        if self.checking or self.started:
            self.storagewrapper.sync()
            self.storage.close()
            self.rerequest_stopped()
        if self.fileselector and self.started:
            if not self.failed:
                self.fileselector.finish()
                torrentdata['resume data'] = self.fileselector.pickle()
            try:
                self.appdataobj.writeTorrentData(self.infohash, torrentdata)
            except Exception as e:
                print(e)
                self.appdataobj.deleteTorrentData(self.infohash)  # clear it
        return not self.failed and not self.excflag.is_set()
        # if returns false, you may wish to auto-restart the torrent

    def setConns(self, conns, conns2=None):
        if not conns2:
            conns2 = conns
        try:

            def s(self=self, conns=conns, conns2=conns2):
                self.config['min_uploads'] = conns
                self.config['max_uploads'] = conns2
                if conns > 30:
                    self.config['max_initiate'] = conns + 10

            self.rawserver.add_task(s)
        except AttributeError:
            pass

    def set_super_seed(self):
        try:
            self.superseedflag.set()

            def s(self=self):
                if self.finflag.is_set():
                    self._set_super_seed()

            self.rawserver.add_task(s)
        except AttributeError:
            pass

    def _set_super_seed(self):
        if not self.super_seeding_active:
            self.super_seeding_active = True
            self.errorfunc('        ** SUPER-SEED OPERATION ACTIVE **\n  '
                           'please set Max uploads so each peer gets 6-8 kB/s')

            def s(self=self):
                self.downloader.set_super_seed()
                self.choker.set_super_seed()

            self.rawserver.add_task(s)
            # mode started when already finished
            if self.finflag.is_set():

                def r(self=self):
                    # so after kicking everyone off, reannounce
                    self.rerequest.announce(3)

                self.rawserver.add_task(r)

    def am_I_finished(self):
        return self.finflag.is_set()
Exemple #4
0
class BT1Download:
    def __init__(self, statusfunc, finfunc, errorfunc, excfunc, doneflag,
                 config, metainfo, infohash, peerid, rawserver, port,
                 appdataobj=None):
        self.statusfunc = statusfunc
        self.finfunc = finfunc
        self.errorfunc = errorfunc
        self.excfunc = excfunc
        self.doneflag = doneflag
        self.config = config
        self.metainfo = metainfo    # MetaInfo
        self.infohash = infohash    # bytes[20]
        self.myid = peerid          # bytes
        self.rawserver = rawserver
        self.port = port

        self.pieces = self.metainfo['info'].hasher.pieces   # [bytes[20]]
        self.len_pieces = len(self.pieces)
        self.argslistheader = argslistheader
        self.unpauseflag = threading.Event()
        self.unpauseflag.set()
        self.downloader = None
        self.storagewrapper = None
        self.fileselector = None
        self.super_seeding_active = False
        self.filedatflag = threading.Event()
        self.spewflag = threading.Event()
        self.superseedflag = threading.Event()
        self.whenpaused = None
        self.finflag = threading.Event()
        self.rerequest = None
        self.tcp_ack_fudge = config['tcp_ack_fudge']

        self.selector_enabled = config['selector_enabled']
        if appdataobj:
            self.appdataobj = appdataobj
        elif self.selector_enabled:
            self.appdataobj = ConfigDir()
            self.appdataobj.deleteOldCacheData(config['expire_cache_data'],
                                               [self.infohash])

        self.excflag = self.rawserver.get_exception_flag()
        self.failed = False
        self.checking = False
        self.started = False

        self.picker = PiecePicker(self.len_pieces,
                                  config['rarest_first_cutoff'],
                                  config['rarest_first_priority_cutoff'])
        self.choker = Choker(config, rawserver.add_task,
                             self.picker, self.finflag.is_set)

    def saveAs(self, filefunc, pathfunc=None):
        try:
            def make(f, forcedir=False):
                if not forcedir:
                    f = os.path.split(f)[0]
                if f != '' and not os.path.exists(f):
                    os.makedirs(f)

            info = self.metainfo['info']
            if 'length' in info:
                file_length = info['length']
                fname = filefunc(info['name'], file_length,
                                 self.config['saveas'], False)
                if fname is None:
                    return None
                make(fname)
                files = [(fname, file_length)]
            else:
                file_length = sum(x['length'] for x in info['files'])
                fname = filefunc(info['name'], file_length,
                                 self.config['saveas'], True)
                if fname is None:
                    return None

                # if this path exists, and no files from the info dict exist,
                # we assume it's a new download and the user wants to create a
                # new directory with the default name
                existing = 0
                if os.path.exists(fname):
                    if not os.path.isdir(fname):
                        self.errorfunc(fname + 'is not a dir')
                        return None
                    if len(os.listdir(fname)) > 0:  # if it's not empty
                        existing = any(
                            os.path.exists(os.path.join(fname, x['path'][0]))
                            for x in info['files'])
                        if not existing:
                            fname = os.path.join(fname, info['name'])
                            if os.path.exists(fname) and \
                                    not os.path.isdir(fname):
                                if fname[-8:] == '.torrent':
                                    fname = fname[:-8]
                                if os.path.exists(fname) and \
                                        not os.path.isdir(fname):
                                    self.errorfunc("Can't create dir - " +
                                                   info['name'])
                                    return None
                make(fname, True)

                # alert the UI to any possible change in path
                if pathfunc is not None:
                    pathfunc(fname)

                files = []
                for x in info['files']:
                    n = os.path.join(fname, *x['path'])
                    files.append((n, x['length']))
                    make(n)
        except OSError as e:
            self.errorfunc("Couldn't allocate dir - " + str(e))
            return None

        self.filename = fname
        self.files = files
        self.datalength = file_length

        return fname

    def _finished(self):
        self.finflag.set()
        try:
            self.storage.set_readonly()
        except (IOError, OSError) as e:
            self.errorfunc('trouble setting readonly at end - ' + str(e))
        if self.superseedflag.is_set():
            self._set_super_seed()
        self.choker.set_round_robin_period(
            max(self.config['round_robin_period'],
                self.config['round_robin_period'] *
                self.metainfo['info']['piece length'] / 200000))
        self.rerequest_complete()
        self.finfunc()

    def _data_flunked(self, amount, index):
        self.ratemeasure_datarejected(amount)
        if not self.doneflag.is_set():
            self.errorfunc('piece {:d} failed hash check, re-downloading it'
                           ''.format(index))

    def _failed(self, reason):
        self.failed = True
        self.doneflag.set()
        if reason is not None:
            self.errorfunc(reason)

    def initFiles(self, old_style=False, statusfunc=None):
        if self.doneflag.is_set():
            return None
        if not statusfunc:
            statusfunc = self.statusfunc

        disabled_files = None
        if self.selector_enabled:
            self.priority = self.config['priority']
            if self.priority:
                try:
                    self.priority = self.priority.split(',')
                    assert len(self.priority) == len(self.files)
                    self.priority = [int(p) for p in self.priority]
                    for p in self.priority:
                        assert p >= -1
                        assert p <= 2
                except (AssertionError, ValueError):
                    self.errorfunc('bad priority list given, ignored')
                    self.priority = None

            data = self.appdataobj.getTorrentData(self.infohash)
            try:
                d = data['resume data']['priority']
                assert len(d) == len(self.files)
                disabled_files = [x == -1 for x in d]
            except (KeyError, TypeError, AssertionError):
                try:
                    disabled_files = [x == -1 for x in self.priority]
                except TypeError:
                    pass

        piece_length = self.metainfo['info']['piece length']
        try:
            try:
                self.storage = Storage(self.files, piece_length, self.doneflag,
                                       self.config, disabled_files)
            except IOError as e:
                self.errorfunc('trouble accessing files - ' + str(e))
                return None
            if self.doneflag.is_set():
                return None

            self.storagewrapper = StorageWrapper(
                self.storage, self.config['download_slice_size'],
                self.pieces, piece_length, self._finished,
                self._failed, statusfunc, self.doneflag,
                self.config['check_hashes'], self._data_flunked,
                self.rawserver.add_task, self.config, self.unpauseflag)

        except ValueError as e:
            self._failed('bad data - ' + str(e))
        except IOError as e:
            self._failed('IOError - ' + str(e))
        if self.doneflag.is_set():
            return None

        if self.selector_enabled:
            self.fileselector = FileSelector(
                self.files, piece_length,
                self.appdataobj.getPieceDir(self.infohash), self.storage,
                self.storagewrapper, self.rawserver.add_task, self._failed)
            if data:
                data = data.get('resume data')
                if data:
                    self.fileselector.unpickle(data)

        self.checking = True
        if old_style:
            return self.storagewrapper.old_style_init()
        return self.storagewrapper.initialize

    def _make_upload(self, connection, ratelimiter, totalup):
        return Upload(connection, ratelimiter, totalup,
                      self.choker, self.storagewrapper, self.picker,
                      self.config)

    def _kick_peer(self, connection):
        self.rawserver.add_task(connection.close, 0)

    def _ban_peer(self, ip):
        self.encoder.ban(ip)

    def _received_raw_data(self, x):
        if self.tcp_ack_fudge:
            x = int(x * self.tcp_ack_fudge)
            self.ratelimiter.adjust_sent(x)

    def _received_data(self, x):
        self.downmeasure.update_rate(x)
        self.ratemeasure.data_came_in(x)

    def _received_http_data(self, x):
        self.downmeasure.update_rate(x)
        self.ratemeasure.data_came_in(x)
        self.downloader.external_data_received(x)

    def _cancelfunc(self, pieces):
        self.downloader.cancel_piece_download(pieces)
        self.httpdownloader.cancel_piece_download(pieces)

    def _reqmorefunc(self, pieces):
        self.downloader.requeue_piece_download(pieces)

    def startEngine(self, ratelimiter=None, statusfunc=None):
        if self.doneflag.is_set():
            return False
        if not statusfunc:
            statusfunc = self.statusfunc

        self.checking = False

        if not CRYPTO_OK:
            if self.config['crypto_allowed']:
                self.errorfunc('warning - crypto library not installed')
            self.config['crypto_allowed'] = 0
            self.config['crypto_only'] = 0
            self.config['crypto_stealth'] = 0

        for i in range(self.len_pieces):
            if self.storagewrapper.do_I_have(i):
                self.picker.complete(i)
        self.upmeasure = Measure(self.config['max_rate_period'],
                                 self.config['upload_rate_fudge'])
        self.downmeasure = Measure(self.config['max_rate_period'])

        if ratelimiter:
            self.ratelimiter = ratelimiter
        else:
            self.ratelimiter = RateLimiter(self.rawserver.add_task,
                                           self.config['upload_unit_size'],
                                           self.setConns)
            self.ratelimiter.set_upload_rate(self.config['max_upload_rate'])

        self.ratemeasure = RateMeasure()
        self.ratemeasure_datarejected = self.ratemeasure.data_rejected

        self.downloader = Downloader(
            self.storagewrapper, self.picker, self.config['request_backlog'],
            self.config['max_rate_period'], self.len_pieces,
            self.config['download_slice_size'], self._received_data,
            self.config['snub_time'], self.config['auto_kick'],
            self._kick_peer, self._ban_peer)
        self.downloader.set_download_rate(self.config['max_download_rate'])
        self.connecter = Connecter(
            self._make_upload, self.downloader, self.choker, self.len_pieces,
            self.upmeasure, self.config, self.ratelimiter,
            self.rawserver.add_task)
        self.encoder = Encoder(
            self.connecter, self.rawserver, self.myid,
            self.config['max_message_length'], self.rawserver.add_task,
            self.config['keepalive_interval'], self.infohash,
            self._received_raw_data, self.config)

        self.httpdownloader = HTTPDownloader(
            self.storagewrapper, self.picker, self.rawserver, self.finflag,
            self.errorfunc, self.downloader, self.config['max_rate_period'],
            self.infohash, self._received_http_data, self.connecter.got_piece)
        if 'httpseeds' in self.metainfo and not self.finflag.is_set():
            for u in self.metainfo['httpseeds']:
                self.httpdownloader.make_download(u)

        if self.selector_enabled:
            self.fileselector.tie_in(self.picker, self._cancelfunc,
                                     self._reqmorefunc,
                                     self.rerequest_ondownloadmore)
            if self.priority:
                self.fileselector.set_priorities_now(self.priority)

            # erase old data once you've started modifying it
            self.appdataobj.deleteTorrentData(self.infohash)

        if self.config['super_seeder']:
            self.set_super_seed()

        self.started = True
        return True

    def rerequest_complete(self):
        if self.rerequest:
            self.rerequest.announce(2)

    def rerequest_stopped(self):
        if self.rerequest:
            self.rerequest.announce(3)

    def rerequest_lastfailed(self):
        if self.rerequest:
            return self.rerequest.last_failed
        return False

    def rerequest_ondownloadmore(self):
        if self.rerequest:
            self.rerequest.hit()

    def startRerequester(self, force_rapid_update=False):
        tracker_urls = self.metainfo.get('announce-list',
                                         [[self.metainfo['announce']]])
        kwargs = {'port': self.port,
                  'ip': IPv4(self.config['ip']),
                  'seed_id': self.config['dedicated_seed_id'],
                  'supportcrypto': self.config['crypto_allowed'],
                  'requirecrypto': self.config['crypto_only'],
                  'cryptostealth': self.config['crypto_stealth'],
                  'no_peer_id': True,
                  'compact': True}
        announcers = urls_to_announcers(tracker_urls, **kwargs)

        self.rerequest = Rerequester(
            self.myid, self.infohash, announcers, self.config,
            self.rawserver.add_task, self.errorfunc, self.excfunc,
            self.encoder.start_connections,
            self.connecter.how_many_connections,
            self.storagewrapper.get_amount_left, self.upmeasure.get_total,
            self.downmeasure.get_total, self.upmeasure.get_rate,
            self.downmeasure.get_rate, self.doneflag, self.unpauseflag,
            force_rapid_update)

        self.rerequest.start()

    def _init_stats(self):
        self.statistics = Statistics(
            self.upmeasure, self.downmeasure, self.connecter,
            self.httpdownloader, self.ratelimiter, self.rerequest_lastfailed,
            self.filedatflag)
        if 'files' in self.metainfo['info']:
            self.statistics.set_dirstats(self.files,
                                         self.metainfo['info']['piece length'])
        if self.config['spew']:
            self.spewflag.set()

    def autoStats(self, displayfunc=None):
        if not displayfunc:
            displayfunc = self.statusfunc

        self._init_stats()
        DownloaderFeedback(
            self.choker, self.httpdownloader, self.rawserver.add_task,
            self.upmeasure.get_rate, self.downmeasure.get_rate,
            self.ratemeasure, self.storagewrapper.get_stats, self.datalength,
            self.finflag, self.spewflag, self.statistics, displayfunc,
            self.config['display_interval'])

    def startStats(self):
        self._init_stats()
        d = DownloaderFeedback(
            self.choker, self.httpdownloader, self.rawserver.add_task,
            self.upmeasure.get_rate, self.downmeasure.get_rate,
            self.ratemeasure, self.storagewrapper.get_stats, self.datalength,
            self.finflag, self.spewflag, self.statistics)
        return d.gather

    def getPortHandler(self):
        return self.encoder

    def shutdown(self, torrentdata={}):
        if self.checking or self.started:
            self.storagewrapper.sync()
            self.storage.close()
            self.rerequest_stopped()
        if self.fileselector and self.started:
            if not self.failed:
                self.fileselector.finish()
                torrentdata['resume data'] = self.fileselector.pickle()
            try:
                self.appdataobj.writeTorrentData(self.infohash, torrentdata)
            except Exception as e:
                print(e)
                self.appdataobj.deleteTorrentData(self.infohash)  # clear it
        return not self.failed and not self.excflag.is_set()
        # if returns false, you may wish to auto-restart the torrent

    def setConns(self, conns, conns2=None):
        if not conns2:
            conns2 = conns
        try:
            def s(self=self, conns=conns, conns2=conns2):
                self.config['min_uploads'] = conns
                self.config['max_uploads'] = conns2
                if conns > 30:
                    self.config['max_initiate'] = conns + 10
            self.rawserver.add_task(s)
        except AttributeError:
            pass

    def set_super_seed(self):
        try:
            self.superseedflag.set()

            def s(self=self):
                if self.finflag.is_set():
                    self._set_super_seed()
            self.rawserver.add_task(s)
        except AttributeError:
            pass

    def _set_super_seed(self):
        if not self.super_seeding_active:
            self.super_seeding_active = True
            self.errorfunc('        ** SUPER-SEED OPERATION ACTIVE **\n  '
                           'please set Max uploads so each peer gets 6-8 kB/s')

            def s(self=self):
                self.downloader.set_super_seed()
                self.choker.set_super_seed()
            self.rawserver.add_task(s)
            # mode started when already finished
            if self.finflag.is_set():
                def r(self=self):
                    # so after kicking everyone off, reannounce
                    self.rerequest.announce(0)
                self.rawserver.add_task(r)

    def am_I_finished(self):
        return self.finflag.is_set()