Beispiel #1
0
    def _copy_chunks(self):
        init = self.info.get_chunk_name(0)  # initial chunk name

        if self.info.get_count() > 1:
            with io.open(init, 'rb+') as fpo:  # first chunkfile
                for i in range(1, self.info.get_count()):
                    # input file
                    # seek to beginning of chunk,
                    # to get rid of overlapping chunks
                    fpo.seek(self.info.get_chunk_range(i - 1)[1] + 1)
                    filename = '{0}.chunk{1:d}'.format(self.path, i)
                    buf = 32 << 10
                    with io.open(filename, mode='rb') as fpi:
                        while True:  # copy in chunks, consumes less memory
                            data = fpi.read(buf)
                            if not data:
                                break
                            fpo.write(data)

                    if fpo.tell() < self.info.get_chunk_range(i)[1]:
                        remove(init)
                        self.info.remove()  # there are probably invalid chunks
                        raise Exception(
                            'Downloaded content was smaller than expected')
                    remove(filename)  # remove chunk

        if self.name:
            filepath = os.path.join(os.path.dirname(self.path), self.name)
            self.set_path(filepath)

        shutil.move(init, self.path)
    def setUp(self):
        PluginTester.setUp(self)
        for f in self.files:
            if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))

        # folder for reports
        report = join("tmp", self.__class__.__name__)
        if exists(report):
            for f in listdir(report):
                remove(join(report, f))
Beispiel #3
0
    def setUp(self):
        PluginTester.setUp(self)
        for fname in self.files:
            path = os.path.join(DL_DIR, fname)
            remove(path, trash=True, ignore_errors=True)

        # folder for reports
        report = os.path.join(self.__class__.__name__)
        if not os.path.exists(report):
            return None
        for fname in os.listdir(report):
            path = os.path.join(report, fname)
            remove(path, trash=True)
Beispiel #4
0
    def _decrypt(self, urls):
        """Internal method to select decrypting method

        :param urls: List of urls/content
        :return:
        """
        cls = self.__class__

        # separate local and remote files
        content, urls = self.getLocalContent(urls)
        result = []

        if urls and has_method(cls, "decrypt"):
            self.logDebug("Deprecated .decrypt() method in Crypter plugin")
            result = []
            for url in urls:
                self.pyfile = PyFileMockup(url)
                self.setup()
                self.decrypt(self.pyfile)
                result.extend(self.convertPackages())
        elif urls:
            method = True
            try:
                self.setup()
                result = to_list(self.decryptURLs(urls))
            except NotImplementedError:
                method = False

            # this will raise error if not implemented
            if not method:
                for url in urls:
                    self.setup()
                    result.extend(to_list(self.decryptURL(url)))

        for f, c in content:
            self.setup()
            result.extend(to_list(self.decryptFile(c)))
            try:
                if f.startswith("tmp_"):
                    remove(f)
            except IOError:
                self.logWarning(_("Could not remove file '%s'") % f)
                self.core.print_exc()

        return result
Beispiel #5
0
    def _decrypt(self, urls):
        """Internal method to select decrypting method

        :param urls: List of urls/content
        :return:
        """
        cls = self.__class__

        # separate local and remote files
        content, urls = self.getLocalContent(urls)

        if has_method(cls, "decryptURLs"):
            self.setup()
            result = to_list(self.decryptURLs(urls))
        elif has_method(cls, "decryptURL"):
            result = []
            for url in urls:
                self.setup()
                result.extend(to_list(self.decryptURL(url)))
        elif has_method(cls, "decrypt"):
            self.logDebug("Deprecated .decrypt() method in Crypter plugin")
            result = []
            for url in urls:
                self.pyfile = PyFileMockup(url)
                self.setup()
                self.decrypt(self.pyfile)
                result.extend(self.convertPackages())
        else:
            if not has_method(cls, "decryptFile") or urls:
                self.logDebug("No suited decrypting method was overwritten in plugin")
            result = []

        if has_method(cls, "decryptFile"):
            for f, c in content:
                self.setup()
                result.extend(to_list(self.decryptFile(c)))
                try:
                    if f.startswith("tmp_"): remove(f)
                except IOError:
                    self.logWarning(_("Could not remove file '%s'") % f)
                    self.core.print_exc()

        return result
Beispiel #6
0
    def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0):
        """ checks the content of the last downloaded file, re match is saved to `lastCheck`
        
        :param rules: dict with names and rules to match (compiled regexp or strings)
        :param api_size: expected file size
        :param max_size: if the file is larger then it wont be checked
        :param delete: delete if matched
        :param read_size: amount of bytes to read from files larger then max_size
        :return: dictionary key of the first rule that matched
        """
        lastDownload = fs_encode(self.lastDownload)
        if not exists(lastDownload):
            return None

        size = stat(lastDownload)
        size = size.st_size

        if api_size and api_size <= size:
            return None
        elif size > max_size and not read_size:
            return None
        self.log.debug("Download Check triggered")
        f = open(lastDownload, "rb")
        content = f.read(read_size if read_size else -1)
        f.close()
        # produces encoding errors, better log to other file in the future?
        # self.log.debug("Content: %s" % content)
        for name, rule in rules.iteritems():
            if type(rule) in (str, unicode):
                if rule in content:
                    if delete:
                        remove(lastDownload)
                    return name
            elif hasattr(rule, "search"):
                m = rule.search(content)
                if m:
                    if delete:
                        remove(lastDownload)
                    self.lastCheck = m
                    return name
Beispiel #7
0
    def check_download(self, rules, api_size=0,
                       max_size=50000, delete=True, read_size=0):
        """Checks the content of the last downloaded file, re match is saved to
        `last_check`

        :param rules: dict with names and rules to match (compiled regexp or strings)
        :param api_size: expected file size
        :param max_size: if the file is larger then it wont be checked
        :param delete: delete if matched
        :param read_size: amount of bytes to read from files larger then max_size
        :return: dictionary key of the first rule that matched

        """
        if not os.path.isfile(self.last_download):
            return

        size = os.stat(self.last_download).st_size

        if api_size and api_size <= size:
            return
        elif size > max_size and not read_size:
            return
        self.pyload.log.debug('Download Check triggered')
        with io.open(self.last_download, mode='rb') as fp:
            content = fp.read(read_size if read_size else -1)
        # produces encoding errors, better log to other file in the future?
        # self.pyload.log.debug("Content: {0}".format(content))
        for name, rule in rules.items():
            if isinstance(rule, str):
                if rule in content:
                    if delete:
                        remove(self.last_download, trash=True)
                    return name
            elif hasattr(rule, 'search'):
                m = rule.search(content)
                if m is not None:
                    if delete:
                        remove(self.last_download, trash=True)
                    self.last_check = m
                    return name
Beispiel #8
0
    def run_tesser(self, subset=False, digits=True,
                   lowercase=True, uppercase=True):
        # self.log.debug("create tmp tif")

        # tmp = tempfile.NamedTemporaryFile(suffix=".tif")
        tmp_path = os.path.join('tmpTif_{0}.tif'.format(self.__name__))
        tmp = io.open(tmp_path, mode='wb')
        tmp.close()
        # self.log.debug("create tmp txt")
        # tmp_txt = tempfile.NamedTemporaryFile(suffix=".txt")
        tmp_txt_path = os.path.join('tmp_txt_{0}.txt'.format(
            self.__name__))
        tmp_txt = io.open(tmp_txt_path, mode='wb')
        tmp_txt.close()

        self.log.debug('save tiff')
        self.image.save(tmp.name, 'TIFF')

        if os.name == 'nt':
            tessparams = [resource_filename(
                __package__, 'tesseract/tesseract.exe')]
        else:
            tessparams = ['tesseract']

        tessparams.extend([tmp.name, tmp_txt.name.replace('.txt', '')])

        if subset and (digits or lowercase or uppercase):
            # self.log.debug("create temp subset config")
            # tmp_sub = tempfile.NamedTemporaryFile(suffix=".subset")
            with io.open(os.path.join('tmp_sub_{0}.subset'.format(self.__name__)), mode='wb') as tmp_sub:
                tmp_sub.write('tessedit_char_whitelist ')
                if digits:
                    tmp_sub.write('0123456789')
                if lowercase:
                    tmp_sub.write('abcdefghijklmnopqrstuvwxyz')
                if uppercase:
                    tmp_sub.write('ABCDEFGHIJKLMNOPQRSTUVWXYZ')
                tmp_sub.write(os.linesep)
                tessparams.append('nobatch')
                tessparams.append(tmp_sub.name)

        self.log.debug('run tesseract')
        self.run(tessparams)
        self.log.debug('read txt')

        try:
            with io.open(tmp_txt.name) as fp:
                self.result_captcha = fp.read().replace(os.linesep, '')
        except Exception:
            self.result_captcha = ''

        self.log.debug(self.result_captcha)
        try:
            remove(tmp.name)
            remove(tmp_txt.name)
            if subset and (digits or lowercase or uppercase):
                remove(tmp_sub.name)
        except OSError:
            pass
Beispiel #9
0
 def setUpClass(cls):
     cls.core = Core()
     name = '{0}.{1}'.format(cls.__module__, cls.__name__)
     for fname in glob(os.path.join(name, 'debug_*')):
         remove(fname, trash=True)
Beispiel #10
0
    def decryptCaptcha(
        self, url, get={}, post={}, cookies=False, forceUser=False, imgtype="jpg", result_type="textual"
    ):
        """ Loads a captcha and decrypts it with ocr, plugin, user input

        :param url: url of captcha image
        :param get: get part for request
        :param post: post part for request
        :param cookies: True if cookies should be enabled
        :param forceUser: if True, ocr is not used
        :param imgtype: Type of the Image
        :param result_type: 'textual' if text is written on the captcha\
        or 'positional' for captcha where the user have to click\
        on a specific region on the captcha

        :return: result of decrypting
        """

        img = self.load(url, get=get, post=post, cookies=cookies)

        id = ("%.2f" % time())[-6:].replace(".", "")
        temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb")
        temp_file.write(img)
        temp_file.close()

        name = "%sOCR" % self.__name__
        has_plugin = name in self.core.pluginManager.getPlugins("internal")

        if self.core.captcha:
            OCR = self.core.pluginManager.loadClass("internal", name)
        else:
            OCR = None

        if OCR and not forceUser:
            sleep(randint(3000, 5000) / 1000.0)
            self.checkAbort()

            ocr = OCR()
            result = ocr.get_captcha(temp_file.name)
        else:
            task = self.im.createCaptchaTask(img, imgtype, temp_file.name, self.__name__, result_type)
            self.task = task

            while task.isWaiting():
                if self.abort():
                    self.im.removeTask(task)
                    raise Abort()
                sleep(1)

            # TODO task handling
            self.im.removeTask(task)

            if task.error and has_plugin:  # ignore default error message since the user could use OCR
                self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting"))
            elif task.error:
                self.fail(task.error)
            elif not task.result:
                self.fail(_("No captcha result obtained in appropriate time."))

            result = task.result
            self.log.debug("Received captcha result: %s" % str(result))

        if not self.core.debug:
            try:
                remove(temp_file.name)
            except:
                pass

        return result
Beispiel #11
0
    def _download(self, chunks, resume):
        if not resume:
            self.info.clear()
            self.info.add_chunk('{0}.chunk0'.format(
                self.path), (0, 0))  # create an initial entry

        self.chunks = []

        # initial chunk that will load complete file (if needed)
        init = CurlChunk(0, self, None, resume)

        self.chunks.append(init)
        self.manager.add_handle(init.get_handle())

        last_finish_check = 0
        last_time_check = 0
        chunks_done = set()  # list of curl handles that are finished
        chunks_created = False
        done = False
        # This is a resume, if we were chunked originally assume still can
        if self.info.get_count() > 1:
            self.chunk_support = True

        while True:
            # need to create chunks
            # will be set later by first chunk
            if not chunks_created and self.chunk_support and self.size:

                self.flags ^= Connection.Resumable  # TODO: Recheck...
                if not resume:
                    self.info.set_size(self.size)
                    self.info.create_chunks(chunks)
                    self.info.save()

                chunks = self.info.get_count()

                init.set_range(self.info.get_chunk_range(0))

                for i in range(1, chunks):
                    c = CurlChunk(
                        i, self, self.info.get_chunk_range(i), resume)

                    handle = c.get_handle()
                    if handle:
                        self.chunks.append(c)
                        self.manager.add_handle(handle)
                    else:
                        # close immediately
                        self.log.debug('Invalid curl handle -> closed')
                        c.close()

                chunks_created = True

            while True:
                ret, _ = self.manager.perform()
                if ret != pycurl.E_CALL_MULTI_PERFORM:
                    break

            t = time.time()

            # reduce these calls
            # when num_q is 0, the loop is exited
            while last_finish_check + 0.5 < t:
                # list of failed curl handles
                failed = []

                # TODO: Rewrite...
                # save only last exception, we can only raise one anyway
                exc = Exception()

                num_q, ok_list, err_list = self.manager.info_read()
                for c in ok_list:
                    chunk = self.find_chunk(c)
                    # check if the header implies success,
                    # else add it to failed list
                    try:
                        chunk.verify_header()
                    except ResponseException as exc:
                        self.log.debug(
                            'Chunk {0:d} failed'.format(
                                chunk.id + 1))
                        self.log.debug(exc, exc_info=True)
                        failed.append(chunk)
                    else:
                        chunks_done.add(c)

                for c in err_list:
                    curl, errno, msg = c
                    chunk = self.find_chunk(curl)
                    # test if chunk was finished
                    if errno != 23 or '0 !=' not in msg:
                        failed.append(chunk)
                        exc = pycurl.error(errno, msg)
                        self.log.debug(
                            'Chunk {0:d} failed'.format(chunk.id + 1))
                        self.log.debug(exc, exc_info=True)
                        continue
                    # check if the header implies success,
                    # else add it to failed list
                    try:
                        chunk.verify_header()
                    except ResponseException as exc:
                        self.log.debug(
                            'Chunk {0:d} failed'.format(
                                chunk.id + 1))
                        self.log.debug(exc, exc_info=True)
                        failed.append(chunk)
                    else:
                        chunks_done.add(curl)
                if not num_q:  # no more info to get

                    # check if init is not finished so we reset download
                    # connections
                    # note that other chunks are closed and everything
                    # downloaded with initial connection
                    if failed:
                        if init in failed or init.curl in chunks_done:
                            raise exc
                        self.log.error(
                            'Download chunks failed, fallback to '
                            'single connection | {0}'.format(exc))

                        # list of chunks to clean and remove
                        to_clean = [x for x in self.chunks if x is not init]
                        for chunk in to_clean:
                            self.close_chunk(chunk)
                            self.chunks.remove(chunk)
                            remove(self.info.get_chunk_name(chunk.id))

                        # let first chunk load the rest and update the
                        # info file
                        init.reset_range()
                        self.info.clear()
                        self.info.add_chunk('{0}.chunk0'.format(
                            self.path), (0, self.size))
                        self.info.save()

                    last_finish_check = t

                    if len(chunks_done) >= len(self.chunks):
                        if len(chunks_done) > len(self.chunks):
                            self.log.warning(
                                'Finished download chunks size incorrect')
                        done = True  # all chunks loaded

                    break

            if done:
                break  # all chunks loaded

            # calc speed once per second, averaging over 3 seconds
            if last_time_check + 1 < t:
                len_la = len(self.last_arrived)
                diff = [c.arrived - (self.last_arrived[i] if len_la > i else 0)
                        for i, c in enumerate(self.chunks)]

                self.last_speeds[1] = self.last_speeds[0]
                self.last_speeds[0] = self.speeds
                self.speeds = [float(a) // (t - last_time_check) for a in diff]
                self.last_arrived = [c.arrived for c in self.chunks]
                last_time_check = t

            if self._abort:
                raise Abort

            self.manager.select(1)

        for chunk in self.chunks:
            chunk.flush_file()  # make sure downloads are written to disk

        self._copy_chunks()
Beispiel #12
0
    def run_tesser(self,
                   subset=False,
                   digits=True,
                   lowercase=True,
                   uppercase=True):
        # self.log.debug("create tmp tif")

        # tmp = tempfile.NamedTemporaryFile(suffix=".tif")
        tmp_path = os.path.join("tmpTif_{0}.tif".format(self.__name__))
        tmp = lopen(tmp_path, mode='wb')
        tmp.close()
        # self.log.debug("create tmp txt")
        # tmp_txt = tempfile.NamedTemporaryFile(suffix=".txt")
        tmp_txt_path = os.path.join("tmp_txt_{0}.txt".format(self.__name__))
        tmp_txt = lopen(tmp_txt_path, mode='wb')
        tmp_txt.close()

        self.log.debug("save tiff")
        self.image.save(tmp.name, 'TIFF')

        if os.name == 'nt':
            tessparams = [
                resource_filename(__package__, 'tesseract/tesseract.exe')
            ]
        else:
            tessparams = ['tesseract']

        tessparams.extend([tmp.name, tmp_txt.name.replace(".txt", "")])

        if subset and (digits or lowercase or uppercase):
            # self.log.debug("create temp subset config")
            # tmp_sub = tempfile.NamedTemporaryFile(suffix=".subset")
            with lopen(os.path.join("tmp_sub_{0}.subset".format(
                    self.__name__)),
                       mode='wb') as tmp_sub:
                tmp_sub.write("tessedit_char_whitelist ")
                if digits:
                    tmp_sub.write("0123456789")
                if lowercase:
                    tmp_sub.write("abcdefghijklmnopqrstuvwxyz")
                if uppercase:
                    tmp_sub.write("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
                tmp_sub.write(os.linesep)
                tessparams.append("nobatch")
                tessparams.append(tmp_sub.name)

        self.log.debug("run tesseract")
        self.run(tessparams)
        self.log.debug("read txt")

        try:
            with lopen(tmp_txt.name) as fp:
                self.result_captcha = fp.read().replace(os.linesep, "")
        except Exception:
            self.result_captcha = ""

        self.log.debug(self.result_captcha)
        try:
            remove(tmp.name)
            remove(tmp_txt.name)
            if subset and (digits or lowercase or uppercase):
                remove(tmp_sub.name)
        except Exception:
            pass
Beispiel #13
0
    def decryptCaptcha(self,
                       url,
                       get={},
                       post={},
                       cookies=False,
                       forceUser=False,
                       imgtype='jpg',
                       result_type='textual'):
        """ Loads a captcha and decrypts it with ocr, plugin, user input

        :param url: url of captcha image
        :param get: get part for request
        :param post: post part for request
        :param cookies: True if cookies should be enabled
        :param forceUser: if True, ocr is not used
        :param imgtype: Type of the Image
        :param result_type: 'textual' if text is written on the captcha\
        or 'positional' for captcha where the user have to click\
        on a specific region on the captcha

        :return: result of decrypting
        """

        img = self.load(url, get=get, post=post, cookies=cookies)

        id = ("%.2f" % time())[-6:].replace(".", "")
        temp_file = open(
            join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)),
            "wb")
        temp_file.write(img)
        temp_file.close()

        name = "%sOCR" % self.__name__
        has_plugin = name in self.core.pluginManager.getPlugins("internal")

        if self.core.captcha:
            OCR = self.core.pluginManager.loadClass("internal", name)
        else:
            OCR = None

        if OCR and not forceUser:
            sleep(randint(3000, 5000) / 1000.0)
            self.checkAbort()

            ocr = OCR()
            result = ocr.get_captcha(temp_file.name)
        else:
            task = self.im.createCaptchaTask(img, imgtype, temp_file.name,
                                             self.__name__, result_type)
            self.task = task

            while task.isWaiting():
                if self.abort():
                    self.im.removeTask(task)
                    raise Abort()
                sleep(1)

            #TODO task handling
            self.im.removeTask(task)

            if task.error and has_plugin:  #ignore default error message since the user could use OCR
                self.fail(
                    _("Pil and tesseract not installed and no Client connected for captcha decrypting"
                      ))
            elif task.error:
                self.fail(task.error)
            elif not task.result:
                self.fail(_("No captcha result obtained in appropriate time."))

            result = task.result
            self.log.debug("Received captcha result: %s" % str(result))

        if not self.core.debug:
            try:
                remove(temp_file.name)
            except:
                pass

        return result
Beispiel #14
0
    def decrypt_captcha(self,
                        url,
                        get={},
                        post={},
                        cookies=True,
                        forceuser=False,
                        imgtype='jpg',
                        result_type='textual'):
        """
        Loads a captcha and decrypts it with ocr, plugin, user input

        :param url: url of captcha image
        :param get: get part for request
        :param post: post part for request
        :param cookies: True if cookies should be enabled
        :param forceuser: if True, ocr is not used
        :param imgtype: Type of the Image
        :param result_type: 'textual' if text is written on the captcha
        or 'positional' for captcha where the user have to click
        on a specific region on the captcha

        :return: result of decrypting
        """
        img = self.load(url, get=get, post=post, cookies=cookies)

        id = "{0:.2f}".format(time.time())[-6:].replace(".", "")
        with lopen(os.path.join("tmp_captcha_{0}_{1}.{2}".format(
                self.__name__, id, imgtype)),
                   mode='wb') as fp:
            fp.write(img)

            name = "{0}OCR".format(self.__name__)
            has_plugin = name in self.__pyload.pgm.get_plugins("internal")

            if self.__pyload.captcha:
                OCR = self.__pyload.pgm.load_class("internal", name)
            else:
                OCR = None

            if OCR and not forceuser:
                time.sleep(random.randint(3000, 5000) // 1000)
                self.check_abort()

                ocr = OCR()
                result = ocr.get_captcha(fp.name)
            else:
                task = self.__pyload.exm.create_captcha_task(
                    img, imgtype, fp.name, self.__name__, result_type)
                self.task = task

                while task.is_waiting():
                    if self.abort():
                        self.__pyload.exm.remove_task(task)
                        raise Abort
                    time.sleep(1)

                # TODO: task handling
                self.__pyload.exm.remove_task(task)

                if task.error and has_plugin:  # ignore default error message since the user could use OCR
                    self.fail(
                        self.
                        _("Pil and tesseract not installed and no Client connected for captcha decrypting"
                          ))
                elif task.error:
                    self.fail(task.error)
                elif not task.result:
                    self.fail(
                        self._(
                            "No captcha result obtained in appropriate time"))

                result = task.result
                self.__pyload.log.debug(
                    "Received captcha result: {0}".format(result))

            if not self.__pyload.debug:
                try:
                    remove(fp.name)
                except Exception:
                    pass

        return result
Beispiel #15
0
    def _download(self, chunks, resume):
        if not resume:
            self.info.clear()
            self.info.add_chunk("{0}.chunk0".format(
                self.path), (0, 0))  # create an initial entry

        self.chunks = []

        # initial chunk that will load complete file (if needed)
        init = CurlChunk(0, self, None, resume)

        self.chunks.append(init)
        self.__manager.add_handle(init.get_handle())

        last_finish_check = 0
        last_time_check = 0
        chunks_done = set()  # list of curl handles that are finished
        chunks_created = False
        done = False
        # This is a resume, if we were chunked originally assume still can
        if self.info.get_count() > 1:
            self.chunk_support = True

        while True:
            # need to create chunks
            # will be set later by first chunk
            if not chunks_created and self.chunk_support and self.size:

                self.flags ^= Connection.Resumable  # TODO: Recheck...
                if not resume:
                    self.info.set_size(self.size)
                    self.info.create_chunks(chunks)
                    self.info.save()

                chunks = self.info.get_count()

                init.set_range(self.info.get_chunk_range(0))

                for i in range(1, chunks):
                    c = CurlChunk(
                        i, self, self.info.get_chunk_range(i), resume)

                    handle = c.get_handle()
                    if handle:
                        self.chunks.append(c)
                        self.__manager.add_handle(handle)
                    else:
                        # close immediately
                        self.log.debug("Invalid curl handle -> closed")
                        c.close()

                chunks_created = True

            while True:
                ret, num_handles = self.__manager.perform()
                if ret != pycurl.E_CALL_MULTI_PERFORM:
                    break

            t = time.time()

            # reduce these calls
            # when num_q is 0, the loop is exited
            while last_finish_check + 0.5 < t:
                # list of failed curl handles
                failed = []

                # TODO: Rewrite...
                # save only last exception, we can only raise one anyway
                ex = Exception()

                num_q, ok_list, err_list = self.__manager.info_read()
                for c in ok_list:
                    chunk = self.find_chunk(c)
                    # check if the header implies success,
                    # else add it to failed list
                    try:
                        chunk.verify_header()
                    except ResponseException as e:
                        self.log.debug(
                            "Chunk {0:d} failed: {1}".format(
                                chunk.id + 1, str(e)))
                        failed.append(chunk)
                        ex = e
                    else:
                        chunks_done.add(c)

                for c in err_list:
                    curl, errno, msg = c
                    chunk = self.find_chunk(curl)
                    # test if chunk was finished
                    if errno != 23 or "0 !=" not in msg:
                        failed.append(chunk)
                        ex = pycurl.error(errno, msg)
                        self.log.debug(
                            "Chunk {0:d} failed: {1}".format(chunk.id + 1, ex))
                        continue
                    # check if the header implies success,
                    # else add it to failed list
                    try:
                        chunk.verify_header()
                    except ResponseException as e:
                        self.log.debug(
                            "Chunk {0:d} failed: {1}".format(
                                chunk.id + 1, str(e)))
                        failed.append(chunk)
                        ex = e
                    else:
                        chunks_done.add(curl)
                if not num_q:  # no more info to get

                    # check if init is not finished so we reset download
                    # connections
                    # note that other chunks are closed and everything
                    # downloaded with initial connection
                    if failed:
                        if init in failed or init.c in chunks_done:
                            raise ex
                        self.log.error(
                            "Download chunks failed, fallback to "
                            "single connection | {0}".format(ex))

                        # list of chunks to clean and remove
                        to_clean = [x for x in self.chunks if x is not init]
                        for chunk in to_clean:
                            self.close_chunk(chunk)
                            self.chunks.remove(chunk)
                            remove(self.info.get_chunk_name(chunk.id))

                        # let first chunk load the rest and update the
                        # info file
                        init.reset_range()
                        self.info.clear()
                        self.info.add_chunk("{0}.chunk0".format(
                            self.path), (0, self.size))
                        self.info.save()

                    last_finish_check = t

                    if len(chunks_done) >= len(self.chunks):
                        if len(chunks_done) > len(self.chunks):
                            self.log.warning(
                                "Finished download chunks size incorrect")
                        done = True  # all chunks loaded

                    break

            if done:
                break  # all chunks loaded

            # calc speed once per second, averaging over 3 seconds
            if last_time_check + 1 < t:
                len_la = len(self.last_arrived)
                diff = [c.arrived - (self.last_arrived[i] if len_la > i else 0)
                        for i, c in enumerate(self.chunks)]

                self.last_speeds[1] = self.last_speeds[0]
                self.last_speeds[0] = self.speeds
                self.speeds = [float(a) // (t - last_time_check) for a in diff]
                self.last_arrived = [c.arrived for c in self.chunks]
                last_time_check = t

            if self.__abort:
                raise Abort

            self.__manager.select(1)

        for chunk in self.chunks:
            chunk.flush_file()  # make sure downloads are written to disk

        self._copy_chunks()