Ejemplo n.º 1
0
    def setUp(self):
        PluginTester.setUp(self)
        for f in self.files:
            if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))

        # folder for reports
        report = join("tmp", self.__class__.__name__)
        if exists(report):
            for f in listdir(report):
                remove(join(report, f))
Ejemplo n.º 2
0
    def setUp(self):
        PluginTester.setUp(self)
        for f in self.files:
            if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))

        # folder for reports
        report = join("tmp", self.__class__.__name__)
        if exists(report):
            for f in listdir(report):
                remove(join(report, f))
Ejemplo n.º 3
0
    def test_plugin(self, name, url, flag):
        # Print to stdout to see whats going on
        print "%s: %s, %s" % (name, url.encode("utf8"), flag)
        log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)

        # url and plugin should be only important thing
        pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
        pyfile.initPlugin()

        self.thread.pyfile = pyfile
        self.thread.plugin = pyfile.plugin

        try:
            a = time()
            pyfile.plugin.preprocessing(self.thread)

            log(DEBUG, "downloading took %ds" % (time() - a))
            log(DEBUG, "size %d kb" % (pyfile.size / 1024))

            if flag == "offline":
                raise Exception("No offline Exception raised.")

            if pyfile.name not in self.files:
                raise Exception("Filename %s not recognized." % pyfile.name)

            if not exists(save_join(DL_DIR, pyfile.name)):
                raise Exception("File %s does not exists." % pyfile.name)

            hash = md5()
            f = open(save_join(DL_DIR, pyfile.name), "rb")
            while True:
                buf = f.read(4096)
                if not buf: break
                hash.update(buf)
            f.close()

            if hash.hexdigest() != self.files[pyfile.name]:
                log(DEBUG, "Hash is %s" % hash.hexdigest())

                size = stat(f.name).st_size
                if size < 1024 * 1024 * 10:  # 10MB
                    # Copy for debug report
                    log(DEBUG, "Downloaded file copied to report")
                    move(f.name, join("tmp", plugin, f.name))

                raise Exception("Hash does not match.")

        except Exception, e:
            if isinstance(e, Fail) and flag == "fail":
                pass
            elif isinstance(
                    e, Fail) and flag == "offline" and e.message == "offline":
                pass
            else:
                raise
Ejemplo n.º 4
0
    def test_plugin(self, name, url, flag):
        # Print to stdout to see whats going on
        print "%s: %s, %s" % (name, url.encode("utf8"), flag)
        log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)

        # url and plugin should be only important thing
        pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
        pyfile.initPlugin()

        self.thread.pyfile = pyfile
        self.thread.plugin = pyfile.plugin

        try:
            a = time()
            pyfile.plugin.preprocessing(self.thread)

            log(DEBUG, "downloading took %ds" % (time() - a))
            log(DEBUG, "size %d kb" % (pyfile.size / 1024))

            if flag == "offline":
                raise Exception("No offline Exception raised.")

            if pyfile.name not in self.files:
                raise Exception("Filename %s not recognized." % pyfile.name)

            if not exists(save_join(DL_DIR, pyfile.name)):
                raise Exception("File %s does not exists." % pyfile.name)

            hash = md5()
            f = open(save_join(DL_DIR, pyfile.name), "rb")
            while True:
                buf = f.read(4096)
                if not buf:
                    break
                hash.update(buf)
            f.close()

            if hash.hexdigest() != self.files[pyfile.name]:
                log(DEBUG, "Hash is %s" % hash.hexdigest())

                size = stat(f.name).st_size
                if size < 1024 * 1024 * 10:  # 10MB
                    # Copy for debug report
                    log(DEBUG, "Downloaded file copied to report")
                    move(f.name, join("tmp", plugin, f.name))

                raise Exception("Hash does not match.")

        except Exception, e:
            if isinstance(e, Fail) and flag == "fail":
                pass
            elif isinstance(e, Fail) and flag == "offline" and e.message == "offline":
                pass
            else:
                raise
Ejemplo n.º 5
0
    def _copyChunks(self):
        init = fs_encode(self.info.getChunkName(0)) #initial chunk name

        if self.info.getCount() > 1:
            fo = open(init, "rb+") #first chunkfile
            for i in range(1, self.info.getCount()):
                #input file
                fo.seek(
                    self.info.getChunkRange(i - 1)[1] + 1) #seek to beginning of chunk, to get rid of overlapping chunks
                fname = fs_encode("%s.chunk%d" % (self.path, i))
                fi = open(fname, "rb")
                buf = 32 * 1024
                while True: #copy in chunks, consumes less memory
                    data = fi.read(buf)
                    if not data:
                        break
                    fo.write(data)
                fi.close()
                if fo.tell() < self.info.getChunkRange(i)[1]:
                    fo.close()
                    remove(init)
                    self.info.remove() #there are probably invalid chunks
                    raise Exception("Downloaded content was smaller than expected. Try to reduce download connections.")
                remove(fname) #remove chunk
            fo.close()

        if self.name:
            self.path = save_join(dirname(self.path), self.name)

        move(init, fs_encode(self.path))
        self.info.remove() #remove info file
Ejemplo n.º 6
0
    def download(self,
                 url,
                 get={},
                 post={},
                 ref=True,
                 cookies=True,
                 disposition=False):
        """Downloads the content at url to download folder

        :param disposition: if True and server provides content-disposition header\
        the filename will be changed if needed
        :return: The location where the file was saved
        """
        self.checkForSameFiles()
        self.checkAbort()

        self.pyfile.setStatus("downloading")

        download_folder = self.config['general']['download_folder']

        location = save_join(download_folder, self.pyfile.package().folder)

        if not exists(location):
            makedirs(location, int(self.core.config["permission"]["folder"],
                                   8))

            if self.core.config["permission"]["change_dl"] and os.name != "nt":
                try:
                    uid = getpwnam(self.config["permission"]["user"])[2]
                    gid = getgrnam(self.config["permission"]["group"])[2]

                    chown(location, uid, gid)
                except Exception, e:
                    self.log.warning(
                        _("Setting User and Group failed: %s") % str(e))
Ejemplo n.º 7
0
    def checkForSameFiles(self, starting=False):
        """ checks if same file was/is downloaded within same package

        :param starting: indicates that the current download is going to start
        :raises SkipDownload:
        """

        pack = self.pyfile.package()

        for pyfile in self.core.files.cachedFiles():
            if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
                if pyfile.status in (0, 12): #finished or downloading
                    raise SkipDownload(pyfile.pluginname)
                elif pyfile.status in (
                    5, 7) and starting: #a download is waiting/starting and was apparently started before
                    raise SkipDownload(pyfile.pluginname)

        download_folder = self.config['general']['download_folder']
        location = save_join(download_folder, pack.folder, self.pyfile.name)

        if starting and self.core.config['download']['skip_existing'] and exists(location):
            size = os.stat(location).st_size
            if size >= self.pyfile.size:
                raise SkipDownload("File exists.")

        pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
        if pyfile:
            if exists(location):
                raise SkipDownload(pyfile[0])

            self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
Ejemplo n.º 8
0
    def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
        """Downloads the content at url to download folder

        :param disposition: if True and server provides content-disposition header\
        the filename will be changed if needed
        :return: The location where the file was saved
        """
        self.checkForSameFiles()
        self.checkAbort()

        self.pyfile.setStatus("downloading")

        download_folder = self.config['general']['download_folder']

        location = save_join(download_folder, self.pyfile.package().folder)

        if not exists(location):
            makedirs(location, int(self.core.config["permission"]["folder"], 8))

            if self.core.config["permission"]["change_dl"] and os.name != "nt":
                try:
                    uid = getpwnam(self.config["permission"]["user"])[2]
                    gid = getgrnam(self.config["permission"]["group"])[2]

                    chown(location, uid, gid)
                except Exception, e:
                    self.log.warning(_("Setting User and Group failed: %s") % str(e))
Ejemplo n.º 9
0
    def _copyChunks(self):
        init = fs_encode(self.info.getChunkName(0))  #initial chunk name

        if self.info.getCount() > 1:
            fo = open(init, "rb+")  #first chunkfile
            for i in range(1, self.info.getCount()):
                #input file
                fo.seek(
                    self.info.getChunkRange(i - 1)[1] + 1
                )  #seek to beginning of chunk, to get rid of overlapping chunks
                fname = fs_encode("%s.chunk%d" % (self.path, i))
                fi = open(fname, "rb")
                buf = 32 * 1024
                while True:  #copy in chunks, consumes less memory
                    data = fi.read(buf)
                    if not data:
                        break
                    fo.write(data)
                fi.close()
                if fo.tell() < self.info.getChunkRange(i)[1]:
                    fo.close()
                    remove(init)
                    self.info.remove()  #there are probably invalid chunks
                    raise Exception(
                        "Downloaded content was smaller than expected. Try to reduce download connections."
                    )
                remove(fname)  #remove chunk
            fo.close()

        if self.name:
            self.path = save_join(dirname(self.path), self.name)

        move(init, fs_encode(self.path))
        self.info.remove()  #remove info file
Ejemplo n.º 10
0
    def checkForSameFiles(self, starting=False):
        """ checks if same file was/is downloaded within same package

        :param starting: indicates that the current download is going to start
        :raises SkipDownload:
        """

        pack = self.pyfile.package()

        for pyfile in self.core.files.cachedFiles():
            if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
                if pyfile.status in (0, 12): #finished or downloading
                    raise SkipDownload(pyfile.pluginname)
                elif pyfile.status in (
                    5, 7) and starting: #a download is waiting/starting and was apparently started before
                    raise SkipDownload(pyfile.pluginname)

        download_folder = self.config['general']['download_folder']
        location = save_join(download_folder, pack.folder, self.pyfile.name)

        if starting and self.core.config['download']['skip_existing'] and exists(location):
            size = os.stat(location).st_size
            if size >= self.pyfile.size:
                raise SkipDownload("File exists.")

        pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
        if pyfile:
            if exists(location):
                raise SkipDownload(pyfile[0])

            self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
Ejemplo n.º 11
0
    def writeDebugReport(self, name, pyfile=None, plugin=None):
        """ writes a debug report to disk  """

        dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
        if pyfile:
            dump = self.getPluginDump(pyfile.plugin) + "\n"
            dump += self.getFileDump(pyfile)
        else:
            dump = self.getPluginDump(plugin)

        try:
            import zipfile
            zip = zipfile.ZipFile(dump_name, "w")

            if exists(join("tmp", name)):
                for f in listdir(join("tmp", name)):
                    try:
                        # avoid encoding errors
                        zip.write(join("tmp", name, f), save_join(name, f))
                    except:
                        pass

            info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"),
                                   gmtime())
            info.external_attr = 0644 << 16L  # change permissions
            zip.writestr(info, dump)

            info = zipfile.ZipInfo(save_join(name, "system_Report.txt"),
                                   gmtime())
            info.external_attr = 0644 << 16L
            zip.writestr(info, self.getSystemDump())

            zip.close()

            if not stat(dump_name).st_size:
                raise Exception("Empty Zipfile")

        except Exception, e:
            self.log.debug("Error creating zip file: %s" % e)

            dump_name = dump_name.replace(".zip", ".txt")
            f = open(dump_name, "wb")
            f.write(dump)
            f.close()
Ejemplo n.º 12
0
    def writeDebugReport(self, name, pyfile=None, plugin=None):
        """ writes a debug report to disk  """

        dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
        if pyfile:
            dump = self.getPluginDump(pyfile.plugin) + "\n"
            dump += self.getFileDump(pyfile)
        else:
            dump = self.getPluginDump(plugin)

        try:
            import zipfile
            zip = zipfile.ZipFile(dump_name, "w")

            if exists(join("tmp", name)):
                for f in listdir(join("tmp", name)):
                    try:
                        # avoid encoding errors
                        zip.write(join("tmp", name, f), save_join(name, f))
                    except:
                        pass

            info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"), gmtime())
            info.external_attr = 0644 << 16L # change permissions
            zip.writestr(info, dump)

            info = zipfile.ZipInfo(save_join(name, "system_Report.txt"), gmtime())
            info.external_attr = 0644 << 16L
            zip.writestr(info, self.getSystemDump())

            zip.close()

            if not stat(dump_name).st_size:
                raise Exception("Empty Zipfile")

        except Exception, e:
            self.log.debug("Error creating zip file: %s" % e)

            dump_name = dump_name.replace(".zip", ".txt")
            f = open(dump_name, "wb")
            f.write(dump)
            f.close()
Ejemplo n.º 13
0
    def extract(self, ids, thread=None):
        # reload from txt file
        self.reloadPasswords()

        # dl folder
        dl = self.config['general']['download_folder']

        extracted = []

        #iterate packages -> plugins -> targets
        for pid in ids:
            p = self.core.files.getPackage(pid)
            self.logInfo(_("Check package %s") % p.name)
            if not p:
                continue

            # determine output folder
            out = save_join(dl, p.folder, "")
            # force trailing slash

            if self.getConfig("destination") and self.getConfig(
                    "destination").lower() != "none":

                out = save_join(dl, p.folder, self.getConfig("destination"),
                                "")
                #relative to package folder if destination is relative, otherwise absolute path overwrites them

                if self.getConfig("subfolder"):
                    out = join(out, fs_encode(p.folder))

                if not exists(out):
                    makedirs(out)

            files_ids = [(save_join(dl, p.folder, f.name), f.fid)
                         for f in p.getFiles().itervalues()]
            matched = False

            # check as long there are unseen files
            while files_ids:
                new_files_ids = []

                for plugin in self.plugins:
                    targets = plugin.getTargets(files_ids)
                    if targets:
                        self.logDebug("Targets for %s: %s" %
                                      (plugin.__name__, targets))
                        matched = True
                    for target, fid in targets:
                        if target in extracted:
                            self.logDebug(basename(target), "skipped")
                            continue
                        extracted.append(
                            target)  # prevent extracting same file twice

                        klass = plugin(self, target, out,
                                       self.getConfig("fullpath"),
                                       self.getConfig("overwrite"),
                                       self.getConfig("excludefiles"),
                                       self.getConfig("renice"))
                        klass.init()

                        self.logInfo(basename(target),
                                     _("Extract to %s") % out)
                        new_files = self.startExtracting(
                            klass, fid,
                            p.password.strip().splitlines(), thread)
                        self.logDebug("Extracted: %s" % new_files)
                        self.setPermissions(new_files)

                        for file in new_files:
                            if not exists(file):
                                self.logDebug("new file %s does not exists" %
                                              file)
                                continue
                            if self.getConfig("recursive") and isfile(file):
                                new_files_ids.append(
                                    (file, fid))  # append as new target

                files_ids = new_files_ids  # also check extracted files

            if not matched:
                self.logInfo(_("No files found to extract"))
Ejemplo n.º 14
0
    def extract(self, ids, thread=None):
        # reload from txt file
        self.reloadPasswords()

        # dl folder
        dl = self.config['general']['download_folder']

        extracted = []

        #iterate packages -> plugins -> targets
        for pid in ids:
            p = self.core.files.getPackage(pid)
            self.logInfo(_("Check package %s") % p.name)
            if not p:
                continue

            # determine output folder
            out = save_join(dl, p.folder, "")
            # force trailing slash

            if self.getConfig("destination") and self.getConfig("destination").lower() != "none":

                out = save_join(dl, p.folder, self.getConfig("destination"), "")
                #relative to package folder if destination is relative, otherwise absolute path overwrites them

                if self.getConfig("subfolder"):
                    out = join(out, fs_encode(p.folder))

                if not exists(out):
                    makedirs(out)

            files_ids = [(save_join(dl, p.folder, f.name), f.fid) for f in p.getFiles().itervalues()]
            matched = False

            # check as long there are unseen files
            while files_ids:
                new_files_ids = []

                for plugin in self.plugins:
                    targets = plugin.getTargets(files_ids)
                    if targets:
                        self.logDebug("Targets for %s: %s" % (plugin.__name__, targets))
                        matched = True
                    for target, fid in targets:
                        if target in extracted:
                            self.logDebug(basename(target), "skipped")
                            continue
                        extracted.append(target)  # prevent extracting same file twice

                        klass = plugin(self, target, out, self.getConfig("fullpath"), self.getConfig("overwrite"), self.getConfig("excludefiles"),
                                       self.getConfig("renice"))
                        klass.init()

                        self.logInfo(basename(target), _("Extract to %s") % out)
                        new_files = self.startExtracting(klass, fid, p.password.strip().splitlines(), thread)
                        self.logDebug("Extracted: %s" % new_files)
                        self.setPermissions(new_files)

                        for file in new_files:
                            if not exists(file):
                                self.logDebug("new file %s does not exists" % file)
                                continue
                            if self.getConfig("recursive") and isfile(file):
                                new_files_ids.append((file, fid))  # append as new target

                files_ids = new_files_ids  # also check extracted files

            if not matched:
                self.logInfo(_("No files found to extract"))