Пример #1
0
def downloads():
    root = API.get_config_value("general", "download_folder")

    if not os.path.isdir(root):
        return base([_('Download directory not found.')])
    data = {
        'folder': [],
        'files': []
    }

    items = os.listdir(fs_encode(root))

    for item in sorted([fs_decode(x) for x in items]):
        if os.path.isdir(fs_join(root, item)):
            folder = {
                'name': item,
                'path': item,
                'files': []
            }
            files = os.listdir(fs_join(root, item))
            for file in sorted([fs_decode(x) for x in files]):
                try:
                    if os.path.isfile(fs_join(root, item, file)):
                        folder['files'].append(file)
                except Exception:
                    pass

            data['folder'].append(folder)
        elif os.path.isfile(os.path.join(root, item)):
            data['files'].append(item)

    return render_to_response('downloads.html', {'files': data}, [pre_processor])
Пример #2
0
    def write_debug_report(self, pyfile):
        dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, time.strftime("%d-%m-%Y_%H-%M-%S"))
        dump = self.get_debug_dump(pyfile)

        try:
            import zipfile

            zip = zipfile.ZipFile(dump_name, "w")

            for f in os.listdir(os.path.join("tmp", pyfile.pluginname)):
                try:
                    # avoid encoding errors
                    zip.write(os.path.join("tmp", pyfile.pluginname, f), fs_join(pyfile.pluginname, f))
                except Exception:
                    pass

            info = zipfile.ZipInfo(fs_join(pyfile.pluginname, "debug_Report.txt"), time.gmtime())
            info.external_attr = 0644 << 16L  #: change permissions

            zip.writestr(info, dump)
            zip.close()

            if not os.stat(dump_name).st_size:
                raise Exception("Empty Zipfile")

        except Exception, e:
            self.pyload.log.debug("Error creating zip file: %s" % e)

            dump_name = dump_name.replace(".zip", ".txt")
            with open(dump_name, "wb") as f:
                f.write(dump)
Пример #3
0
    def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True, pagesegmode=None):
        # tmpTif = tempfile.NamedTemporaryFile(suffix=".tif")
        try:
            tmpTif = open(fs_join("tmp", "tmpTif_%s.tif" % self.get_class_name()), "wb")
            tmpTif.close()

            # tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt")
            tmpTxt = open(fs_join("tmp", "tmpTxt_%s.txt" % self.get_class_name()), "wb")
            tmpTxt.close()

        except IOError, e:
            self.log_error(e)
            return
Пример #4
0
    def _copy_chunks(self):
        init = fs_encode(self.info.get_chunk_name(0))  #: initial chunk name

        if self.info.get_count() > 1:
            with open(init, "rb+") as fo:  #: first chunkfile
                for i in xrange(1, self.info.get_count()):
                    # input file
                    fo.seek(
                        self.info.get_chunk_range(i - 1)[1] + 1)  #: seek to beginning of chunk, to get rid of overlapping chunks
                    fname = fs_encode("%s.chunk%d" % (self.filename, i))
                    with open(fname, "rb") as fi:
                        buf = 32 * 1024
                        while True:  #: copy in chunks, consumes less memory
                            data = fi.read(buf)
                            if not data:
                                break
                            fo.write(data)
                    if fo.tell() < self.info.get_chunk_range(i)[1]:
                        os.remove(init)
                        self.info.remove()  #: there are probably invalid chunks
                        raise Exception("Downloaded content was smaller than expected. Try to reduce download connections.")
                    os.remove(fname)  #: remove chunk

        if self.nameDisposition and self.disposition:
            self.filename = fs_join(os.path.dirname(self.filename), self.nameDisposition)

        shutil.move(init, fs_encode(self.filename))
        self.info.remove()  #: remove info file
Пример #5
0
    def check_for_same_files(self, starting=False):
        """ checks if same file was/is downloaded within same package

        :param starting: indicates that the current download is going to start
        :raises Skip:
        """

        pack = self.pyfile.package()

        for pyfile in self.core.files.cache.values():
            if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
                if pyfile.status in (0, 12):  #: finished or downloading
                    raise Skip(pyfile.pluginname)
                elif pyfile.status in (5, 7) and starting:  #: a download is waiting/starting and was appenrently started before
                    raise Skip(pyfile.pluginname)

        download_folder = self.core.config.get("general", "download_folder")
        location = fs_join(download_folder, pack.folder, self.pyfile.name)

        if starting and self.core.config.get("download", "skip_existing") and os.path.exists(location):
            size = os.stat(location).st_size
            if size >= self.pyfile.size:
                raise Skip("File exists")

        pyfile = self.core.db.find_duplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
        if pyfile:
            if os.path.exists(location):
                raise Skip(pyfile[0])

            self.log_debug("File %s not skipped, because it does not exists." % self.pyfile.name)
Пример #6
0
    def load_to_disk(self):
        """Loads container to disk if its stored remotely and overwrite url,
        or check existent on several places at disk"""

        if self.pyfile.url.startswith("http"):
            self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1]
            content = self.load(self.pyfile.url)
            self.pyfile.url = fs_join(self.pyload.config.get("general", "download_folder"), self.pyfile.name)
            try:
                with open(self.pyfile.url, "wb") as f:
                    f.write(content)
            except IOError, e:
                self.fail(str(e))
Пример #7
0
    def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
        """Downloads the content at url to download folder

        :param url:
        :param get:
        :param post:
        :param ref:
        :param cookies:
        :param disposition: if True and server provides content-disposition header\
        the filename will be changed if needed
        :return: The location where the file was saved
        """
        if self.pyfile.abort:
            self.abort()

        if not url:
            self.fail(_("No url given"))

        url = urllib.unquote(encode(url).strip())

        if self.core.debug:
            self.log_debug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])

        self.check_for_same_files()

        self.pyfile.set_status("downloading")

        if disposition:
            self.pyfile.name = urlparse.urlparse(url).path.split('/')[-1] or self.pyfile.name

        download_folder = self.core.config.get("general", "download_folder")

        location = fs_join(download_folder, self.pyfile.package().folder)

        if not os.path.exists(location):
            try:
                os.makedirs(location, int(self.core.config.get("permission", "folder"), 8))

                if self.core.config.get("permission", "change_dl") and os.name != "nt":
                    uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2]
                    gid = grp.getgrnam(self.core.config.get("permission", "group"))[2]
                    os.chown(location, uid, gid)

            except Exception, e:
                self.fail(e)
Пример #8
0
    def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False, follow_location=True, save_cookies=True):
        """Load content at url and returns it

        :param url:
        :param get:
        :param post:
        :param ref:
        :param cookies:
        :param just_header: If True only the header will be retrieved and returned as dict
        :param decode: Wether to decode the output according to http header, should be True in most cases
        :param follow_location: If True follow location else not
        :param save_cookies: If True saves received cookies else discard them
        :return: Loaded content
        """
        if self.pyfile.abort:
            self.abort()

        if not url:
            self.fail(_("No url given"))

        url = urllib.unquote(encode(url).strip())  #@NOTE: utf8 vs decode -> please use decode attribute in all future plugins

        if self.core.debug:
            self.log_debug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])

        res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode, follow_location=follow_location, save_cookies=save_cookies)

        if decode:
            res = encode(res)

        if self.core.debug:
            import inspect

            frame = inspect.currentframe()
            framefile = fs_join("tmp", self.get_class_name(), "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
            try:
                if not os.path.exists(os.path.join("tmp", self.get_class_name())):
                    os.makedirs(os.path.join("tmp", self.get_class_name()))

                with open(framefile, "wb") as f:
                    del frame  #: delete the frame or it wont be cleaned
                    f.write(res)
            except IOError, e:
                self.log_error(e)
Пример #9
0
    def remove_plugins(self, type_plugins):
        """Delete plugins from disk"""

        if not type_plugins:
            return

        removed = set()

        self.log_debug("Requested deletion of plugins: %s" % type_plugins)

        for type, name in type_plugins:
            rootplugins = os.path.join(pypath, "module", "plugins")

            for dir in ("userplugins", rootplugins):
                py_filename = fs_join(dir, type, name + ".py")
                pyc_filename = py_filename + "c"

                if type == "addon":
                    try:
                        self.manager.deactivate_addon(name)

                    except Exception, e:
                        self.log_debug(e)

                for filename in (py_filename, pyc_filename):
                    if not exists(filename):
                        continue

                    try:
                        os.remove(filename)

                    except OSError, e:
                        self.log_error(_("Error removing: %s") % filename, e)

                    else:
                        id = (type, name)
                        removed.add(id)
Пример #10
0
        self.create_packages()


    def load_to_disk(self):
        """Loads container to disk if its stored remotely and overwrite url,
        or check existent on several places at disk"""

        if self.pyfile.url.startswith("http"):
            self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1]
            content = self.load(self.pyfile.url)
            self.pyfile.url = fs_join(self.pyload.config.get("general", "download_folder"), self.pyfile.name)
            try:
                with open(self.pyfile.url, "wb") as f:
                    f.write(content)
            except IOError, e:
                self.fail(str(e))

        else:
            self.pyfile.name = os.path.basename(self.pyfile.url)
            if not os.path.exists(self.pyfile.url):
                if os.path.exists(fs_join(pypath, self.pyfile.url)):
                    self.pyfile.url = fs_join(pypath, self.pyfile.url)
                else:
                    self.fail(_("File not exists"))


    def delete_tmp(self):
        if self.pyfile.name.startswith("tmp_"):
            os.remove(self.pyfile.url)