示例#1
0
文件: Plugin.py 项目: Bobbaone/pyload
    def writeDebugReport(self, pyfile):
        """ writes a
        :return:
        """

        dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, time.strftime("%d-%m-%Y_%H-%M-%S"))
        dump = self.getDebugDump(pyfile)

        try:
            import zipfile

            zip = zipfile.ZipFile(dump_name, "w")

            for f in os.listdir(os.path.join("tmp", pyfile.pluginname)):
                try:
                    # avoid encoding errors
                    zip.write(os.path.join("tmp", pyfile.pluginname, f), fs_join(pyfile.pluginname, f))
                except Exception:
                    pass

            info = zipfile.ZipInfo(fs_join(pyfile.pluginname, "debug_Report.txt"), time.gmtime())
            info.external_attr = 0644 << 16L  #: change permissions

            zip.writestr(info, dump)
            zip.close()

            if not os.stat(dump_name).st_size:
                raise Exception("Empty Zipfile")

        except Exception, e:
            self.m.log.debug("Error creating zip file: %s" % e)

            dump_name = dump_name.replace(".zip", ".txt")
            with open(dump_name, "wb") as f:
                f.write(dump)
示例#2
0
def downloads():
    root = PYLOAD.getConfigValue("general", "download_folder")

    if not os.path.isdir(root):
        return base([_('Download directory not found.')])
    data = {'folder': [], 'files': []}

    items = os.listdir(fs_encode(root))

    for item in sorted([fs_decode(x) for x in items]):
        if os.path.isdir(fs_join(root, item)):
            folder = {'name': item, 'path': item, 'files': []}
            files = os.listdir(fs_join(root, item))
            for file in sorted([fs_decode(x) for x in files]):
                try:
                    if os.path.isfile(fs_join(root, item, file)):
                        folder['files'].append(file)
                except Exception:
                    pass

            data['folder'].append(folder)
        elif os.path.isfile(os.path.join(root, item)):
            data['files'].append(item)

    return render_to_response('downloads.html', {'files': data},
                              [pre_processor])
示例#3
0
def downloads():
    root = PYLOAD.getConfigValue("general", "download_folder")

    if not isdir(root):
        return base([_('Download directory not found.')])
    data = {
        'folder': [],
        'files': []
    }

    items = listdir(fs_encode(root))

    for item in sorted([fs_decode(x) for x in items]):
        if isdir(fs_join(root, item)):
            folder = {
                'name': item,
                'path': item,
                'files': []
            }
            files = listdir(fs_join(root, item))
            for file in sorted([fs_decode(x) for x in files]):
                try:
                    if isfile(fs_join(root, item, file)):
                        folder['files'].append(file)
                except Exception:
                    pass

            data['folder'].append(folder)
        elif isfile(join(root, item)):
            data['files'].append(item)

    return render_to_response('downloads.html', {'files': data}, [pre_processor])
示例#4
0
文件: UnRar.py 项目: PaddyPat/pyload
    def list(self, password=None):
        command = "vb" if self.fullpath else "lb"

        p = self.call_cmd(command,
                          "-v",
                          fs_encode(self.filename),
                          password=password)
        out, err = p.communicate()

        if "Cannot open" in err:
            raise ArchiveError(_("Cannot open file"))

        if err.strip():  #: only log error at this point
            self.manager.logError(err.strip())

        result = set()
        if not self.fullpath and self.VERSION.startswith('5'):
            # NOTE: Unrar 5 always list full path
            for f in fs_decode(out).splitlines():
                f = fs_join(self.out, os.path.basename(f.strip()))
                if os.path.isfile(f):
                    result.add(fs_join(self.out, os.path.basename(f)))
        else:
            for f in fs_decode(out).splitlines():
                f = f.strip()
                result.add(fs_join(self.out, f))

        return list(result)
示例#5
0
    def packageFinished(self, pypack):
        download_folder = fs_join(
            self.config.get("general", "download_folder"), pypack.folder, "")

        for link in pypack.getChildren().itervalues():
            file_type = splitext(link['name'])[1][1:].lower()

            if file_type not in self.formats:
                continue

            hash_file = fs_encode(fs_join(download_folder, link['name']))
            if not isfile(hash_file):
                self.logWarning(_("File not found"), link['name'])
                continue

            with open(hash_file) as f:
                text = f.read()

            for m in re.finditer(
                    self.regexps.get(file_type, self.regexps['default']),
                    text):
                data = m.groupdict()
                self.logDebug(link['name'], data)

                local_file = fs_encode(fs_join(download_folder, data['NAME']))
                algorithm = self.methods.get(file_type, file_type)
                checksum = computeChecksum(local_file, algorithm)
                if checksum == data['HASH']:
                    self.logInfo(
                        _('File integrity of "%s" verified by %s checksum (%s)'
                          ) % (data['NAME'], algorithm, checksum))
                else:
                    self.logWarning(
                        _("%s checksum for file %s does not match (%s != %s)")
                        % (algorithm, data['NAME'], checksum, data['HASH']))
示例#6
0
    def list(self, password=None):
        command = "vb" if self.fullpath else "lb"

        p = self.call_cmd(command, "-v", fs_encode(self.filename), password=password)
        out, err = p.communicate()

        if "Cannot open" in err:
            raise ArchiveError(_("Cannot open file"))

        if err.strip():  #: only log error at this point
            self.manager.logError(err.strip())

        result = set()
        if not self.fullpath and self.VERSION.startswith('5'):
            # NOTE: Unrar 5 always list full path
            for f in fs_decode(out).splitlines():
                f = fs_join(self.out, os.path.basename(f.strip()))
                if os.path.isfile(f):
                    result.add(fs_join(self.out, os.path.basename(f)))
        else:
            for f in fs_decode(out).splitlines():
                f = f.strip()
                result.add(fs_join(self.out, f))

        return list(result)
示例#7
0
    def packageFinished(self, pypack):
        download_folder = fs_join(self.config.get("general", "download_folder"), pypack.folder, "")

        for link in pypack.getChildren().itervalues():
            file_type = splitext(link['name'])[1][1:].lower()

            if file_type not in self.formats:
                continue

            hash_file = fs_encode(fs_join(download_folder, link['name']))
            if not isfile(hash_file):
                self.logWarning(_("File not found"), link['name'])
                continue

            with open(hash_file) as f:
                text = f.read()

            for m in re.finditer(self.regexps.get(file_type, self.regexps['default']), text):
                data = m.groupdict()
                self.logDebug(link['name'], data)

                local_file = fs_encode(fs_join(download_folder, data['NAME']))
                algorithm = self.methods.get(file_type, file_type)
                checksum = computeChecksum(local_file, algorithm)
                if checksum == data['HASH']:
                    self.logInfo(_('File integrity of "%s" verified by %s checksum (%s)') %
                                (data['NAME'], algorithm, checksum))
                else:
                    self.logWarning(_("%s checksum for file %s does not match (%s != %s)") %
                                   (algorithm, data['NAME'], checksum, data['HASH']))
    def downloadFinished(self, pyfile):
        if self.core.config.get("general", "folder_per_package"):
            download_folder = fs_join(self.core.config.get("general", "download_folder"), pyfile.package().folder)
        else:
            download_folder = self.core.config.get("general", "download_folder")

        for script in self.scripts['download_finished']:
            file = fs_join(download_folder, pyfile.name)
            self.callScript(script, pyfile.id, pyfile.name, file, pyfile.pluginname, pyfile.url)
示例#9
0
    def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True, pagesegmode=None):
        # tmpTif = tempfile.NamedTemporaryFile(suffix=".tif")
        try:
            tmpTif = open(fs_join("tmp", "tmpTif_%s.tif" % self.__class__.__name__), "wb")
            tmpTif.close()

            # tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt")
            tmpTxt = open(fs_join("tmp", "tmpTxt_%s.txt" % self.__class__.__name__), "wb")
            tmpTxt.close()

        except IOError, e:
            self.logError(e)
            return
示例#10
0
    def packageFinished(self, pack):
        files = {}
        fid_dict = {}
        for fid, data in pack.getChildren().iteritems():
            if re.search("\.\d{3}$", data['name']):
                if data['name'][:-4] not in files:
                    files[data['name'][:-4]] = []
                files[data['name'][:-4]].append(data['name'])
                files[data['name'][:-4]].sort()
                fid_dict[data['name']] = fid

        download_folder = self.config.get("general", "download_folder")

        if self.config.get("general", "folder_per_package"):
            download_folder = fs_join(download_folder, pack.folder)

        for name, file_list in files.iteritems():
            self.logInfo(_("Starting merging of"), name)

            with open(fs_join(download_folder, name), "wb") as final_file:
                for splitted_file in file_list:
                    self.logDebug("Merging part", splitted_file)

                    pyfile = self.core.files.getFile(fid_dict[splitted_file])

                    pyfile.setStatus("processing")

                    try:
                        with open(fs_join(download_folder, splitted_file), "rb") as s_file:
                            size_written = 0
                            s_file_size = int(os.path.getsize(os.path.join(download_folder, splitted_file)))

                            while True:
                                f_buffer = s_file.read(self.BUFFER_SIZE)
                                if f_buffer:
                                    final_file.write(f_buffer)
                                    size_written += self.BUFFER_SIZE
                                    pyfile.setProgress((size_written * 100) / s_file_size)
                                else:
                                    break

                        self.logDebug("Finished merging part", splitted_file)

                    except Exception, e:
                        traceback.print_exc()

                    finally:
                        pyfile.setProgress(100)
                        pyfile.setStatus("finished")
                        pyfile.release()
示例#11
0
文件: Plugin.py 项目: PaddyPat/pyload
    def download(self,
                 url,
                 get={},
                 post={},
                 ref=True,
                 cookies=True,
                 disposition=False):
        """Downloads the content at url to download folder

        :param url:
        :param get:
        :param post:
        :param ref:
        :param cookies:
        :param disposition: if True and server provides content-disposition header\
        the filename will be changed if needed
        :return: The location where the file was saved
        """
        if self.pyfile.abort:
            self.abort()

        if not url:
            self.fail(_("No url given"))

        url = urllib.unquote(encode(url).strip())

        if self.core.debug:
            self.logDebug(
                "Download url: " + url, *[
                    "%s=%s" % (key, val) for key, val in locals().iteritems()
                    if key not in ("self", "url")
                ])

        self.checkForSameFiles()

        self.pyfile.setStatus("downloading")

        if disposition:
            self.pyfile.name = urlparse.urlparse(url).path.split(
                '/')[-1] or self.pyfile.name

        download_folder = self.core.config.get("general", "download_folder")

        location = fs_join(download_folder, self.pyfile.package().folder)

        if not exists(location):
            try:
                makedirs(location,
                         int(self.core.config.get("permission", "folder"), 8))

                if self.core.config.get("permission",
                                        "change_dl") and os.name != "nt":
                    uid = getpwnam(self.core.config.get("permission",
                                                        "user"))[2]
                    gid = getgrnam(self.core.config.get("permission",
                                                        "group"))[2]
                    chown(location, uid, gid)

            except Exception, e:
                self.fail(e)
示例#12
0
    def _copyChunks(self):
        init = fs_encode(self.info.getChunkName(0))  #: initial chunk name

        if self.info.getCount() > 1:
            with open(init, "rb+") as fo:  #: first chunkfile
                for i in xrange(1, self.info.getCount()):
                    # input file
                    fo.seek(
                        self.info.getChunkRange(i - 1)[1] + 1
                    )  #: seek to beginning of chunk, to get rid of overlapping chunks
                    fname = fs_encode("%s.chunk%d" % (self.filename, i))
                    with open(fname, "rb") as fi:
                        buf = 32 * 1024
                        while True:  #: copy in chunks, consumes less memory
                            data = fi.read(buf)
                            if not data:
                                break
                            fo.write(data)
                    if fo.tell() < self.info.getChunkRange(i)[1]:
                        reshutil.move(init)
                        self.info.remove(
                        )  #: there are probably invalid chunks
                        raise Exception(
                            "Downloaded content was smaller than expected. Try to reduce download connections."
                        )
                    reshutil.move(fname)  #: remove chunk

        if self.nameDisposition and self.disposition:
            self.filename = fs_join(os.path.dirname(self.filename),
                                    self.nameDisposition)

        shutil.move(init, fs_encode(self.filename))
        self.info.remove()  #: remove info file
示例#13
0
    def decrypt(self, pyfile):
        fs_filename = fs_encode(pyfile.url.strip())
        opener = urllib2.build_opener(
            MultipartPostHandler.MultipartPostHandler)

        dlc_content = opener.open(
            'http://service.jdownloader.net/dlcrypt/getDLC.php', {
                'src': "ccf",
                'filename': "test.ccf",
                'upload': open(fs_filename, "rb")
            }).read()

        download_folder = self.config.get("general", "download_folder")
        dlc_file = fs_join(download_folder, "tmp_%s.dlc" % pyfile.name)

        try:
            dlc = re.search(r'<dlc>(.+)</dlc>', dlc_content,
                            re.S).group(1).decode('base64')

        except AttributeError:
            self.fail(_("Container is corrupted"))

        with open(dlc_file, "w") as tempdlc:
            tempdlc.write(dlc)

        self.urls = [dlc_file]
    def decrypt(self, pyfile):
        m = re.match(self.__pattern, pyfile.url)
        m_id = m.group('ID')
        m_type = m.group('TYPE')

        if m_type == "playlist":
            self.logDebug("Url recognized as Playlist")
            p_info = self.getPlaylistInfo(m_id)
            playlists = [(m_id, ) + p_info] if p_info else None
        else:
            self.logDebug("Url recognized as Channel")
            playlists = self.getPlaylists(m_id)
            self.logDebug("%s playlist\s found on channel \"%s\"" %
                          (len(playlists), m_id))

        if not playlists:
            self.fail(_("No playlist available"))

        for p_id, p_name, p_owner in playlists:
            p_videos = self.getVideos(p_id)
            p_folder = fs_join(self.config.get("general", "download_folder"),
                               p_owner, p_name)
            self.logDebug("%s video\s found on playlist \"%s\"" %
                          (len(p_videos), p_name))
            self.packages.append(
                (p_name, p_videos,
                 p_folder))  #: folder is NOT recognized by pyload 0.4.9!
示例#15
0
    def _copyChunks(self):
        init = fs_encode(self.info.getChunkName(0))  #: initial chunk name

        if self.info.getCount() > 1:
            with open(init, "rb+") as fo:  #: first chunkfile
                for i in xrange(1, self.info.getCount()):
                    # input file
                    fo.seek(
                        self.info.getChunkRange(i - 1)[1] + 1)  #: seek to beginning of chunk, to get rid of overlapping chunks
                    fname = fs_encode("%s.chunk%d" % (self.filename, i))
                    with open(fname, "rb") as fi:
                        buf = 32 * 1024
                        while True:  #: copy in chunks, consumes less memory
                            data = fi.read(buf)
                            if not data:
                                break
                            fo.write(data)
                    if fo.tell() < self.info.getChunkRange(i)[1]:
                        reshutil.move(init)
                        self.info.remove()  #: there are probably invalid chunks
                        raise Exception("Downloaded content was smaller than expected. Try to reduce download connections.")
                    reshutil.move(fname)  #: remove chunk

        if self.nameDisposition and self.disposition:
            self.filename = fs_join(os.path.dirname(self.filename), self.nameDisposition)

        shutil.move(init, fs_encode(self.filename))
        self.info.remove()  #: remove info file
示例#16
0
文件: Plugin.py 项目: Bobbaone/pyload
    def checkForSameFiles(self, starting=False):
        """ checks if same file was/is downloaded within same package

        :param starting: indicates that the current download is going to start
        :raises SkipDownload:
        """

        pack = self.pyfile.package()

        for pyfile in self.core.files.cache.values():
            if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
                if pyfile.status in (0, 12):  #: finished or downloading
                    raise SkipDownload(pyfile.pluginname)
                elif pyfile.status in (5, 7) and starting:  #: a download is waiting/starting and was appenrently started before
                    raise SkipDownload(pyfile.pluginname)

        download_folder = self.core.config.get("general", "download_folder")
        location = fs_join(download_folder, pack.folder, self.pyfile.name)

        if starting and self.core.config.get("download", "skip_existing") and os.path.exists(location):
            size = os.stat(location).st_size
            if size >= self.pyfile.size:
                raise SkipDownload("File exists")

        pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
        if pyfile:
            if os.path.exists(location):
                raise SkipDownload(pyfile[0])

            self.logDebug("File %s not skipped, because it does not exists." % self.pyfile.name)
    def package_extracted(self, pypack):
        if self.core.config.get("general", "folder_per_package"):
            download_folder = fs_join(self.core.config.get("general", "download_folder"), pypack.folder)
        else:
            download_folder = self.core.config.get("general", "download_folder")

        for script in self.scripts['package_extracted']:
            self.callScript(script, pypack.id, pypack.name, download_folder)
    def packageDeleted(self, pid):
        pack = self.core.api.getPackageInfo(pid)

        if self.core.config.get("general", "folder_per_package"):
            download_folder = fs_join(self.core.config.get("general", "download_folder"), pack.folder)
        else:
            download_folder = self.core.config.get("general", "download_folder")

        for script in self.scripts['package_deleted']:
            self.callScript(script, pack.id, pack.name, download_folder, pack.password)
示例#19
0
    def run_tesser(self,
                   subset=False,
                   digits=True,
                   lowercase=True,
                   uppercase=True,
                   pagesegmode=None):
        # tmpTif = tempfile.NamedTemporaryFile(suffix=".tif")
        try:
            tmpTif = open(
                fs_join("tmp", "tmpTif_%s.tif" % self.getClassName()), "wb")
            tmpTif.close()

            # tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt")
            tmpTxt = open(
                fs_join("tmp", "tmpTxt_%s.txt" % self.getClassName()), "wb")
            tmpTxt.close()

        except IOError, e:
            self.logError(e)
            return
示例#20
0
    def getDeleteFiles(self):
        dir, name = os.path.split(self.filename)

        # actually extracted file
        files = [self.filename]

        # eventually Multipart Files
        files.extend(fs_join(dir, os.path.basename(file)) for file in filter(self.isMultipart, os.listdir(dir))
                     if re.sub(self.re_multipart,".rar",name) == re.sub(self.re_multipart,".rar",file))

        return files
示例#21
0
    def decrypt(self, pyfile):
        m = re.match(self.__pattern, pyfile.url)
        m_id = m.group('ID')
        m_type = m.group('TYPE')

        if m_type == "user":
            self.logDebug("Url recognized as Channel")
            user = m_id
            channel = self.getChannel(user)

            if channel:
                playlists = self.getPlaylists(channel['id'])
                self.logDebug("%s playlist\s found on channel \"%s\"" % (len(playlists), channel['title']))

                relatedplaylist = dict((p_name, self.getPlaylist(p_id)) for p_name, p_id in channel['relatedPlaylists'].iteritems())
                self.logDebug("Channel's related playlists found = %s" % relatedplaylist.keys())

                relatedplaylist['uploads']['title'] = "Unplaylisted videos"
                relatedplaylist['uploads']['checkDups'] = True  #: checkDups flag

                for p_name, p_data in relatedplaylist.iteritems():
                    if self.getConfig(p_name):
                        p_data['title'] += " of " + user
                        playlists.append(p_data)
            else:
                playlists = []
        else:
            self.logDebug("Url recognized as Playlist")
            playlists = [self.getPlaylist(m_id)]

        if not playlists:
            self.fail(_("No playlist available"))

        addedvideos = []
        urlize = lambda x: "https://www.youtube.com/watch?v=" + x
        for p in playlists:
            p_name = p['title']
            p_videos = self.getVideosId(p['id'])
            p_folder = fs_join(self.config.get("general", "download_folder"), p['channelTitle'], p_name)
            self.logDebug("%s video\s found on playlist \"%s\"" % (len(p_videos), p_name))

            if not p_videos:
                continue
            elif "checkDups" in p:
                p_urls = [urlize(v_id) for v_id in p_videos if v_id not in addedvideos]
                self.logDebug("%s video\s available on playlist \"%s\" after duplicates cleanup" % (len(p_urls), p_name))
            else:
                p_urls = map(urlize, p_videos)

            self.packages.append((p_name, p_urls, p_folder))  #: folder is NOT recognized by pyload 0.4.9!

            addedvideos.extend(p_videos)
示例#22
0
    def writeDebugReport(self, pyfile):
        """ writes a
        :return:
        """

        dump_name = "debug_%s_%s.zip" % (pyfile.pluginname,
                                         strftime("%d-%m-%Y_%H-%M-%S"))
        dump = self.getDebugDump(pyfile)

        try:
            import zipfile

            zip = zipfile.ZipFile(dump_name, "w")

            for f in listdir(join("tmp", pyfile.pluginname)):
                try:
                    # avoid encoding errors
                    zip.write(join("tmp", pyfile.pluginname, f),
                              fs_join(pyfile.pluginname, f))
                except Exception:
                    pass

            info = zipfile.ZipInfo(
                fs_join(pyfile.pluginname, "debug_Report.txt"), gmtime())
            info.external_attr = 0644 << 16L  #: change permissions

            zip.writestr(info, dump)
            zip.close()

            if not stat(dump_name).st_size:
                raise Exception("Empty Zipfile")

        except Exception, e:
            self.m.log.debug("Error creating zip file: %s" % e)

            dump_name = dump_name.replace(".zip", ".txt")
            f = open(dump_name, "wb")
            f.write(dump)
            f.close()
示例#23
0
    def loadToDisk(self):
        """loads container to disk if its stored remotely and overwrite url,
        or check existent on several places at disk"""

        if self.pyfile.url.startswith("http"):
            self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1]
            content = self.load(self.pyfile.url)
            self.pyfile.url = fs_join(self.core.config.get("general", "download_folder"), self.pyfile.name)
            try:
                with open(self.pyfile.url, "wb") as f:
                    f.write(content)
            except IOError, e:
                self.fail(str(e))
示例#24
0
    def loadToDisk(self):
        """loads container to disk if its stored remotely and overwrite url,
        or check existent on several places at disk"""

        if self.pyfile.url.startswith("http"):
            self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1]
            content = self.load(self.pyfile.url)
            self.pyfile.url = fs_join(
                self.core.config.get("general", "download_folder"),
                self.pyfile.name)
            try:
                with open(self.pyfile.url, "wb") as f:
                    f.write(content)
            except IOError, e:
                self.fail(str(e))
示例#25
0
文件: Plugin.py 项目: Bobbaone/pyload
    def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
        """Downloads the content at url to download folder

        :param url:
        :param get:
        :param post:
        :param ref:
        :param cookies:
        :param disposition: if True and server provides content-disposition header\
        the filename will be changed if needed
        :return: The location where the file was saved
        """
        if self.pyfile.abort:
            self.abort()

        if not url:
            self.fail(_("No url given"))

        url = urllib.unquote(encode(url).strip())

        if self.core.debug:
            self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])

        self.checkForSameFiles()

        self.pyfile.setStatus("downloading")

        if disposition:
            self.pyfile.name = urlparse.urlparse(url).path.split('/')[-1] or self.pyfile.name

        download_folder = self.core.config.get("general", "download_folder")

        location = fs_join(download_folder, self.pyfile.package().folder)

        if not os.path.exists(location):
            try:
                os.makedirs(location, int(self.core.config.get("permission", "folder"), 8))

                if self.core.config.get("permission", "change_dl") and os.name != "nt":
                    uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2]
                    gid = grp.getgrnam(self.core.config.get("permission", "group"))[2]
                    os.chown(location, uid, gid)

            except Exception, e:
                self.fail(e)
示例#26
0
    def list(self, password=None):
        command = "l" if self.fullpath else "l"

        p = self.call_cmd(command, fs_encode(self.filename), password=password)
        out, err = p.communicate()

        if "Can not open" in err:
            raise ArchiveError(_("Cannot open file"))

        if p.returncode > 1:
            raise ArchiveError(_("Process return code: %d") % p.returncode)

        result = set()
        for groups in self.re_filelist.findall(out):
            f = groups[-1].strip()
            result.add(fs_join(self.out, f))

        return list(result)
示例#27
0
    def list(self, password=None):
        command = "l" if self.fullpath else "l"

        p = self.call_cmd(command, fs_encode(self.filename), password=password)
        out, err = p.communicate()

        if "Can not open" in err:
            raise ArchiveError(_("Cannot open file"))

        if p.returncode > 1:
            raise ArchiveError(_("Process return code: %d") % p.returncode)

        result = set()
        for groups in self.re_filelist.findall(out):
            f = groups[-1].strip()
            result.add(fs_join(self.out, f))

        return list(result)
示例#28
0
文件: Plugin.py 项目: Bobbaone/pyload
    def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False, follow_location=True, save_cookies=True):
        """Load content at url and returns it

        :param url:
        :param get:
        :param post:
        :param ref:
        :param cookies:
        :param just_header: If True only the header will be retrieved and returned as dict
        :param decode: Wether to decode the output according to http header, should be True in most cases
        :param follow_location: If True follow location else not
        :param save_cookies: If True saves received cookies else discard them
        :return: Loaded content
        """
        if self.pyfile.abort:
            self.abort()

        if not url:
            self.fail(_("No url given"))

        url = urllib.unquote(encode(url).strip())  #@NOTE: utf8 vs decode -> please use decode attribute in all future plugins

        if self.core.debug:
            self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])

        res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode, follow_location=follow_location, save_cookies=save_cookies)

        if decode:
            res = encode(res)

        if self.core.debug:
            import inspect

            frame = inspect.currentframe()
            framefile = fs_join("tmp", self.getClassName(), "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
            try:
                if not os.path.exists(os.path.join("tmp", self.getClassName())):
                    os.makedirs(os.path.join("tmp", self.getClassName()))

                with open(framefile, "wb") as f:
                    del frame  #: delete the frame or it wont be cleaned
                    f.write(res)
            except IOError, e:
                self.logError(e)
示例#29
0
    def handleDLCLinks(self):
        self.logDebug("Search for DLC links")
        package_links = []
        m = re.search(self.DLC_LINK_REGEX, self.html)
        if m:
            container_url = self.DLC_DOWNLOAD_URL + "?id=%s&dlc=1" % self.fileid
            self.logDebug("Downloading DLC container link [%s]" % container_url)
            try:
                dlc = self.load(container_url)
                dlc_filename = self.fileid + ".dlc"
                dlc_filepath = fs_join(self.config.get("general", "download_folder"), dlc_filename)
                with open(dlc_filepath, "wb") as f:
                    f.write(dlc)
                package_links.append(dlc_filepath)

            except Exception:
                self.fail(_("Unable to download DLC container"))

        return package_links
示例#30
0
    def periodical(self):
        folder = fs_encode(self.getConfig('folder'))
        file = fs_encode(self.getConfig('file'))

        try:
            if not os.path.isdir(os.path.join(folder, "finished")):
                os.makedirs(os.path.join(folder, "finished"))

            if self.getConfig('watch_file'):
                with open(file, "a+") as f:
                    f.seek(0)
                    content = f.read().strip()

                if content:
                    f = open(file, "wb")
                    f.close()

                    name = "%s_%s.txt" % (file,
                                          time.strftime("%H-%M-%S_%d%b%Y"))

                    with open(fs_join(folder, "finished", name), "wb") as f:
                        f.write(content)

                    self.core.api.addPackage(f.name, [f.name], 1)

            for f in os.listdir(folder):
                path = os.path.join(folder, f)

                if not os.path.isfile(path) or f.endswith("~") or f.startswith(
                        "#") or f.startswith("."):
                    continue

                newpath = os.path.join(
                    folder, "finished",
                    f if self.getConfig('keep') else "tmp_" + f)
                move(path, newpath)

                self.logInfo(_("Added %s from HotFolder") % f)
                self.core.api.addPackage(f, [newpath], 1)

        except (IOError, OSError), e:
            self.logError(e)
示例#31
0
文件: CCF.py 项目: Bobbaone/pyload
    def decrypt(self, pyfile):
        fs_filename = fs_encode(pyfile.url.strip())
        opener      = urllib2.build_opener(MultipartPostHandler.MultipartPostHandler)

        dlc_content = opener.open('http://service.jdownloader.net/dlcrypt/getDLC.php',
                                  {'src'     : "ccf",
                                   'filename': "test.ccf",
                                   'upload'  : open(fs_filename, "rb")}).read()

        download_folder = self.config.get("general", "download_folder")
        dlc_file        = fs_join(download_folder, "tmp_%s.dlc" % pyfile.name)

        try:
            dlc = re.search(r'<dlc>(.+)</dlc>', dlc_content, re.S).group(1).decode('base64')

        except AttributeError:
            self.fail(_("Container is corrupted"))

        with open(dlc_file, "w") as tempdlc:
            tempdlc.write(dlc)

        self.urls = [dlc_file]
示例#32
0
    def decrypt(self, pyfile):
        m = re.match(self.__pattern, pyfile.url)
        m_id = m.group('ID')
        m_type = m.group('TYPE')

        if m_type == "playlist":
            self.logDebug("Url recognized as Playlist")
            p_info = self.getPlaylistInfo(m_id)
            playlists = [(m_id,) + p_info] if p_info else None
        else:
            self.logDebug("Url recognized as Channel")
            playlists = self.getPlaylists(m_id)
            self.logDebug("%s playlist\s found on channel \"%s\"" % (len(playlists), m_id))

        if not playlists:
            self.fail(_("No playlist available"))

        for p_id, p_name, p_owner in playlists:
            p_videos = self.getVideos(p_id)
            p_folder = fs_join(self.config.get("general", "download_folder"), p_owner, p_name)
            self.logDebug("%s video\s found on playlist \"%s\"" % (len(p_videos), p_name))
            self.packages.append((p_name, p_videos, p_folder))  #: folder is NOT recognized by pyload 0.4.9!
示例#33
0
    def periodical(self):
        folder = fs_encode(self.getConfig('folder'))
        file   = fs_encode(self.getConfig('file'))

        try:
            if not os.path.isdir(os.path.join(folder, "finished")):
                os.makedirs(os.path.join(folder, "finished"))

            if self.getConfig('watch_file'):
                with open(file, "a+") as f:
                    f.seek(0)
                    content = f.read().strip()

                if content:
                    f = open(file, "wb")
                    f.close()

                    name = "%s_%s.txt" % (file, time.strftime("%H-%M-%S_%d%b%Y"))

                    with open(fs_join(folder, "finished", name), "wb") as f:
                        f.write(content)

                    self.core.api.addPackage(f.name, [f.name], 1)

            for f in os.listdir(folder):
                path = os.path.join(folder, f)

                if not os.path.isfile(path) or f.endswith("~") or f.startswith("#") or f.startswith("."):
                    continue

                newpath = os.path.join(folder, "finished", f if self.getConfig('keep') else "tmp_" + f)
                shutil.move(path, newpath)

                self.logInfo(_("Added %s from HotFolder") % f)
                self.core.api.addPackage(f, [newpath], 1)

        except (IOError, OSError), e:
            self.logError(e)
示例#34
0
    def removePlugins(self, type_plugins):
        """ delete plugins from disk """

        if not type_plugins:
            return

        removed = set()

        self.logDebug("Requested deletion of plugins: %s" % type_plugins)

        for type, name in type_plugins:
            rootplugins = os.path.join(pypath, "module", "plugins")

            for dir in ("userplugins", rootplugins):
                py_filename = fs_join(dir, type, name + ".py")
                pyc_filename = py_filename + "c"

                if type == "addon":
                    try:
                        self.manager.deactivateAddon(name)

                    except Exception, e:
                        self.logDebug(e)

                for filename in (py_filename, pyc_filename):
                    if not exists(filename):
                        continue

                    try:
                        os.remove(filename)

                    except OSError, e:
                        self.logError(_("Error removing: %s") % filename, e)

                    else:
                        id = (type, name)
                        removed.add(id)
示例#35
0
    def removePlugins(self, type_plugins):
        """ delete plugins from disk """

        if not type_plugins:
            return

        removed = set()

        self.logDebug("Requested deletion of plugins: %s" % type_plugins)

        for type, name in type_plugins:
            rootplugins = os.path.join(pypath, "module", "plugins")

            for dir in ("userplugins", rootplugins):
                py_filename  = fs_join(dir, type, name + ".py")
                pyc_filename = py_filename + "c"

                if type == "addon":
                    try:
                        self.manager.deactivateAddon(name)

                    except Exception, e:
                        self.logDebug(e)

                for filename in (py_filename, pyc_filename):
                    if not exists(filename):
                        continue

                    try:
                        os.remove(filename)

                    except OSError, e:
                        self.logError(_("Error removing: %s") % filename, e)

                    else:
                        id = (type, name)
                        removed.add(id)
示例#36
0
    def checkForSameFiles(self, starting=False):
        """ checks if same file was/is downloaded within same package

        :param starting: indicates that the current download is going to start
        :raises SkipDownload:
        """

        pack = self.pyfile.package()

        for pyfile in self.core.files.cache.values():
            if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package(
            ).folder == pack.folder:
                if pyfile.status in (0, 12):  #: finished or downloading
                    raise SkipDownload(pyfile.pluginname)
                elif pyfile.status in (
                        5, 7
                ) and starting:  #: a download is waiting/starting and was appenrently started before
                    raise SkipDownload(pyfile.pluginname)

        download_folder = self.core.config.get("general", "download_folder")
        location = fs_join(download_folder, pack.folder, self.pyfile.name)

        if starting and self.core.config.get(
                "download", "skip_existing") and os.path.exists(location):
            size = os.stat(location).st_size
            if size >= self.pyfile.size:
                raise SkipDownload("File exists")

        pyfile = self.core.db.findDuplicates(self.pyfile.id,
                                             self.pyfile.package().folder,
                                             self.pyfile.name)
        if pyfile:
            if os.path.exists(location):
                raise SkipDownload(pyfile[0])

            self.logDebug("File %s not skipped, because it does not exists." %
                          self.pyfile.name)
class ExternalScripts(Addon):
    __name    = "ExternalScripts"
    __type    = "addon"
    __version = "0.39"

    __config = [("activated", "bool", "Activated"         , True),
                ("waitend"  , "bool", "Wait script ending", False)]

    __description = """Run external scripts"""
    __license     = "GPLv3"
    __authors     = [("mkaay"         , "*****@*****.**"),
                     ("RaNaN"         , "*****@*****.**"),
                     ("spoob"         , "*****@*****.**"),
                     ("Walter Purcaro", "*****@*****.**")]

    event_list = ["archive_extract_failed", "archive_extracted"     ,
                  "package_extract_failed", "package_extracted"     ,
                  "all_archives_extracted", "all_archives_processed",
                  "allDownloadsFinished"  , "allDownloadsProcessed" ,
                  "packageDeleted"]


    def setup(self):
        self.info['oldip'] = None
        self.scripts = {}

        folders = ["pyload_start", "pyload_restart", "pyload_stop",
                   "before_reconnect", "after_reconnect",
                   "download_preparing", "download_failed", "download_finished",
                   "archive_extract_failed", "archive_extracted",
                   "package_finished", "package_deleted", "package_extract_failed", "package_extracted",
                   "all_downloads_processed", "all_downloads_finished",  #@TODO: Invert `all_downloads_processed`, `all_downloads_finished` order in 0.4.10
                   "all_archives_extracted", "all_archives_processed"]

        for folder in folders:
            self.scripts[folder] = []
            for dir in (pypath, ''):
                self.initPluginType(folder, os.path.join(dir, 'scripts', folder))

        for script_type, names in self.scripts.iteritems():
            if names:
                self.logInfo(_("Installed scripts for: ") + script_type, ", ".join(map(os.path.basename, names)))

        self.pyload_start()


    def initPluginType(self, name, dir):
        if not os.path.isdir(dir):
            try:
                os.makedirs(dir)

            except OSError, e:
                self.logDebug(e)
                return

        for filename in os.listdir(dir):
            file = fs_join(dir, filename)

            if not os.path.isfile(file):
                continue

            if filename[0] in ("#", "_") or filename.endswith("~") or filename.endswith(".swp"):
                continue

            if not os.access(file, os.X_OK):
                self.logWarning(_("Script not executable:") + " %s/%s" % (name, filename))

            self.scripts[name].append(file)
示例#38
0
        self.createPackages()


    def loadToDisk(self):
        """loads container to disk if its stored remotely and overwrite url,
        or check existent on several places at disk"""

        if self.pyfile.url.startswith("http"):
            self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1]
            content = self.load(self.pyfile.url)
            self.pyfile.url = fs_join(self.core.config.get("general", "download_folder"), self.pyfile.name)
            try:
                with open(self.pyfile.url, "wb") as f:
                    f.write(content)
            except IOError, e:
                self.fail(str(e))

        else:
            self.pyfile.name = os.path.basename(self.pyfile.url)
            if not os.path.exists(self.pyfile.url):
                if os.path.exists(fs_join(pypath, self.pyfile.url)):
                    self.pyfile.url = fs_join(pypath, self.pyfile.url)
                else:
                    self.fail(_("File not exists"))


    def deleteTmp(self):
        if self.pyfile.name.startswith("tmp_"):
            os.remove(self.pyfile.url)
示例#39
0
    def load(self,
             url,
             get={},
             post={},
             ref=True,
             cookies=True,
             just_header=False,
             decode=False,
             follow_location=True,
             save_cookies=True):
        """Load content at url and returns it

        :param url:
        :param get:
        :param post:
        :param ref:
        :param cookies:
        :param just_header: If True only the header will be retrieved and returned as dict
        :param decode: Wether to decode the output according to http header, should be True in most cases
        :param follow_location: If True follow location else not
        :param save_cookies: If True saves received cookies else discard them
        :return: Loaded content
        """
        if self.pyfile.abort:
            self.abort()

        if not url:
            self.fail(_("No url given"))

        url = urllib.unquote(
            encode(url).strip()
        )  #@NOTE: utf8 vs decode -> please use decode attribute in all future plugins

        if self.core.debug:
            self.logDebug(
                "Load url: " + url, *[
                    "%s=%s" % (key, val) for key, val in locals().iteritems()
                    if key not in ("self", "url")
                ])

        res = self.req.load(url,
                            get,
                            post,
                            ref,
                            cookies,
                            just_header,
                            decode=decode,
                            follow_location=follow_location,
                            save_cookies=save_cookies)

        if decode:
            res = encode(res)

        if self.core.debug:
            import inspect

            frame = inspect.currentframe()
            framefile = fs_join(
                "tmp", self.getClassName(), "%s_line%s.dump.html" %
                (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
            try:
                if not os.path.exists(os.path.join("tmp",
                                                   self.getClassName())):
                    os.makedirs(os.path.join("tmp", self.getClassName()))

                with open(framefile, "wb") as f:
                    del frame  #: delete the frame or it wont be cleaned
                    f.write(res)
            except IOError, e:
                self.logError(e)
示例#40
0
    def _updatePlugins(self, data):
        """ check for plugin updates """

        exitcode = 0
        updated  = []

        url    = data[0]
        schema = data[1].split('|')

        VERSION = re.compile(r'__version.*=.*("|\')([\d.]+)')

        if "BLACKLIST" in data:
            blacklist  = data[data.index('BLACKLIST') + 1:]
            updatelist = data[2:data.index('BLACKLIST')]
        else:
            blacklist  = []
            updatelist = data[2:]

        updatelist = [dict(zip(schema, x.split('|'))) for x in updatelist]
        blacklist  = [dict(zip(schema, x.split('|'))) for x in blacklist]

        if blacklist:
            type_plugins = [(plugin['type'], plugin['name'].rsplit('.', 1)[0]) for plugin in blacklist]

            # Protect UpdateManager from self-removing
            try:
                type_plugins.remove(("addon", "UpdateManager"))
            except ValueError:
                pass

            for t, n in type_plugins:
                for idx, plugin in enumerate(updatelist):
                    if n == plugin['name'] and t == plugin['type']:
                        updatelist.pop(idx)
                        break

            for t, n in self.removePlugins(sorted(type_plugins)):
                self.logInfo(_("Removed blacklisted plugin: [%(type)s] %(name)s") % {
                    'type': t,
                    'name': n,
                })

        for plugin in sorted(updatelist, key=operator.itemgetter("type", "name")):
            filename = plugin['name']
            type     = plugin['type']
            version  = plugin['version']

            if filename.endswith(".pyc"):
                name = filename[:filename.find("_")]
            else:
                name = filename.replace(".py", "")

            plugins = getattr(self.core.pluginManager, "%sPlugins" % type)

            oldver = float(plugins[name]['version']) if name in plugins else None
            newver = float(version)

            if not oldver:
                msg = "New plugin: [%(type)s] %(name)s (v%(newver).2f)"
            elif newver > oldver:
                msg = "New version of plugin: [%(type)s] %(name)s (v%(oldver).2f -> v%(newver).2f)"
            else:
                continue

            self.logInfo(_(msg) % {'type': type,
                                   'name': name,
                                   'oldver': oldver,
                                   'newver': newver})
            try:
                content = getURL(url % plugin)
                m = VERSION.search(content)

                if m and m.group(2) == version:
                    with open(fs_join("userplugins", type, filename), "wb") as f:
                        f.write(content)

                    updated.append((type, name))
                else:
                    raise Exception, _("Version mismatch")

            except Exception, e:
                self.logError(_("Error updating plugin: %s") % filename, e)
示例#41
0
    def _updatePlugins(self, data):
        """ check for plugin updates """

        exitcode = 0
        updated = []

        url = data[0]
        schema = data[1].split('|')

        VERSION = re.compile(r'__version.*=.*("|\')([\d.]+)')

        if "BLACKLIST" in data:
            blacklist = data[data.index('BLACKLIST') + 1:]
            updatelist = data[2:data.index('BLACKLIST')]
        else:
            blacklist = []
            updatelist = data[2:]

        updatelist = [dict(zip(schema, x.split('|'))) for x in updatelist]
        blacklist = [dict(zip(schema, x.split('|'))) for x in blacklist]

        if blacklist:
            type_plugins = [(plugin['type'], plugin['name'].rsplit('.', 1)[0])
                            for plugin in blacklist]

            # Protect UpdateManager from self-removing
            try:
                type_plugins.remove(("addon", "UpdateManager"))
            except ValueError:
                pass

            for t, n in type_plugins:
                for idx, plugin in enumerate(updatelist):
                    if n == plugin['name'] and t == plugin['type']:
                        updatelist.pop(idx)
                        break

            for t, n in self.removePlugins(sorted(type_plugins)):
                self.logInfo(
                    _("Removed blacklisted plugin: [%(type)s] %(name)s") % {
                        'type': t,
                        'name': n,
                    })

        for plugin in sorted(updatelist,
                             key=operator.itemgetter("type", "name")):
            filename = plugin['name']
            type = plugin['type']
            version = plugin['version']

            if filename.endswith(".pyc"):
                name = filename[:filename.find("_")]
            else:
                name = filename.replace(".py", "")

            plugins = getattr(self.core.pluginManager, "%sPlugins" % type)

            oldver = float(
                plugins[name]['version']) if name in plugins else None
            newver = float(version)

            if not oldver:
                msg = "New plugin: [%(type)s] %(name)s (v%(newver).2f)"
            elif newver > oldver:
                msg = "New version of plugin: [%(type)s] %(name)s (v%(oldver).2f -> v%(newver).2f)"
            else:
                continue

            self.logInfo(
                _(msg) % {
                    'type': type,
                    'name': name,
                    'oldver': oldver,
                    'newver': newver
                })
            try:
                content = getURL(url % plugin)
                m = VERSION.search(content)

                if m and m.group(2) == version:
                    with open(fs_join("userplugins", type, filename),
                              "wb") as f:
                        f.write(content)

                    updated.append((type, name))
                else:
                    raise Exception, _("Version mismatch")

            except Exception, e:
                self.logError(_("Error updating plugin: %s") % filename, e)
示例#42
0
    def decrypt(self, pyfile):
        m = re.match(self.__pattern, pyfile.url)
        m_id = m.group('ID')
        m_type = m.group('TYPE')

        if m_type == "user":
            self.logDebug("Url recognized as Channel")
            user = m_id
            channel = self.getChannel(user)

            if channel:
                playlists = self.getPlaylists(channel['id'])
                self.logDebug("%s playlist\s found on channel \"%s\"" %
                              (len(playlists), channel['title']))

                relatedplaylist = {
                    p_name: self.getPlaylist(p_id)
                    for p_name, p_id in
                    channel['relatedPlaylists'].iteritems()
                }
                self.logDebug("Channel's related playlists found = %s" %
                              relatedplaylist.keys())

                relatedplaylist['uploads']['title'] = "Unplaylisted videos"
                relatedplaylist['uploads'][
                    'checkDups'] = True  #: checkDups flag

                for p_name, p_data in relatedplaylist.iteritems():
                    if self.getConfig(p_name):
                        p_data['title'] += " of " + user
                        playlists.append(p_data)
            else:
                playlists = []
        else:
            self.logDebug("Url recognized as Playlist")
            playlists = [self.getPlaylist(m_id)]

        if not playlists:
            self.fail(_("No playlist available"))

        addedvideos = []
        urlize = lambda x: "https://www.youtube.com/watch?v=" + x
        for p in playlists:
            p_name = p['title']
            p_videos = self.getVideosId(p['id'])
            p_folder = fs_join(self.config.get("general", "download_folder"),
                               p['channelTitle'], p_name)
            self.logDebug("%s video\s found on playlist \"%s\"" %
                          (len(p_videos), p_name))

            if not p_videos:
                continue
            elif "checkDups" in p:
                p_urls = [
                    urlize(v_id) for v_id in p_videos
                    if v_id not in addedvideos
                ]
                self.logDebug(
                    "%s video\s available on playlist \"%s\" after duplicates cleanup"
                    % (len(p_urls), p_name))
            else:
                p_urls = map(urlize, p_videos)

            self.packages.append(
                (p_name, p_urls,
                 p_folder))  #: folder is NOT recognized by pyload 0.4.9!

            addedvideos.extend(p_videos)
示例#43
0
文件: Xdcc.py 项目: PaddyPat/pyload
    def doDownload(self, url):
        self.pyfile.setStatus("waiting")  #: real link

        m = re.match(r'xdcc://(.*?)/#?(.*?)/(.*?)/#?(\d+)/?', url)
        server = m.group(1)
        chan = m.group(2)
        bot = m.group(3)
        pack = m.group(4)
        nick = self.getConfig('nick')
        ident = self.getConfig('ident')
        real = self.getConfig('realname')

        temp = server.split(':')
        ln = len(temp)
        if ln == 2:
            host, port = temp
        elif ln == 1:
            host, port = temp[0], 6667
        else:
            self.fail(_("Invalid hostname for IRC Server: %s") % server)

        #######################
        # CONNECT TO IRC AND IDLE FOR REAL LINK
        dl_time = time.time()

        sock = socket.socket()
        sock.connect((host, int(port)))
        if nick == "pyload":
            nick = "pyload-%d" % (time.time() % 1000)  #: last 3 digits
        sock.send("NICK %s\r\n" % nick)
        sock.send("USER %s %s bla :%s\r\n" % (ident, host, real))

        self.setWait(3)
        self.wait()

        sock.send("JOIN #%s\r\n" % chan)
        sock.send("PRIVMSG %s :xdcc send #%s\r\n" % (bot, pack))

        # IRC recv loop
        readbuffer = ""
        done = False
        retry = None
        m = None
        while True:

            # done is set if we got our real link
            if done:
                break

            if retry:
                if time.time() > retry:
                    retry = None
                    dl_time = time.time()
                    sock.send("PRIVMSG %s :xdcc send #%s\r\n" % (bot, pack))

            else:
                if (dl_time + self.timeout) < time.time():  #@TODO: add in config
                    sock.send("QUIT :byebye\r\n")
                    sock.close()
                    self.fail(_("XDCC Bot did not answer"))

            fdset = select([sock], [], [], 0)
            if sock not in fdset[0]:
                continue

            readbuffer += sock.recv(1024)
            temp = readbuffer.split("\n")
            readbuffer = temp.pop()

            for line in temp:
                if self.debug is 2:
                    print "*> " + unicode(line, errors='ignore')
                line = line.rstrip()
                first = line.split()

                if first[0] == "PING":
                    sock.send("PONG %s\r\n" % first[1])

                if first[0] == "ERROR":
                    self.fail(_("IRC-Error: %s") % line)

                msg = line.split(None, 3)
                if len(msg) != 4:
                    continue

                msg = {
                    "origin": msg[0][1:],
                    "action": msg[1],
                    "target": msg[2],
                    "text": msg[3][1:]
                }

                if nick == msg['target'][0:len(nick)] and "PRIVMSG" == msg['action']:
                    if msg['text'] == "\x01VERSION\x01":
                        self.logDebug("Sending CTCP VERSION")
                        sock.send("NOTICE %s :%s\r\n" % (msg['origin'], "pyLoad! IRC Interface"))
                    elif msg['text'] == "\x01TIME\x01":
                        self.logDebug("Sending CTCP TIME")
                        sock.send("NOTICE %s :%d\r\n" % (msg['origin'], time.time()))
                    elif msg['text'] == "\x01LAG\x01":
                        pass  #: don't know how to answer

                if not (bot == msg['origin'][0:len(bot)]
                        and nick == msg['target'][0:len(nick)]
                        and msg['action'] in ("PRIVMSG", "NOTICE")):
                    continue

                if self.debug is 1:
                    print "%s: %s" % (msg['origin'], msg['text'])

                if "You already requested that pack" in msg['text']:
                    retry = time.time() + 300

                if "you must be on a known channel to request a pack" in msg['text']:
                    self.fail(_("Wrong channel"))

                m = re.match('\x01DCC SEND (.*?) (\d+) (\d+)(?: (\d+))?\x01', msg['text'])
                if m:
                    done = True

        # get connection data
        ip = socket.inet_ntoa(struct.pack('L', socket.ntohl(int(m.group(2)))))
        port = int(m.group(3))
        packname = m.group(1)

        if len(m.groups()) > 3:
            self.req.filesize = int(m.group(4))

        self.pyfile.name = packname

        download_folder = self.config.get("general", "download_folder")
        filename = fs_join(download_folder, packname)

        self.logInfo(_("Downloading %s from %s:%d") % (packname, ip, port))

        self.pyfile.setStatus("downloading")
        newname = self.req.download(ip, port, filename, sock, self.pyfile.setProgress)
        if newname and newname != filename:
            self.logInfo(_("%(name)s saved as %(newname)s") % {"name": self.pyfile.name, "newname": newname})
            filename = newname

        # kill IRC socket
        # sock.send("QUIT :byebye\r\n")
        sock.close()

        self.lastDownload = filename
        return self.lastDownload
示例#44
0
文件: Xdcc.py 项目: Bobbaone/pyload
    def doDownload(self, url):
        self.pyfile.setStatus("waiting")  #: real link

        m = re.match(r'xdcc://(.*?)/#?(.*?)/(.*?)/#?(\d+)/?', url)
        server = m.group(1)
        chan = m.group(2)
        bot = m.group(3)
        pack = m.group(4)
        nick = self.getConfig('nick')
        ident = self.getConfig('ident')
        real = self.getConfig('realname')

        temp = server.split(':')
        ln = len(temp)
        if ln == 2:
            host, port = temp
        elif ln == 1:
            host, port = temp[0], 6667
        else:
            self.fail(_("Invalid hostname for IRC Server: %s") % server)

        #######################
        # CONNECT TO IRC AND IDLE FOR REAL LINK
        dl_time = time.time()

        sock = socket.socket()
        sock.connect((host, int(port)))
        if nick == "pyload":
            nick = "pyload-%d" % (time.time() % 1000)  #: last 3 digits
        sock.send("NICK %s\r\n" % nick)
        sock.send("USER %s %s bla :%s\r\n" % (ident, host, real))

        self.setWait(3)
        self.wait()

        sock.send("JOIN #%s\r\n" % chan)
        sock.send("PRIVMSG %s :xdcc send #%s\r\n" % (bot, pack))

        # IRC recv loop
        readbuffer = ""
        done = False
        retry = None
        m = None
        while True:

            # done is set if we got our real link
            if done:
                break

            if retry:
                if time.time() > retry:
                    retry = None
                    dl_time = time.time()
                    sock.send("PRIVMSG %s :xdcc send #%s\r\n" % (bot, pack))

            else:
                if (dl_time + self.timeout) < time.time():  #@TODO: add in config
                    sock.send("QUIT :byebye\r\n")
                    sock.close()
                    self.fail(_("XDCC Bot did not answer"))

            fdset = select.select([sock], [], [], 0)
            if sock not in fdset[0]:
                continue

            readbuffer += sock.recv(1024)
            temp = readbuffer.split("\n")
            readbuffer = temp.pop()

            for line in temp:
                if self.debug is 2:
                    print "*> " + unicode(line, errors='ignore')
                line = line.rstrip()
                first = line.split()

                if first[0] == "PING":
                    sock.send("PONG %s\r\n" % first[1])

                if first[0] == "ERROR":
                    self.fail(_("IRC-Error: %s") % line)

                msg = line.split(None, 3)
                if len(msg) != 4:
                    continue

                msg = {
                    "origin": msg[0][1:],
                    "action": msg[1],
                    "target": msg[2],
                    "text": msg[3][1:]
                }

                if nick == msg['target'][0:len(nick)] and "PRIVMSG" == msg['action']:
                    if msg['text'] == "\x01VERSION\x01":
                        self.logDebug("Sending CTCP VERSION")
                        sock.send("NOTICE %s :%s\r\n" % (msg['origin'], "pyLoad! IRC Interface"))
                    elif msg['text'] == "\x01TIME\x01":
                        self.logDebug("Sending CTCP TIME")
                        sock.send("NOTICE %s :%d\r\n" % (msg['origin'], time.time()))
                    elif msg['text'] == "\x01LAG\x01":
                        pass  #: don't know how to answer

                if not (bot == msg['origin'][0:len(bot)]
                        and nick == msg['target'][0:len(nick)]
                        and msg['action'] in ("PRIVMSG", "NOTICE")):
                    continue

                if self.debug is 1:
                    print "%s: %s" % (msg['origin'], msg['text'])

                if "You already requested that pack" in msg['text']:
                    retry = time.time() + 300

                if "you must be on a known channel to request a pack" in msg['text']:
                    self.fail(_("Wrong channel"))

                m = re.match('\x01DCC SEND (.*?) (\d+) (\d+)(?: (\d+))?\x01', msg['text'])
                if m:
                    done = True

        # get connection data
        ip = socket.inet_ntoa(struct.pack('L', socket.ntohl(int(m.group(2)))))
        port = int(m.group(3))
        packname = m.group(1)

        if len(m.groups()) > 3:
            self.req.filesize = int(m.group(4))

        self.pyfile.name = packname

        download_folder = self.config.get("general", "download_folder")
        filename = fs_join(download_folder, packname)

        self.logInfo(_("Downloading %s from %s:%d") % (packname, ip, port))

        self.pyfile.setStatus("downloading")
        newname = self.req.download(ip, port, filename, sock, self.pyfile.setProgress)
        if newname and newname != filename:
            self.logInfo(_("%(name)s saved as %(newname)s") % {"name": self.pyfile.name, "newname": newname})
            filename = newname

        # kill IRC socket
        # sock.send("QUIT :byebye\r\n")
        sock.close()

        self.lastDownload = filename
        return self.lastDownload
示例#45
0
文件: OCR.py 项目: torrero007/pyload
class OCR(Base):
    __name = "OCR"
    __type = "ocr"
    __version = "0.12"

    __description = """OCR base plugin"""
    __license = "GPLv3"
    __authors = [("pyLoad Team", "*****@*****.**")]

    def __init__(self):
        self.logger = logging.getLogger("log")

    def load_image(self, image):
        self.image = Image.open(image)
        self.pixels = self.image.load()
        self.result_captcha = ''

    def deactivate(self):
        """delete all tmp images"""
        pass

    def threshold(self, value):
        self.image = self.image.point(lambda a: a * value + 10)

    def run(self, command):
        """Run a command"""

        popen = subprocess.Popen(command,
                                 bufsize=-1,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
        popen.wait()
        output = popen.stdout.read() + " | " + popen.stderr.read()
        popen.stdout.close()
        popen.stderr.close()
        self.logger.debug("Tesseract ReturnCode %s Output: %s" %
                          (popen.returncode, output))

    def run_tesser(self,
                   subset=False,
                   digits=True,
                   lowercase=True,
                   uppercase=True,
                   pagesegmode=None):
        # tmpTif = tempfile.NamedTemporaryFile(suffix=".tif")
        try:
            tmpTif = open(
                fs_join("tmp", "tmpTif_%s.tif" % self.__class__.__name__),
                "wb")
            tmpTif.close()

            # tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt")
            tmpTxt = open(
                fs_join("tmp", "tmpTxt_%s.txt" % self.__class__.__name__),
                "wb")
            tmpTxt.close()

        except IOError, e:
            self.logError(e)
            return

        self.logger.debug("save tiff")
        self.image.save(tmpTif.name, 'TIFF')

        if os.name == "nt":
            tessparams = [os.path.join(pypath, "tesseract", "tesseract.exe")]
        else:
            tessparams = ["tesseract"]

        tessparams.extend([
            os.path.abspath(tmpTif.name),
            os.path.abspath(tmpTxt.name).replace(".txt", "")
        ])

        if pagesegmode:
            tessparams.extend(["-psm", str(pagesegmode)])

        if subset and (digits or lowercase or uppercase):
            # tmpSub = tempfile.NamedTemporaryFile(suffix=".subset")
            with open(
                    fs_join("tmp",
                            "tmpSub_%s.subset" % self.__class__.__name__),
                    "wb") as tmpSub:
                tmpSub.write("tessedit_char_whitelist ")

                if digits:
                    tmpSub.write("0123456789")
                if lowercase:
                    tmpSub.write("abcdefghijklmnopqrstuvwxyz")
                if uppercase:
                    tmpSub.write("ABCDEFGHIJKLMNOPQRSTUVWXYZ")

                tmpSub.write("\n")
                tessparams.append("nobatch")
                tessparams.append(os.path.abspath(tmpSub.name))

        self.logger.debug("run tesseract")
        self.run(tessparams)
        self.logger.debug("read txt")

        try:
            with open(tmpTxt.name, 'r') as f:
                self.result_captcha = f.read().replace("\n", "")
        except Exception:
            self.result_captcha = ""

        self.logger.debug(self.result_captcha)
        try:
            os.remove(tmpTif.name)
            os.remove(tmpTxt.name)
            if subset and (digits or lowercase or uppercase):
                os.remove(tmpSub.name)
        except Exception:
            pass
示例#46
0
    def extract(self, ids, thread=None):  #@TODO: Use pypack, not pid to improve method usability
        if not ids:
            return False

        self.extracting = True

        processed = []
        extracted = []
        failed    = []

        toList = lambda string: string.replace(' ', '').replace(',', '|').replace(';', '|').split('|')

        destination = self.getConfig('destination')
        subfolder   = self.getConfig('subfolder')
        fullpath    = self.getConfig('fullpath')
        overwrite   = self.getConfig('overwrite')
        renice      = self.getConfig('renice')
        recursive   = self.getConfig('recursive')
        delete      = self.getConfig('delete')
        keepbroken  = self.getConfig('keepbroken')

        extensions   = [x.lstrip('.').lower() for x in toList(self.getConfig('extensions'))]
        excludefiles = toList(self.getConfig('excludefiles'))

        if extensions:
            self.logDebug("Use for extensions: %s" % "|.".join(extensions))

        # reload from txt file
        self.reloadPasswords()

        download_folder = self.config.get("general", "download_folder")

        # iterate packages -> extractors -> targets
        for pid in ids:
            pypack = self.core.files.getPackage(pid)

            if not pypack:
                self.queue.remove(pid)
                continue

            self.logInfo(_("Check package: %s") % pypack.name)

            # determine output folder
            out = fs_join(download_folder, pypack.folder, destination, "")  #: force trailing slash

            if subfolder:
                out = fs_join(out, pypack.folder)

            if not os.path.exists(out):
                os.makedirs(out)

            matched   = False
            success   = True
            files_ids = dict((pylink['name'], ((fs_join(download_folder, pypack.folder, pylink['name'])), pylink['id'], out)) for pylink
                             in sorted(pypack.getChildren().itervalues(), key=lambda k: k['name'])).values()  #: remove duplicates

            # check as long there are unseen files
            while files_ids:
                new_files_ids = []

                if extensions:
                    files_ids = [(fname, fid, fout) for fname, fid, fout in files_ids
                                 if filter(lambda ext: fname.lower().endswith(ext), extensions)]

                for Extractor in self.extractors:
                    targets = Extractor.getTargets(files_ids)
                    if targets:
                        self.logDebug("Targets for %s: %s" % (Extractor.__name__, targets))
                        matched = True

                    for fname, fid, fout in targets:
                        name = os.path.basename(fname)

                        if not os.path.exists(fname):
                            self.logDebug(name, "File not found")
                            continue

                        self.logInfo(name, _("Extract to: %s") % fout)
                        try:
                            pyfile  = self.core.files.getFile(fid)
                            archive = Extractor(self,
                                                fname,
                                                fout,
                                                fullpath,
                                                overwrite,
                                                excludefiles,
                                                renice,
                                                delete,
                                                keepbroken,
                                                fid)

                            thread.addActive(pyfile)
                            archive.init()

                            try:
                                new_files = self._extract(pyfile, archive, pypack.password)

                            finally:
                                pyfile.setProgress(100)
                                thread.finishFile(pyfile)

                        except Exception, e:
                            self.logError(name, e)
                            success = False
                            continue

                        # remove processed file and related multiparts from list
                        files_ids = [(fname, fid, fout) for fname, fid, fout in files_ids
                                     if fname not in archive.getDeleteFiles()]
                        self.logDebug("Extracted files: %s" % new_files)
                        self.setPermissions(new_files)

                        for filename in new_files:
                            file = fs_encode(fs_join(os.path.dirname(archive.filename), filename))
                            if not os.path.exists(file):
                                self.logDebug("New file %s does not exists" % filename)
                                continue

                            if recursive and os.path.isfile(file):
                                new_files_ids.append((filename, fid, os.path.dirname(filename)))  #: append as new target

                        self.manager.dispatchEvent("archive_extracted", pyfile, archive)

                files_ids = new_files_ids  #: also check extracted files

            if matched:
                if success:
                    extracted.append(pid)
                    self.manager.dispatchEvent("package_extracted", pypack)

                else:
                    failed.append(pid)
                    self.manager.dispatchEvent("package_extract_failed", pypack)

                    self.failed.add(pid)
            else:
                self.logInfo(_("No files found to extract"))

            if not matched or not success and subfolder:
                try:
                    os.rmdir(out)

                except OSError:
                    pass

            self.queue.remove(pid)
示例#47
0
        self.createPackages()

    def loadToDisk(self):
        """loads container to disk if its stored remotely and overwrite url,
        or check existent on several places at disk"""

        if self.pyfile.url.startswith("http"):
            self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1]
            content = self.load(self.pyfile.url)
            self.pyfile.url = fs_join(
                self.core.config.get("general", "download_folder"),
                self.pyfile.name)
            try:
                with open(self.pyfile.url, "wb") as f:
                    f.write(content)
            except IOError, e:
                self.fail(str(e))

        else:
            self.pyfile.name = os.path.basename(self.pyfile.url)
            if not os.path.exists(self.pyfile.url):
                if os.path.exists(fs_join(pypath, self.pyfile.url)):
                    self.pyfile.url = fs_join(pypath, self.pyfile.url)
                else:
                    self.fail(_("File not exists"))

    def deleteTmp(self):
        if self.pyfile.name.startswith("tmp_"):
            os.remove(self.pyfile.url)
示例#48
0
    def extract(self, ids, thread=None):  #@TODO: Use pypack, not pid to improve method usability
        if not ids:
            return False

        processed = []
        extracted = []
        failed    = []

        toList = lambda string: string.replace(' ', '').replace(',', '|').replace(';', '|').split('|')

        destination = self.getConfig('destination')
        subfolder   = self.getConfig('subfolder')
        fullpath    = self.getConfig('fullpath')
        overwrite   = self.getConfig('overwrite')
        renice      = self.getConfig('renice')
        recursive   = self.getConfig('recursive')
        delete      = self.getConfig('delete')
        keepbroken  = self.getConfig('keepbroken')

        extensions   = [x.lstrip('.').lower() for x in toList(self.getConfig('extensions'))]
        excludefiles = toList(self.getConfig('excludefiles'))

        if extensions:
            self.logDebug("Use for extensions: %s" % "|.".join(extensions))

        # reload from txt file
        self.reloadPasswords()

        download_folder = self.config.get("general", "download_folder")

        # iterate packages -> extractors -> targets
        for pid in ids:
            pypack = self.core.files.getPackage(pid)

            if not pypack:
                self.queue.remove(pid)
                continue

            self.logInfo(_("Check package: %s") % pypack.name)

            # determine output folder
            out = fs_join(download_folder, pypack.folder, destination, "")  #: force trailing slash

            if subfolder:
                out = fs_join(out, pypack.folder)

            if not os.path.exists(out):
                os.makedirs(out)

            matched   = False
            success   = True
            files_ids = dict((pylink['name'], ((fs_join(download_folder, pypack.folder, pylink['name'])), pylink['id'], out)) for pylink
                             in sorted(pypack.getChildren().itervalues(), key=lambda k: k['name'])).values()  #: remove duplicates

            # check as long there are unseen files
            while files_ids:
                new_files_ids = []

                if extensions:
                    files_ids = [(fname, fid, fout) for fname, fid, fout in files_ids
                                 if filter(lambda ext: fname.lower().endswith(ext), extensions)]

                for Extractor in self.extractors:
                    targets = Extractor.getTargets(files_ids)
                    if targets:
                        self.logDebug("Targets for %s: %s" % (Extractor.__name__, targets))
                        matched = True

                    for fname, fid, fout in targets:
                        name = os.path.basename(fname)

                        if not os.path.exists(fname):
                            self.logDebug(name, "File not found")
                            continue

                        self.logInfo(name, _("Extract to: %s") % fout)
                        try:
                            pyfile  = self.core.files.getFile(fid)
                            archive = Extractor(self,
                                                fname,
                                                fout,
                                                fullpath,
                                                overwrite,
                                                excludefiles,
                                                renice,
                                                delete,
                                                keepbroken,
                                                fid)

                            thread.addActive(pyfile)
                            archive.init()

                            try:
                                new_files = self._extract(pyfile, archive, pypack.password)

                            finally:
                                pyfile.setProgress(100)
                                thread.finishFile(pyfile)

                        except Exception, e:
                            self.logError(name, e)
                            success = False
                            continue

                        # remove processed file and related multiparts from list
                        files_ids = [(fname, fid, fout) for fname, fid, fout in files_ids
                                     if fname not in archive.getDeleteFiles()]
                        self.logDebug("Extracted files: %s" % new_files)
                        self.setPermissions(new_files)

                        for filename in new_files:
                            file = fs_encode(fs_join(os.path.dirname(archive.filename), filename))
                            if not os.path.exists(file):
                                self.logDebug("New file %s does not exists" % filename)
                                continue

                            if recursive and os.path.isfile(file):
                                new_files_ids.append((filename, fid, os.path.dirname(filename)))  #: append as new target

                        self.manager.dispatchEvent("archive_extracted", pyfile, archive)

                files_ids = new_files_ids  #: also check extracted files

            if matched:
                if success:
                    extracted.append(pid)
                    self.manager.dispatchEvent("package_extracted", pypack)

                else:
                    failed.append(pid)
                    self.manager.dispatchEvent("package_extract_failed", pypack)

                    self.failed.add(pid)
            else:
                self.logInfo(_("No files found to extract"))

            if not matched or not success and subfolder:
                try:
                    os.rmdir(out)

                except OSError:
                    pass

            self.queue.remove(pid)