def _init_filelogger(self): fmt = "%(asctime)s %(levelname)-8s %(message)s" datefmt = "%Y-%m-%d %H:%M:%S" fileform = logging.Formatter(fmt, datefmt) logfile_folder = self.config.get('log', 'logfile_folder') if not logfile_folder: logfile_folder = self.DEFAULT_LOGDIRNAME makedirs(logfile_folder, exist_ok=True) logfile_name = self.config.get('log', 'logfile_name') if not logfile_name: logfile_name = self.DEFAULT_LOGFILENAME logfile = os.path.join(logfile_folder, logfile_name) if self.config.get('log', 'rotate'): logfile_size = self.config.get('log', 'logfile_size') << 10 max_logfiles = self.config.get('log', 'max_logfiles') filehdlr = logging.handlers.RotatingFileHandler( logfile, maxBytes=logfile_size, backupCount=max_logfiles, encoding=locale.getpreferredencoding(do_setlocale=False)) else: filehdlr = logging.FileHandler( logfile, encoding=locale.getpreferredencoding(do_setlocale=False)) filehdlr.setFormatter(fileform) self.log.addHandler(filehdlr)
def _setup_filelog(logger, config): fmt = '%(asctime)s %(levelname)-8s %(message)s' datefmt = '%Y-%m-%d %H:%M:%S' fileform = logging.Formatter(fmt, datefmt) filelog_folder = config.get('log', 'filelog_folder') makedirs(filelog_folder, exist_ok=True) filelog_name = config.get('log', 'filelog_name') filelog = os.path.join(filelog_folder, filelog_name) if config.get('log', 'rotate'): filelog_size = config.get('log', 'filelog_size') << 10 max_logfiles = config.get('log', 'max_logfiles') filehdlr = logging.handlers.RotatingFileHandler( filelog, maxBytes=filelog_size, backupCount=max_logfiles, encoding=locale.getpreferredencoding(do_setlocale=False)) else: filehdlr = logging.FileHandler( filelog, encoding=locale.getpreferredencoding(do_setlocale=False)) filehdlr.setFormatter(fileform) logger.addHandler(filehdlr)
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder :param disposition: if True and server provides content-disposition header\ the filename will be changed if needed :return: The location where the file was saved """ self.checkForSameFiles() self.checkAbort() self.pyfile.setStatus("downloading") download_folder = self.config['general']['download_folder'] location = save_join(download_folder, self.pyfile.package().folder) if not exists(location): makedirs(location, int(self.core.config["permission"]["folder"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(location, uid, gid) except Exception, e: self.log.warning(_("Setting User and Group failed: %s") % str(e))
def conf_path(self, trans=False): if trans: gettext.setpaths( [join(os.sep, "usr", "share", "pyload", "locale"), None]) translation = gettext.translation( "setup", join(self.path, "locale"), languages=[self.config["general"]["language"], "en"], fallback=True) translation.install(True) print _( "Setting new configpath, current configuration will not be transferred!" ) path = self.ask(_("Config path"), abspath("")) try: path = join(pypath, path) if not exists(path): makedirs(path) f = open(join(pypath, "pyload", "config", "configdir"), "wb") f.write(path) f.close() print _( "Config path changed, setup will now close, please restart to go on." ) print _("Press Enter to exit.") raw_input() exit() except Exception, e: print _("Setting config path failed: %s") % str(e)
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder :param disposition: if True and server provides content-disposition header\ the filename will be changed if needed :return: The location where the file was saved """ self.checkForSameFiles() self.checkAbort() self.pyfile.setStatus("downloading") download_folder = self.config['general']['download_folder'] location = save_join(download_folder, self.pyfile.package().folder) if not exists(location): makedirs(location, int(self.core.config["permission"]["folder"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(location, uid, gid) except Exception, e: self.log.warning( _("Setting User and Group failed: %s") % str(e))
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): """Load content at url and returns it :param url: url as string :param get: GET as dict :param post: POST as dict, list or string :param ref: Set HTTP_REFERER header :param cookies: use saved cookies :param just_header: if True only the header will be retrieved and returned as dict :param decode: Whether to decode the output according to http header, should be True in most cases :return: Loaded content """ if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.") self.checkAbort() res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) if self.core.debug: from inspect import currentframe frame = currentframe() if not exists(join("tmp", self.__name__)): makedirs(join("tmp", self.__name__)) f = open( join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) , "wb") del frame # delete the frame or it wont be cleaned try: tmp = res.encode("utf8") except: tmp = res f.write(tmp) f.close() if just_header: #parse header header = {"code": self.req.code} for line in res.splitlines(): line = line.strip() if not line or ":" not in line: continue key, none, value = line.partition(":") key = key.lower().strip() value = value.strip() if key in header: if type(header[key]) == list: header[key].append(value) else: header[key] = [header[key], value] else: header[key] = value res = header return res
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): """ Load content at url and returns it :param url: url as string :param get: GET as dict :param post: POST as dict, list or string :param ref: Set HTTP_REFERER header :param cookies: use saved cookies :param just_header: if True only the header will be retrieved and returned as dict :param decode: Whether to decode the output according to http header, should be True in most cases :return: Loaded content """ if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute") self.check_abort() res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) if self.pyload_core.debug: from inspect import currentframe frame = currentframe() dumpdir = os.path.join(self.pyload_core.cachedir, 'plugins', self.__name__) makedirs(dumpdir, exist_ok=True) filepath = os.path.join(dumpdir, "dump_{0}_line{1}.html".format( frame.f_back.f_code.co_name, frame.f_back.f_lineno)) with lopen(filepath, mode='wb') as fp: fp.write(res) del frame # delete the frame or it wont be cleaned if just_header: # parse header header = {'code': self.req.code} for line in res.splitlines(): line = line.strip() if not line or ":" not in line: continue key, none, value = line.partition(":") key = key.lower().strip() value = value.strip() if key in header: if isinstance(header[key], list): header[key].append(value) else: header[key] = [header[key], value] else: header[key] = value res = header return res
def _setup_storage(self): storage_folder = self.config.get('general', 'storage_folder') if not storage_folder: storage_folder = os.path.join(USERDIR, self.DEFAULT_STORAGEDIRNAME) self.log.debug("Storage: {0}".format(storage_folder)) makedirs(storage_folder, exist_ok=True) avail_space = format.size(availspace(storage_folder)) self.log.info( self._("Available storage space: {0}").format(avail_space))
def _init_cache(self): # Re-use cache tempdir = self.__tempdir if tempdir is None: tempdir = self.session.get('previous', 'cache', 'path') if tempdir is None or not os.path.isdir(tempdir): pydir = os.path.join(TMPDIR, __namespace__) makedirs(pydir, exist_ok=True) tempdir = tempfile.mkdtemp(dir=pydir) self.session.set('current', 'cache', 'path', tempdir) self.cachedir = tempdir
def _setup_storage(self): self.log.debug('Setup storage...') storage_folder = self.config.get('general', 'storage_folder') if storage_folder is None: storage_folder = os.path.join(builtins.USERDIR, self.DEFAULT_STORAGENAME) self.log.info(self._('Storage: {0}'.format(storage_folder))) makedirs(storage_folder, exist_ok=True) avail_space = format.size(availspace(storage_folder)) self.log.info( self._('Available storage space: {0}').format(avail_space))
def _setup_storage(self): self.log.debug('Setup storage...') storage_folder = self.config.get('general', 'storage_folder') if storage_folder is None: storage_folder = os.path.join( builtins.USERDIR, self.DEFAULT_STORAGENAME) self.log.info(self._('Storage: {0}'.format(storage_folder))) makedirs(storage_folder, exist_ok=True) avail_space = format.size(availspace(storage_folder)) self.log.info( self._('Available storage space: {0}').format(avail_space))
def _mkdprofile(profile=None, rootdir=None): DEFAULT_PROFILE = 'default' if not profile: profile = DEFAULT_PROFILE if rootdir is None: dirname = '.' + __namespace__ if os.name != 'nt' else __namespace__ configdir = os.path.join(DATADIR, dirname) else: configdir = os.path.expanduser(rootdir) profiledir = os.path.realpath(os.path.join(configdir, profile)) makedirs(profiledir, exist_ok=True) return profiledir
def debug_report(self, file): dumpdir = os.path.join(self.pyload.tmpdir, 'plugins', file.pluginname) makedirs(dumpdir, exist_ok=True) # NOTE: Relpath to configdir reportdir = os.path.join('crashes', 'plugins', file.pluginname) makedirs(reportdir, exist_ok=True) filename = 'debug-report_{0}_{1}.zip'.format( file.pluginname, time.strftime('%d-%m-%Y_%H-%M-%S')) filepath = os.path.join(reportdir, filename) reports = self._gen_reports(file) self._zip(filepath, reports, dumpdir) self.pyload.log.info( self._('Debug Report written to file {0}').format(filename))
def conf_path(self, trans=False): if trans: translation = gettext.translation("setup", join(self.path, "locale"), languages=[self.config["general"]["language"], "en"], fallback=True) translation.install(True) print _("Setting new configpath, current configuration will not be transferred!") path = self.ask(_("Config path"), abspath("")) try: path = join(pypath, path) if not exists(path): makedirs(path) f = open(join(pypath, "pyload", "config", "configdir"), "wb") f.write(path) f.close() print _("Config path changed, setup will now close, please restart to go on.") print _("Press Enter to exit.") raw_input() exit() except Exception, e: print _("Setting config path failed: %s") % str(e)
def _setup_filelog(logger, config): fmt = '%(asctime)s %(levelname)-8s %(message)s' datefmt = '%Y-%m-%d %H:%M:%S' fileform = logging.Formatter(fmt, datefmt) filelog_folder = config.get('log', 'filelog_folder') makedirs(filelog_folder, exist_ok=True) filelog_name = config.get('log', 'filelog_name') filelog = os.path.join(filelog_folder, filelog_name) if config.get('log', 'rotate'): filelog_size = config.get('log', 'filelog_size') << 10 max_logfiles = config.get('log', 'max_logfiles') filehdlr = logging.handlers.RotatingFileHandler( filelog, maxBytes=filelog_size, backupCount=max_logfiles, encoding=locale.getpreferredencoding(do_setlocale=False)) else: filehdlr = logging.FileHandler( filelog, encoding=locale.getpreferredencoding( do_setlocale=False)) filehdlr.setFormatter(fileform) logger.addHandler(filehdlr)
def tearDownClass(cls): name = '{0}.{1}'.format(cls.__module__, cls.__name__) makedirs(name, exist_ok=True) for fname in glob('debug_*'): shutil.move(fname, os.path.join(name, fname))
def download(self, url, get=None, post=None, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder. :param disposition: if True and server provides content-disposition header the filename will be changed if needed :return: The location where the file was saved """ self.check_for_same_files() self.check_abort() self.file.set_status('downloading') download_folder = self.pyload.config.get('general', 'storage_folder') location = os.path.join(download_folder, self.file.package().folder) if not os.path.isdir(location): mode = self.pyload.config.get('permission', 'foldermode') makedirs(location, mode, exist_ok=True) if self.pyload.config.get( 'permission', 'change_fileowner') and os.name != 'nt': try: uid = pwd.getpwnam(self.pyload.config.get( 'permission', 'user'))[2] gid = grp.getgrnam(self.pyload.config.get( 'permission', 'group'))[2] os.chown(location, uid, gid) except Exception as exc: self.pyload.log.warning( self._('Setting User and Group failed')) self.pyload.log.error(exc, exc_info=self.pyload.debug) name = self.file.name filepath = os.path.join(location, name) self.pyload.adm.fire('download:start', self.file, url, filepath) # Create the class used for downloading self.dl = self.pyload.req.get_download_request( self.req, self.DOWNLOAD_CLASS) try: # TODO: hardcoded arguments newname = self.dl.download( url, filepath, get=get, post=post, referer=ref, chunks=self.get_chunk_count(), resume=self.resume_download, cookies=cookies, disposition=disposition) finally: self.dl.close() self.file.size = self.dl.size if disposition and newname and newname != name: # triple check, just to be sure self.pyload.log.info( self._('{0} saved as {1}').format(name, newname)) self.file.name = newname filepath = os.path.join(location, newname) fs_filename = filepath if self.pyload.config.get('permission', 'change_filemode'): os.chmod(fs_filename, self.pyload.config.get( 'permission', 'filemode')) if self.pyload.config.get( 'permission', 'change_fileowner') and os.name != 'nt': try: uid = pwd.getpwnam( self.pyload.config.get( 'permission', 'user'))[2] gid = grp.getgrnam(self.pyload.config.get( 'permission', 'group'))[2] os.chown(fs_filename, uid, gid) except Exception as exc: self.pyload.log.warning( self._('Setting User and Group failed')) self.pyload.log.error(exc, exc_info=self.pyload.debug) self.last_download = fs_filename return self.last_download
def download(self, url, get=None, post=None, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder. :param disposition: if True and server provides content-disposition header the filename will be changed if needed :return: The location where the file was saved """ self.check_for_same_files() self.check_abort() self.file.set_status('downloading') download_folder = self.pyload.config.get('general', 'storage_folder') location = os.path.join(download_folder, self.file.package().folder) if not os.path.isdir(location): mode = self.pyload.config.get('permission', 'foldermode') makedirs(location, mode, exist_ok=True) if self.pyload.config.get('permission', 'change_fileowner') and os.name != 'nt': try: uid = pwd.getpwnam( self.pyload.config.get('permission', 'user'))[2] gid = grp.getgrnam( self.pyload.config.get('permission', 'group'))[2] os.chown(location, uid, gid) except Exception as exc: self.pyload.log.warning( self._('Setting User and Group failed')) self.pyload.log.error(exc, exc_info=self.pyload.debug) name = self.file.name filepath = os.path.join(location, name) self.pyload.adm.fire('download:start', self.file, url, filepath) # Create the class used for downloading self.dl = self.pyload.req.get_download_request(self.req, self.DOWNLOAD_CLASS) try: # TODO: hardcoded arguments newname = self.dl.download(url, filepath, get=get, post=post, referer=ref, chunks=self.get_chunk_count(), resume=self.resume_download, cookies=cookies, disposition=disposition) finally: self.dl.close() self.file.size = self.dl.size if disposition and newname and newname != name: # triple check, just to be sure self.pyload.log.info( self._('{0} saved as {1}').format(name, newname)) self.file.name = newname filepath = os.path.join(location, newname) fs_filename = filepath if self.pyload.config.get('permission', 'change_filemode'): os.chmod(fs_filename, self.pyload.config.get('permission', 'filemode')) if self.pyload.config.get('permission', 'change_fileowner') and os.name != 'nt': try: uid = pwd.getpwnam(self.pyload.config.get( 'permission', 'user'))[2] gid = grp.getgrnam( self.pyload.config.get('permission', 'group'))[2] os.chown(fs_filename, uid, gid) except Exception as exc: self.pyload.log.warning( self._('Setting User and Group failed')) self.pyload.log.error(exc, exc_info=self.pyload.debug) self.last_download = fs_filename return self.last_download