def conf_path(self, trans=False): if trans: translation = gettext.translation( "setup", os.path.join(PACKDIR, "locale"), languages=[self.config.get('general', 'language'), "en"], fallback=True) translation.install(True) print( _("Setting new configpath, current configuration will not be transferred!" )) path = self.ask(_("Config path"), os.path.abspath("")) try: path = os.path.join(COREDIR, path) if not os.path.exists(path): makedirs(path) with io.open(os.path.join(COREDIR, "pyload", "config", "configdir"), mode='wb') as fp: fp.write(path) print( _("Config path changed, setup will now close, please restart to go on" )) print(_("Press Enter to exit")) eval(input()) exit() except Exception as e: print(_("Setting config path failed: {0}").format(e.message))
def parse(self, folder): """ Analyze and parses all plugins in folder. """ plugins = {} pfolder = os.path.join(self.path, folder) if not os.path.exists(pfolder): makedirs(pfolder) if not os.path.exists(os.path.join(pfolder, "__init__.py")): fp = io.open(os.path.join(pfolder, "__init__.py"), mode='wb') fp.close() for f in os.listdir(pfolder): if os.path.isfile(os.path.join(pfolder, f)) and f.endswith( ".py") and not f.startswith("_"): # replace suffix and version tag name = f[:-3] if name[-1] == ".": name = name[:-4] plugin = self.parse_plugin( os.path.join(pfolder, f), folder, name) if plugin: plugins[name] = plugin return plugins
def _init_storage(self): storage_folder = self.config.get('general', 'storage_folder') if not storage_folder: storage_folder = os.path.join(USERDIR, 'Downloads') self.log.debug("Storage: {0}".format(storage_folder)) makedirs(storage_folder) space_size = format.size(availspace(storage_folder)) self.log.info(_("Available storage space: {0}").format(space_size))
def _gen_profiledir(profile=None, configdir=None): if not profile: profile = 'default' if configdir: configdir = os.path.expanduser(configdir) else: configdir = os.path.join( DATADIR, 'pyLoad' if os.name == 'nt' else '.pyload') profiledir = os.path.abspath(os.path.join(configdir, profile)) makedirs(profiledir) return profiledir
def _gen_profiledir(profile=None, configdir=None): if not profile: profile = 'default' if configdir: configdir = os.path.expanduser(configdir) else: configdir = os.path.join(DATADIR, 'pyLoad' if os.name == 'nt' else '.pyload') profiledir = os.path.abspath(os.path.join(configdir, profile)) makedirs(profiledir) return profiledir
def _init_config(self, profile, configdir): self.profiledir = _gen_profiledir(profile, configdir) self.configdir, self.profile = os.path.split(self.profiledir) tmproot = os.path.join(TMPDIR, os.path.basename(self.configdir)) makedirs(tmproot) self.tmpdir = tempfile.mkdtemp(dir=tmproot) self._init_pid() if self.profiledir not in sys.path: sys.path.append(self.profiledir) os.chdir(self.profiledir) self.configfile = os.path.join(self.profiledir, 'pyload.conf') self.config = self.cfg = ConfigParser(self.configfile, self.version)
def create_index(self): """ Create information for all plugins available. """ if not os.path.exists(self.path): makedirs(self.path) if not os.path.exists(os.path.join(self.path, "__init__.py")): fp = io.open(os.path.join(self.path, "__init__.py"), mode='wb') fp.close() a = time() for _type in self.TYPES: self.plugins[_type] = self.parse(_type) self.log.debug("Created index of plugins for {0} in {1:.2f} ms".format( self.path, (time() - a) * 1000))
def create_index(self): """ Create information for all plugins available. """ if not os.path.exists(self.path): makedirs(self.path) if not os.path.exists(os.path.join(self.path, "__init__.py")): fp = io.open(os.path.join(self.path, "__init__.py"), mode='wb') fp.close() a = time() for _type in self.TYPES: self.plugins[_type] = self.parse(_type) self.log.debug( "Created index of plugins for {0} in {1:.2f} ms".format( self.path, (time() - a) * 1000) )
def _write_report(self, report, path): with zipfile.ZipFile(path, 'w') as zip: reportdir = os.path.join(self.pyload.profiledir, 'crashes', 'reports', name) # NOTE: Relpath to configdir makedirs(reportdir) for fname in os.listdir(reportdir): try: zip.write(os.path.join(reportdir, fname), os.path.join(name, fname)) except Exception: pass for name, data in (('debug', report), ('system', system)): info = zipfile.ZipInfo( os.path.join(name, "{0}_Report.txt".format(name)), time.gmtime()) info.external_attr = 0o644 << 16 #: change permissions zip.writestr(info, data) if not filesize(path): raise Exception("Empty Zipfile")
def tearDownClass(cls): name = "{0}.{1}".format(cls.__module__, cls.__name__) makedirs(name) for fname in glob("debug_*"): shutil.move(fname, os.path.join(name, fname))
def tearDownClass(cls): name = "{0}.{1}".format(cls.__module__, cls.__name__) if not os.path.exists(name): makedirs(name) for f in glob("debug_*"): shutil.move(f, os.path.join(name, f))
def _mklogdir(self): logfile_folder = self.config.get('log', 'logfile_folder') if not logfile_folder: logfile_folder = os.path.abspath("logs") makedirs(logfile_folder)
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """ Downloads the content at url to download folder :param disposition: if True and server provides content-disposition header the filename will be changed if needed :return: The location where the file was saved """ self.check_for_same_files() self.check_abort() self.pyfile.set_status("downloading") download_folder = self.pyload.config.get('general', 'storage_folder') location = os.path.join(download_folder, self.pyfile.package().folder) if not os.path.exists(location): makedirs(location, int(self.pyload.config.get( 'permission', 'foldermode'), 8)) if self.pyload.config.get( 'permission', 'change_fileowner') and os.name != 'nt': try: uid = pwd.getpwnam(self.pyload.config.get( 'permission', 'user'))[2] gid = grp.getgrnam(self.pyload.config.get( 'permission', 'group'))[2] os.chown(location, uid, gid) except Exception as e: self.pyload.log.warning( _("Setting User and Group failed: {0}").format(e.message)) name = self.pyfile.name filename = os.path.join(location, name) self.pyload.adm.fire("download:start", self.pyfile, url, filename) # Create the class used for downloading self.dl = self.pyload.req.get_download_request( self.req, self.DOWNLOAD_CLASS) try: # TODO: hardcoded arguments newname = self.dl.download(url, filename, get=get, post=post, referer=ref, chunks=self.get_chunk_count(), resume=self.resume_download, cookies=cookies, disposition=disposition) finally: self.dl.close() self.pyfile.size = self.dl.size if disposition and newname and newname != name: #: triple check, just to be sure self.pyload.log.info(_("{0} saved as {1}").format(name, newname)) self.pyfile.name = newname filename = os.path.join(location, newname) fs_filename = format.path(filename) if self.pyload.config.get('permission', 'change_filemode'): os.chmod(fs_filename, int(self.pyload.config.get( 'permission', 'filemode'), 8)) if self.pyload.config.get( 'permission', 'change_fileowner') and os.name != 'nt': try: uid = pwd.getpwnam( self.pyload.config.get( 'permission', 'user'))[2] gid = grp.getgrnam(self.pyload.config.get( 'permission', 'group'))[2] os.chown(fs_filename, uid, gid) except Exception as e: self.pyload.log.warning( _("Setting User and Group failed: {0}").format(e.message)) self.last_download = fs_filename return self.last_download
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): """ Load content at url and returns it :param url: url as string :param get: GET as dict :param post: POST as dict, list or string :param ref: Set HTTP_REFERER header :param cookies: use saved cookies :param just_header: if True only the header will be retrieved and returned as dict :param decode: Whether to decode the output according to http header, should be True in most cases :return: Loaded content """ if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute") self.check_abort() res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) if self.pyload.debug: from inspect import currentframe frame = currentframe() if not os.path.exists(os.path.join( self.pyload.profiledir, 'crashes', 'dumps', self.__name__)): makedirs( os.path.join( self.pyload.profiledir, 'crashes', 'dumps', self.__name__)) file = os.path.join(self.pyload.profiledir, 'crashes', 'dumps', self.__name__, "{0}_line{1}.dump.html".format( frame.f_back.f_code.co_name, frame.f_back.f_lineno)) with io.open(file, mode='wb') as fp: del frame #: delete the frame or it wont be cleaned fp.write(res) if just_header: # parse header header = {'code': self.req.code} for line in res.splitlines(): line = line.strip() if not line or ":" not in line: continue key, none, value = line.partition(":") key = key.lower().strip() value = value.strip() if key in header: if isinstance(header[key], list): header[key].append(value) else: header[key] = [header[key], value] else: header[key] = value res = header return res
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): """ Load content at url and returns it :param url: url as string :param get: GET as dict :param post: POST as dict, list or string :param ref: Set HTTP_REFERER header :param cookies: use saved cookies :param just_header: if True only the header will be retrieved and returned as dict :param decode: Whether to decode the output according to http header, should be True in most cases :return: Loaded content """ if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute") self.check_abort() res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) if self.pyload.debug: from inspect import currentframe frame = currentframe() if not os.path.exists( os.path.join(self.pyload.profiledir, 'crashes', 'dumps', self.__name__)): makedirs( os.path.join(self.pyload.profiledir, 'crashes', 'dumps', self.__name__)) path = os.path.join( self.pyload.profiledir, 'crashes', 'dumps', self.__name__, "{0}_line{1}.dump.html".format(frame.f_back.f_code.co_name, frame.f_back.f_lineno)) with io.open(path, mode='wb') as fp: del frame #: delete the frame or it wont be cleaned fp.write(res) if just_header: # parse header header = {'code': self.req.code} for line in res.splitlines(): line = line.strip() if not line or ":" not in line: continue key, none, value = line.partition(":") key = key.lower().strip() value = value.strip() if key in header: if isinstance(header[key], list): header[key].append(value) else: header[key] = [header[key], value] else: header[key] = value res = header return res
def _init_logger(self, level): # Init logger self.log = logging.getLogger('pyload') self.log.setLevel(level) # Set console handler if self.config.get('log', 'color_console') and ismodule('colorlog'): fmt = "%(label)s %(levelname)-8s %(reset)s %(log_color)s%(asctime)s %(message)s" datefmt = "%Y-%m-%d %H:%M:%S" log_colors = { 'DEBUG': "bold,cyan", 'WARNING': "bold,yellow", 'ERROR': "bold,red", 'CRITICAL': "bold,purple", } log_colors_2 = { 'label': { 'DEBUG': "bold,white,bg_cyan", 'INFO': "bold,white,bg_green", 'WARNING': "bold,white,bg_yellow", 'ERROR': "bold,white,bg_red", 'CRITICAL': "bold,white,bg_purple", } } consoleform = colorlog.ColoredFormatter( fmt, datefmt, log_colors, secondary_log_colors=log_colors_2) else: fmt = "%(asctime)s %(levelname)-8s %(message)s" datefmt = "%Y-%m-%d %H:%M:%S" consoleform = logging.Formatter(fmt, datefmt) consolehdlr = logging.StreamHandler(sys.stdout) consolehdlr.setFormatter(consoleform) self.log.addHandler(consolehdlr) # Set syslog handler syslog = self.config.get('log', 'syslog') if syslog != 'no': #: try to mimic to normal syslog messages fmt = "%(asctime)s %(name)s: %(message)s" datefmt = "%b %e %H:%M:%S" syslogform = logging.Formatter(fmt, datefmt) syslogaddr = None if syslog == 'remote': syslog_host = self.config.get('log', 'syslog_host') syslog_port = self.config.get('log', 'syslog_port') syslogaddr = (syslog_host, syslog_port) else: syslog_folder = self.config.get('log', 'syslog_folder') if syslogaddr: syslogaddr = syslog_folder elif sys.platform == 'darwin': syslogaddr = '/var/run/syslog' elif os.name != 'nt': syslogaddr = '/dev/log' sysloghdlr = logging.handlers.SysLogHandler(syslogaddr) sysloghdlr.setFormatter(syslogform) self.log.addHandler(sysloghdlr) # Create logfile folder logfile_folder = self.config.get('log', 'logfile_folder') if not logfile_folder: logfile_folder = os.path.abspath("logs") makedirs(logfile_folder) # Set file handler if not self.config.get('log', 'logfile'): return None fmt = "%(asctime)s %(levelname)-8s %(message)s" datefmt = "%Y-%m-%d %H:%M:%S" fileform = logging.Formatter(fmt, datefmt) logfile = os.path.join(logfile_folder, 'log.txt') if self.config.get('log', 'rotate'): logfile_size = self.config.get('log', 'logfile_size') << 10 max_logfiles = self.config.get('log', 'max_logfiles') filehdlr = logging.handlers.RotatingFileHandler( logfile, maxBytes=logfile_size, backupCount=max_logfiles, encoding=getpreferredencoding()) else: filehdlr = logging.FileHandler(logfile, encoding=getpreferredencoding()) filehdlr.setFormatter(fileform) self.log.addHandler(filehdlr)
def do_download(self, url): self.pyfile.set_status("waiting") #: real link download_folder = self.pyload.config.get('general', 'storage_folder') location = os.path.join(download_folder, self.pyfile.package( ).folder.decode(sys.getfilesystemencoding())) if not os.path.exists(location): makedirs(location) m = re.match(r'xdcc://(.*?)/#?(.*?)/(.*?)/#?(\d+)/?', url) server = m.group(1) chan = m.group(2) bot = m.group(3) pack = m.group(4) nick = self.get_config('nick') ident = self.get_config('ident') real = self.get_config('realname') temp = server.split(':') ln = len(temp) if ln == 2: host, port = temp elif ln == 1: host, port = temp[0], 6667 else: self.fail(_("Invalid hostname for IRC Server ({0})").format(server)) ####################### # CONNECT TO IRC AND IDLE FOR REAL LINK dl_time = time.time() with closing(socket.socket()) as sock: sock.connect((host, int(port))) if nick == "pyload": # last 3 digits nick = "pyload-{0:d}".format(time.time() % 1000) sock.send("NICK {0}\r\n".format(nick)) sock.send("USER {0} {1} bla :{2}\r\n".format(ident, host, real)) time.sleep(3) sock.send("JOIN #{0}\r\n".format(chan)) sock.send("PRIVMSG {0} :xdcc send #{1}\r\n".format(bot, pack)) # IRC recv loop readbuffer = "" done = False retry = None m = None while True: # done is set if we got our real link if done: break if retry: if time.time() > retry: retry = None dl_time = time.time() sock.send( "PRIVMSG {0} :xdcc send #{1}\r\n".format(bot, pack)) else: if (dl_time + self.timeout) < time.time(): # TODO: add in config sock.send("QUIT :byebye\r\n") # sock.close() self.fail(_("XDCC Bot did not answer")) fdset = select([sock], [], [], 0) if sock not in fdset[0]: continue readbuffer += sock.recv(1024) temp = readbuffer.split("\n") readbuffer = temp.pop() for line in temp: if self.debug == 2: print("*> {0}".format(line, errors='ignore')) line = line.rstrip() first = line.split() if first[0] == "PING": sock.send("PONG {0}\r\n".format(first[1])) if first[0] == "ERROR": self.fail(_("IRC-Error: {0}").format(line)) msg = line.split(None, 3) if len(msg) != 4: continue msg = { 'origin': msg[0][1:], 'action': msg[1], 'target': msg[2], 'text': msg[3][1:] } if nick == msg['target'][0:len(nick)] and "PRIVMSG" == msg[ 'action']: if msg['text'] == "\x01VERSION\x01": self.log_debug("XDCC: Sending CTCP VERSION") sock.send("NOTICE {0} :{1}\r\n".format( msg['origin'], "pyLoad IRC Interface")) elif msg['text'] == "\x01TIME\x01": self.log_debug("Sending CTCP TIME") sock.send("NOTICE {0} :{1:d}\r\n".format( msg['origin'], time.time())) elif msg['text'] == "\x01LAG\x01": pass #: do not know how to answer if not (bot == msg['origin'][0:len(bot)] and nick == msg['target'][0:len(nick)] and msg['action'] in ("PRIVMSG", "NOTICE")): continue if self.debug == 1: print("{0}: {1}".format(msg['origin'], msg['text'])) if "You already requested that pack" in msg['text']: retry = time.time() + 300 if "you must be on a known channel to request a pack" in msg[ 'text']: self.fail(_("Wrong channel")) m = re.match( '\x01DCC SEND (.*?) (\d+) (\d+)(?: (\d+))?\x01', msg['text']) if m: done = True # get connection data ip = socket.inet_ntoa(struct.pack( 'L', socket.ntohl(int(m.group(2))))) port = int(m.group(3)) packname = m.group(1) if len(m.groups()) > 3: self.req.filesize = int(m.group(4)) self.pyfile.name = packname filename = format.path(location, packname) self.log_info( _("XDCC: Downloading {0} from {1}:{2:d}").format(packname, ip, port)) self.pyfile.set_status("downloading") newname = self.req.download( ip, port, filename, sock, self.pyfile.set_progress) if newname and newname != filename: self.log_info(_("{0} saved as {1}").format( self.pyfile.name, newname)) filename = newname # kill IRC socket # sock.send("QUIT :byebye\r\n") self.last_download = filename return self.last_download
def _init_logger(self, level): # Init logger self.log = logging.getLogger('pyload') self.log.setLevel(level) # Set console handler if self.config.get('log', 'color_console') and ismodule('colorlog'): fmt = "%(label)s %(levelname)-8s %(reset)s %(log_color)s%(asctime)s %(message)s" datefmt = "%Y-%m-%d %H:%M:%S" log_colors = { 'DEBUG': "bold,cyan", 'WARNING': "bold,yellow", 'ERROR': "bold,red", 'CRITICAL': "bold,purple", } log_colors_2 = { 'label': { 'DEBUG': "bold,white,bg_cyan", 'INFO': "bold,white,bg_green", 'WARNING': "bold,white,bg_yellow", 'ERROR': "bold,white,bg_red", 'CRITICAL': "bold,white,bg_purple", } } consoleform = colorlog.ColoredFormatter(fmt, datefmt, log_colors, secondary_log_colors=log_colors_2) else: fmt = "%(asctime)s %(levelname)-8s %(message)s" datefmt = "%Y-%m-%d %H:%M:%S" consoleform = logging.Formatter(fmt, datefmt) consolehdlr = logging.StreamHandler(sys.stdout) consolehdlr.setFormatter(consoleform) self.log.addHandler(consolehdlr) # Set syslog handler syslog = self.config.get('log', 'syslog') if syslog != 'no': #: try to mimic to normal syslog messages fmt = "%(asctime)s %(name)s: %(message)s" datefmt = "%b %e %H:%M:%S" syslogform = logging.Formatter(fmt, datefmt) syslogaddr = None if syslog == 'remote': syslog_host = self.config.get('log', 'syslog_host') syslog_port = self.config.get('log', 'syslog_port') syslogaddr = (syslog_host, syslog_port) else: syslog_folder = self.config.get('log', 'syslog_folder') if syslogaddr: syslogaddr = syslog_folder elif sys.platform == 'darwin': syslogaddr = '/var/run/syslog' elif os.name != 'nt': syslogaddr = '/dev/log' sysloghdlr = logging.handlers.SysLogHandler(syslogaddr) sysloghdlr.setFormatter(syslogform) self.log.addHandler(sysloghdlr) # Create logfile folder logfile_folder = self.config.get('log', 'logfile_folder') if not logfile_folder: logfile_folder = os.path.abspath("logs") makedirs(logfile_folder) # Set file handler if not self.config.get('log', 'logfile'): return None fmt = "%(asctime)s %(levelname)-8s %(message)s" datefmt = "%Y-%m-%d %H:%M:%S" fileform = logging.Formatter(fmt, datefmt) logfile = os.path.join(logfile_folder, 'log.txt') if self.config.get('log', 'rotate'): logfile_size = self.config.get('log', 'logfile_size') << 10 max_logfiles = self.config.get('log', 'max_logfiles') filehdlr = logging.handlers.RotatingFileHandler(logfile, maxBytes=logfile_size, backupCount=max_logfiles, encoding=getpreferredencoding()) else: filehdlr = logging.FileHandler( logfile, encoding=getpreferredencoding()) filehdlr.setFormatter(fileform) self.log.addHandler(filehdlr)