def __init__(self, cfgdir, tmpdir, debug=None, restore=False):
self.__running = Event()
self.__do_restart = False
self.__do_exit = False
self._ = lambda x: x
self.cfgdir = fullpath(cfgdir)
self.tmpdir = fullpath(tmpdir)
os.chdir(self.cfgdir)
# if self.tmpdir not in sys.path:
# sys.path.append(self.tmpdir)
# if refresh:
# cleanpy(PACKDIR)
self.config = ConfigParser(self.DEFAULT_CONFIGNAME)
self.debug = self.config.get('log',
'debug') if debug is None else debug
self.log = LoggerFactory(self, self.debug)
self._init_database(restore)
self._init_managers()
self.request = self.req = RequestFactory(self)
self._init_api()
atexit.register(self.exit)
def __init__(self, file, version):
self.config = ConfigParser(file, version)
self.stdin_encoding = console_encoding(sys.stdin.encoding)
self.lang = None
self.db = None
# We will create a timestamp so that the setup will be completed in a
# specific interval
self.timestamp = time()
# TODO: probably unneeded
self.yes = "yes"
self.no = "no"
def __init__(self, cfgdir, tmpdir, debug=None, restore=False):
self.__running = Event()
self.__do_restart = False
self.__do_exit = False
self._ = lambda x: x
self.cfgdir = fullpath(cfgdir)
self.tmpdir = fullpath(tmpdir)
os.chdir(self.cfgdir)
# if self.tmpdir not in sys.path:
# sys.path.append(self.tmpdir)
# if refresh:
# cleanpy(PACKDIR)
self.config = ConfigParser(self.DEFAULT_CONFIGNAME)
self.debug = self.config.get(
'log', 'debug') if debug is None else debug
self.log = LoggerFactory(self, self.debug)
self._init_database(restore)
self._init_managers()
self.request = self.req = RequestFactory(self)
self._init_api()
atexit.register(self.exit)
def _init_config(self, profile, configdir):
self.profiledir = _gen_profiledir(profile, configdir)
self.configdir, self.profile = os.path.split(self.profiledir)
tmproot = os.path.join(TMPDIR, os.path.basename(self.configdir))
makedirs(tmproot)
self.tmpdir = tempfile.mkdtemp(dir=tmproot)
self._init_pid()
if self.profiledir not in sys.path:
sys.path.append(self.profiledir)
os.chdir(self.profiledir)
self.configfile = os.path.join(self.profiledir, 'pyload.conf')
self.config = self.cfg = ConfigParser(self.configfile, self.version)
class Core(Process):
__SESSIONFILENAME = 'session.ini'
__CONFIGFILENAME = 'config.ini'
DEFAULT_LANGUAGE = 'english'
DEFAULT_USERNAME = '******'
DEFAULT_PASSWORD = '******'
DEFAULT_STORAGEDIRNAME = 'downloads'
DEFAULT_LOGDIRNAME = 'logs'
DEFAULT_LOGFILENAME = 'log.txt'
def _init_consolelogger(self):
if self.config.get('log', 'color_console') and ismodule('colorlog'):
fmt = "%(label)s %(levelname)-8s %(reset)s %(log_color)s%(asctime)s %(message)s"
datefmt = "%Y-%m-%d %H:%M:%S"
primary_colors = {
'DEBUG': "bold,cyan",
'WARNING': "bold,yellow",
'ERROR': "bold,red",
'CRITICAL': "bold,purple",
}
secondary_colors = {
'label': {
'DEBUG': "bold,white,bg_cyan",
'INFO': "bold,white,bg_green",
'WARNING': "bold,white,bg_yellow",
'ERROR': "bold,white,bg_red",
'CRITICAL': "bold,white,bg_purple",
}
}
consoleform = colorlog.ColoredFormatter(
fmt,
datefmt,
primary_colors,
secondary_log_colors=secondary_colors)
else:
fmt = "%(asctime)s %(levelname)-8s %(message)s"
datefmt = "%Y-%m-%d %H:%M:%S"
consoleform = logging.Formatter(fmt, datefmt)
consolehdlr = logging.StreamHandler(sys.stdout)
consolehdlr.setFormatter(consoleform)
self.log.addHandler(consolehdlr)
def _init_syslogger(self):
# try to mimic to normal syslog messages
fmt = "%(asctime)s %(name)s: %(message)s"
datefmt = "%b %e %H:%M:%S"
syslogform = logging.Formatter(fmt, datefmt)
syslogaddr = None
syslog = self.config.get('log', 'syslog')
if syslog == 'remote':
syslog_host = self.config.get('log', 'syslog_host')
syslog_port = self.config.get('log', 'syslog_port')
syslogaddr = (syslog_host, syslog_port)
else:
syslog_folder = self.config.get('log', 'syslog_folder')
if syslogaddr:
syslogaddr = syslog_folder
elif sys.platform == 'darwin':
syslogaddr = '/var/run/syslog'
elif os.name != 'nt':
syslogaddr = '/dev/log'
sysloghdlr = logging.handlers.SysLogHandler(syslogaddr)
sysloghdlr.setFormatter(syslogform)
self.log.addHandler(sysloghdlr)
def _init_filelogger(self):
fmt = "%(asctime)s %(levelname)-8s %(message)s"
datefmt = "%Y-%m-%d %H:%M:%S"
fileform = logging.Formatter(fmt, datefmt)
logfile_folder = self.config.get('log', 'logfile_folder')
if not logfile_folder:
logfile_folder = self.DEFAULT_LOGDIRNAME
makedirs(logfile_folder, exist_ok=True)
logfile_name = self.config.get('log', 'logfile_name')
if not logfile_name:
logfile_name = self.DEFAULT_LOGFILENAME
logfile = os.path.join(logfile_folder, logfile_name)
if self.config.get('log', 'rotate'):
logfile_size = self.config.get('log', 'logfile_size') << 10
max_logfiles = self.config.get('log', 'max_logfiles')
filehdlr = logging.handlers.RotatingFileHandler(
logfile,
maxBytes=logfile_size,
backupCount=max_logfiles,
encoding=locale.getpreferredencoding(do_setlocale=False))
else:
filehdlr = logging.FileHandler(
logfile,
encoding=locale.getpreferredencoding(do_setlocale=False))
filehdlr.setFormatter(fileform)
self.log.addHandler(filehdlr)
# TODO: Extend `logging.Logger` like `..plugin.Log`
def _init_logger(self):
level = logging.DEBUG if self.debug else logging.INFO
# Init logger
self.log = logging.getLogger()
self.log.setLevel(level)
# Set console handler
self._init_consolelogger()
# Set syslog handler
if self.config.get('log', 'syslog') != 'no':
self._init_syslogger()
# Set file handler
if self.config.get('log', 'logfile'):
self._init_filelogger()
def _setup_permissions(self):
if os.name == 'nt':
return None
change_group = self.config.get('permission', 'change_group')
change_user = self.config.get('permission', 'change_user')
if change_group:
try:
group = self.config.get('permission', 'group')
set_process_group(group)
except Exception as e:
self.log.error(self._("Unable to change gid"), str(e))
if change_user:
try:
user = self.config.get('permission', 'user')
set_process_user(user)
except Exception as e:
self.log.error(self._("Unable to change uid"), str(e))
def set_language(self, lang):
localedir = resource_filename(__package__, 'locale')
lc = locale.locale_alias[lang.lower()].split('_', 1)[0]
trans = get_translation('core', localedir, (lc, ))
try:
self._ = trans.ugettext
except AttributeError:
self._ = trans.gettext
def _setup_language(self):
self.log.debug("Loading language ...")
lang = self.config.get('general', 'language')
default = self.DEFAULT_LANGUAGE
if not lang:
code = locale.getlocale()[0] or locale.getdefaultlocale()[0]
lang = default if code is None else code.lower().split('_', 1)[0]
try:
self.set_language(lang)
except Exception as e:
if lang == default:
raise
self.log.warning(
self._("Unable to load `{0}` language, "
"use default `{1}`").format(lang, default), str(e))
self.set_language(default)
def _setup_debug(self):
if self.__debug is None:
debug_log = self.config.get('log', 'debug')
verbose_log = self.config.get('log', 'verbose')
self.__debug = 0 if not debug_log else 2 if verbose_log else 1
# def start_interface(self, webui=None, rpc=None):
# if webui is None:
# webui = self.__webui
# if rpc is None:
# rpc = self.__rpc
# # TODO: Parse `remote`
# if rpc or self.config.get('rpc', 'activated'):
# self.log.debug("Activating RPC interface ...")
# self.rem.start()
# elif not webui:
# webui = True
# TODO: Parse remote host:port
# if isinstance(webui, str):
# host, port = map(str.strip, webui.rsplit(':', 1))
# webui = True
# else:
# host, port = (None, None)
# kwgs = {
# 'server': self.config.get('webui', 'server'),
# 'host': host or self.config.get('webui', 'host'),
# 'port': port or self.config.get('webui', 'port'),
# 'key': self.config.get('ssl', 'key'),
# 'cert': self.config.get('ssl', 'cert'),
# 'ssl': self.config.get('ssl', 'activated')
# }
# if webui or self.config.get('webui', 'activated'):
# from .thread.webserver import WebServer
# self.webserver = WebServer(self)
# self.webserver.start()
# self.svm.add('webui', **kwgs)
# self.svm.start()
def _init_api(self):
from .api import Api
self.api = Api(self)
def _init_database(self):
from .database import DatabaseBackend
from .datatype import Permission, Role
# TODO: Move inside DatabaseBackend
newdb = not os.path.isfile(DatabaseBackend.DB_FILE)
self.db = DatabaseBackend(self)
self.db.setup()
if self.__restore or newdb:
self.db.add_user(self.DEFAULT_USERNAME, self.DEFAULT_PASSWORD,
Role.Admin, Permission.All)
if self.__restore:
self.log.warning(
self._("Restored default login credentials `admin|pyload`"))
def _init_managers(self):
from .manager import (AccountManager, AddonManager, EventManager,
ExchangeManager, FileManager, InfoManager,
PluginManager, TransferManager)
self.scheduler = sched.scheduler(time.time, time.sleep)
self.filemanager = self.files = FileManager(self)
self.pluginmanager = self.pgm = PluginManager(self)
self.exchangemanager = self.exm = ExchangeManager(self)
self.eventmanager = self.evm = EventManager(self)
self.accountmanager = self.acm = AccountManager(self)
self.infomanager = self.iom = InfoManager(self)
self.transfermanager = self.tsm = TransferManager(self)
self.addonmanager = self.adm = AddonManager(self)
# self.remotemanager = self.rem = RemoteManager(self)
# self.servermanager = self.svm = ServerManager(self)
self.db.manager = self.files # ugly?
def _init_requests(self):
self.request = self.req = RequestFactory(self)
def _init_config(self):
session = ConfigParser(self.__SESSIONFILENAME, session_defaults)
flags = portalocker.LOCK_EX | portalocker.LOCK_NB
portalocker.lock(session.fp, flags)
profiledir = os.path.join(self.configdir, self.profile)
psp = psutil.Process()
session.set('current', 'id', time.time())
session.set('current', 'profile', 'path', profiledir)
session.set('current', 'profile', 'pid', psp.pid)
session.set('current', 'profile', 'ctime', psp.create_time())
self.config = ConfigParser(self.__CONFIGFILENAME, config_defaults)
self.session = session
def _init_cache(self):
# Re-use cache
tempdir = self.__tempdir
if tempdir is None:
tempdir = self.session.get('previous', 'cache', 'path')
if tempdir is None or not os.path.isdir(tempdir):
pydir = os.path.join(TMPDIR, __namespace__)
makedirs(pydir, exist_ok=True)
tempdir = tempfile.mkdtemp(dir=pydir)
self.session.set('current', 'cache', 'path', tempdir)
self.cachedir = tempdir
# if tempdir not in sys.path:
# sys.path.append(tempdir)
def _register_signals(self):
shutfn = lambda s, f: self.shutdown()
quitfn = lambda s, f: self.terminate()
try:
if os.name == 'nt':
# signal.signal(signal.CTRL_C_EVENT, shutfn)
signal.signal(signal.CTRL_BREAK_EVENT, shutfn)
else:
signal.signal(signal.SIGTERM, shutfn)
# signal.signal(signal.SIGINT, shutfn)
signal.signal(signal.SIGQUIT, quitfn)
# signal.signal(signal.SIGTSTP, lambda s, f: self.stop())
# signal.signal(signal.SIGCONT, lambda s, f: self.run())
except Exception:
pass
def __init__(self,
profiledir=None,
tempdir=None,
debug=None,
restore=None):
self.__running = Event()
self.__do_restart = False
self.__do_shutdown = False
self.__debug = debug if debug is None else int(debug)
self.__restore = bool(restore)
self.__tempdir = tempdir
self._ = lambda x: x
self._init_profile(profiledir)
# if refresh:
# cleanpy(PACKDIR)
Process.__init__(self)
@property
def version(self):
return __version__
@property
def version_info(self):
return __version_info__
@property
def running(self):
return self.__running.is_set()
@property
def debug(self):
return self.__debug
def _init_profile(self, profiledir):
profiledir = fullpath(profiledir)
os.chdir(profiledir)
self.configdir, self.profile = os.path.split(profiledir)
def _setup_process(self):
try:
set_process_name('pyLoad')
except AttributeError:
pass
niceness = self.config.get('general', 'niceness')
renice(niceness=niceness)
ioniceness = int(self.config.get('general', 'ioniceness'))
ionice(niceness=ioniceness)
def _setup_storage(self):
storage_folder = self.config.get('general', 'storage_folder')
if not storage_folder:
storage_folder = os.path.join(USERDIR, self.DEFAULT_STORAGEDIRNAME)
self.log.debug("Storage: {0}".format(storage_folder))
makedirs(storage_folder, exist_ok=True)
avail_space = format.size(availspace(storage_folder))
self.log.info(
self._("Available storage space: {0}").format(avail_space))
def _workloop(self):
self.__running.set()
self.tsm.pause = False # NOTE: Recheck...
while True:
self.__running.wait()
self.tsm.work()
self.iom.work()
self.exm.work()
if self.__do_restart:
raise Restart
if self.__do_shutdown:
raise Shutdown
self.scheduler.run()
def _start_plugins(self):
# TODO: Move to accountmanager
self.log.info(self._("Activating accounts ..."))
self.acm.load_accounts()
# self.scheduler.enter(0, 0, self.acm.load_accounts)
self.adm.activate_addons()
def _show_info(self):
self.log.info(self._("Welcome to pyLoad v{0}").format(self.version))
self.log.info(self._("Profile: {0}").format(self.profile))
self.log.info(self._("Config directory: {0}").format(self.configdir))
self.log.debug("Cache directory: {0}".format(self.cachedir))
def run(self):
self._init_config()
self._init_cache()
self._setup_debug()
self._init_logger()
try:
self.log.debug("Running pyLoad ...")
self._setup_language()
self._setup_permissions()
self._init_database()
self._init_managers()
self._init_requests()
self._init_api()
self._show_info()
self._setup_storage()
self._start_plugins()
self._setup_process()
self.log.info(self._("pyLoad is up and running"))
self.evm.fire('pyload:started')
# # some memory stats
# from guppy import hpy
# hp=hpy()
# print(hp.heap())
# import objgraph
# objgraph.show_most_common_types(limit=30)
# import memdebug
# memdebug.start(8002)
# from meliae import scanner
# scanner.dump_all_objects(os.path.join(PACKDIR, 'objs.json'))
self._workloop()
except Restart:
self.restart()
except Shutdown:
self.shutdown()
except (KeyboardInterrupt, SystemExit):
self.shutdown()
except Exception as e:
self.log.critical(str(e))
self.terminate()
raise
else:
self.shutdown()
def _remove_loggers(self):
for handler in self.log.handlers:
with closing(handler) as hdlr:
self.log.removeHandler(hdlr)
def restart(self):
self.stop()
self.log.info(self._("Restarting pyLoad ..."))
self.evm.fire('pyload:restarting')
self.start()
def _register_instance(self):
profiledir = os.path.join(self.configdir, self.profile)
if profiledir in _pmap:
raise RuntimeError("A pyLoad instance using profile `{0}` "
"is already running".format(profiledir))
_pmap[profiledir] = self
def _unregister_instance(self):
profiledir = os.path.join(self.configdir, self.profile)
_pmap.pop(profiledir, None)
def _close_session(self):
id = self.session.get('previous', 'id')
self.session[id] = self.session['previous']
self.session['previous'] = self.session['current']
self.session['current'].reset()
self.session.close()
def terminate(self):
try:
self.log.debug("Killing pyLoad ...")
self._unregister_instance()
self._close_session()
finally:
Process.terminate(self)
def shutdown(self):
try:
self.stop()
self.log.info(self._("Exiting pyLoad ..."))
self.tsm.shutdown()
self.db.shutdown() # NOTE: Why here?
self.config.close()
self._remove_loggers()
# if cleanup:
# self.log.info(self._("Deleting temp files ..."))
# remove(self.tempdir, ignore_errors=True)
finally:
self.terminate()
def start(self):
if not self.is_alive():
self._register_instance()
self._register_signals()
Process.start(self)
elif not self.running:
self.log.info(self._("Starting pyLoad ..."))
self.evm.fire('pyload:starting')
self.__running.set()
def stop(self):
if not self.running:
return None
try:
self.log.info(self._("Stopping pyLoad ..."))
self.evm.fire('pyload:stopping')
self.adm.deactivate_addons()
self.api.stop_all_downloads()
finally:
self.files.sync_save()
self.__running.clear()
self.evm.fire('pyload:stopped')
def _init_config(self):
session = ConfigParser(self.__SESSIONFILENAME, session_defaults)
flags = portalocker.LOCK_EX | portalocker.LOCK_NB
portalocker.lock(session.fp, flags)
profiledir = os.path.join(self.configdir, self.profile)
psp = psutil.Process()
session.set('current', 'id', time.time())
session.set('current', 'profile', 'path', profiledir)
session.set('current', 'profile', 'pid', psp.pid)
session.set('current', 'profile', 'ctime', psp.create_time())
self.config = ConfigParser(self.__CONFIGFILENAME, config_defaults)
self.session = session
class Core(object):
DEFAULT_CONFIGNAME = 'config.ini'
DEFAULT_LANGUAGE = 'english'
DEFAULT_USERNAME = '******'
DEFAULT_PASSWORD = '******'
DEFAULT_STORAGENAME = 'downloads'
@property
def version(self):
return __version__
@property
def version_info(self):
return __version_info__
@property
def running(self):
return self.__running.is_set()
def __init__(self, cfgdir, tmpdir, debug=None, restore=False):
self.__running = Event()
self.__do_restart = False
self.__do_exit = False
self._ = lambda x: x
self.cfgdir = fullpath(cfgdir)
self.tmpdir = fullpath(tmpdir)
os.chdir(self.cfgdir)
# if self.tmpdir not in sys.path:
# sys.path.append(self.tmpdir)
# if refresh:
# cleanpy(PACKDIR)
self.config = ConfigParser(self.DEFAULT_CONFIGNAME)
self.debug = self.config.get('log',
'debug') if debug is None else debug
self.log = LoggerFactory(self, self.debug)
self._init_database(restore)
self._init_managers()
self.request = self.req = RequestFactory(self)
self._init_api()
atexit.register(self.exit)
def _init_api(self):
from pyload.api import Api
self.api = Api(self)
def _init_database(self, restore):
from pyload.core.database import DatabaseBackend
from pyload.core.datatype import Permission, Role
# TODO: Move inside DatabaseBackend
newdb = not os.path.isfile(DatabaseBackend.DB_FILE)
self.db = DatabaseBackend(self)
self.db.setup()
if restore or newdb:
self.db.add_user(self.DEFAULT_USERNAME, self.DEFAULT_PASSWORD,
Role.Admin, Permission.All)
if restore:
self.log.warning(
self._('Restored default login credentials `admin|pyload`'))
def _init_managers(self):
from pyload.core.manager import (AccountManager, AddonManager,
EventManager, ExchangeManager,
FileManager, InfoManager,
PluginManager, TransferManager)
self.scheduler = sched.scheduler(time.time, time.sleep)
self.filemanager = self.files = FileManager(self)
self.pluginmanager = self.pgm = PluginManager(self)
self.exchangemanager = self.exm = ExchangeManager(self)
self.eventmanager = self.evm = EventManager(self)
self.accountmanager = self.acm = AccountManager(self)
self.infomanager = self.iom = InfoManager(self)
self.transfermanager = self.tsm = TransferManager(self)
# TODO: Remove builtins.ADDONMANAGER
builtins.ADDONMANAGER = self.addonmanager = self.adm = AddonManager(
self)
# self.remotemanager = self.rem = RemoteManager(self)
# self.servermanager = self.svm = ServerManager(self)
self.db.manager = self.files # ugly?
def _setup_permissions(self):
self.log.debug('Setup permissions...')
if os.name == 'nt':
return
change_group = self.config.get('permission', 'change_group')
change_user = self.config.get('permission', 'change_user')
if change_group:
try:
group = self.config.get('permission', 'group')
set_process_group(group)
except Exception as exc:
self.log.error(self._('Unable to change gid'))
self.log.error(exc, exc_info=self.debug)
if change_user:
try:
user = self.config.get('permission', 'user')
set_process_user(user)
except Exception as exc:
self.log.error(self._('Unable to change uid'))
self.log.error(exc, exc_info=self.debug)
def set_language(self, lang):
domain = 'core'
localedir = resource_filename(__package__, 'locale')
languages = (locale.locale_alias[lang.lower()].split('_', 1)[0], )
self._set_language(domain, localedir, languages)
def _set_language(self, *args, **kwargs):
trans = gettext.translation(*args, **kwargs)
try:
self._ = trans.ugettext
except AttributeError:
self._ = trans.gettext
def _setup_language(self):
self.log.debug('Setup language...')
lang = self.config.get('general', 'language')
if not lang:
lc = locale.getlocale()[0] or locale.getdefaultlocale()[0]
lang = lc.split('_', 1)[0] if lc else 'en'
try:
self.set_language(lang)
except IOError as exc:
self.log.error(exc, exc_info=self.debug)
self._set_language('core', fallback=True)
# def _setup_niceness(self):
# niceness = self.config.get('general', 'niceness')
# renice(niceness=niceness)
# ioniceness = int(self.config.get('general', 'ioniceness'))
# ionice(niceness=ioniceness)
def _setup_storage(self):
self.log.debug('Setup storage...')
storage_folder = self.config.get('general', 'storage_folder')
if storage_folder is None:
storage_folder = os.path.join(builtins.USERDIR,
self.DEFAULT_STORAGENAME)
self.log.info(self._('Storage: {0}'.format(storage_folder)))
makedirs(storage_folder, exist_ok=True)
avail_space = format.size(availspace(storage_folder))
self.log.info(
self._('Available storage space: {0}').format(avail_space))
def _setup_network(self):
self.log.debug('Setup network...')
# TODO: Move to accountmanager
self.log.info(self._('Activating accounts...'))
self.acm.load_accounts()
# self.scheduler.enter(0, 0, self.acm.load_accounts)
self.adm.activate_addons()
def run(self):
self.log.info('Welcome to pyLoad v{0}'.format(self.version))
if self.debug:
self.log.warning('*** DEBUG MODE ***')
try:
self.log.debug('Starting pyLoad...')
self.evm.fire('pyload:starting')
self.__running.set()
self._setup_language()
self._setup_permissions()
self.log.info(self._('Config directory: {0}').format(self.cfgdir))
self.log.info(self._('Cache directory: {0}').format(self.tmpdir))
self._setup_storage()
self._setup_network()
# self._setup_niceness()
# # some memory stats
# from guppy import hpy
# hp=hpy()
# print(hp.heap())
# import objgraph
# objgraph.show_most_common_types(limit=30)
# import memdebug
# memdebug.start(8002)
# from meliae import scanner
# scanner.dump_all_objects(os.path.join(PACKDIR, 'objs.json'))
self.log.debug('pyLoad is up and running')
self.evm.fire('pyload:started')
self.tsm.pause = False # NOTE: Recheck...
while True:
self.__running.wait()
self.tsm.work()
self.iom.work()
self.exm.work()
if self.__do_restart:
raise Restart
if self.__do_exit:
raise Exit
self.scheduler.run()
time.sleep(1)
except Restart:
self.restart()
except (Exit, KeyboardInterrupt, SystemExit):
self.exit()
except Exception as exc:
self.log.critical(exc, exc_info=True)
self.exit()
def _remove_loggers(self):
for handler in self.log.handlers:
with closing(handler) as hdlr:
self.log.removeHandler(hdlr)
def restart(self):
self.stop()
self.log.info(self._('Restarting pyLoad...'))
self.evm.fire('pyload:restarting')
self.run()
def exit(self):
self.stop()
self.log.info(self._('Exiting pyLoad...'))
self.tsm.exit()
self.db.exit() # NOTE: Why here?
self._remove_loggers()
# if cleanup:
# self.log.info(self._("Deleting temp files..."))
# remove(self.tmpdir, ignore_errors=True)
def stop(self):
try:
self.log.debug('Stopping pyLoad...')
self.evm.fire('pyload:stopping')
self.adm.deactivate_addons()
self.api.stop_all_downloads()
finally:
self.files.sync_save()
self.__running.clear()
self.evm.fire('pyload:stopped')
class SetupAssistant(object):
"""
pyLoads initial setup configuration assistant.
"""
__slots__ = [
'config', 'db', 'lang', 'no', 'stdin_encoding', 'timestamp', 'yes'
]
def __init__(self, file, version):
self.config = ConfigParser(file, version)
self.stdin_encoding = console_encoding(sys.stdin.encoding)
self.lang = None
self.db = None
# We will create a timestamp so that the setup will be completed in a
# specific interval
self.timestamp = time()
# TODO: probably unneeded
self.yes = "yes"
self.no = "no"
def start(self):
web = WebServer(pysetup=self)
web.start()
error = web.check_error()
# TODO: start cli in this case
if error: # TODO: errno 44 port already in use
print(error)
url = "http://{0}:{1:d}/".format(
socket.gethostbyname(socket.gethostname()), web.port)
print("Setup is running at {0}".format(url))
opened = webbrowser.open_new_tab(url)
if not opened:
print("Please point your browser to the url above")
eval(input())
return True
def start_cli(self):
self.ask_lang()
print(_("Welcome to the pyLoad Configuration Assistent"))
print(
_("It will check your system and make a basic setup in order to run pyLoad"
))
print("")
print(_("The value in brackets [] always is the default value,"))
print(
_("in case you do not want to change it or you are unsure what to choose, just hit enter"
))
print(
_("Don't forget: You can always rerun this assistent with --setup or -s parameter, when you start pyLoadCore"
))
print(_("If you have any problems with this assistent hit CTRL+C,"))
print(
_("to abort and do not let him start with pyLoadCore automatically anymore"
))
print("")
print(_("When you are ready for system check, hit enter"))
eval(input())
# TODO: new system check + deps
con = self.ask(_("Continue with setup?"), self.yes, bool=True)
if not con:
return False
print("")
print(
_("Do you want to change the config path? Current is {0}").format(
os.path.abspath("")))
print(
_("If you use pyLoad on a server or the home partition lives on an internal flash it may be a good idea to change it"
))
path = self.ask(_("Change config path?"), self.no, bool=True)
if path:
self.conf_path()
# calls exit when changed
print("")
print(_("Do you want to configure login data and basic settings?"))
print(_("This is recommend for first run"))
con = self.ask(_("Make basic setup?"), self.yes, bool=True)
if con:
self.conf_basic()
print("")
print(_("Do you want to configure ssl?"))
ssl = self.ask(_("Configure ssl?"), self.no, bool=True)
if ssl:
self.conf_ssl()
print("")
print(_("Do you want to configure the Web UI?"))
web = self.ask(_("Configure the Web UI?"), self.yes, bool=True)
if web:
self.conf_web()
print("")
print(_("Setup finished successfully"))
print(_("Hit enter to exit and restart pyLoad"))
eval(input())
return True
def conf_basic(self):
print("")
print(_("## Basic Setup ##"))
from pyload.core.database import DatabaseBackend
db = DatabaseBackend(None)
db.setup()
username = self.ask(_("Username"), "User")
password = self.ask("", "", password=True)
db.add_user(username, password)
db.shutdown()
print("")
langs = self.config.get_meta_data("general", "language")
self.config.set('general', 'language',
self.ask(_("Language"), "en", langs.type.split(";")))
self.config.set('general', 'storage_folder',
self.ask(_("Storage folder"), "Storage"))
self.config.set('connection', 'max_transfers',
self.ask(_("Max parallel transfers"), "3"))
reconnect = self.ask(_("Use Reconnect?"), self.no, bool=True)
self.config.set('reconnect', 'activated', reconnect)
if reconnect:
self.config.set(
'reconnect', 'script',
self.ask(_("Reconnection script"), "./reconnect.sh"))
def conf_web(self):
print("")
print(_("## Web UI Setup ##"))
print("")
self.config.set(
'webui', 'activated',
self.ask(_("Activate the Web UI?"), self.yes, bool=True))
print("")
print(
_("Listen address, if you use 127.0.0.1 or localhost, the Web UI will only accessible locally"
))
self.config.set('webui', 'host', self.ask(_("Address"), "localhost"))
self.config.set('webui', 'port', self.ask(_("Port"), "8010"))
print("")
print(
_("pyLoad offers several server backends, now following a short explanation"
))
print(
"threaded:",
_("Default server, this server offers SSL and is a good alternative to builtin"
))
print(
"fastcgi:",
_("Can be used by apache, lighttpd, requires you to configure them, which is not too easy job"
))
print(
"lightweight:",
_("Very fast alternative written in C, requires libev and linux knowledge"
))
print(
"\t",
_("Get it from here: https://github.com/jonashaag/bjoern, compile it"
))
print("\t", _("and copy bjoern.so to pyload/lib"))
print()
print(
_("Attention: In some rare cases the builtin server is not working, if you notice problems with the Web UI"
))
print(
_("come back here and change the builtin server to the threaded one here"
))
self.config.set(
'webui', 'server',
self.ask(_("Server"), "threaded",
["builtin", "threaded", "fastcgi", "lightweight"]))
def conf_ssl(self):
print("")
print(_("## SSL Setup ##"))
print("")
print(
_("Execute these commands from pyLoad config folder to make ssl certificates:"
))
print("")
print("openssl genrsa -out ssl.key 1024")
print("openssl req -new -key ssl.key -out ssl.csr")
print(
"openssl req -days 36500 -x509 -key ssl.key -in ssl.csr > ssl.crt "
)
print("")
print(
_("If you're done and everything went fine, you can activate ssl now"
))
self.config.set('ssl', 'activated',
self.ask(_("Activate SSL?"), self.yes, bool=True))
def set_user(self):
translation = gettext.translation(
"setup",
os.path.join(PACKDIR, "locale"),
languages=[self.config.get('general', 'language'), "en"],
fallback=True)
translation.install(True)
self.open_db()
try:
while True:
print(_("Select action"))
print(_("1 - Create/Edit user"))
print(_("2 - List users"))
print(_("3 - Remove user"))
print(_("4 - Quit"))
action = eval(input("[1]/2/3/4: "))
if action not in ("1", "2", "3", "4"):
continue
elif action == "1":
print("")
username = self.ask(_("Username"), "User")
password = self.ask("", "", password=True)
admin = self.ask("Admin?", self.yes, bool=True)
self.db.add_user(username, password,
Role.Admin if admin else Role.User,
int('1111111', 2))
elif action == "2":
print("")
print(_("Users"))
print("-----")
users = self.db.get_all_user_data()
for user in users.values():
print(user.name)
print("-----")
print("")
elif action == "3":
print("")
username = self.ask(_("Username"), "")
if username:
self.db.remove_user_by_name(username)
elif action == "4":
self.db.sync_save()
break
finally:
self.close_db()
def add_user(self, username, password, role=Role.Admin):
self.open_db()
try:
self.db.add_user(username, password, role, int('1111111', 2))
finally:
self.close_db()
def open_db(self):
from pyload.core.database import DatabaseBackend
if self.db is None:
self.db = DatabaseBackend(None)
self.db.setup()
def close_db(self):
if self.db is not None:
self.db.sync_save()
self.db.shutdown()
def save(self):
self.config.save()
self.close_db()
def conf_path(self, trans=False):
if trans:
translation = gettext.translation(
"setup",
os.path.join(PACKDIR, "locale"),
languages=[self.config.get('general', 'language'), "en"],
fallback=True)
translation.install(True)
print(
_("Setting new configpath, current configuration will not be transferred!"
))
path = self.ask(_("Config path"), os.path.abspath(""))
try:
path = os.path.join(COREDIR, path)
if not os.path.exists(path):
makedirs(path)
with io.open(os.path.join(COREDIR, "pyload", "config",
"configdir"),
mode='wb') as fp:
fp.write(path)
print(
_("Config path changed, setup will now close, please restart to go on"
))
print(_("Press Enter to exit"))
eval(input())
exit()
except Exception as e:
print(_("Setting config path failed: {0}").format(e.message))
def ask_lang(self):
langs = self.config.get_meta_data("general",
"language").type.split(";")
self.lang = self.ask(u"Choose your Language / Wähle deine Sprache",
"en", langs)
translation = gettext.translation("setup",
os.path.join(PACKDIR, "locale"),
languages=[self.lang, "en"],
fallback=True)
translation.install(True)
# l10n Input shorthand for yes
self.yes = _("y")
# l10n Input shorthand for no
self.no = _("n")
def ask(self, qst, default, answers=[], bool=False, password=False):
"""
Generate dialog on command line.
"""
if answers:
info = "("
for i, answer in enumerate(answers):
info += (", " if i != 0 else "") + str(
(answer == default and "[{0}]".format(answer)) or answer)
info += ")"
elif bool:
if default == self.yes:
info = "([{0}]/{1})".format(self.yes, self.no)
else:
info = "({0}/[{1}])".format(self.yes, self.no)
else:
info = "[{0}]".format(default)
if password:
p1 = True
p2 = False
while p1 != p2:
# getpass(_("Password: "******"Password: "******"")
if len(p1) < 4:
print(_("Password too short. Use at least 4 symbols"))
continue
sys.stdout.write(_("Password (again): "))
p2 = getpass("")
if p1 == p2:
return p1
else:
print(_("Passwords did not match"))
while True:
input = eval(input(qst + " {0}: ".format(info)))
input = input.decode(self.stdin_encoding)
if input.strip() == "":
input = default
if bool:
# l10n yes, true,t are inputs for booleans with value true
if input.lower().strip() in [
self.yes, _("yes"),
_("true"),
_("t"), "yes"
]:
return True
# l10n no, false,f are inputs for booleans with value false
elif input.lower().strip() in [
self.no, _("no"),
_("false"), _("f"), "no"
]:
return False
else:
print(_("Invalid Input"))
continue
if not answers:
return input
else:
if input in answers:
return input
else:
print(_("Invalid Input"))
class Core(object):
DEFAULT_CONFIGNAME = 'config.ini'
DEFAULT_LANGUAGE = 'english'
DEFAULT_USERNAME = '******'
DEFAULT_PASSWORD = '******'
DEFAULT_STORAGENAME = 'downloads'
@property
def version(self):
return __version__
@property
def version_info(self):
return __version_info__
@property
def running(self):
return self.__running.is_set()
def __init__(self, cfgdir, tmpdir, debug=None, restore=False):
self.__running = Event()
self.__do_restart = False
self.__do_exit = False
self._ = lambda x: x
self.cfgdir = fullpath(cfgdir)
self.tmpdir = fullpath(tmpdir)
os.chdir(self.cfgdir)
# if self.tmpdir not in sys.path:
# sys.path.append(self.tmpdir)
# if refresh:
# cleanpy(PACKDIR)
self.config = ConfigParser(self.DEFAULT_CONFIGNAME)
self.debug = self.config.get(
'log', 'debug') if debug is None else debug
self.log = LoggerFactory(self, self.debug)
self._init_database(restore)
self._init_managers()
self.request = self.req = RequestFactory(self)
self._init_api()
atexit.register(self.exit)
def _init_api(self):
from pyload.api import Api
self.api = Api(self)
def _init_database(self, restore):
from pyload.core.database import DatabaseBackend
from pyload.core.datatype import Permission, Role
# TODO: Move inside DatabaseBackend
newdb = not os.path.isfile(DatabaseBackend.DB_FILE)
self.db = DatabaseBackend(self)
self.db.setup()
if restore or newdb:
self.db.add_user(
self.DEFAULT_USERNAME, self.DEFAULT_PASSWORD, Role.Admin,
Permission.All)
if restore:
self.log.warning(
self._('Restored default login credentials `admin|pyload`'))
def _init_managers(self):
from pyload.core.manager import (
AccountManager, AddonManager, EventManager, ExchangeManager,
FileManager, InfoManager, PluginManager, TransferManager)
self.scheduler = sched.scheduler(time.time, time.sleep)
self.filemanager = self.files = FileManager(self)
self.pluginmanager = self.pgm = PluginManager(self)
self.exchangemanager = self.exm = ExchangeManager(self)
self.eventmanager = self.evm = EventManager(self)
self.accountmanager = self.acm = AccountManager(self)
self.infomanager = self.iom = InfoManager(self)
self.transfermanager = self.tsm = TransferManager(self)
# TODO: Remove builtins.ADDONMANAGER
builtins.ADDONMANAGER = self.addonmanager = self.adm = AddonManager(
self)
# self.remotemanager = self.rem = RemoteManager(self)
# self.servermanager = self.svm = ServerManager(self)
self.db.manager = self.files # ugly?
def _setup_permissions(self):
self.log.debug('Setup permissions...')
if os.name == 'nt':
return
change_group = self.config.get('permission', 'change_group')
change_user = self.config.get('permission', 'change_user')
if change_group:
try:
group = self.config.get('permission', 'group')
set_process_group(group)
except Exception as exc:
self.log.error(self._('Unable to change gid'))
self.log.error(exc, exc_info=self.debug)
if change_user:
try:
user = self.config.get('permission', 'user')
set_process_user(user)
except Exception as exc:
self.log.error(self._('Unable to change uid'))
self.log.error(exc, exc_info=self.debug)
def set_language(self, lang):
domain = 'core'
localedir = resource_filename(__package__, 'locale')
languages = (locale.locale_alias[lang.lower()].split('_', 1)[0],)
self._set_language(domain, localedir, languages)
def _set_language(self, *args, **kwargs):
trans = gettext.translation(*args, **kwargs)
try:
self._ = trans.ugettext
except AttributeError:
self._ = trans.gettext
def _setup_language(self):
self.log.debug('Setup language...')
lang = self.config.get('general', 'language')
if not lang:
lc = locale.getlocale()[0] or locale.getdefaultlocale()[0]
lang = lc.split('_', 1)[0] if lc else 'en'
try:
self.set_language(lang)
except IOError as exc:
self.log.error(exc, exc_info=self.debug)
self._set_language('core', fallback=True)
# def _setup_niceness(self):
# niceness = self.config.get('general', 'niceness')
# renice(niceness=niceness)
# ioniceness = int(self.config.get('general', 'ioniceness'))
# ionice(niceness=ioniceness)
def _setup_storage(self):
self.log.debug('Setup storage...')
storage_folder = self.config.get('general', 'storage_folder')
if storage_folder is None:
storage_folder = os.path.join(
builtins.USERDIR, self.DEFAULT_STORAGENAME)
self.log.info(self._('Storage: {0}'.format(storage_folder)))
makedirs(storage_folder, exist_ok=True)
avail_space = format.size(availspace(storage_folder))
self.log.info(
self._('Available storage space: {0}').format(avail_space))
def _setup_network(self):
self.log.debug('Setup network...')
# TODO: Move to accountmanager
self.log.info(self._('Activating accounts...'))
self.acm.load_accounts()
# self.scheduler.enter(0, 0, self.acm.load_accounts)
self.adm.activate_addons()
def run(self):
self.log.info('Welcome to pyLoad v{0}'.format(self.version))
if self.debug:
self.log.warning('*** DEBUG MODE ***')
try:
self.log.debug('Starting pyLoad...')
self.evm.fire('pyload:starting')
self.__running.set()
self._setup_language()
self._setup_permissions()
self.log.info(self._('Config directory: {0}').format(self.cfgdir))
self.log.info(self._('Cache directory: {0}').format(self.tmpdir))
self._setup_storage()
self._setup_network()
# self._setup_niceness()
# # some memory stats
# from guppy import hpy
# hp=hpy()
# print(hp.heap())
# import objgraph
# objgraph.show_most_common_types(limit=30)
# import memdebug
# memdebug.start(8002)
# from meliae import scanner
# scanner.dump_all_objects(os.path.join(PACKDIR, 'objs.json'))
self.log.debug('pyLoad is up and running')
self.evm.fire('pyload:started')
self.tsm.pause = False # NOTE: Recheck...
while True:
self.__running.wait()
self.tsm.work()
self.iom.work()
self.exm.work()
if self.__do_restart:
raise Restart
if self.__do_exit:
raise Exit
self.scheduler.run()
time.sleep(1)
except Restart:
self.restart()
except (Exit, KeyboardInterrupt, SystemExit):
self.exit()
except Exception as exc:
self.log.critical(exc, exc_info=True)
self.exit()
def _remove_loggers(self):
for handler in self.log.handlers:
with closing(handler) as hdlr:
self.log.removeHandler(hdlr)
def restart(self):
self.stop()
self.log.info(self._('Restarting pyLoad...'))
self.evm.fire('pyload:restarting')
self.run()
def exit(self):
self.stop()
self.log.info(self._('Exiting pyLoad...'))
self.tsm.exit()
self.db.exit() # NOTE: Why here?
self._remove_loggers()
# if cleanup:
# self.log.info(self._("Deleting temp files..."))
# remove(self.tmpdir, ignore_errors=True)
def stop(self):
try:
self.log.debug('Stopping pyLoad...')
self.evm.fire('pyload:stopping')
self.adm.deactivate_addons()
self.api.stop_all_downloads()
finally:
self.files.sync_save()
self.__running.clear()
self.evm.fire('pyload:stopped')