class DatabaseJob(object):
__slots__ = ['args', 'done', 'exception', 'fn', 'kwargs', 'result']
def __init__(self, fn, *args, **kwargs):
self.done = Event()
self.fn = fn
self.args = args
self.kwargs = kwargs
self.result = None
self.exception = False
# import inspect
# self.frame = inspect.currentframe()
def __repr__(self):
# frame = self.frame.f_back
output = ""
# for i in range(5):
# output += "\t{0}:{1}, {2}\n".format(
# os.path.basename(frame.f_code.co_filename), frame.f_lineno, frame.f_code.co_name)
# frame = frame.f_back
# del frame
# del self.frame
return "DataBase Job {0}:{1}\n{2}Result: {3}".format(
self.fn.__name__, self.args[1:], output, self.result)
def process_job(self):
try:
self.result = self.fn(*self.args, **self.kwargs)
except Exception as e:
print_exc()
try:
print("Database Error @", self.fn.__name__, self.args[1:],
self.kwargs, str(e))
except Exception:
pass
self.exception = e
finally:
self.done.set()
def wait(self):
self.done.wait()
class DatabaseJob(object):
def __init__(self, func, *args, **kwargs):
self.done = Event()
self.func = func
self.args = args
self.kwgs = kwargs
self.result = None
self.exception = False
self.pyload = self.args[0].pyload # self.args[0] is DatabaseBackend
# import inspect
# self.frame = inspect.currentframe()
def __repr__(self):
# frame = self.frame.f_back
output = ''
# for i in range(5):
# output += "\t{0}:{1}, {2}\n".format(
# os.path.basename(
# frame.f_code.co_filename), frame.f_lineno, frame.f_code.co_name)
# frame = frame.f_back
# del frame
# del self.frame
return 'DataBase Job {0}:{1}{2}{3}Result: {4}'.format(
self.func.__name__, self.args[1:], os.linesep, output, self.result)
def process_job(self):
try:
self.result = self.func(*self.args, **self.kwgs)
except Exception as exc:
self.pyload.log.error(exc, exc_info=self.pyload.debug)
self.exception = exc
finally:
self.done.set()
def wait(self):
self.done.wait()
class WebSocketServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
"""
HTTPServer specialized for WebSocket.
"""
# Overrides SocketServer.ThreadingMixIn.daemon_threads
daemon_threads = True
# Overrides BaseHTTPServer.HTTPServer.allow_reuse_address
allow_reuse_address = True
def __init__(self, options):
"""
Override SocketServer.TCPServer.__init__ to set SSL enabled
socket object to self.socket before server_bind and server_activate,
if necessary
"""
# Removed dispatcher init here
self._logger = logging.getLogger('pyload')
self.request_queue_size = options.request_queue_size
self.__ws_is_shut_down = Event()
self.__ws_serving = False
socketserver.BaseServer.__init__(self,
(options.server_host, options.port),
WebSocketRequestHandler)
# Expose the options object to allow handler objects access it. We name
# it with websocket_ prefix to avoid conflict.
self.websocket_server_options = options
self._create_sockets()
self.server_bind()
self.server_activate()
def _create_sockets(self):
self.server_name, self.server_port = self.server_address
self._sockets = []
if not self.server_name:
# On platforms that does not support IPv6, the first bind fails.
# On platforms that supports IPv6
# - If it binds both IPv4 and IPv6 on call with AF_INET6, the
# first bind succeeds and the second fails (we'll see 'Address
# already in use' error).
# - If it binds only IPv6 on call with AF_INET6, both call are
# expected to succeed to listen both protocol.
addrinfo_array = [(socket.AF_INET6, socket.SOCK_STREAM, '', '',
''),
(socket.AF_INET, socket.SOCK_STREAM, '', '', '')]
else:
addrinfo_array = socket.getaddrinfo(self.server_name,
self.server_port,
socket.AF_UNSPEC,
socket.SOCK_STREAM,
socket.IPPROTO_TCP)
for addrinfo in addrinfo_array:
family, socktype, proto, canonname, sockaddr = addrinfo
try:
socket_ = socket.socket(family, socktype)
except Exception as e:
self._logger.info(_("Skip by failure: {0}").format(e.message))
continue
server_options = self.websocket_server_options
if server_options.use_tls:
# For the case of _HAS_OPEN_SSL, we do wrapper setup after
# accept.
if server_options.tls_module == _TLS_BY_STANDARD_MODULE:
if server_options.tls_client_auth:
if server_options.tls_client_cert_optional:
client_cert_ = ssl.CERT_OPTIONAL
else:
client_cert_ = ssl.CERT_REQUIRED
else:
client_cert_ = ssl.CERT_NONE
socket_ = ssl.layer_socket(
socket_,
keyfile=server_options.private_key,
certfile=server_options.certificate,
ssl_version=ssl.PROTOCOL_SSLv23,
ca_certs=server_options.tls_client_ca,
cert_reqs=client_cert_,
do_handshake_on_connect=False)
self._sockets.append((socket_, addrinfo))
def server_bind(self):
"""
Override SocketServer.TCPServer.server_bind to enable multiple
sockets bind
"""
failed_sockets = []
for socketinfo in self._sockets:
socket_, addrinfo = socketinfo
if self.allow_reuse_address:
socket_.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
socket_.bind(self.server_address)
except Exception as e:
self._logger.info(_("Skip by failure: {0}").format(e.message))
socket_.close()
failed_sockets.append(socketinfo)
if self.server_address[1] == 0:
# The operating system assigns the actual port number for port
# number 0. This case, the second and later sockets should use
# the same port number. Also self.server_port is rewritten
# because it is exported, and will be used by external code.
self.server_address = (self.server_name,
socket_.getsockname()[1])
self.server_port = self.server_address[1]
self._logger.info(
_('Port {0:d} is assigned').format(self.server_port))
for socketinfo in failed_sockets:
self._sockets.remove(socketinfo)
def server_activate(self):
"""
Override SocketServer.TCPServer.server_activate to enable multiple
sockets listen
"""
failed_sockets = []
for socketinfo in self._sockets:
socket_, addrinfo = socketinfo
self._logger.debug("Listen on: {0}".format(addrinfo))
try:
socket_.listen(self.request_queue_size)
except Exception as e:
self._logger.info(_("Skip by failure: {0}").format(e.message))
socket_.close()
failed_sockets.append(socketinfo)
for socketinfo in failed_sockets:
self._sockets.remove(socketinfo)
if len(self._sockets) == 0:
self._logger.critical(
_('No sockets activated. Use info log level to see the reason')
)
def server_close(self):
"""
Override SocketServer.TCPServer.server_close to enable multiple
sockets close
"""
for socketinfo in self._sockets:
socket_, addrinfo = socketinfo
self._logger.info(_("Close on: {0}").format(addrinfo))
socket_.close()
def fileno(self):
"""
Override SocketServer.TCPServer.fileno.
"""
self._logger.critical(_('Not supported: fileno'))
return self._sockets[0][0].fileno()
# NOTE: client_address is a tuple
def handle_error(self, request, client_address):
"""
Override SocketServer.handle_error.
"""
self._logger.error(
"Exception in processing request from: {0}\n{1}".format(
client_address, util.get_stack_trace()))
def get_request(self):
"""
Override TCPServer.get_request to lib OpenSSL.SSL.Connection
object with _StandaloneSSLConnection to provide makefile method. We
cannot substitute OpenSSL.SSL.Connection.makefile since it's readonly
attribute
"""
accepted_socket, client_address = self.socket.accept()
server_options = self.websocket_server_options
if server_options.use_tls:
if server_options.tls_module == _TLS_BY_STANDARD_MODULE:
try:
accepted_socket.do_handshake()
except ssl.SSLError as e:
self._logger.debug("{0}".format(e.message))
raise
# Print cipher in use. Handshake is done on accept.
self._logger.debug("Cipher: {0}".format(
accepted_socket.cipher()))
self._logger.debug("Client cert: {0}".format(
accepted_socket.getpeercert()))
elif server_options.tls_module == _TLS_BY_PYOPENSSL:
# We cannot print(the cipher in use. pyOpenSSL does not provide)
# any method to fetch that.
ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
ctx.use_privatekey_file(server_options.private_key)
ctx.use_certificate_file(server_options.certificate)
def default_callback(conn, cert, errnum, errdepth, ok):
return ok == 1
# See the OpenSSL document for SSL_CTX_set_verify.
if server_options.tls_client_auth:
verify_mode = OpenSSL.SSL.VERIFY_PEER
if not server_options.tls_client_cert_optional:
verify_mode |= OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT
ctx.set_verify(verify_mode, default_callback)
ctx.load_verify_locations(server_options.tls_client_ca,
None)
else:
ctx.set_verify(OpenSSL.SSL.VERIFY_NONE, default_callback)
accepted_socket = OpenSSL.SSL.Connection(ctx, accepted_socket)
accepted_socket.set_accept_state()
# Convert SSL related error into socket.error so that
# SocketServer ignores them and keeps running.
#
# TODO(tyoshino): Convert all kinds of errors.
try:
accepted_socket.do_handshake()
except OpenSSL.SSL.Error as e:
# Set errno part to 1 (SSL_ERROR_SSL) like the ssl module
# does.
self._logger.debug('{0!r}'.format(e))
raise socket.error(1, repr(e))
cert = accepted_socket.get_peer_certificate()
self._logger.debug("Client cert subject: {0}".format(
cert.get_subject().get_components()))
accepted_socket = _StandaloneSSLConnection(accepted_socket)
else:
raise ValueError('No TLS support module is available')
return accepted_socket, client_address
def serve_forever(self, poll_interval=0.5):
"""
Override SocketServer.BaseServer.serve_forever.
"""
self.__ws_serving = True
self.__ws_is_shut_down.clear()
handle_request = self.handle_request
if hasattr(self, '_handle_request_noblock'):
handle_request = self._handle_request_noblock
else:
self._logger.warning(_('Fallback to blocking request handler'))
try:
while self.__ws_serving:
r, w, e = select.select(
[socket_[0] for socket_ in self._sockets], [], [],
poll_interval)
for socket_ in r:
self.socket = socket_
handle_request()
self.socket = None
finally:
self.__ws_is_shut_down.set()
def shutdown(self):
"""
Override SocketServer.BaseServer.shutdown.
"""
self.__ws_serving = False
self.__ws_is_shut_down.wait()
class DatabaseBackend(Thread):
subs = []
DB_FILE = 'pyload.db'
VERSION_FILE = 'db.version'
def __init__(self, core):
super(DatabaseBackend, self).__init__()
self.setDaemon(True)
self.pyload = core
self._ = core._
self.manager = None # set later
self.error = None # TODO: Recheck...
self.__running = Event()
self.jobs = Queue()
set_db(self)
@property
def running(self):
return self.__running.is_set()
def setup(self):
"""
*MUST* be called before db can be used !.
"""
self.start()
self.__running.wait()
def init(self):
"""Main loop, which executes commands."""
version = self._check_version()
self.conn = sqlite3.connect(self.DB_FILE)
os.chmod(self.DB_FILE, 0o600)
self.c = self.conn.cursor()
if version is not None and version < DB_VERSION:
success = self._convert_db(version)
# delete database
if not success:
self.c.close()
self.conn.close()
remove(self.VERSION_FILE)
shutil.move(self.DB_FILE, self.DB_FILE + '.bak')
self.pyload.log.warning(
self._('Database was deleted due to incompatible version'))
with io.open(self.VERSION_FILE, mode='wb') as fp:
fp.write(to_str(DB_VERSION))
self.conn = sqlite3.connect(self.DB_FILE)
os.chmod(self.DB_FILE, 0o600)
self.c = self.conn.cursor()
self._create_tables()
self.conn.commit()
def run(self):
try:
self.init()
except Exception as exc:
self.error = exc
finally:
self.__running.set()
while True:
j = self.jobs.get()
if j == 'quit':
self.c.close()
self.conn.commit()
self.conn.close()
self.closing.set()
break
j.process_job()
# TODO: Recheck...
def exit(self):
self.__running.clear()
self.closing = Event()
self.jobs.put('quit')
self.closing.wait(1)
def _check_version(self):
"""Get db version."""
if not os.path.isfile(self.VERSION_FILE) or not os.path.getsize(
self.VERSION_FILE):
with io.open(self.VERSION_FILE, mode='w') as fp:
fp.write(to_str(DB_VERSION))
with io.open(self.VERSION_FILE, mode='r') as fp:
v = int(fp.read().strip())
return v
def _convert_db(self, v):
try:
return getattr(self, '_convertV{0:d}'.format(v))()
except Exception:
return False
# -- convert scripts start --
def _convert_v6(self):
return False
# -- convert scripts end --
def _create_tables(self):
"""Create tables for database."""
self.c.execute(
'CREATE TABLE IF NOT EXISTS "packages" ('
'"pid" INTEGER PRIMARY KEY AUTOINCREMENT, '
'"name" TEXT NOT NULL, '
'"folder" TEXT DEFAULT "" NOT NULL, '
'"site" TEXT DEFAULT "" NOT NULL, '
'"comment" TEXT DEFAULT "" NOT NULL, '
'"password" TEXT DEFAULT "" NOT NULL, '
'"added" INTEGER DEFAULT 0 NOT NULL,' # set by trigger
'"status" INTEGER DEFAULT 0 NOT NULL,'
'"tags" TEXT DEFAULT "" NOT NULL,'
'"shared" INTEGER DEFAULT 0 NOT NULL,'
'"packageorder" INTEGER DEFAULT -1 NOT NULL,' # inc by trigger
'"root" INTEGER DEFAULT -1 NOT NULL, '
'"owner" INTEGER NOT NULL, '
'FOREIGN KEY(owner) REFERENCES users(uid), '
'CHECK (root != pid)'
')')
self.c.execute('CREATE TRIGGER IF NOT EXISTS "insert_package" '
'AFTER INSERT ON "packages"'
'BEGIN '
'UPDATE packages SET added = strftime("%s", "now"), '
'packageorder = (SELECT max(p.packageorder) + 1 FROM '
'packages p WHERE p.root=new.root) '
'WHERE rowid = new.rowid;'
'END')
self.c.execute(
'CREATE TRIGGER IF NOT EXISTS "delete_package" '
'AFTER DELETE ON "packages"'
'BEGIN '
'DELETE FROM files WHERE package = old.pid;'
'UPDATE packages SET packageorder=packageorder-1 '
'WHERE packageorder > old.packageorder AND root=old.pid;'
'END')
self.c.execute('CREATE INDEX IF NOT EXISTS "package_index" ON '
'packages(root, owner)')
self.c.execute(
'CREATE INDEX IF NOT EXISTS "package_owner" ON packages(owner)')
self.c.execute('CREATE TABLE IF NOT EXISTS "files" ('
'"fid" INTEGER PRIMARY KEY AUTOINCREMENT, '
'"name" TEXT NOT NULL, '
'"size" INTEGER DEFAULT 0 NOT NULL, '
'"status" INTEGER DEFAULT 0 NOT NULL, '
'"media" INTEGER DEFAULT 1 NOT NULL,'
'"added" INTEGER DEFAULT 0 NOT NULL,'
'"fileorder" INTEGER DEFAULT -1 NOT NULL, '
'"url" TEXT DEFAULT "" NOT NULL, '
'"plugin" TEXT DEFAULT "" NOT NULL, '
'"hash" TEXT DEFAULT "" NOT NULL, '
'"dlstatus" INTEGER DEFAULT 0 NOT NULL, '
'"error" TEXT DEFAULT "" NOT NULL, '
'"package" INTEGER NOT NULL, '
'"owner" INTEGER NOT NULL, '
'FOREIGN KEY(owner) REFERENCES users(uid), '
'FOREIGN KEY(package) REFERENCES packages(id)'
')')
self.c.execute(
'CREATE INDEX IF NOT EXISTS "file_index" ON files(package, owner)')
self.c.execute(
'CREATE INDEX IF NOT EXISTS "file_owner" ON files(owner)')
self.c.execute(
'CREATE INDEX IF NOT EXISTS "file_plugin" ON files(plugin)')
self.c.execute('CREATE TRIGGER IF NOT EXISTS "insert_file" '
'AFTER INSERT ON "files"'
'BEGIN '
'UPDATE files SET added = strftime("%s", "now"), '
'fileorder = (SELECT max(f.fileorder) + 1 FROM files f '
'WHERE f.package=new.package) '
'WHERE rowid = new.rowid;'
'END')
self.c.execute('CREATE TABLE IF NOT EXISTS "collector" ('
'"owner" INTEGER NOT NULL, '
'"data" TEXT NOT NULL, '
'FOREIGN KEY(owner) REFERENCES users(uid), '
'PRIMARY KEY(owner) ON CONFLICT REPLACE'
') ')
self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ('
'"identifier" TEXT NOT NULL, '
'"key" TEXT NOT NULL, '
'"value" TEXT DEFAULT "", '
'PRIMARY KEY (identifier, key) ON CONFLICT REPLACE'
')')
self.c.execute(
'CREATE TABLE IF NOT EXISTS "users" ('
'"uid" INTEGER PRIMARY KEY AUTOINCREMENT, '
'"name" TEXT NOT NULL UNIQUE, '
'"email" TEXT DEFAULT "" NOT NULL, '
'"password" TEXT NOT NULL, '
'"role" INTEGER DEFAULT 0 NOT NULL, '
'"permission" INTEGER DEFAULT 0 NOT NULL, '
'"folder" TEXT DEFAULT "" NOT NULL, '
'"traffic" INTEGER DEFAULT -1 NOT NULL, '
'"dllimit" INTEGER DEFAULT -1 NOT NULL, '
'"dlquota" TEXT DEFAULT "" NOT NULL, '
'"hddquota" INTEGER DEFAULT -1 NOT NULL, '
'"template" TEXT DEFAULT "default" NOT NULL, '
'"user" INTEGER DEFAULT -1 NOT NULL, ' # set by trigger to self
'FOREIGN KEY(user) REFERENCES users(uid)'
')')
self.c.execute(
'CREATE INDEX IF NOT EXISTS "username_index" ON users(name)')
self.c.execute('CREATE TRIGGER IF NOT EXISTS "insert_user" AFTER '
'INSERT ON "users"'
'BEGIN '
'UPDATE users SET user = new.uid, folder=new.name '
'WHERE rowid = new.rowid;'
'END')
self.c.execute('CREATE TABLE IF NOT EXISTS "settings" ('
'"plugin" TEXT NOT NULL, '
'"user" INTEGER DEFAULT -1 NOT NULL, '
'"config" TEXT NOT NULL, '
'FOREIGN KEY(user) REFERENCES users(uid), '
'PRIMARY KEY (plugin, user) ON CONFLICT REPLACE'
')')
self.c.execute('CREATE TABLE IF NOT EXISTS "accounts" ('
'"aid" INTEGER PRIMARY KEY AUTOINCREMENT, '
'"plugin" TEXT NOT NULL, '
'"loginname" TEXT NOT NULL, '
'"owner" INTEGER NOT NULL, '
'"activated" INTEGER NOT NULL DEFAULT 1, '
'"password" TEXT DEFAULT "", '
'"shared" INTEGER NOT NULL DEFAULT 0, '
'"options" TEXT DEFAULT "", '
'FOREIGN KEY(owner) REFERENCES users(uid)'
')')
self.c.execute('CREATE INDEX IF NOT EXISTS "accounts_login" ON '
'accounts(plugin, loginname)')
self.c.execute('CREATE TABLE IF NOT EXISTS "stats" ('
'"id" INTEGER PRIMARY KEY AUTOINCREMENT, '
'"user" INTEGER NOT NULL, '
'"plugin" TEXT NOT NULL, '
'"time" INTEGER NOT NULL, '
'"premium" INTEGER DEFAULT 0 NOT NULL, '
'"amount" INTEGER DEFAULT 0 NOT NULL, '
'FOREIGN KEY(user) REFERENCES users(uid)'
')')
self.c.execute(
'CREATE INDEX IF NOT EXISTS "stats_time" ON stats(user, time)')
# try to lower ids
self.c.execute('SELECT max(fid) FROM files')
fid = self.c.fetchone()[0]
fid = int(fid) if fid else 0
self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?',
(fid, 'files'))
self.c.execute('SELECT max(pid) FROM packages')
pid = self.c.fetchone()[0]
pid = int(pid) if pid else 0
self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?',
(pid, 'packages'))
self.c.execute('VACUUM')
def create_cursor(self):
return self.conn.cursor()
@async
def commit(self):
self.conn.commit()
@queue
def sync_save(self):
self.conn.commit()
@async
def rollback(self):
self.conn.rollback()
def async (self, f, *args, **kwargs):
args = (self, ) + args
job = DatabaseJob(f, *args, **kwargs)
self.jobs.put(job)
def queue(self, f, *args, **kwargs):
# Raise previous error of initialization
if isinstance(self.error, Exception):
raise self.error
args = (self, ) + args
job = DatabaseJob(f, *args, **kwargs)
self.jobs.put(job)
# only wait when db is running
if self.running:
job.wait()
return job.result
@classmethod
def register_sub(cls, klass):
cls.subs.append(klass)
@classmethod
def unregister_sub(cls, klass):
cls.subs.remove(klass)
def __getattr__(self, attr):
for sub in DatabaseBackend.subs:
if hasattr(sub, attr):
return getattr(sub, attr)
raise AttributeError(attr)
class Core(object):
DEFAULT_CONFIGNAME = 'config.ini'
DEFAULT_LANGUAGE = 'english'
DEFAULT_USERNAME = '******'
DEFAULT_PASSWORD = '******'
DEFAULT_STORAGENAME = 'downloads'
@property
def version(self):
return __version__
@property
def version_info(self):
return __version_info__
@property
def running(self):
return self.__running.is_set()
def __init__(self, cfgdir, tmpdir, debug=None, restore=False):
self.__running = Event()
self.__do_restart = False
self.__do_exit = False
self._ = lambda x: x
self.cfgdir = fullpath(cfgdir)
self.tmpdir = fullpath(tmpdir)
os.chdir(self.cfgdir)
# if self.tmpdir not in sys.path:
# sys.path.append(self.tmpdir)
# if refresh:
# cleanpy(PACKDIR)
self.config = ConfigParser(self.DEFAULT_CONFIGNAME)
self.debug = self.config.get('log',
'debug') if debug is None else debug
self.log = LoggerFactory(self, self.debug)
self._init_database(restore)
self._init_managers()
self.request = self.req = RequestFactory(self)
self._init_api()
atexit.register(self.exit)
def _init_api(self):
from pyload.api import Api
self.api = Api(self)
def _init_database(self, restore):
from pyload.core.database import DatabaseBackend
from pyload.core.datatype import Permission, Role
# TODO: Move inside DatabaseBackend
newdb = not os.path.isfile(DatabaseBackend.DB_FILE)
self.db = DatabaseBackend(self)
self.db.setup()
if restore or newdb:
self.db.add_user(self.DEFAULT_USERNAME, self.DEFAULT_PASSWORD,
Role.Admin, Permission.All)
if restore:
self.log.warning(
self._('Restored default login credentials `admin|pyload`'))
def _init_managers(self):
from pyload.core.manager import (AccountManager, AddonManager,
EventManager, ExchangeManager,
FileManager, InfoManager,
PluginManager, TransferManager)
self.scheduler = sched.scheduler(time.time, time.sleep)
self.filemanager = self.files = FileManager(self)
self.pluginmanager = self.pgm = PluginManager(self)
self.exchangemanager = self.exm = ExchangeManager(self)
self.eventmanager = self.evm = EventManager(self)
self.accountmanager = self.acm = AccountManager(self)
self.infomanager = self.iom = InfoManager(self)
self.transfermanager = self.tsm = TransferManager(self)
# TODO: Remove builtins.ADDONMANAGER
builtins.ADDONMANAGER = self.addonmanager = self.adm = AddonManager(
self)
# self.remotemanager = self.rem = RemoteManager(self)
# self.servermanager = self.svm = ServerManager(self)
self.db.manager = self.files # ugly?
def _setup_permissions(self):
self.log.debug('Setup permissions...')
if os.name == 'nt':
return
change_group = self.config.get('permission', 'change_group')
change_user = self.config.get('permission', 'change_user')
if change_group:
try:
group = self.config.get('permission', 'group')
set_process_group(group)
except Exception as exc:
self.log.error(self._('Unable to change gid'))
self.log.error(exc, exc_info=self.debug)
if change_user:
try:
user = self.config.get('permission', 'user')
set_process_user(user)
except Exception as exc:
self.log.error(self._('Unable to change uid'))
self.log.error(exc, exc_info=self.debug)
def set_language(self, lang):
domain = 'core'
localedir = resource_filename(__package__, 'locale')
languages = (locale.locale_alias[lang.lower()].split('_', 1)[0], )
self._set_language(domain, localedir, languages)
def _set_language(self, *args, **kwargs):
trans = gettext.translation(*args, **kwargs)
try:
self._ = trans.ugettext
except AttributeError:
self._ = trans.gettext
def _setup_language(self):
self.log.debug('Setup language...')
lang = self.config.get('general', 'language')
if not lang:
lc = locale.getlocale()[0] or locale.getdefaultlocale()[0]
lang = lc.split('_', 1)[0] if lc else 'en'
try:
self.set_language(lang)
except IOError as exc:
self.log.error(exc, exc_info=self.debug)
self._set_language('core', fallback=True)
# def _setup_niceness(self):
# niceness = self.config.get('general', 'niceness')
# renice(niceness=niceness)
# ioniceness = int(self.config.get('general', 'ioniceness'))
# ionice(niceness=ioniceness)
def _setup_storage(self):
self.log.debug('Setup storage...')
storage_folder = self.config.get('general', 'storage_folder')
if storage_folder is None:
storage_folder = os.path.join(builtins.USERDIR,
self.DEFAULT_STORAGENAME)
self.log.info(self._('Storage: {0}'.format(storage_folder)))
makedirs(storage_folder, exist_ok=True)
avail_space = format.size(availspace(storage_folder))
self.log.info(
self._('Available storage space: {0}').format(avail_space))
def _setup_network(self):
self.log.debug('Setup network...')
# TODO: Move to accountmanager
self.log.info(self._('Activating accounts...'))
self.acm.load_accounts()
# self.scheduler.enter(0, 0, self.acm.load_accounts)
self.adm.activate_addons()
def run(self):
self.log.info('Welcome to pyLoad v{0}'.format(self.version))
if self.debug:
self.log.warning('*** DEBUG MODE ***')
try:
self.log.debug('Starting pyLoad...')
self.evm.fire('pyload:starting')
self.__running.set()
self._setup_language()
self._setup_permissions()
self.log.info(self._('Config directory: {0}').format(self.cfgdir))
self.log.info(self._('Cache directory: {0}').format(self.tmpdir))
self._setup_storage()
self._setup_network()
# self._setup_niceness()
# # some memory stats
# from guppy import hpy
# hp=hpy()
# print(hp.heap())
# import objgraph
# objgraph.show_most_common_types(limit=30)
# import memdebug
# memdebug.start(8002)
# from meliae import scanner
# scanner.dump_all_objects(os.path.join(PACKDIR, 'objs.json'))
self.log.debug('pyLoad is up and running')
self.evm.fire('pyload:started')
self.tsm.pause = False # NOTE: Recheck...
while True:
self.__running.wait()
self.tsm.work()
self.iom.work()
self.exm.work()
if self.__do_restart:
raise Restart
if self.__do_exit:
raise Exit
self.scheduler.run()
time.sleep(1)
except Restart:
self.restart()
except (Exit, KeyboardInterrupt, SystemExit):
self.exit()
except Exception as exc:
self.log.critical(exc, exc_info=True)
self.exit()
def _remove_loggers(self):
for handler in self.log.handlers:
with closing(handler) as hdlr:
self.log.removeHandler(hdlr)
def restart(self):
self.stop()
self.log.info(self._('Restarting pyLoad...'))
self.evm.fire('pyload:restarting')
self.run()
def exit(self):
self.stop()
self.log.info(self._('Exiting pyLoad...'))
self.tsm.exit()
self.db.exit() # NOTE: Why here?
self._remove_loggers()
# if cleanup:
# self.log.info(self._("Deleting temp files..."))
# remove(self.tmpdir, ignore_errors=True)
def stop(self):
try:
self.log.debug('Stopping pyLoad...')
self.evm.fire('pyload:stopping')
self.adm.deactivate_addons()
self.api.stop_all_downloads()
finally:
self.files.sync_save()
self.__running.clear()
self.evm.fire('pyload:stopped')
class Core(object):
DEFAULT_CONFIGNAME = 'config.ini'
DEFAULT_LANGUAGE = 'english'
DEFAULT_USERNAME = '******'
DEFAULT_PASSWORD = '******'
DEFAULT_STORAGENAME = 'downloads'
@property
def version(self):
return __version__
@property
def version_info(self):
return __version_info__
@property
def running(self):
return self.__running.is_set()
def __init__(self, cfgdir, tmpdir, debug=None, restore=False):
self.__running = Event()
self.__do_restart = False
self.__do_exit = False
self._ = lambda x: x
self.cfgdir = fullpath(cfgdir)
self.tmpdir = fullpath(tmpdir)
os.chdir(self.cfgdir)
# if self.tmpdir not in sys.path:
# sys.path.append(self.tmpdir)
# if refresh:
# cleanpy(PACKDIR)
self.config = ConfigParser(self.DEFAULT_CONFIGNAME)
self.debug = self.config.get(
'log', 'debug') if debug is None else debug
self.log = LoggerFactory(self, self.debug)
self._init_database(restore)
self._init_managers()
self.request = self.req = RequestFactory(self)
self._init_api()
atexit.register(self.exit)
def _init_api(self):
from pyload.api import Api
self.api = Api(self)
def _init_database(self, restore):
from pyload.core.database import DatabaseBackend
from pyload.core.datatype import Permission, Role
# TODO: Move inside DatabaseBackend
newdb = not os.path.isfile(DatabaseBackend.DB_FILE)
self.db = DatabaseBackend(self)
self.db.setup()
if restore or newdb:
self.db.add_user(
self.DEFAULT_USERNAME, self.DEFAULT_PASSWORD, Role.Admin,
Permission.All)
if restore:
self.log.warning(
self._('Restored default login credentials `admin|pyload`'))
def _init_managers(self):
from pyload.core.manager import (
AccountManager, AddonManager, EventManager, ExchangeManager,
FileManager, InfoManager, PluginManager, TransferManager)
self.scheduler = sched.scheduler(time.time, time.sleep)
self.filemanager = self.files = FileManager(self)
self.pluginmanager = self.pgm = PluginManager(self)
self.exchangemanager = self.exm = ExchangeManager(self)
self.eventmanager = self.evm = EventManager(self)
self.accountmanager = self.acm = AccountManager(self)
self.infomanager = self.iom = InfoManager(self)
self.transfermanager = self.tsm = TransferManager(self)
# TODO: Remove builtins.ADDONMANAGER
builtins.ADDONMANAGER = self.addonmanager = self.adm = AddonManager(
self)
# self.remotemanager = self.rem = RemoteManager(self)
# self.servermanager = self.svm = ServerManager(self)
self.db.manager = self.files # ugly?
def _setup_permissions(self):
self.log.debug('Setup permissions...')
if os.name == 'nt':
return
change_group = self.config.get('permission', 'change_group')
change_user = self.config.get('permission', 'change_user')
if change_group:
try:
group = self.config.get('permission', 'group')
set_process_group(group)
except Exception as exc:
self.log.error(self._('Unable to change gid'))
self.log.error(exc, exc_info=self.debug)
if change_user:
try:
user = self.config.get('permission', 'user')
set_process_user(user)
except Exception as exc:
self.log.error(self._('Unable to change uid'))
self.log.error(exc, exc_info=self.debug)
def set_language(self, lang):
domain = 'core'
localedir = resource_filename(__package__, 'locale')
languages = (locale.locale_alias[lang.lower()].split('_', 1)[0],)
self._set_language(domain, localedir, languages)
def _set_language(self, *args, **kwargs):
trans = gettext.translation(*args, **kwargs)
try:
self._ = trans.ugettext
except AttributeError:
self._ = trans.gettext
def _setup_language(self):
self.log.debug('Setup language...')
lang = self.config.get('general', 'language')
if not lang:
lc = locale.getlocale()[0] or locale.getdefaultlocale()[0]
lang = lc.split('_', 1)[0] if lc else 'en'
try:
self.set_language(lang)
except IOError as exc:
self.log.error(exc, exc_info=self.debug)
self._set_language('core', fallback=True)
# def _setup_niceness(self):
# niceness = self.config.get('general', 'niceness')
# renice(niceness=niceness)
# ioniceness = int(self.config.get('general', 'ioniceness'))
# ionice(niceness=ioniceness)
def _setup_storage(self):
self.log.debug('Setup storage...')
storage_folder = self.config.get('general', 'storage_folder')
if storage_folder is None:
storage_folder = os.path.join(
builtins.USERDIR, self.DEFAULT_STORAGENAME)
self.log.info(self._('Storage: {0}'.format(storage_folder)))
makedirs(storage_folder, exist_ok=True)
avail_space = format.size(availspace(storage_folder))
self.log.info(
self._('Available storage space: {0}').format(avail_space))
def _setup_network(self):
self.log.debug('Setup network...')
# TODO: Move to accountmanager
self.log.info(self._('Activating accounts...'))
self.acm.load_accounts()
# self.scheduler.enter(0, 0, self.acm.load_accounts)
self.adm.activate_addons()
def run(self):
self.log.info('Welcome to pyLoad v{0}'.format(self.version))
if self.debug:
self.log.warning('*** DEBUG MODE ***')
try:
self.log.debug('Starting pyLoad...')
self.evm.fire('pyload:starting')
self.__running.set()
self._setup_language()
self._setup_permissions()
self.log.info(self._('Config directory: {0}').format(self.cfgdir))
self.log.info(self._('Cache directory: {0}').format(self.tmpdir))
self._setup_storage()
self._setup_network()
# self._setup_niceness()
# # some memory stats
# from guppy import hpy
# hp=hpy()
# print(hp.heap())
# import objgraph
# objgraph.show_most_common_types(limit=30)
# import memdebug
# memdebug.start(8002)
# from meliae import scanner
# scanner.dump_all_objects(os.path.join(PACKDIR, 'objs.json'))
self.log.debug('pyLoad is up and running')
self.evm.fire('pyload:started')
self.tsm.pause = False # NOTE: Recheck...
while True:
self.__running.wait()
self.tsm.work()
self.iom.work()
self.exm.work()
if self.__do_restart:
raise Restart
if self.__do_exit:
raise Exit
self.scheduler.run()
time.sleep(1)
except Restart:
self.restart()
except (Exit, KeyboardInterrupt, SystemExit):
self.exit()
except Exception as exc:
self.log.critical(exc, exc_info=True)
self.exit()
def _remove_loggers(self):
for handler in self.log.handlers:
with closing(handler) as hdlr:
self.log.removeHandler(hdlr)
def restart(self):
self.stop()
self.log.info(self._('Restarting pyLoad...'))
self.evm.fire('pyload:restarting')
self.run()
def exit(self):
self.stop()
self.log.info(self._('Exiting pyLoad...'))
self.tsm.exit()
self.db.exit() # NOTE: Why here?
self._remove_loggers()
# if cleanup:
# self.log.info(self._("Deleting temp files..."))
# remove(self.tmpdir, ignore_errors=True)
def stop(self):
try:
self.log.debug('Stopping pyLoad...')
self.evm.fire('pyload:stopping')
self.adm.deactivate_addons()
self.api.stop_all_downloads()
finally:
self.files.sync_save()
self.__running.clear()
self.evm.fire('pyload:stopped')