def _gen_fileconfig(self):
config = OrderedDict({self.SELF_SECTION: OrderedDict()})
for name, item in self.loweritems():
if self.is_section(name):
fc = self._to_fileconfig(item, name)
config.update(fc)
else:
fv = self._to_filevalue(item.get())
config[self.SELF_SECTION][name] = fv
return config
def __init__(self, file, version):
self.filename = file
self.version = convert.to_version(version, version)
# Meta data information
self.config = OrderedDict()
# The actual config values
self.values = {}
self.check_version()
self.load_default()
self.parse(self.filename)
def __init__(self, core, parser):
# No __init__ call to super class is needed!
self.pyload = core
# The config parser, holding the core config
self.parser = parser
# similar to parser, separated meta data and values
self.config = OrderedDict()
# Value cache for multiple user configs
# Values are populated from db on first access
# Entries are saved as (user, section) keys
self.values = {}
def get_all_packages(self, root=None, owner=None, tags=None):
"""
Return dict with package information
:param root: optional root to filter
:param owner: optional user id
:param tags: optional tag list
"""
qry = (
'SELECT pid, name, folder, root, owner, site, comment, password, '
'added, tags, status, shared, packageorder FROM packages{} '
'ORDER BY root, packageorder')
if root is None:
stats = self.get_package_stats(owner=owner)
if owner is None:
self.c.execute(qry.format(""))
else:
self.c.execute(qry.format(" WHERE owner=?"), (owner, ))
else:
stats = self.get_package_stats(root=root, owner=owner)
if owner is None:
self.c.execute(qry.format(' WHERE root=? OR pid=?'),
(root, root))
else:
self.c.execute(
qry.format(' WHERE (root=? OR pid=?) AND owner=?'),
(root, root, owner))
data = OrderedDict()
for r in self.c.fetchall():
data[r[0]] = PackageInfo(r[0], r[1], r[2], r[3], r[4], r[5], r[6],
r[7], r[8], r[9].split(","), r[10], r[11],
r[12], stats.get(r[0], _zero_stats))
return data
def get_all_files(self, package=None, search=None, state=None, owner=None):
"""
Return dict with file information
:param package: optional package to filter out
:param search: or search string for file name
:param unfinished: filter by dlstatus not finished
:param owner: only specific owner
"""
qry = ('SELECT fid, name, owner, size, status, media, added, '
'fileorder, url, plugin, hash, dlstatus, error, package '
'FROM files WHERE ')
arg = []
if state is not None and state != DownloadState.All:
qry += "dlstatus IN ({0}) AND ".format(statestring(state))
if owner is not None:
qry += 'owner=? AND '
arg.append(owner)
if package is not None:
arg.append(package)
qry += 'package=? AND '
if search is not None:
search = "%%{0}%%".format(search.strip("%"))
arg.append(search)
qry += "name LIKE ? "
# make qry valid
if qry.endswith("WHERE "):
qry = qry[:-6]
if qry.endswith("AND "):
qry = qry[:-4]
self.c.execute(qry + "ORDER BY package, fileorder", arg)
data = OrderedDict()
for r in self.c.fetchall():
finfo = FileInfo(r[0], r[1], r[13], r[2],
r[3], r[4], r[5], r[6], r[7])
if r[11] > 0: # dl status != NA
finfo.download = DownloadInfo(
r[8],
r[9],
r[10],
r[11],
self.__manager.status_msg[r[11]],
r[12])
data[r[0]] = finfo
return data
def _to_fileconfig(self, section, section_name):
config = OrderedDict()
for name, item in section.loweritems():
if section.is_section(name):
sub_name = '{0}{1}{2}'.format(section_name, self.SECTION_SEP,
name)
fc = self._to_fileconfig(item, sub_name)
config.update(fc)
else:
fv = self._to_filevalue(item.get())
config.setdefault(section_name, OrderedDict())[name] = fv
return config
def add_config_section(self, section, label, desc, expl, config):
"""
Adds a section to the config.
`config` is a list of config tuple as used in plugin api defined as
the order of the config elements is preserved with OrderedDict
"""
d = OrderedDict()
for entry in config:
name, data = to_configdata(entry)
d[name] = data
data = SectionTuple(gettext(label), gettext(desc), gettext(expl), d)
self.config[section] = data
def __init__(self, core, parser):
# No __init__ call to super class is needed!
self.pyload = core
# The config parser, holding the core config
self.parser = parser
# similar to parser, separated meta data and values
self.config = OrderedDict()
# Value cache for multiple user configs
# Values are populated from db on first access
# Entries are saved as (user, section) keys
self.values = {}
def __init__(self, core):
BaseManager.__init__(self, core)
self.tasks = OrderedDict() # task store, for all outgoing tasks
self.last_clients = {}
self.ids = 0 # uniue interaction ids
class ExchangeManager(BaseManager):
"""
Class that gives ability to interact with the user.
Arbitrary tasks with predefined output and input types can be set off
"""
# number of seconds a client is classified as active
CLIENT_THRESHOLD = 60
NOTIFICATION_TIMEOUT = 60 * 60 * 30
MAX_NOTIFICATIONS = 50
def __init__(self, core):
BaseManager.__init__(self, core)
self.tasks = OrderedDict() # task store, for all outgoing tasks
self.last_clients = {}
self.ids = 0 # uniue interaction ids
def is_client_connected(self, user):
return self.last_clients.get(user,
0) + self.CLIENT_THRESHOLD > time.time()
@lock
def work(self):
# old notifications will be removed
for n in (k for k, v in self.tasks.items() if v.timed_out()):
del self.tasks[n]
# keep notifications count limited
n = [
k for k, v in self.tasks.items()
if v.type == Interaction.Notification
][::-1]
for v in n[:self.MAX_NOTIFICATIONS]:
del self.tasks[v]
@lock
def create_notification(self,
title,
content,
desc="",
plugin="",
owner=None):
"""
Creates and queues a new Notification
:param title: short title
:param content: text content
:param desc: short form of the notification
:param plugin: plugin name
:return: :class:`InteractionTask`
"""
task = InteractionTask(self.ids,
Interaction.Notification,
Input(InputType.Str, None, content),
title,
desc,
plugin,
owner=owner)
self.ids += 1
self.queue_task(task)
return task
@lock
def create_query_task(self, input, desc, plugin="", owner=None):
# input type was given, create a input widget
if isinstance(input, int):
input = Input(input)
if not isinstance(input, Input):
raise TypeError("'Input' class expected not '{0}'".format(
type(input)))
task = InteractionTask(self.ids,
Interaction.Query,
input,
self._("Query"),
desc,
plugin,
owner=owner)
self.ids += 1
self.queue_task(task)
return task
@lock
def create_captcha_task(self,
img,
format,
filename,
plugin="",
type_=InputType.Str,
owner=None):
"""
Createss a new captcha task.
:param img: image content (not base encoded)
:param format: img format
:param type_: :class:`InputType`
:return:
"""
if type_ == 'textual':
type_ = InputType.Str
elif type_ == 'positional':
type_ = InputType.Click
input = Input(type_, data=[standard_b64encode(img), format, filename])
# TODO: title desc plugin
task = InteractionTask(self.ids,
Interaction.Captcha,
input,
self._("Captcha request"),
self._("Please solve the captcha"),
plugin,
owner=owner)
self.ids += 1
self.queue_task(task)
return task
@lock
def remove_task(self, task):
if task.iid in self.tasks:
del self.tasks[task.iid]
self.__pyload.evm.fire("interaction:deleted", task.iid)
@lock
def get_task_by_id(self, iid):
return self.tasks.get(iid, None)
@lock
def get_tasks(self, user, mode=Interaction.All):
# update last active clients
self.last_clients[user] = time.time()
# filter current mode
tasks = [
tsk for tsk in self.tasks.values()
if mode == Interaction.All or bitset(tsk.type, mode)
]
# filter correct user / or shared
tasks = [
tsk for tsk in tasks
if user is None or user == tsk.owner or tsk.shared
]
return tasks
def is_task_waiting(self, user, mode=Interaction.All):
tasks = [
tsk for tsk in self.get_tasks(user, mode)
if not tsk.type == Interaction.Notification or not tsk.seen
]
return len(tasks) > 0
def queue_task(self, task):
cli = self.is_client_connected(task.owner)
# set waiting times based on threshold
if cli:
task.set_waiting(self.CLIENT_THRESHOLD)
else: # TODO: higher threshold after client connects?
task.set_waiting(self.CLIENT_THRESHOLD // 3)
if task.type == Interaction.Notification:
# notifications are valid for 30h
task.set_waiting(self.NOTIFICATION_TIMEOUT)
for plugin in self.__pyload.adm.active_plugins():
try:
plugin.new_interaction_task(task)
except Exception:
# self.__pyload.print_exc()
pass
self.tasks[task.iid] = task
self.__pyload.evm.fire("interaction:added", task)
class ConfigManager(ConfigParser):
"""
Manages the core config and configs for addons and single user.
Has similar interface to ConfigParser.
"""
__slots__ = ['config', 'parser', 'pyload', 'values']
def __init__(self, core, parser):
# No __init__ call to super class is needed!
self.pyload = core
# The config parser, holding the core config
self.parser = parser
# similar to parser, separated meta data and values
self.config = OrderedDict()
# Value cache for multiple user configs
# Values are populated from db on first access
# Entries are saved as (user, section) keys
self.values = {}
# TODO: similar to a cache, could be deleted periodically
def save(self):
self.parser.save()
@convertkeyerror
def get(self, section, option, user=None):
"""
Get config value, core config only available for admins.
if user is not valid default value will be returned.
"""
# Core config loaded from parser, when no user is given or he is admin
if section in self.parser and user is None:
return self.parser.get(section, option)
else:
# We need the id and not the instance
# Will be None for admin user and so the same as internal access
try:
# Check if this config exists
# Configs without meta data can not be loaded!
assert self.config[section].config[option]
return self.load_values(user, section)[option]
except KeyError:
pass #: Returns default value later
return self.config[section].config[option].input.default_value
def load_values(self, user, section):
if (user, section) not in self.values:
conf = self.pyload.db.load_config(section, user)
try:
self.values[user, section] = json.loads(conf) if conf else {}
except ValueError: #: Something did go wrong when parsing
self.values[user, section] = {}
# self.pyload.print_exc()
return self.values[user, section]
@convertkeyerror
def set(self, section, option, value, sync=True, user=None):
"""
Set config value.
"""
changed = False
if section in self.parser and user is None:
changed = self.parser.set(section, option, value, sync)
else:
data = self.config[section].config[option]
value = from_string(value, data.input.type)
old_value = self.get(section, option)
# Values will always be saved to db, sync is ignored
if value != old_value:
changed = True
self.values[user, section][option] = value
if sync:
self.save_values(user, section)
if changed:
self.pyload.evm.fire("config:changed", section, option, value)
return changed
def save_values(self, user, section):
if section in self.parser and user is None:
self.save()
elif (user, section) in self.values:
self.pyload.db.save_config(section,
json.dumps(self.values[user, section]),
user)
def delete(self, section, user=None):
"""
Deletes values saved in db and cached values for given user, NOT meta data.
Does not trigger an error when nothing was deleted.
"""
if (user, section) in self.values:
del self.values[user, section]
self.pyload.db.delete_config(section, user)
self.pyload.evm.fire("config:deleted", section, user)
def iter_core_sections(self):
return self.parser.iter_sections()
def iter_sections(self, user=None):
"""
Yields: section, metadata, values.
"""
values = self.pyload.db.load_configs_for_user(user)
# Every section needs to be json decoded
for section, data in values.items():
try:
values[section] = json.loads(data) if data else {}
except ValueError:
values[section] = {}
# self.pyload.print_exc()
for name, config in self.config.items():
yield name, config, values[name] if name in values else {}
def get_section(self, section, user=None):
if section in self.parser and user is None:
return self.parser.get_section(section)
values = self.load_values(user, section)
return self.config.get(section), values
def __init__(self, core):
self.lock = Lock()
self.pyload = core
self.tasks = OrderedDict() #: task store, for all outgoing tasks
self.last_clients = {}
self.ids = 0 #: uniue interaction ids
class InteractionManager(object):
"""
Class that gives ability to interact with the user.
Arbitrary tasks with predefined output and input types can be set off
"""
__slots__ = [
'CLIENT_THRESHOLD',
'MAX_NOTIFICATIONS',
'NOTIFICATION_TIMEOUT',
'ids',
'last_clients',
'lock',
'pyload',
'tasks']
# number of seconds a client is classified as active
CLIENT_THRESHOLD = 60
NOTIFICATION_TIMEOUT = 60 * 60 * 30
MAX_NOTIFICATIONS = 50
def __init__(self, core):
self.lock = Lock()
self.pyload = core
self.tasks = OrderedDict() #: task store, for all outgoing tasks
self.last_clients = {}
self.ids = 0 #: uniue interaction ids
def is_client_connected(self, user):
return self.last_clients.get(user, 0) + self.CLIENT_THRESHOLD > time()
@lock
def work(self):
# old notifications will be removed
for n in [k for k, v in self.tasks.items() if v.timed_out()]:
del self.tasks[n]
# keep notifications count limited
n = [k for k, v in self.tasks.items() if v.type == Interaction.Notification][
::-1]
for v in n[:self.MAX_NOTIFICATIONS]:
del self.tasks[v]
@lock
def create_notification(self, title, content,
desc="", plugin="", owner=None):
"""
Creates and queues a new Notification
:param title: short title
:param content: text content
:param desc: short form of the notification
:param plugin: plugin name
:return: :class:`InteractionTask`
"""
task = InteractionTask(self.ids, Interaction.Notification, Input(InputType.Text, None, content), title, desc, plugin,
owner=owner)
self.ids += 1
self.queue_task(task)
return task
@lock
def create_query_task(self, input, desc, plugin="", owner=None):
# input type was given, create a input widget
if isinstance(input, int):
input = Input(input)
if not isinstance(input, Input):
raise TypeError(
"'Input' class expected not '{0}'".format(type(input)))
task = InteractionTask(self.ids, Interaction.Query, input, _(
"Query"), desc, plugin, owner=owner)
self.ids += 1
self.queue_task(task)
return task
@lock
def create_captcha_task(self, img, format, filename,
plugin="", type_=InputType.Text, owner=None):
"""
Createss a new captcha task.
:param img: image content (not base encoded)
:param format: img format
:param type_: :class:`InputType`
:return:
"""
if type_ == 'textual':
type_ = InputType.Text
elif type_ == 'positional':
type_ = InputType.Click
input = Input(type_, data=[standard_b64encode(img), format, filename])
# TODO: title desc plugin
task = InteractionTask(self.ids, Interaction.Captcha, input,
_("Captcha request"), _("Please solve the captcha"), plugin, owner=owner)
self.ids += 1
self.queue_task(task)
return task
@lock
def remove_task(self, task):
if task.iid in self.tasks:
del self.tasks[task.iid]
self.pyload.evm.fire("interaction:deleted", task.iid)
@lock
def get_task_by_id(self, iid):
return self.tasks.get(iid, None)
@lock
def get_tasks(self, user, mode=Interaction.All):
# update last active clients
self.last_clients[user] = time()
# filter current mode
tasks = [t for t in self.tasks.values() if mode ==
Interaction.All or bitset(t.type, mode)]
# filter correct user / or shared
tasks = [t for t in tasks if user is None or user == t.owner or t.shared]
return tasks
def is_task_waiting(self, user, mode=Interaction.All):
tasks = [t for t in self.get_tasks(
user, mode) if not t.type == Interaction.Notification or not t.seen]
return len(tasks) > 0
def queue_task(self, task):
cli = self.is_client_connected(task.owner)
# set waiting times based on threshold
if cli:
task.set_waiting(self.CLIENT_THRESHOLD)
else: # TODO: higher threshold after client connects?
task.set_waiting(self.CLIENT_THRESHOLD // 3)
if task.type == Interaction.Notification:
# notifications are valid for 30h
task.set_waiting(self.NOTIFICATION_TIMEOUT)
for plugin in self.pyload.adm.active_plugins():
try:
plugin.new_interaction_task(task)
except Exception:
# self.pyload.print_exc()
pass
self.tasks[task.iid] = task
self.pyload.evm.fire("interaction:added", task)
class ConfigManager(ConfigParser):
"""
Manages the core config and configs for addons and single user.
Has similar interface to ConfigParser.
"""
__slots__ = ['config', 'parser', 'pyload', 'values']
def __init__(self, core, parser):
# No __init__ call to super class is needed!
self.pyload = core
# The config parser, holding the core config
self.parser = parser
# similar to parser, separated meta data and values
self.config = OrderedDict()
# Value cache for multiple user configs
# Values are populated from db on first access
# Entries are saved as (user, section) keys
self.values = {}
# TODO: similar to a cache, could be deleted periodically
def save(self):
self.parser.save()
@convertkeyerror
def get(self, section, option, user=None):
"""
Get config value, core config only available for admins.
if user is not valid default value will be returned.
"""
# Core config loaded from parser, when no user is given or he is admin
if section in self.parser and user is None:
return self.parser.get(section, option)
else:
# We need the id and not the instance
# Will be None for admin user and so the same as internal access
try:
# Check if this config exists
# Configs without meta data can not be loaded!
assert self.config[section].config[option]
return self.load_values(user, section)[option]
except KeyError:
pass #: Returns default value later
return self.config[section].config[option].input.default_value
def load_values(self, user, section):
if (user, section) not in self.values:
conf = self.pyload.db.load_config(section, user)
try:
self.values[user, section] = json.loads(conf) if conf else {}
except ValueError: #: Something did go wrong when parsing
self.values[user, section] = {}
# self.pyload.print_exc()
return self.values[user, section]
@convertkeyerror
def set(self, section, option, value, sync=True, user=None):
"""
Set config value.
"""
changed = False
if section in self.parser and user is None:
changed = self.parser.set(section, option, value, sync)
else:
data = self.config[section].config[option]
value = from_string(value, data.input.type)
old_value = self.get(section, option)
# Values will always be saved to db, sync is ignored
if value != old_value:
changed = True
self.values[user, section][option] = value
if sync:
self.save_values(user, section)
if changed:
self.pyload.evm.fire("config:changed", section, option, value)
return changed
def save_values(self, user, section):
if section in self.parser and user is None:
self.save()
elif (user, section) in self.values:
self.pyload.db.save_config(section, json.dumps(
self.values[user, section]), user)
def delete(self, section, user=None):
"""
Deletes values saved in db and cached values for given user, NOT meta data.
Does not trigger an error when nothing was deleted.
"""
if (user, section) in self.values:
del self.values[user, section]
self.pyload.db.delete_config(section, user)
self.pyload.evm.fire("config:deleted", section, user)
def iter_core_sections(self):
return self.parser.iter_sections()
def iter_sections(self, user=None):
"""
Yields: section, metadata, values.
"""
values = self.pyload.db.load_configs_for_user(user)
# Every section needs to be json decoded
for section, data in values.items():
try:
values[section] = json.loads(data) if data else {}
except ValueError:
values[section] = {}
# self.pyload.print_exc()
for name, config in self.config.items():
yield name, config, values[name] if name in values else {}
def get_section(self, section, user=None):
if section in self.parser and user is None:
return self.parser.get_section(section)
values = self.load_values(user, section)
return self.config.get(section), values
class ConfigParser(object):
"""
Holds and manages the configuration + meta data for config read from file.
"""
__slots__ = ['_RE_VERSION', 'config', 'filename', 'values', 'version']
_RE_VERSION = re.compile(r'\s*version\s*:\s*(\d+\.\d+\.\d+)', flags=re.I)
def __init__(self, file, version):
self.filename = file
self.version = convert.to_version(version, version)
# Meta data information
self.config = OrderedDict()
# The actual config values
self.values = {}
self.check_version()
self.load_default()
self.parse(self.filename)
def load_default(self):
make_config(self)
def check_version(self):
"""
Determines if config needs to be deleted.
"""
if not os.path.exists(self.filename):
with io.open(self.filename, mode='wb') as fp:
fp.write("version: {0}".format(
convert.from_version(self.version)))
return None
version = None
try:
with io.open(self.filename, mode='rb') as fp:
m = self._RE_VERSION.match(fp.readline())
version = convert.to_version(m.group(1))
except (AttributeError, TypeError):
pass
if not self.version or version[:-1] != self.version[:-1]:
os.rename(self.filename, "{0}.old".format(self.filename))
with io.open(self.filename, mode='wb') as fp:
fp.write("version: {0}".format(convert.from_version(self.version)))
def parse(self, filename):
"""
Read config values from file.
"""
with io.open(filename, mode='rb') as fp:
# save the current section
section = ""
fp.readline()
for line in iter(fp.readline, ""):
line = line.strip()
# comment line, different variants
if not line or line.startswith("#") or line.startswith(
"//") or line.startswith("|"):
continue
if line.startswith("["):
section = line.replace("[", "").replace("]", "")
if section not in self.config:
print("Unrecognized section", section)
section = ""
else:
name, non, value = line.rpartition("=")
name = name.strip()
value = value.strip()
if not section:
print("Value without section", name)
continue
if name in self.config[section].config:
self.set(section, name, value, sync=False)
else:
print("Unrecognized option", section, name)
def save(self):
"""
Saves config to filename.
"""
configs = []
with io.open(self.filename, mode='wb') as fp:
configs.append(fp)
os.chmod(self.filename, 0o600)
fp.write("version: {0}\n\n".format(
convert.from_version(self.version)))
for section, data in self.config.items():
fp.write("[{0}]\n".format(section))
for option, data in data.config.items():
value = self.get(section, option)
fp.write("{0} = {1}\n".format(option, value))
fp.write("\n")
def __getitem__(self, section):
"""
Provides dictionary like access: c['section']['option'].
"""
return ConfigSection(self, section)
def __contains__(self, section):
"""
Checks if parser contains section.
"""
return section in self.config
def get(self, section, option):
"""
Get value or default.
"""
try:
return self.values[section][option]
except KeyError:
return self.config[section].config[option].input.default_value
def set(self, section, option, value, sync=True):
"""
Set value.
"""
data = self.config[section].config[option]
value = from_string(value, data.input.type)
old_value = self.get(section, option)
# only save when different values
if value != old_value:
if section not in self.values:
self.values[section] = {}
self.values[section][option] = value
if sync:
self.save()
return True
return False
def get_meta_data(self, section, option):
"""
Get all config data for an option.
"""
return self.config[section].config[option]
def iter_sections(self):
"""
Yields section, config info, values, for all sections.
"""
for name, config in self.config.items():
yield name, config, self.values[
name] if name in self.values else {}
def get_section(self, section):
"""
Retrieves single config as tuple (section, values).
"""
return self.config[
section], self.values[section] if section in self.values else {}
def add_config_section(self, section, label, desc, expl, config):
"""
Adds a section to the config.
`config` is a list of config tuple as used in plugin api defined as
the order of the config elements is preserved with OrderedDict
"""
d = OrderedDict()
for entry in config:
name, data = to_configdata(entry)
d[name] = data
data = SectionTuple(gettext(label), gettext(desc), gettext(expl), d)
self.config[section] = data