def _gen_fileconfig(self): config = OrderedDict({self.SELF_SECTION: OrderedDict()}) for name, item in self.loweritems(): if self.is_section(name): fc = self._to_fileconfig(item, name) config.update(fc) else: fv = self._to_filevalue(item.get()) config[self.SELF_SECTION][name] = fv return config
def __init__(self, file, version): self.filename = file self.version = convert.to_version(version, version) # Meta data information self.config = OrderedDict() # The actual config values self.values = {} self.check_version() self.load_default() self.parse(self.filename)
def __init__(self, core, parser): # No __init__ call to super class is needed! self.pyload = core # The config parser, holding the core config self.parser = parser # similar to parser, separated meta data and values self.config = OrderedDict() # Value cache for multiple user configs # Values are populated from db on first access # Entries are saved as (user, section) keys self.values = {}
def get_all_packages(self, root=None, owner=None, tags=None): """ Return dict with package information :param root: optional root to filter :param owner: optional user id :param tags: optional tag list """ qry = ( 'SELECT pid, name, folder, root, owner, site, comment, password, ' 'added, tags, status, shared, packageorder FROM packages{} ' 'ORDER BY root, packageorder') if root is None: stats = self.get_package_stats(owner=owner) if owner is None: self.c.execute(qry.format("")) else: self.c.execute(qry.format(" WHERE owner=?"), (owner, )) else: stats = self.get_package_stats(root=root, owner=owner) if owner is None: self.c.execute(qry.format(' WHERE root=? OR pid=?'), (root, root)) else: self.c.execute( qry.format(' WHERE (root=? OR pid=?) AND owner=?'), (root, root, owner)) data = OrderedDict() for r in self.c.fetchall(): data[r[0]] = PackageInfo(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9].split(","), r[10], r[11], r[12], stats.get(r[0], _zero_stats)) return data
def get_all_files(self, package=None, search=None, state=None, owner=None): """ Return dict with file information :param package: optional package to filter out :param search: or search string for file name :param unfinished: filter by dlstatus not finished :param owner: only specific owner """ qry = ('SELECT fid, name, owner, size, status, media, added, ' 'fileorder, url, plugin, hash, dlstatus, error, package ' 'FROM files WHERE ') arg = [] if state is not None and state != DownloadState.All: qry += "dlstatus IN ({0}) AND ".format(statestring(state)) if owner is not None: qry += 'owner=? AND ' arg.append(owner) if package is not None: arg.append(package) qry += 'package=? AND ' if search is not None: search = "%%{0}%%".format(search.strip("%")) arg.append(search) qry += "name LIKE ? " # make qry valid if qry.endswith("WHERE "): qry = qry[:-6] if qry.endswith("AND "): qry = qry[:-4] self.c.execute(qry + "ORDER BY package, fileorder", arg) data = OrderedDict() for r in self.c.fetchall(): finfo = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) if r[11] > 0: # dl status != NA finfo.download = DownloadInfo( r[8], r[9], r[10], r[11], self.__manager.status_msg[r[11]], r[12]) data[r[0]] = finfo return data
def _to_fileconfig(self, section, section_name): config = OrderedDict() for name, item in section.loweritems(): if section.is_section(name): sub_name = '{0}{1}{2}'.format(section_name, self.SECTION_SEP, name) fc = self._to_fileconfig(item, sub_name) config.update(fc) else: fv = self._to_filevalue(item.get()) config.setdefault(section_name, OrderedDict())[name] = fv return config
def add_config_section(self, section, label, desc, expl, config): """ Adds a section to the config. `config` is a list of config tuple as used in plugin api defined as the order of the config elements is preserved with OrderedDict """ d = OrderedDict() for entry in config: name, data = to_configdata(entry) d[name] = data data = SectionTuple(gettext(label), gettext(desc), gettext(expl), d) self.config[section] = data
def __init__(self, core, parser): # No __init__ call to super class is needed! self.pyload = core # The config parser, holding the core config self.parser = parser # similar to parser, separated meta data and values self.config = OrderedDict() # Value cache for multiple user configs # Values are populated from db on first access # Entries are saved as (user, section) keys self.values = {}
def __init__(self, core): BaseManager.__init__(self, core) self.tasks = OrderedDict() # task store, for all outgoing tasks self.last_clients = {} self.ids = 0 # uniue interaction ids
class ExchangeManager(BaseManager): """ Class that gives ability to interact with the user. Arbitrary tasks with predefined output and input types can be set off """ # number of seconds a client is classified as active CLIENT_THRESHOLD = 60 NOTIFICATION_TIMEOUT = 60 * 60 * 30 MAX_NOTIFICATIONS = 50 def __init__(self, core): BaseManager.__init__(self, core) self.tasks = OrderedDict() # task store, for all outgoing tasks self.last_clients = {} self.ids = 0 # uniue interaction ids def is_client_connected(self, user): return self.last_clients.get(user, 0) + self.CLIENT_THRESHOLD > time.time() @lock def work(self): # old notifications will be removed for n in (k for k, v in self.tasks.items() if v.timed_out()): del self.tasks[n] # keep notifications count limited n = [ k for k, v in self.tasks.items() if v.type == Interaction.Notification ][::-1] for v in n[:self.MAX_NOTIFICATIONS]: del self.tasks[v] @lock def create_notification(self, title, content, desc="", plugin="", owner=None): """ Creates and queues a new Notification :param title: short title :param content: text content :param desc: short form of the notification :param plugin: plugin name :return: :class:`InteractionTask` """ task = InteractionTask(self.ids, Interaction.Notification, Input(InputType.Str, None, content), title, desc, plugin, owner=owner) self.ids += 1 self.queue_task(task) return task @lock def create_query_task(self, input, desc, plugin="", owner=None): # input type was given, create a input widget if isinstance(input, int): input = Input(input) if not isinstance(input, Input): raise TypeError("'Input' class expected not '{0}'".format( type(input))) task = InteractionTask(self.ids, Interaction.Query, input, self._("Query"), desc, plugin, owner=owner) self.ids += 1 self.queue_task(task) return task @lock def create_captcha_task(self, img, format, filename, plugin="", type_=InputType.Str, owner=None): """ Createss a new captcha task. :param img: image content (not base encoded) :param format: img format :param type_: :class:`InputType` :return: """ if type_ == 'textual': type_ = InputType.Str elif type_ == 'positional': type_ = InputType.Click input = Input(type_, data=[standard_b64encode(img), format, filename]) # TODO: title desc plugin task = InteractionTask(self.ids, Interaction.Captcha, input, self._("Captcha request"), self._("Please solve the captcha"), plugin, owner=owner) self.ids += 1 self.queue_task(task) return task @lock def remove_task(self, task): if task.iid in self.tasks: del self.tasks[task.iid] self.__pyload.evm.fire("interaction:deleted", task.iid) @lock def get_task_by_id(self, iid): return self.tasks.get(iid, None) @lock def get_tasks(self, user, mode=Interaction.All): # update last active clients self.last_clients[user] = time.time() # filter current mode tasks = [ tsk for tsk in self.tasks.values() if mode == Interaction.All or bitset(tsk.type, mode) ] # filter correct user / or shared tasks = [ tsk for tsk in tasks if user is None or user == tsk.owner or tsk.shared ] return tasks def is_task_waiting(self, user, mode=Interaction.All): tasks = [ tsk for tsk in self.get_tasks(user, mode) if not tsk.type == Interaction.Notification or not tsk.seen ] return len(tasks) > 0 def queue_task(self, task): cli = self.is_client_connected(task.owner) # set waiting times based on threshold if cli: task.set_waiting(self.CLIENT_THRESHOLD) else: # TODO: higher threshold after client connects? task.set_waiting(self.CLIENT_THRESHOLD // 3) if task.type == Interaction.Notification: # notifications are valid for 30h task.set_waiting(self.NOTIFICATION_TIMEOUT) for plugin in self.__pyload.adm.active_plugins(): try: plugin.new_interaction_task(task) except Exception: # self.__pyload.print_exc() pass self.tasks[task.iid] = task self.__pyload.evm.fire("interaction:added", task)
class ConfigManager(ConfigParser): """ Manages the core config and configs for addons and single user. Has similar interface to ConfigParser. """ __slots__ = ['config', 'parser', 'pyload', 'values'] def __init__(self, core, parser): # No __init__ call to super class is needed! self.pyload = core # The config parser, holding the core config self.parser = parser # similar to parser, separated meta data and values self.config = OrderedDict() # Value cache for multiple user configs # Values are populated from db on first access # Entries are saved as (user, section) keys self.values = {} # TODO: similar to a cache, could be deleted periodically def save(self): self.parser.save() @convertkeyerror def get(self, section, option, user=None): """ Get config value, core config only available for admins. if user is not valid default value will be returned. """ # Core config loaded from parser, when no user is given or he is admin if section in self.parser and user is None: return self.parser.get(section, option) else: # We need the id and not the instance # Will be None for admin user and so the same as internal access try: # Check if this config exists # Configs without meta data can not be loaded! assert self.config[section].config[option] return self.load_values(user, section)[option] except KeyError: pass #: Returns default value later return self.config[section].config[option].input.default_value def load_values(self, user, section): if (user, section) not in self.values: conf = self.pyload.db.load_config(section, user) try: self.values[user, section] = json.loads(conf) if conf else {} except ValueError: #: Something did go wrong when parsing self.values[user, section] = {} # self.pyload.print_exc() return self.values[user, section] @convertkeyerror def set(self, section, option, value, sync=True, user=None): """ Set config value. """ changed = False if section in self.parser and user is None: changed = self.parser.set(section, option, value, sync) else: data = self.config[section].config[option] value = from_string(value, data.input.type) old_value = self.get(section, option) # Values will always be saved to db, sync is ignored if value != old_value: changed = True self.values[user, section][option] = value if sync: self.save_values(user, section) if changed: self.pyload.evm.fire("config:changed", section, option, value) return changed def save_values(self, user, section): if section in self.parser and user is None: self.save() elif (user, section) in self.values: self.pyload.db.save_config(section, json.dumps(self.values[user, section]), user) def delete(self, section, user=None): """ Deletes values saved in db and cached values for given user, NOT meta data. Does not trigger an error when nothing was deleted. """ if (user, section) in self.values: del self.values[user, section] self.pyload.db.delete_config(section, user) self.pyload.evm.fire("config:deleted", section, user) def iter_core_sections(self): return self.parser.iter_sections() def iter_sections(self, user=None): """ Yields: section, metadata, values. """ values = self.pyload.db.load_configs_for_user(user) # Every section needs to be json decoded for section, data in values.items(): try: values[section] = json.loads(data) if data else {} except ValueError: values[section] = {} # self.pyload.print_exc() for name, config in self.config.items(): yield name, config, values[name] if name in values else {} def get_section(self, section, user=None): if section in self.parser and user is None: return self.parser.get_section(section) values = self.load_values(user, section) return self.config.get(section), values
def __init__(self, core): self.lock = Lock() self.pyload = core self.tasks = OrderedDict() #: task store, for all outgoing tasks self.last_clients = {} self.ids = 0 #: uniue interaction ids
class InteractionManager(object): """ Class that gives ability to interact with the user. Arbitrary tasks with predefined output and input types can be set off """ __slots__ = [ 'CLIENT_THRESHOLD', 'MAX_NOTIFICATIONS', 'NOTIFICATION_TIMEOUT', 'ids', 'last_clients', 'lock', 'pyload', 'tasks'] # number of seconds a client is classified as active CLIENT_THRESHOLD = 60 NOTIFICATION_TIMEOUT = 60 * 60 * 30 MAX_NOTIFICATIONS = 50 def __init__(self, core): self.lock = Lock() self.pyload = core self.tasks = OrderedDict() #: task store, for all outgoing tasks self.last_clients = {} self.ids = 0 #: uniue interaction ids def is_client_connected(self, user): return self.last_clients.get(user, 0) + self.CLIENT_THRESHOLD > time() @lock def work(self): # old notifications will be removed for n in [k for k, v in self.tasks.items() if v.timed_out()]: del self.tasks[n] # keep notifications count limited n = [k for k, v in self.tasks.items() if v.type == Interaction.Notification][ ::-1] for v in n[:self.MAX_NOTIFICATIONS]: del self.tasks[v] @lock def create_notification(self, title, content, desc="", plugin="", owner=None): """ Creates and queues a new Notification :param title: short title :param content: text content :param desc: short form of the notification :param plugin: plugin name :return: :class:`InteractionTask` """ task = InteractionTask(self.ids, Interaction.Notification, Input(InputType.Text, None, content), title, desc, plugin, owner=owner) self.ids += 1 self.queue_task(task) return task @lock def create_query_task(self, input, desc, plugin="", owner=None): # input type was given, create a input widget if isinstance(input, int): input = Input(input) if not isinstance(input, Input): raise TypeError( "'Input' class expected not '{0}'".format(type(input))) task = InteractionTask(self.ids, Interaction.Query, input, _( "Query"), desc, plugin, owner=owner) self.ids += 1 self.queue_task(task) return task @lock def create_captcha_task(self, img, format, filename, plugin="", type_=InputType.Text, owner=None): """ Createss a new captcha task. :param img: image content (not base encoded) :param format: img format :param type_: :class:`InputType` :return: """ if type_ == 'textual': type_ = InputType.Text elif type_ == 'positional': type_ = InputType.Click input = Input(type_, data=[standard_b64encode(img), format, filename]) # TODO: title desc plugin task = InteractionTask(self.ids, Interaction.Captcha, input, _("Captcha request"), _("Please solve the captcha"), plugin, owner=owner) self.ids += 1 self.queue_task(task) return task @lock def remove_task(self, task): if task.iid in self.tasks: del self.tasks[task.iid] self.pyload.evm.fire("interaction:deleted", task.iid) @lock def get_task_by_id(self, iid): return self.tasks.get(iid, None) @lock def get_tasks(self, user, mode=Interaction.All): # update last active clients self.last_clients[user] = time() # filter current mode tasks = [t for t in self.tasks.values() if mode == Interaction.All or bitset(t.type, mode)] # filter correct user / or shared tasks = [t for t in tasks if user is None or user == t.owner or t.shared] return tasks def is_task_waiting(self, user, mode=Interaction.All): tasks = [t for t in self.get_tasks( user, mode) if not t.type == Interaction.Notification or not t.seen] return len(tasks) > 0 def queue_task(self, task): cli = self.is_client_connected(task.owner) # set waiting times based on threshold if cli: task.set_waiting(self.CLIENT_THRESHOLD) else: # TODO: higher threshold after client connects? task.set_waiting(self.CLIENT_THRESHOLD // 3) if task.type == Interaction.Notification: # notifications are valid for 30h task.set_waiting(self.NOTIFICATION_TIMEOUT) for plugin in self.pyload.adm.active_plugins(): try: plugin.new_interaction_task(task) except Exception: # self.pyload.print_exc() pass self.tasks[task.iid] = task self.pyload.evm.fire("interaction:added", task)
class ConfigManager(ConfigParser): """ Manages the core config and configs for addons and single user. Has similar interface to ConfigParser. """ __slots__ = ['config', 'parser', 'pyload', 'values'] def __init__(self, core, parser): # No __init__ call to super class is needed! self.pyload = core # The config parser, holding the core config self.parser = parser # similar to parser, separated meta data and values self.config = OrderedDict() # Value cache for multiple user configs # Values are populated from db on first access # Entries are saved as (user, section) keys self.values = {} # TODO: similar to a cache, could be deleted periodically def save(self): self.parser.save() @convertkeyerror def get(self, section, option, user=None): """ Get config value, core config only available for admins. if user is not valid default value will be returned. """ # Core config loaded from parser, when no user is given or he is admin if section in self.parser and user is None: return self.parser.get(section, option) else: # We need the id and not the instance # Will be None for admin user and so the same as internal access try: # Check if this config exists # Configs without meta data can not be loaded! assert self.config[section].config[option] return self.load_values(user, section)[option] except KeyError: pass #: Returns default value later return self.config[section].config[option].input.default_value def load_values(self, user, section): if (user, section) not in self.values: conf = self.pyload.db.load_config(section, user) try: self.values[user, section] = json.loads(conf) if conf else {} except ValueError: #: Something did go wrong when parsing self.values[user, section] = {} # self.pyload.print_exc() return self.values[user, section] @convertkeyerror def set(self, section, option, value, sync=True, user=None): """ Set config value. """ changed = False if section in self.parser and user is None: changed = self.parser.set(section, option, value, sync) else: data = self.config[section].config[option] value = from_string(value, data.input.type) old_value = self.get(section, option) # Values will always be saved to db, sync is ignored if value != old_value: changed = True self.values[user, section][option] = value if sync: self.save_values(user, section) if changed: self.pyload.evm.fire("config:changed", section, option, value) return changed def save_values(self, user, section): if section in self.parser and user is None: self.save() elif (user, section) in self.values: self.pyload.db.save_config(section, json.dumps( self.values[user, section]), user) def delete(self, section, user=None): """ Deletes values saved in db and cached values for given user, NOT meta data. Does not trigger an error when nothing was deleted. """ if (user, section) in self.values: del self.values[user, section] self.pyload.db.delete_config(section, user) self.pyload.evm.fire("config:deleted", section, user) def iter_core_sections(self): return self.parser.iter_sections() def iter_sections(self, user=None): """ Yields: section, metadata, values. """ values = self.pyload.db.load_configs_for_user(user) # Every section needs to be json decoded for section, data in values.items(): try: values[section] = json.loads(data) if data else {} except ValueError: values[section] = {} # self.pyload.print_exc() for name, config in self.config.items(): yield name, config, values[name] if name in values else {} def get_section(self, section, user=None): if section in self.parser and user is None: return self.parser.get_section(section) values = self.load_values(user, section) return self.config.get(section), values
class ConfigParser(object): """ Holds and manages the configuration + meta data for config read from file. """ __slots__ = ['_RE_VERSION', 'config', 'filename', 'values', 'version'] _RE_VERSION = re.compile(r'\s*version\s*:\s*(\d+\.\d+\.\d+)', flags=re.I) def __init__(self, file, version): self.filename = file self.version = convert.to_version(version, version) # Meta data information self.config = OrderedDict() # The actual config values self.values = {} self.check_version() self.load_default() self.parse(self.filename) def load_default(self): make_config(self) def check_version(self): """ Determines if config needs to be deleted. """ if not os.path.exists(self.filename): with io.open(self.filename, mode='wb') as fp: fp.write("version: {0}".format( convert.from_version(self.version))) return None version = None try: with io.open(self.filename, mode='rb') as fp: m = self._RE_VERSION.match(fp.readline()) version = convert.to_version(m.group(1)) except (AttributeError, TypeError): pass if not self.version or version[:-1] != self.version[:-1]: os.rename(self.filename, "{0}.old".format(self.filename)) with io.open(self.filename, mode='wb') as fp: fp.write("version: {0}".format(convert.from_version(self.version))) def parse(self, filename): """ Read config values from file. """ with io.open(filename, mode='rb') as fp: # save the current section section = "" fp.readline() for line in iter(fp.readline, ""): line = line.strip() # comment line, different variants if not line or line.startswith("#") or line.startswith( "//") or line.startswith("|"): continue if line.startswith("["): section = line.replace("[", "").replace("]", "") if section not in self.config: print("Unrecognized section", section) section = "" else: name, non, value = line.rpartition("=") name = name.strip() value = value.strip() if not section: print("Value without section", name) continue if name in self.config[section].config: self.set(section, name, value, sync=False) else: print("Unrecognized option", section, name) def save(self): """ Saves config to filename. """ configs = [] with io.open(self.filename, mode='wb') as fp: configs.append(fp) os.chmod(self.filename, 0o600) fp.write("version: {0}\n\n".format( convert.from_version(self.version))) for section, data in self.config.items(): fp.write("[{0}]\n".format(section)) for option, data in data.config.items(): value = self.get(section, option) fp.write("{0} = {1}\n".format(option, value)) fp.write("\n") def __getitem__(self, section): """ Provides dictionary like access: c['section']['option']. """ return ConfigSection(self, section) def __contains__(self, section): """ Checks if parser contains section. """ return section in self.config def get(self, section, option): """ Get value or default. """ try: return self.values[section][option] except KeyError: return self.config[section].config[option].input.default_value def set(self, section, option, value, sync=True): """ Set value. """ data = self.config[section].config[option] value = from_string(value, data.input.type) old_value = self.get(section, option) # only save when different values if value != old_value: if section not in self.values: self.values[section] = {} self.values[section][option] = value if sync: self.save() return True return False def get_meta_data(self, section, option): """ Get all config data for an option. """ return self.config[section].config[option] def iter_sections(self): """ Yields section, config info, values, for all sections. """ for name, config in self.config.items(): yield name, config, self.values[ name] if name in self.values else {} def get_section(self, section): """ Retrieves single config as tuple (section, values). """ return self.config[ section], self.values[section] if section in self.values else {} def add_config_section(self, section, label, desc, expl, config): """ Adds a section to the config. `config` is a list of config tuple as used in plugin api defined as the order of the config elements is preserved with OrderedDict """ d = OrderedDict() for entry in config: name, data = to_configdata(entry) d[name] = data data = SectionTuple(gettext(label), gettext(desc), gettext(expl), d) self.config[section] = data