def setUpClass(cls): os.environ['PYSEMANTIC_CONFIG'] = "test.conf" pr.CONF_FILE_NAME = "test.conf" cls.testenv = os.environ cls.test_config_path = op.join(os.getcwd(), "test.conf") shutil.copy(TEST_CONFIG_FILE_PATH, cls.test_config_path) # Change the relative paths in the config file to absolute paths parser = RawConfigParser() parser.read(cls.test_config_path) for section in parser.sections(): schema_path = parser.get(section, "specfile") parser.remove_option(section, "specfile") parser.set(section, "specfile", op.join(op.abspath(op.dirname(__file__)), schema_path)) with open(cls.test_config_path, "w") as fileobj: parser.write(fileobj) # change the relative paths in the test dictionary to absolute paths with open(TEST_DATA_DICT, "r") as fileobj: cls.org_specs = yaml.load(fileobj, Loader=yaml.CLoader) new_specs = deepcopy(cls.org_specs) for _, specs in new_specs.iteritems(): path = specs['path'] specs['path'] = op.join(op.abspath(op.dirname(__file__)), path) # Rewrite this to the file with open(TEST_DATA_DICT, "w") as fileobj: yaml.dump(new_specs, fileobj, Dumper=yaml.CDumper, default_flow_style=False)
def iter_backends(self): """ Iterate on backends. :returns: each tuple contains the backend name, module name and module options :rtype: :class:`tuple` """ config = RawConfigParser() config.read(self.confpath) changed = False for backend_name in config.sections(): params = dict(config.items(backend_name)) try: module_name = params.pop('_module') except KeyError: try: module_name = params.pop('_backend') config.set(backend_name, '_module', module_name) config.remove_option(backend_name, '_backend') changed = True except KeyError: warning('Missing field "_module" for configured backend "%s"', backend_name) continue yield backend_name, module_name, params if changed: with open(self.confpath, 'wb') as f: config.write(f)
def tearDownClass(cls): try: # modify the testdata back with open(TEST_DATA_DICT, "r") as fileobj: test_data = yaml.load(fileobj, Loader=yaml.CLoader) test_data['iris']['path'] = op.join("testdata", "iris.csv") test_data['person_activity']['path'] = op.join( "testdata", "person_activity.tsv") del test_data['multi_iris'] with open(TEST_DATA_DICT, "w") as fileobj: test_data = yaml.dump(test_data, fileobj, Dumper=yaml.CDumper, default_flow_style=False) # Change the config files back parser = RawConfigParser() parser.read(cls.test_conf_file) parser.remove_option("pysemantic", "specfile") parser.set("pysemantic", "specfile", op.join("testdata", "test_dictionary.yaml")) with open(TEST_CONFIG_FILE_PATH, 'w') as fileobj: parser.write(fileobj) finally: os.unlink(cls.test_conf_file) os.unlink(cls.copied_iris_path)
def tearDownClass(cls): try: # modify the testdata back with open(TEST_DATA_DICT, "r") as fileobj: test_data = yaml.load(fileobj, Loader=yaml.CLoader) test_data['iris']['path'] = op.join("testdata", "iris.csv") test_data['person_activity']['path'] = op.join("testdata", "person_activity.tsv") del test_data['multi_iris'] with open(TEST_DATA_DICT, "w") as fileobj: test_data = yaml.dump(test_data, fileobj, Dumper=yaml.CDumper, default_flow_style=False) # Change the config files back parser = RawConfigParser() parser.read(cls.test_conf_file) parser.remove_option("pysemantic", "specfile") parser.set("pysemantic", "specfile", op.join("testdata", "test_dictionary.yaml")) with open(TEST_CONFIG_FILE_PATH, 'w') as fileobj: parser.write(fileobj) finally: os.unlink(cls.test_conf_file) os.unlink(cls.copied_iris_path)
def rc_to_ini(rc_info, default_section=SECTION): """Convert a .rc file to a ConfigParser (.ini-like object) Items are assumed to be in app:assembl section, unless prefixed by "{section}__" . Keys prefixed with an underscore are not passed on. Keys prefixed with a star are put in the global (DEFAULT) section. Value of '__delete_key__' is eliminated if existing. """ p = Parser() for key, val in rc_info.iteritems(): if not key or key.startswith('_'): continue if key[0] == '*': # Global keys section = "DEFAULT" key = key[1:] elif '__' in key: section, key = key.split('__', 1) else: section = default_section ensureSection(p, section) if val == '__delete_key__': # Allow to remove a variable from rc # so we can fall back to underlying ini p.remove_option(section, key) else: p.set(section, key, val) return p
def yaml_to_ini(yaml_conf, default_section='app:assembl'): """Convert a .yaml file to a ConfigParser (.ini-like object) Items are assumed to be in app:assembl section, unless prefixed by "{section}__" . Keys prefixed with an underscore are not passed on. Keys prefixed with a star are put in the global (DEFAULT) section. Value of '__delete_key__' is eliminated if existing. """ p = RawConfigParser() ensureSection(p, default_section) for key, val in yaml_conf.iteritems(): if key.startswith('_'): continue if isinstance(val, dict): if key in _known_invoke_sections: continue ensureSection(p, key) for subk, subv in val.iteritems(): p.set(key, subk, val_to_ini(subv)) else: if val == '__delete_key__': # Allow to remove a variable from rc # so we can fall back to underlying ini p.remove_option(default_section, key) else: p.set(default_section, key, val_to_ini(val)) return p
def remove_option(self, section, option, also_remove_default=False): try: if also_remove_default: DEFAULTS[section].pop(option) RawConfigParser.remove_option(self, section, option) except (NoSectionError, KeyError): pass
def setUpClass(cls): os.environ['PYSEMANTIC_CONFIG'] = "test.conf" pr.CONF_FILE_NAME = "test.conf" cls.testenv = os.environ cls.test_config_path = op.join(os.getcwd(), "test.conf") shutil.copy(TEST_CONFIG_FILE_PATH, cls.test_config_path) # Change the relative paths in the config file to absolute paths parser = RawConfigParser() parser.read(cls.test_config_path) for section in parser.sections(): schema_path = parser.get(section, "specfile") parser.remove_option(section, "specfile") parser.set(section, "specfile", op.join(op.abspath(op.dirname(__file__)), schema_path)) with open(cls.test_config_path, "w") as fileobj: parser.write(fileobj) # change the relative paths in the test dictionary to absolute paths with open(TEST_DATA_DICT, "r") as fileobj: cls.org_specs = yaml.load(fileobj, Loader=Loader) new_specs = deepcopy(cls.org_specs) for _, specs in new_specs.iteritems(): path = specs['path'] specs['path'] = op.join(op.abspath(op.dirname(__file__)), path) # Rewrite this to the file with open(TEST_DATA_DICT, "w") as fileobj: yaml.dump(new_specs, fileobj, Dumper=Dumper, default_flow_style=False)
def _path_fixer(filepath, root=None): """Change all the relative paths in `filepath` to absolute ones. :param filepath: File to be changed :param root: Root path with which the relative paths are prefixed. If None (default), the directory with this file is the root. """ if root is None: root = op.join(op.abspath(op.dirname(__file__))) if filepath.endswith((".yaml", ".yml")): with open(filepath, "r") as fileobj: data = yaml.load(fileobj, Loader=Loader) for specs in data.itervalues(): specs['path'] = op.join(root, specs['path']) with open(filepath, "w") as fileobj: yaml.dump(data, fileobj, Dumper=Dumper, default_flow_style=False) elif filepath.endswith(".conf"): parser = RawConfigParser() parser.read(filepath) for section in parser.sections(): path = parser.get(section, "specfile") parser.remove_option(section, "specfile") parser.set(section, "specfile", op.join(root, path)) with open(filepath, "w") as fileobj: parser.write(fileobj)
def remove_option(self, section, option, also_remove_default=False): try: if also_remove_default: DEFAULTS[section].pop(option) RawConfigParser.remove_option(self, section, option) except (NoSectionError, KeyError): pass
def iter_backends(self): """ Iterate on backends. :returns: each tuple contains the backend name, module name and module options :rtype: :class:`tuple` """ config = RawConfigParser() config.read(self.confpath) changed = False for backend_name in config.sections(): params = dict(config.items(backend_name)) try: module_name = params.pop('_module') except KeyError: try: module_name = params.pop('_backend') config.set(backend_name, '_module', module_name) config.remove_option(backend_name, '_backend') changed = True except KeyError: warning( 'Missing field "_module" for configured backend "%s"', backend_name) continue yield backend_name, module_name, params if changed: with open(self.confpath, 'wb') as f: config.write(f)
def _path_fixer(filepath, root=None): """Change all the relative paths in `filepath` to absolute ones. :param filepath: File to be changed :param root: Root path with which the relative paths are prefixed. If None (default), the directory with this file is the root. """ if root is None: root = op.join(op.abspath(op.dirname(__file__))) if filepath.endswith((".yaml", ".yml")): with open(filepath, "r") as fileobj: data = yaml.load(fileobj, Loader=Loader) for specs in data.itervalues(): specs['path'] = op.join(root, specs['path']) with open(filepath, "w") as fileobj: yaml.dump(data, fileobj, Dumper=Dumper, default_flow_style=False) elif filepath.endswith(".conf"): parser = RawConfigParser() parser.read(filepath) for section in parser.sections(): path = parser.get(section, "specfile") parser.remove_option(section, "specfile") parser.set(section, "specfile", op.join(root, path)) with open(filepath, "w") as fileobj: parser.write(fileobj)
class ParamStore(object): def __init__(self, root_dir, file_name): if not os.path.isdir(root_dir): os.makedirs(root_dir) self._path = os.path.join(root_dir, file_name) self._dirty = False # open config file self._config = RawConfigParser() self._config.read(self._path) def __del__(self): self.flush() def flush(self): if not self._dirty: return self._dirty = False of = open(self._path, "w") self._config.write(of) of.close() def get(self, section, option, default=None): """Get a parameter value and return a string. If default is specified and section or option are not defined in the file, they are created and set to default, which is then the return value. """ if not self._config.has_option(section, option): if default is not None: self.set(section, option, default) return default return self._config.get(section, option) def get_datetime(self, section, option, default=None): result = self.get(section, option, default) if result: return safestrptime(result) return result def set(self, section, option, value): """Set option in section to string value.""" if not self._config.has_section(section): self._config.add_section(section) elif self._config.has_option(section, option) and self._config.get(section, option) == value: return self._config.set(section, option, value) self._dirty = True def unset(self, section, option): """Remove option from section.""" if not self._config.has_section(section): return if self._config.has_option(section, option): self._config.remove_option(section, option) self._dirty = True if not self._config.options(section): self._config.remove_section(section) self._dirty = True
class ConfigStore(object): def __init__(self, name): self.config = RawConfigParser() self.file_opts = {} if sys.version_info[0] >= 3: self.file_opts['encoding'] = 'utf-8' if hasattr(appdirs, 'user_config_dir'): data_dir = appdirs.user_config_dir('photini') else: data_dir = appdirs.user_data_dir('photini') if not os.path.isdir(data_dir): os.makedirs(data_dir, mode=0700) self.file_name = os.path.join(data_dir, '%s.ini' % name) if name == 'editor': for old_file_name in (os.path.expanduser('~/photini.ini'), os.path.join(data_dir, 'photini.ini')): if os.path.exists(old_file_name): self.config.read(old_file_name, **self.file_opts) self.save() os.unlink(old_file_name) self.config.read(self.file_name, **self.file_opts) self.timer = QtCore.QTimer() self.timer.setSingleShot(True) self.timer.setInterval(3000) self.timer.timeout.connect(self.save) self.has_section = self.config.has_section def get(self, section, option, default=None): if self.config.has_option(section, option): result = self.config.get(section, option) if sys.version_info[0] < 3: return result.decode('utf-8') return result if default is not None: self.set(section, option, default) return default def set(self, section, option, value): if not self.config.has_section(section): self.config.add_section(section) if (self.config.has_option(section, option) and self.config.get(section, option) == value): return if sys.version_info[0] < 3: value = value.encode('utf-8') self.config.set(section, option, value) self.timer.start() def remove_section(self, section): if not self.config.has_section(section): return for option in self.config.options(section): self.config.remove_option(section, option) self.config.remove_section(section) self.timer.start() def save(self): self.config.write(open(self.file_name, 'w', **self.file_opts)) os.chmod(self.file_name, 0600)
class FileStorage(object): """docstring for FileStorage""" def __init__(self, filepath): super(FileStorage, self).__init__() self.filepath = filepath self.filetype = 'ini' self.__cp = RawConfigParser() self.__cp.read(self.filepath) self.__section = 'data' if not os.path.isfile(self.filepath): with open(self.filepath, 'w') as f: self.__cp.add_section(self.__section) self.__cp.write(f) def __flush(self): with open(self.filepath, 'w') as f: self.__cp.write(f) def insert(self, key, data): if self.__cp.get(self.__section, key): raise KeyExistsError() if isinstance(data, dict): data = json.dumps(data) self.__cp.set(self.__section, key, data) self.__flush() def delete(self, key): self.__cp.remove_option(self.__section, key) self.__flush() def find(self, key, to_json=False): try: data = self.__cp.get(self.__section, key) except: data = None if not data: return {} if to_json: try: return json.loads(data) except ValueError: print('cannot decode data') print(data) return data def update(self, key, data): if isinstance(data, dict): data = json.dumps(data) self.__cp.set(self.__section, key, data) self.__flush() def upsert(self, key, data): if isinstance(data, dict): data = json.dumps(data) self.__cp.set(self.__section, key, data) self.__flush()
def remove_item_from_section(self, ini_file, section_name, key_value): """ Removes option from *.ini file :parameter ini_file, ini file where the removing should be done :parameter section_name, section where the option wants to be removed :parameter key_value, value which needs to be removed """ confp = RawConfigParser() confp.read(ini_file) confp.remove_option(section_name, key_value) with open(ini_file, 'w') as configfile: confp.write(configfile)
def remove_option(self, option): """ Removes an option (in ``section/key`` syntax), thus will not be saved anymore :param option: the option path :type option: string """ splitvals = option.split('/') section, key = "/".join(splitvals[:-1]), splitvals[-1] RawConfigParser.remove_option(self, section, key)
def remove_item_from_section(self, ini_file, section_name, key_value): """ Removes option from *.ini file :parameter ini_file, ini file where the removing should be done :parameter section_name, section where the option wants to be removed :parameter key_value, value which needs to be removed """ confp = RawConfigParser() confp.read(ini_file) confp.remove_option(section_name, key_value) with open(ini_file, 'w') as configfile: confp.write(configfile)
def merge(master, updates, outfile, **kwargs): if not master: master = _find_master() if not isinstance(updates, (list, tuple)): updates = [updates] print "Merging files: %s and %s" % (master, ', '.join(updates)) parser = RawConfigParser() parser.read(updates) with open(master, 'r') as orig: with open(outfile, 'w') as new: current_section = None for line in orig: sec_m = re.match("^\[([\w\d_\-\s]+)\]\s*", line, re.IGNORECASE) if sec_m: current_section = sec_m.group(1) new.write(line) else: if not parser.has_section(current_section): new.write(line) continue var_m = re.match("^(?:;)?([\w\d\_\-\.]+)\s*=\s*(.*)\n$", line, re.IGNORECASE) if var_m: key, value = var_m.groups() if parser.has_option(current_section, key): new_value = parser.get(current_section, key) # print "REPLACING: %s = %s with value %s" % (key, value, new_value) new.write("%s = %s\n" % (key, new_value % kwargs)) parser.remove_option(current_section, key) if not parser.items(current_section): parser.remove_section(current_section) else: new.write(line) else: new.write(line) if parser.sections(): #print "The following values were not set:" for s in parser.sections(): new.write("") new.write("[%s]\n" % s) for t in parser.items(s): new.write("%s = %s\n" % t)
def merge(master, updates, outfile, **kwargs): if not master: master = _find_master() if not isinstance(updates, (list, tuple)): updates = [updates] print "Merging files: %s and %s" % (master, ', '.join(updates)) parser = RawConfigParser() parser.read(updates) with open(master, 'r') as orig: with open(outfile, 'w') as new: current_section=None for line in orig: sec_m = re.match("^\[([\w\d_\-\s]+)\]\s*", line, re.IGNORECASE) if sec_m: current_section=sec_m.group(1) new.write(line) else: if not parser.has_section(current_section): new.write(line) continue var_m = re.match("^(?:;)?([\w\d\_\-\.]+)\s*=\s*(.*)\n$", line, re.IGNORECASE) if var_m: key, value = var_m.groups() if parser.has_option(current_section, key): new_value = parser.get(current_section, key) # print "REPLACING: %s = %s with value %s" % (key, value, new_value) new.write("%s = %s\n" % (key, new_value % kwargs)) parser.remove_option(current_section, key) if not parser.items(current_section): parser.remove_section(current_section) else: new.write(line) else: new.write(line) if parser.sections(): #print "The following values were not set:" for s in parser.sections(): new.write("") new.write("[%s]\n" % s) for t in parser.items(s): new.write("%s = %s\n" % t)
def ConvertRanks(): print "Converting ranks..." start = time.time() ConfigReader = RawConfigParser() ConfigReader.read("../../ranks.ini") Items = ConfigReader.items("ranks") for Item in Items: Username,Rank = Item if OldToNew.has_key(Rank): ConfigReader.remove_option("ranks",Username) ConfigReader.set("ranks",Username,OldToNew[Rank]) fHandle = open("../../ranks.ini","w") ConfigReader.write(fHandle) fHandle.close() print "Converted ranks in %f seconds" %(time.time()-start)
def remove_options(self, section, data): """Retrieve options from a section """ #print self.data['conf_file'] if 'conf_file' in self.data and self.data['conf_file'] is not None: config = RawConfigParser() config.read([self.data['conf_file']]) for key in data: config.remove_option(section, key) if section in self._cache and key in self._cache[section]: del self._cache[section][key] with open(self.data['conf_file'], 'wb') as configfile: config.write(configfile) return True return False
class SystemProvider(object): def __init__(self, config): self.config = config self.log = kaizen.logging.getLogger(self) self.configparser = RawConfigParser() def load(self, filename=None): self.configparser = RawConfigParser() if not filename: filename = self.config.get("system") if not filename or not os.path.isfile(filename): self.log.debug("no config file found for system povided "\ "dependencies. Config file %r will be created" % \ filename) else: self.log.debug("Reading system provide file %s" % filename) self.configparser.read(filename) if not self.configparser.has_section("provides"): self.log.debug("system config file '%s' has to provides section. "\ "Section will be created." % filename) self.configparser.add_section("provides") def provides(self, name): return self.configparser and self.configparser.has_option("provides", name) def get(self, name): if not self.provides(name): return None version = self.configparser.get("provides", name) return SystemDependency(name, version) def add(self, name, version): if not self.configparser: return if version is None: version = '' self.configparser.set("provides", name, version) def remove(self, name): if not self.configparser: return False return self.configparser.remove_option("provides", name) def save(self, filename=None): if not filename: filename = self.config.get("system") f = open(filename, "w") self.configparser.write(f) f.close() def is_empty(self): return self.configparser is None and \ len(self.configparser.items("provides")) == 0 def list(self): if not self.configparser: return [] return self.configparser.items("provides")
def set_schema_fpath(project_name, schema_fpath): """Set the schema path for a given project. :param project_name: Name of the project :param schema_fpath: path to the yaml file to be used as the schema for \ the project. :type project_name: str :type schema_fpath: str :return: True, if setting the schema path was successful. :Example: >>> set_schema_fpath('skynet', '/path/to/new/schema.yaml') True """ path = locate_config_file() parser = RawConfigParser() parser.read(path) if project_name in parser.sections(): if not parser.remove_option(project_name, "specfile"): raise MissingProject else: parser.set(project_name, "specfile", schema_fpath) with open(path, "w") as f: parser.write(f) return True raise MissingProject
def set_schema_fpath(project_name, schema_fpath): """Set the schema path for a given project. :param project_name: Name of the project :param schema_fpath: path to the yaml file to be used as the schema for \ the project. :type project_name: str :type schema_fpath: str :return: True, if setting the schema path was successful. :Example: >>> set_schema_fpath('skynet', '/path/to/new/schema.yaml') True """ path = locate_config_file() parser = RawConfigParser() parser.read(path) if project_name in parser.sections(): if not parser.remove_option(project_name, "specfile"): raise MissingProject else: parser.set(project_name, "specfile", schema_fpath) with open(path, "w") as f: parser.write(f) return True raise MissingProject
def test_set_schema_fpath(self): """Test if programmatically setting a schema file to an existing project works.""" old_schempath = pr.get_default_specfile("pysemantic") try: self.assertTrue(pr.set_schema_fpath("pysemantic", "/foo/bar")) self.assertEqual(pr.get_default_specfile("pysemantic"), "/foo/bar") self.assertRaises(MissingProject, pr.set_schema_fpath, "foobar", "/foo/bar") finally: conf_path = pr.locate_config_file() parser = RawConfigParser() parser.read(conf_path) parser.remove_option("pysemantic", "specfile") parser.set("pysemantic", "specfile", old_schempath) with open(TEST_CONFIG_FILE_PATH, "w") as fileobj: parser.write(fileobj)
def WriteSignleConfigValue(file, section, entry, value, remove = False): try: config = RawConfigParser() config.read(file) if remove: if config.has_option(section, entry): config.remove_option(section, entry) else: config.set(section, entry, str(value)) # Writing our configuration file disk with open(file, 'wb') as configfile: config.write(configfile) except Exception as e1: log.error("Error Writing Config File (WriteSignleConfigValue): " + str(e1))
def test_set_schema_fpath(self): """Test if programmatically setting a schema file to an existing project works.""" old_schempath = pr.get_default_specfile("pysemantic") try: self.assertTrue(pr.set_schema_fpath("pysemantic", "/foo/bar")) self.assertEqual(pr.get_default_specfile("pysemantic"), "/foo/bar") self.assertRaises(MissingProject, pr.set_schema_fpath, "foobar", "/foo/bar") finally: conf_path = pr.locate_config_file() parser = RawConfigParser() parser.read(conf_path) parser.remove_option("pysemantic", "specfile") parser.set("pysemantic", "specfile", old_schempath) with open(TEST_CONFIG_FILE_PATH, "w") as fileobj: parser.write(fileobj)
def set_options(self, kwargs): config = RawConfigParser() fp = StringIO(self.data) config.readfp(fp) for key, val in kwargs.items(): if key == "filter_by": if val is None: config.remove_option("options", "filter_by") else: config.set("options", key, " ".join(val)) else: config.set("options", key, val) fp = StringIO() config.write(fp) fp.seek(0) self.data = fp.read() self.save()
def setUpClass(cls): cls.maxDiff = None # modify the testdata dict to have absolute paths with open(TEST_DATA_DICT, "r") as fileobj: test_data = yaml.load(fileobj, Loader=Loader) for _, specs in test_data.iteritems(): path = op.join(op.abspath(op.dirname(__file__)), specs['path']) specs['path'] = path # Put in the multifile specs cls.copied_iris_path = test_data['iris']['path'].replace( "iris", "iris2") dframe = pd.read_csv(test_data['iris']['path']) dframe.to_csv(cls.copied_iris_path, index=False) copied_iris_specs = deepcopy(test_data['iris']) copied_iris_specs['path'] = [ copied_iris_specs['path'], cls.copied_iris_path ] copied_iris_specs['nrows'] = [150, 150] test_data['multi_iris'] = copied_iris_specs with open(TEST_DATA_DICT, "w") as fileobj: yaml.dump(test_data, fileobj, Dumper=Dumper, default_flow_style=False) cls.data_specs = test_data _path_fixer(TEST_XL_DICT) # Fix config file to have absolute paths config_fname = op.basename(TEST_CONFIG_FILE_PATH) cls.test_conf_file = op.join(os.getcwd(), config_fname) parser = RawConfigParser() parser.read(TEST_CONFIG_FILE_PATH) for project in ("pysemantic", "test_excel"): specfile = parser.get(project, 'specfile') specfile = op.join(op.abspath(op.dirname(__file__)), specfile) parser.remove_option(project, "specfile") parser.set(project, "specfile", specfile) with open(cls.test_conf_file, 'w') as fileobj: parser.write(fileobj) pr.CONF_FILE_NAME = config_fname
def remove_option(self, section, option): """Removes an option from a section. :param section: the section from which to remove the option :type section: string :param option: the option to be removed :type option: string :rtype: boolean :returns: True if the option existed """ return RCP.remove_option(self, section, option)
def parse_config(self, filename): from ConfigParser import RawConfigParser import io config = RawConfigParser() config.readfp(io.open(filename, 'r', encoding='utf_8_sig')) for s in config.sections(): port = int(config.get(s, 'port')) config.remove_option(s, 'port') xsize, ysize = [int(d) for d in config.get(s, 'size').split(",")] config.remove_option(s, 'size') x_off, y_off = [int(d) for d in config.get(s, 'offset').split(",")] config.remove_option(s, 'offset') self.offsets[s] = (x_off, y_off) for device, offset in config.items(s): x_off, y_off = [int(d) for d in offset.split(",")] if self.offsets.has_key(device): if (x_off, y_off) != self.offsets[device]: raise RuntimeError("conflicting offsets for device %s" % device) self.offsets[device] = (x_off, y_off) if s in self.transtbl: self.transtbl[s].append(device) else: self.transtbl[s] = [device] if device in self.transtbl: self.transtbl[device].append(s) else: self.transtbl[device] = [s] self.add_virtual(s, xsize, ysize, port)
def edit_config(filename, settings, dry_run=False): """Edit a configuration file to include `settings` `settings` is a dictionary of dictionaries or ``None`` values, keyed by command/section name. A ``None`` value means to delete the entire section, while a dictionary lists settings to be changed or deleted in that section. A setting of ``None`` means to delete that setting. """ from ConfigParser import RawConfigParser log.debug("Reading configuration from %s", filename) opts = RawConfigParser() opts.read([filename]) for section, options in settings.items(): if options is None: log.info("Deleting section [%s] from %s", section, filename) opts.remove_section(section) else: if not opts.has_section(section): log.debug("Adding new section [%s] to %s", section, filename) opts.add_section(section) for option,value in options.items(): if value is None: log.debug("Deleting %s.%s from %s", section, option, filename ) opts.remove_option(section,option) if not opts.options(section): log.info("Deleting empty [%s] section from %s", section, filename) opts.remove_section(section) else: log.debug( "Setting %s.%s to %r in %s", section, option, value, filename ) opts.set(section,option,value) log.info("Writing %s", filename) if not dry_run: f = open(filename,'w'); opts.write(f); f.close()
def setUpClass(cls): cls.maxDiff = None # modify the testdata dict to have absolute paths with open(TEST_DATA_DICT, "r") as fileobj: test_data = yaml.load(fileobj, Loader=Loader) for _, specs in test_data.iteritems(): path = op.join(op.abspath(op.dirname(__file__)), specs['path']) specs['path'] = path # Put in the multifile specs cls.copied_iris_path = test_data['iris']['path'].replace("iris", "iris2") dframe = pd.read_csv(test_data['iris']['path']) dframe.to_csv(cls.copied_iris_path, index=False) copied_iris_specs = deepcopy(test_data['iris']) copied_iris_specs['path'] = [copied_iris_specs['path'], cls.copied_iris_path] copied_iris_specs['nrows'] = [150, 150] test_data['multi_iris'] = copied_iris_specs with open(TEST_DATA_DICT, "w") as fileobj: yaml.dump(test_data, fileobj, Dumper=Dumper, default_flow_style=False) cls.data_specs = test_data _path_fixer(TEST_XL_DICT) # Fix config file to have absolute paths config_fname = op.basename(TEST_CONFIG_FILE_PATH) cls.test_conf_file = op.join(os.getcwd(), config_fname) parser = RawConfigParser() parser.read(TEST_CONFIG_FILE_PATH) for project in ("pysemantic", "test_excel"): specfile = parser.get(project, 'specfile') specfile = op.join(op.abspath(op.dirname(__file__)), specfile) parser.remove_option(project, "specfile") parser.set(project, "specfile", specfile) with open(cls.test_conf_file, 'w') as fileobj: parser.write(fileobj) pr.CONF_FILE_NAME = config_fname
def edit_config(filename, settings, dry_run=False): """Edit a configuration file to include `settings` `settings` is a dictionary of dictionaries or ``None`` values, keyed by command/section name. A ``None`` value means to delete the entire section, while a dictionary lists settings to be changed or deleted in that section. A setting of ``None`` means to delete that setting. """ from ConfigParser import RawConfigParser log.debug("Reading configuration from %s", filename) opts = RawConfigParser() opts.read([filename]) for section, options in settings.items(): if options is None: log.info("Deleting section [%s] from %s", section, filename) opts.remove_section(section) else: if not opts.has_section(section): log.debug("Adding new section [%s] to %s", section, filename) opts.add_section(section) for option, value in options.items(): if value is None: log.debug("Deleting %s.%s from %s", section, option, filename) opts.remove_option(section, option) if not opts.options(section): log.info("Deleting empty [%s] section from %s", section, filename) opts.remove_section(section) else: log.debug("Setting %s.%s to %r in %s", section, option, value, filename) opts.set(section, option, value) log.info("Writing %s", filename) if not dry_run: f = open(filename, 'w') opts.write(f) f.close()
def parse_config(self, filename): from ConfigParser import RawConfigParser import io config = RawConfigParser() with io.open(filename, 'r', encoding='utf_8_sig') as fp: config.readfp(fp) for s in config.sections(): port = int(config.get(s, 'port')) config.remove_option(s, 'port') if sys.argv[1] == '-s': self.snoopy_port = int(config.get(s, 'snoopy_port')) config.remove_option(s, 'snoopy_port') self.snoopy_addr = config.get(s, 'snoopy_addr') config.remove_option(s, 'snoopy_addr') xsize, ysize = [int(d) for d in config.get(s, 'size').split(",")] config.remove_option(s, 'size') x_off, y_off = [int(d) for d in config.get(s, 'offset').split(",")] config.remove_option(s, 'offset') self.offsets[s] = (x_off, y_off) for device, offset in config.items(s): x_off, y_off = [int(d) for d in offset.split(",")] if self.offsets.has_key(device): if (x_off, y_off) != self.offsets[device]: raise RuntimeError( "conflicting offsets for device %s" % device) self.offsets[device] = (x_off, y_off) if s in self.transtbl: self.transtbl[s].append(device) else: self.transtbl[s] = [device] if device in self.transtbl: self.transtbl[device].append(s) else: self.transtbl[device] = [s] self.add_virtual(s, xsize, ysize, port)
def iter_backends(self): config = RawConfigParser() config.read(self.confpath) changed = False for backend_name in config.sections(): params = dict(config.items(backend_name)) try: module_name = params.pop('_module') except KeyError: try: module_name = params.pop('_backend') config.set(backend_name, '_module', module_name) config.remove_option(backend_name, '_backend') changed = True except KeyError: warning('Missing field "_module" for configured backend "%s"', backend_name) continue yield backend_name, module_name, params if changed: with open(self.confpath, 'wb') as f: config.write(f)
def unsetFilenameInConfigFile(self, root_path, name): """Remove filename field association in config file @root_path: Path where config file is stored @param name: Field name """ path = os.path.join(root_path, self.cfg_filename) # Config file doesn't exists if not os.path.exists(path): return # Initialize config file config = RawConfigParser() # Read old config file fd = open(path, 'r') try: # Read file config.readfp(fd) finally: fd.close() # Update config file if config.has_section(self.cfg_filename_section): config.remove_option(self.cfg_filename_section, name) if not config.options(self.cfg_filename_section): # Section is empty so remove config file rm_file(path) else: fd = open(path, 'w') try: # Write file config.write(fd) finally: fd.close()
def unsetFilenameInConfigFile(self, root_path, name): """Remove filename field association in config file @root_path: Path where config file is stored @param name: Field name """ path = os.path.join(root_path, self.cfg_filename) # Config file doesn't exists if not os.path.exists(path): return # Initialize config file config = RawConfigParser() # Read old config file fd = open(path, 'r') try: # Read file config.readfp(fd) finally: fd.close() # Update config file if config.has_section(self.cfg_filename_section): config.remove_option(self.cfg_filename_section, name) if not config.options(self.cfg_filename_section): # Section is empty so remove config file rm_file(path) else: fd = open(path, 'w') try: # Write file config.write(fd) finally: fd.close()
def reset(self, name): # If custom conf file is not set then it is only read only configs if self.custom_conf_file is None: raise GconfNotConfigurable() # If a config can not be modified if name != "all" and not self._is_configurable(name): raise GconfNotConfigurable() cnf = RawConfigParser() with open(self.custom_conf_file) as f: cnf.readfp(f) # Nothing to Reset, Not configured if name != "all": if not cnf.has_option("vars", name): return True # Remove option from custom conf file cnf.remove_option("vars", name) else: # Remove and add empty section, do not disturb if config file # already has any other section try: cnf.remove_section("vars") except NoSectionError: pass cnf.add_section("vars") with open(self.tmp_conf_file, "w") as fw: cnf.write(fw) os.rename(self.tmp_conf_file, self.custom_conf_file) self.reload() return True
class Config(object): """A wrapper around RawConfigParser. Provides a ``defaults`` attribute of the same type which can be used to set default values. """ def __init__(self, version=None, _defaults=True): """Use read() to read in an existing config file. version should be an int starting with 0 that gets incremented if you want to register a new upgrade function. If None, upgrade is disabled. """ self._config = ConfigParser(dict_type=_sorted_dict) self.defaults = None if _defaults: self.defaults = Config(_defaults=False) self._version = version self._loaded_version = None self._upgrade_funcs = [] def _do_upgrade(self, func): assert self._loaded_version is not None assert self._version is not None old_version = self._loaded_version new_version = self._version if old_version != new_version: print_d("Config upgrade: %d->%d (%r)" % ( old_version, new_version, func)) func(self, old_version, new_version) def get_version(self): """Get the version of the loaded config file (for testing only) Raises Error if no file was loaded or versioning is disabled. """ if self._version is None: raise Error("Versioning disabled") if self._loaded_version is None: raise Error("No file loaded") return self._loaded_version def register_upgrade_function(self, function): """Register an upgrade function that gets called at each read() if the current config version and the loaded version don't match. Can also be registered after read was called. function(config, old_version: int, new_version: int) -> None """ if self._version is None: raise Error("Versioning disabled") self._upgrade_funcs.append(function) # after read(), so upgrade now if self._loaded_version is not None: self._do_upgrade(function) return function def reset(self, section, option): """Reset the value to the default state""" assert self.defaults is not None self.set(section, option, self.defaults.get(section, option)) def options(self, section): """Returns a list of options available in the specified section.""" try: options = self._config.options(section) except NoSectionError: if self.defaults: return self.defaults.options(section) raise else: if self.defaults: try: options.extend(self.defaults.options(section)) options = list_unique(options) except NoSectionError: pass return options def get(self, section, option, default=_DEFAULT): """get(section, option[, default]) -> str If default is not given or set, raises Error in case of an error """ try: return self._config.get(section, option) except Error: if default is _DEFAULT: if self.defaults is not None: try: return self.defaults.get(section, option) except Error: pass raise return default def gettext(self, *args, **kwargs): value = self.get(*args, **kwargs) if PY2: value = value.decode("utf-8") else: # make sure there are no surrogates value.encode("utf-8") return value def getbytes(self, section, option, default=_DEFAULT): try: value = self._config.get(section, option) if PY3: value = value.encode("utf-8", "surrogateescape") return value except (Error, ValueError) as e: if default is _DEFAULT: if self.defaults is not None: try: return self.defaults.getbytes(section, option) except Error: pass raise Error(e) return default def getboolean(self, section, option, default=_DEFAULT): """getboolean(section, option[, default]) -> bool If default is not given or set, raises Error in case of an error """ try: return self._config.getboolean(section, option) except (Error, ValueError) as e: if default is _DEFAULT: if self.defaults is not None: try: return self.defaults.getboolean(section, option) except Error: pass raise Error(e) return default def getint(self, section, option, default=_DEFAULT): """getint(section, option[, default]) -> int If default is not give or set, raises Error in case of an error """ try: return int(self._config.getfloat(section, option)) except (Error, ValueError) as e: if default is _DEFAULT: if self.defaults is not None: try: return self.defaults.getint(section, option) except Error: pass raise Error(e) return default def getfloat(self, section, option, default=_DEFAULT): """getfloat(section, option[, default]) -> float If default is not give or set, raises Error in case of an error """ try: return self._config.getfloat(section, option) except (Error, ValueError) as e: if default is _DEFAULT: if self.defaults is not None: try: return self.defaults.getfloat(section, option) except Error: pass raise Error(e) return default def getstringlist(self, section, option, default=_DEFAULT): """getstringlist(section, option[, default]) -> list If default is not given or set, raises Error in case of an error. Gets a list of strings, using CSV to parse and delimit. """ try: value = self._config.get(section, option) parser = csv.reader( [value], lineterminator='\n', quoting=csv.QUOTE_MINIMAL) try: if PY2: vals = [v.decode('utf-8') for v in next(parser)] else: vals = next(parser) except (csv.Error, ValueError) as e: raise Error(e) return vals except Error as e: if default is _DEFAULT: if self.defaults is not None: try: return self.defaults.getstringlist(section, option) except Error: pass raise Error(e) return default def setstringlist(self, section, option, values): """Saves a list of unicode strings using the csv module""" if PY2: sw = cBytesIO() values = [text_type(v).encode('utf-8') for v in values] else: sw = StringIO() values = [text_type(v) for v in values] writer = csv.writer(sw, lineterminator='\n', quoting=csv.QUOTE_MINIMAL) writer.writerow(values) self.set(section, option, sw.getvalue()) def setlist(self, section, option, values, sep=","): """Saves a list of str using ',' as a separator and \\ for escaping""" values = [str(v) for v in values] joined = join_escape(values, sep) self.set(section, option, joined) def getlist(self, section, option, default=_DEFAULT, sep=","): """Returns a str list saved with setlist()""" try: value = self._config.get(section, option) return split_escape(value, sep) except (Error, ValueError) as e: if default is _DEFAULT: if self.defaults is not None: try: return self.defaults.getlist(section, option, sep=sep) except Error: pass raise Error(e) return default def set(self, section, option, value): """Saves the string representation for the passed value Don't pass unicode, encode first. """ if PY3 and isinstance(value, bytes): raise TypeError("use setbytes") # RawConfigParser only allows string values but doesn't # scream if they are not (and it only fails before the # first config save..) if not isinstance(value, str): value = str(value) try: self._config.set(section, option, value) except NoSectionError: if self.defaults and self.defaults.has_section(section): self._config.add_section(section) self._config.set(section, option, value) else: raise def settext(self, section, option, value): value = text_type(value) if PY2: value = value.encode("utf-8") else: # make sure there are no surrogates value.encode("utf-8") self.set(section, option, value) def setbytes(self, section, option, value): assert isinstance(value, bytes) if PY3: value = value.decode("utf-8", "surrogateescape") self.set(section, option, value) def write(self, filename): """Write config to filename. Can raise EnvironmentError """ assert isinstance(filename, fsnative) mkdir(os.path.dirname(filename)) # temporary set the new version for saving if self._version is not None: self.add_section("__config__") self.set("__config__", "version", self._version) try: with atomic_save(filename, "wb") as fileobj: if PY2: self._config.write(fileobj) else: temp = StringIO() self._config.write(temp) data = temp.getvalue().encode("utf-8", "surrogateescape") fileobj.write(data) finally: if self._loaded_version is not None: self.set("__config__", "version", self._loaded_version) def clear(self): """Remove all sections.""" for section in self._config.sections(): self._config.remove_section(section) def is_empty(self): """Whether the config has any sections""" return not self._config.sections() def read(self, filename): """Reads the config from `filename` if the file exists, otherwise does nothing Can raise EnvironmentError, Error. """ try: with open(filename, "rb") as fileobj: if PY3: fileobj = StringIO( fileobj.read().decode("utf-8", "surrogateescape")) self._config.readfp(fileobj, filename) except (IOError, OSError): return # don't upgrade if we just created a new config if self._version is not None: self._loaded_version = self.getint("__config__", "version", -1) for func in self._upgrade_funcs: self._do_upgrade(func) def has_option(self, section, option): """If the given section exists, and contains the given option""" return self._config.has_option(section, option) or ( self.defaults and self.defaults.has_option(section, option)) def has_section(self, section): """If the given section exists""" return self._config.has_section(section) or ( self.defaults and self.defaults.has_section(section)) def remove_option(self, section, option): """Remove the specified option from the specified section Can raise Error. """ return self._config.remove_option(section, option) def add_section(self, section): """Add a section named section to the instance if it not already exists.""" if not self._config.has_section(section): self._config.add_section(section)
class AwsCredentials(object): """Wraps a RawConfigParser to treat a section named 'default' as a nomral section.""" __REAL_DEFAULT_SECTION_NAME = 'default' __TEMP_DEFAULT_SECTION_NAME = '__default__' __REAL_DEFAULT_SECTION_HEADING = '[' + __REAL_DEFAULT_SECTION_NAME + ']' __TEMP_DEFAULT_SECTION_HEADING = '[' + __TEMP_DEFAULT_SECTION_NAME + ']' def __init__(self): self.__config = RawConfigParser() def read(self, path): with open(path, 'r') as file: content = file.read() content = content.replace(self.__REAL_DEFAULT_SECTION_HEADING, self.__TEMP_DEFAULT_SECTION_HEADING) content_io = StringIO(content) self.__config.readfp(content_io) def write(self, path): content_io = StringIO() self.__config.write(content_io) content = content_io.getvalue() content = content.replace(self.__TEMP_DEFAULT_SECTION_HEADING, self.__REAL_DEFAULT_SECTION_HEADING) with open(path, 'w') as file: file.write(content) def __to_temp_name(self, name): if name == self.__REAL_DEFAULT_SECTION_NAME: name = self.__TEMP_DEFAULT_SECTION_NAME return name def __to_real_name(self, name): if name == self.__TEMP_DEFAULT_SECTION_NAME: name = self.__REAL_DEFAULT_SECTION_NAME return name def sections(self): sections = self.__config.sections() sections = [self.__to_real_name(section) for section in sections] return sections def add_section(self, section): section = self.__to_temp_name(section) self.__config.add_section(section) def has_section(self, section): section = self.__to_temp_name(section) return self.__config.has_section(section) def options(self, section): section = self.__to_temp_name(section) return self.__config.options(section) def has_option(self, section, option): section = self.__to_temp_name(section) return self.__config.has_option(section, option) def get(self, section, option): section = self.__to_temp_name(section) return self.__config.get(section, option) def items(self, section): section = self.__to_temp_name(section) return self.__config.items(section) def set(self, section, option, value): section = self.__to_temp_name(section) self.__config.set(section, option, value) def remove_option(self, section, option): section = self.__to_temp_name(section) return self.__config.remove_option(section, section, option) def remove_section(self, section): section = self.__to_temp_name(section) return self.__config.remove_section(section)
def remove_option(self, option): return _RawConfigParser.remove_option(self, self._default_section, option)
def remove_option(self, section, option): if self.has_option(section, option): RawConfigParser.remove_option(self, section, option)
class TestPolicyUid(unittest.TestCase): def remove_option(self, section, option): self.config.remove_option(section, option) fp = open(conf.config_file, "w") self.config.write(fp) fp.close() def set(self, section, option, value): self.config.set(section, option, value) fp = open(conf.config_file, "w") self.config.write(fp) fp.close() @classmethod def setup_class(self, *args, **kw): self.config = RawConfigParser() self.config.read(conf.config_file) from tests.functional.purge_users import purge_users purge_users() self.user = { 'local': 'john.doe', 'domain': 'example.org' } self.login = conf.get('ldap', 'bind_dn') self.password = conf.get('ldap', 'bind_pw') self.domain = conf.get('kolab', 'primary_domain') result = wap_client.authenticate(self.login, self.password, self.domain) @classmethod def teardown_class(self, *args, **kw): self.config.remove_option('example.org', 'policy_uid') fp = open(conf.config_file, "w") self.config.write(fp) fp.close() from tests.functional.purge_users import purge_users purge_users() def test_001_default(self): from tests.functional.user_add import user_add user_add("John", "Doe") from tests.functional.synchronize import synchronize_once synchronize_once() auth = Auth() auth.connect() user = auth.find_recipient('*****@*****.**') user_info = wap_client.user_info(user) self.assertEqual(user_info['uid'], "doe") from tests.functional.purge_users import purge_users purge_users() def test_002_givenname_dot_surname(self): self.set('example.org', 'policy_uid', '%(givenname)s.%(surname)s') from tests.functional.user_add import user_add user_add("John", "Doe") from tests.functional.synchronize import synchronize_once synchronize_once() auth = Auth() auth.connect() user = auth.find_recipient('*****@*****.**') user_info = wap_client.user_info(user) self.assertEqual(user_info['uid'], "John.Doe") from tests.functional.purge_users import purge_users purge_users() def test_003_givenname_fc_dot_surname(self): self.set('example.org', 'policy_uid', "'%(givenname)s'[0:1].%(surname)s") from tests.functional.user_add import user_add user_add("John", "Doe") from tests.functional.synchronize import synchronize_once synchronize_once() auth = Auth() auth.connect() user = auth.find_recipient('*****@*****.**') user_info = wap_client.user_info(user) self.assertEqual(user_info['uid'], "J.Doe") from tests.functional.purge_users import purge_users purge_users() def test_004_givenname(self): self.set('example.org', 'policy_uid', '%(givenname)s') from tests.functional.user_add import user_add user_add("John", "Doe") from tests.functional.synchronize import synchronize_once synchronize_once() auth = Auth() auth.connect() user = auth.find_recipient('*****@*****.**') user_info = wap_client.user_info(user) self.assertEqual(user_info['uid'], "John") from tests.functional.purge_users import purge_users purge_users() def test_005_lowercase_givenname(self): self.set('example.org', 'policy_uid', '%(givenname)s.lower()') from tests.functional.user_add import user_add user_add("John", "Doe") from tests.functional.synchronize import synchronize_once synchronize_once() auth = Auth() auth.connect() user = auth.find_recipient('*****@*****.**') user_info = wap_client.user_info(user) self.assertEqual(user_info['uid'], "john") from tests.functional.purge_users import purge_users purge_users() def test_006_lowercase_givenname_surname(self): self.set('example.org', 'policy_uid', "%(givenname)s.lower().%(surname)s.lower()") from tests.functional.user_add import user_add user_add("John", "Doe") from tests.functional.synchronize import synchronize_once synchronize_once() auth = Auth() auth.connect() user = auth.find_recipient('*****@*****.**') user_info = wap_client.user_info(user) self.assertEqual(user_info['uid'], "john.doe") from tests.functional.purge_users import purge_users purge_users()
def main(args): config = ConfigParser({"htrc": False, "sentences": "False"}) config.read(args.config_file) if config.getboolean("main", "sentences"): from vsm.extensions.ldasentences import CorpusSent as Corpus else: from vsm.corpus import Corpus if args.lang is None: args.lang = [] args.corpus_path = config.get("main", "corpus_file") c = Corpus.load(args.corpus_path) # check for htrc metadata if args.htrc or config.get("main", "htrc"): htrc_langs = get_htrc_langs(args) if htrc_langs: args.lang.extend(new_langs) # auto-guess a language """ new_langs = [lang for lang in detect_langs(c) if lang in langs and lang not in args.lang] if new_langs: args.lang.extend(new_langs) """ # add default locale if no other languages are specified # do not add if in quiet mode -- make everything explicit if not args.lang and not args.quiet: import locale locale = locale.getdefaultlocale()[0].split('_')[0].lower() if locale in langs.keys(): args.lang.append(locale) # check for any new candidates args.lang = [lang for lang in args.lang if stop_language(c, langs[lang])] if args.lang and not args.quiet: args.lang = lang_prompt(args.lang) stoplist = set() # Apply stop words print " " for lang in args.lang: print "Applying", langs[lang], "stopwords" candidates = stop_language(c, langs[lang]) if len(candidates): stoplist.update(candidates) # Apply custom stopwords file if args.stopword_file: with open(args.stopword_file, encoding='utf8') as swf: candidates = [unidecode(word.strip()) for word in swf] if len(candidates): print "Applying custom stopword file to remove {} word{}.".format( len(candidates), 's' if len(candidates) > 1 else '') stoplist.update(candidates) if args.min_word_len: candidates = get_small_words(c, args.min_word_len) if len(candidates): print "Filtering {} small word{} with less than {} characters.".format( len(candidates), 's' if len(candidates) > 1 else '', args.min_word_len) stoplist.update(candidates) if not args.special_chars: candidates = get_special_chars(c) if len(candidates): print "Filtering {} word{} with special characters.".format( len(candidates), 's' if len(candidates) > 1 else '') stoplist.update(candidates) if args.high_filter is None and not args.quiet: args.high_filter, candidates = get_high_filter(args, c, words=stoplist) if len(candidates): print "Filtering {} high frequency word{}.".format( len(candidates), 's' if len(candidates) > 1 else '') stoplist.update(candidates) elif args.high_filter > 0: candidates = get_candidate_words(c, args.high_filter, sort=False) if len(candidates): print "Filtering {} high frequency word{}.".format( len(candidates), 's' if len(candidates) > 1 else '') stoplist.update(candidates) if args.low_filter is None and not args.quiet: args.low_filter, candidates = get_low_filter(args, c, words=stoplist) if len(candidates): print "Filtering {} low frequency word{}.".format( len(candidates), 's' if len(candidates) > 1 else '') stoplist.update(candidates) elif args.low_filter > 0: candidates = get_candidate_words(c, -1 * args.low_filter, sort=False) if len(candidates): print "Filtering {} low frequency words.".format(len(candidates)) stoplist.update(candidates) if not stoplist: print "No stopwords applied.\n\n" sys.exit(0) else: print "\n\nApplying {} stopword{}".format( len(stoplist), 's' if len(stoplist) > 1 else '') c.in_place_stoplist(stoplist) print "\n" def name_corpus(dirname, languages, lowfreq=None, highfreq=None): items, counts = get_items_counts(c.corpus) corpus_name = [dirname] if args.lang: corpus_name.append('nltk') corpus_name.append(''.join(args.lang)) if lowfreq > 0: corpus_name.append('freq%s' % lowfreq) else: corpus_name.append('freq%s' % min(counts)) if highfreq > 0: corpus_name.append('N%s' % highfreq) else: corpus_name.append('freq%s' % max(counts)) corpus_name = '-'.join(corpus_name) corpus_name += '.npz' return corpus_name dirname = os.path.basename(args.corpus_path).split('-nltk-')[0].replace( '.npz', '') corpus_name = name_corpus(dirname, ['en'], args.low_filter, args.high_filter) model_path = os.path.dirname(args.corpus_path) args.corpus_path = os.path.join(model_path, corpus_name) c.save(args.corpus_path) config.set("main", "corpus_file", args.corpus_path) config.remove_option("main", "model_pattern") with open(args.config_file, 'wb') as configfh: config.write(configfh)
class Config(object): """A wrapper around RawConfigParser""" def __init__(self, version=None): """Use read() to read in an existing config file. version should be an int starting with 0 that gets incremented if you want to register a new upgrade function. If None, upgrade is disabled. """ self._config = ConfigParser(dict_type=_sorted_dict) self._version = version self._loaded_version = None self._upgrade_funcs = [] self._initial = {} def _do_upgrade(self, func): assert self._loaded_version is not None assert self._version is not None old_version = self._loaded_version new_version = self._version if old_version != new_version: print_d("Config upgrade: %d->%d (%r)" % (old_version, new_version, func)) func(self, old_version, new_version) def get_version(self): """Get the version of the loaded config file (for testing only) Raises Error if no file was loaded or versioning is disabled. """ if self._version is None: raise Error("Versioning disabled") if self._loaded_version is None: raise Error("No file loaded") return self._loaded_version def register_upgrade_function(self, function): """Register an upgrade function that gets called at each read() if the current config version and the loaded version don't match. Can also be registered after read was called. function(config, old_version: int, new_version: int) -> None """ if self._version is None: raise Error("Versioning disabled") self._upgrade_funcs.append(function) # after read(), so upgrade now if self._loaded_version is not None: self._do_upgrade(function) return function def set_inital(self, section, option, value): """Set an initial value for an option. The section must be added with add_section() first. Adds the value to the config and calling reset() will reset the value to it. """ self.set(section, option, value) self._initial.setdefault(section, {}) self._initial[section].setdefault(option, {}) self._initial[section][option] = value def reset(self, section, option): """Reset the value to the initial state""" value = self._initial[section][option] self.set(section, option, value) def options(self, section): """Returns a list of options available in the specified section.""" return self._config.options(section) def get(self, *args): """get(section, option[, default]) -> str If default is not given, raises Error in case of an error """ if len(args) == 3: try: return self._config.get(*args[:2]) except Error: return args[-1] return self._config.get(*args) def getboolean(self, *args): """getboolean(section, option[, default]) -> bool If default is not given, raises Error in case of an error """ if len(args) == 3: if not isinstance(args[-1], bool): raise ValueError try: return self._config.getboolean(*args[:2]) # ValueError if the value found in the config file # does not match any string representation -> so catch it too except (ValueError, Error): return args[-1] return self._config.getboolean(*args) def getint(self, *args): """getint(section, option[, default]) -> int If default is not give, raises Error in case of an error """ if len(args) == 3: if not isinstance(args[-1], int): raise ValueError try: return self._config.getint(*args[:2]) except Error: return args[-1] return self._config.getint(*args) def getfloat(self, *args): """getfloat(section, option[, default]) -> float If default is not give, raises Error in case of an error """ if len(args) == 3: if not isinstance(args[-1], float): raise ValueError try: return self._config.getfloat(*args[:2]) except Error: return args[-1] return self._config.getfloat(*args) def getstringlist(self, *args): """getstringlist(section, option[, default]) -> list If default is not given, raises Error in case of an error. Gets a list of strings, using CSV to parse and delimit. """ if len(args) == 3: if not isinstance(args[-1], list): raise ValueError try: value = self._config.get(*args[:2]) except Error: return args[-1] else: value = self._config.get(*args) parser = csv.reader([value], lineterminator='\n', quoting=csv.QUOTE_MINIMAL) try: vals = [v.decode('utf-8') for v in parser.next()] except (csv.Error, ValueError) as e: raise Error(e) return vals def setstringlist(self, section, option, values): """Saves a list of unicode strings using the csv module""" sw = StringIO() values = [unicode(v).encode('utf-8') for v in values] writer = csv.writer(sw, lineterminator='\n', quoting=csv.QUOTE_MINIMAL) writer.writerow(values) self._config.set(section, option, sw.getvalue()) def set(self, section, option, value): """Saves the string representation for the passed value Don't pass unicode, encode first. """ # RawConfigParser only allows string values but doesn't # scream if they are not (and it only fails before the # first config save..) if not isinstance(value, str): value = str(value) self._config.set(section, option, value) def setdefault(self, section, option, default): """Like set but only sets the new value if the option isn't set before. """ if not self._config.has_option(section, option): self._config.set(section, option, default) def write(self, filename): """Write config to filename. Can raise EnvironmentError """ assert is_fsnative(filename) mkdir(os.path.dirname(filename)) # temporary set the new version for saving if self._version is not None: self.add_section("__config__") self.set("__config__", "version", self._version) try: with atomic_save(filename, ".tmp", "wb") as fileobj: self._config.write(fileobj) finally: if self._loaded_version is not None: self.set("__config__", "version", self._loaded_version) def clear(self): """Remove all sections and initial values""" for section in self._config.sections(): self._config.remove_section(section) self._initial.clear() def is_empty(self): """Whether the config has any sections""" return not self._config.sections() def read(self, filename): """Reads the config from `filename` if the file exists, otherwise does nothing Can raise EnvironmentError, Error. """ parsed_filenames = self._config.read(filename) # don't upgrade if we just created a new config if parsed_filenames and self._version is not None: self._loaded_version = self.getint("__config__", "version", -1) for func in self._upgrade_funcs: self._do_upgrade(func) def sections(self): """Return a list of the sections available""" return self._config.sections() def has_option(self, section, option): """If the given section exists, and contains the given option""" return self._config.has_option(section, option) def remove_option(self, section, option): """Remove the specified option from the specified section Can raise Error. """ return self._config.remove_option(section, option) def add_section(self, section): """Add a section named section to the instance if it not already exists.""" if not self._config.has_section(section): self._config.add_section(section)
def load_config(path): # type: (str) -> RawConfigParser """Load the config file from the given path, applying fixes as needed. @todo: Refactor all this """ first_run = not os.path.exists(path) config = RawConfigParser() # Make keys case-sensitive because keysyms must be config.optionxform = str # type: ignore # (Cannot assign to a method) # TODO: Maybe switch to two config files so I can have only the keys in the # keymap case-sensitive? config.read(path) dirty = False if not config.has_section('general'): config.add_section('general') # Change this if you make backwards-incompatible changes to the # section and key naming in the config file. config.set('general', 'cfg_schema', 1) dirty = True for key, val in DEFAULTS['general'].items(): if not config.has_option('general', key): config.set('general', key, str(val)) dirty = True mk_raw = modkeys = config.get('general', 'ModMask') if ' ' in modkeys.strip() and '<' not in modkeys: modkeys = '<%s>' % '><'.join(modkeys.strip().split()) logging.info("Updating modkeys format:\n %r --> %r", mk_raw, modkeys) config.set('general', 'ModMask', modkeys) dirty = True # Either load the keybindings or use and save the defaults if config.has_section('keys'): keymap = dict(config.items('keys')) else: keymap = DEFAULTS['keys'] config.add_section('keys') for row in keymap.items(): config.set('keys', row[0], row[1]) dirty = True # Migrate from the deprecated syntax for punctuation keysyms for key in keymap: # Look up unrecognized shortkeys in a hardcoded dict and # replace with valid names like ',' -> 'comma' if key in KEYLOOKUP: logging.warn("Updating config file from deprecated keybind syntax:" "\n\t%r --> %r", key, KEYLOOKUP[key]) config.remove_option('keys', key) config.set('keys', KEYLOOKUP[key], keymap[key]) dirty = True if dirty: cfg_file = open(path, 'wb') # TODO: REPORT: Argument 1 to "write" of "RawConfigParser" has # incompatible type BinaryIO"; expected "file" config.write(cfg_file) # type: ignore cfg_file.close() if first_run: logging.info("Wrote default config file to %s", path) return config
class TestPolicyUid(unittest.TestCase): def remove_option(self, section, option): self.config.remove_option(section, option) fp = open(conf.config_file, "w") self.config.write(fp) fp.close() def set(self, section, option, value): self.config.set(section, option, value) fp = open(conf.config_file, "w") self.config.write(fp) fp.close() @classmethod def setup_class(self, *args, **kw): self.config = RawConfigParser() self.config.read(conf.config_file) from tests.functional.purge_users import purge_users purge_users() self.user = { 'local': 'john.doe', 'domain': 'example.org' } self.login = conf.get('ldap', 'bind_dn') self.password = conf.get('ldap', 'bind_pw') self.domain = conf.get('kolab', 'primary_domain') result = wap_client.authenticate(self.login, self.password, self.domain) @classmethod def teardown_class(self, *args, **kw): self.config.remove_option('example.org', 'policy_uid') fp = open(conf.config_file, "w") self.config.write(fp) fp.close() from tests.functional.purge_users import purge_users purge_users() def test_001_default(self): from tests.functional.user_add import user_add user_add("John", "Doe") from tests.functional.synchronize import synchronize_once synchronize_once() auth = Auth() auth.connect() user = auth.find_recipient('*****@*****.**') user_info = wap_client.user_info(user) self.assertEqual(user_info['uid'], "doe") from tests.functional.purge_users import purge_users purge_users() def test_002_givenname_dot_surname(self): self.set('example.org', 'policy_uid', '%(givenname)s.%(surname)s') from tests.functional.user_add import user_add user_add("John", "Doe") from tests.functional.synchronize import synchronize_once synchronize_once() auth = Auth() auth.connect() user = auth.find_recipient('*****@*****.**') user_info = wap_client.user_info(user) self.assertEqual(user_info['uid'], "John.Doe") from tests.functional.purge_users import purge_users purge_users() def test_003_givenname_fc_dot_surname(self): self.set('example.org', 'policy_uid', "'%(givenname)s'[0:1].%(surname)s") from tests.functional.user_add import user_add user_add("John", "Doe") from tests.functional.synchronize import synchronize_once synchronize_once() auth = Auth() auth.connect() user = auth.find_recipient('*****@*****.**') user_info = wap_client.user_info(user) self.assertEqual(user_info['uid'], "J.Doe") from tests.functional.purge_users import purge_users purge_users() def test_004_givenname(self): self.set('example.org', 'policy_uid', '%(givenname)s') from tests.functional.user_add import user_add user_add("John", "Doe") from tests.functional.synchronize import synchronize_once synchronize_once() auth = Auth() auth.connect() user = auth.find_recipient('*****@*****.**') user_info = wap_client.user_info(user) self.assertEqual(user_info['uid'], "John") from tests.functional.purge_users import purge_users purge_users() def test_005_lowercase_givenname(self): self.set('example.org', 'policy_uid', '%(givenname)s.lower()') from tests.functional.user_add import user_add user_add("John", "Doe") from tests.functional.synchronize import synchronize_once synchronize_once() auth = Auth() auth.connect() user = auth.find_recipient('*****@*****.**') user_info = wap_client.user_info(user) self.assertEqual(user_info['uid'], "john") from tests.functional.purge_users import purge_users purge_users() def test_006_lowercase_givenname_surname(self): self.set('example.org', 'policy_uid', "%(givenname)s.lower().%(surname)s.lower()") from tests.functional.user_add import user_add user_add("John", "Doe") from tests.functional.synchronize import synchronize_once synchronize_once() auth = Auth() auth.connect() user = auth.find_recipient('*****@*****.**') user_info = wap_client.user_info(user) self.assertEqual(user_info['uid'], "john.doe") from tests.functional.purge_users import purge_users purge_users()
class ParamStore(object): def __init__(self, root_dir, file_name): self._lock = Lock() with self._lock: if not os.path.isdir(root_dir): raise RuntimeError( 'Directory "' + root_dir + '" does not exist.') self._path = os.path.join(root_dir, file_name) self._dirty = False # open config file self._config = RawConfigParser() self._config.read(self._path) def __del__(self): self.flush() def flush(self): if not self._dirty: return with self._lock: self._dirty = False of = open(self._path, 'w') self._config.write(of) of.close() def get(self, section, option, default=None): """Get a parameter value and return a string. If default is specified and section or option are not defined in the file, they are created and set to default, which is then the return value. """ with self._lock: if not self._config.has_option(section, option): if default is not None: self._set(section, option, default) return default return self._config.get(section, option) def get_datetime(self, section, option, default=None): result = self.get(section, option, default) if result: return safestrptime(result) return result def set(self, section, option, value): """Set option in section to string value.""" with self._lock: self._set(section, option, value) def _set(self, section, option, value): if not self._config.has_section(section): self._config.add_section(section) elif (self._config.has_option(section, option) and self._config.get(section, option) == value): return self._config.set(section, option, value) self._dirty = True def unset(self, section, option): """Remove option from section.""" with self._lock: if not self._config.has_section(section): return if self._config.has_option(section, option): self._config.remove_option(section, option) self._dirty = True if not self._config.options(section): self._config.remove_section(section) self._dirty = True
class TraktForVLC(object): # Check if there's a newer version of TraktForVLC on the project's github, # and print informations on that subject in the logs def __check_version(self): # The leading information for lines printed by this method in logs lead = "VERSION:" # Request the github API to get the releases information github_releases = requests.get( url="https://api.github.com/repos/XaF/TraktForVLC/releases") # If there was a problem getting the releases if not github_releases.ok: self.log.error(lead + "Unable to verify new releases of TraktForVLC") return # Else, we get the json answer releases = github_releases.json() # If we didn't find any release if not releases: self.log.warning(lead + "No releases found on github") return # We get the latest release, all included newest = sorted( releases, key=lambda x: x['tag_name'], reverse=True)[0] if newest['tag_name'][:1] == "v": newest['tag_name'] = newest['tag_name'][1:] newest_V = parse_version(newest['tag_name']) # We get the latest _stable_ release newest_stbl = sorted( [r for r in releases if not r['prerelease']], key=lambda x: x['tag_name'], reverse=True)[0] if newest_stbl['tag_name'][:1] == "v": newest_stbl['tag_name'] = newest_stbl['tag_name'][1:] newest_stbl_V = parse_version(newest_stbl['tag_name']) # We parse the current version current_V = parse_version(__version__) if newest_V <= current_V: self.log.info(lead + "TraktForVLC is up to date") return # We only show the latest stable release if # it's newer than our current release if newest_stbl_V > current_V: # We reformat the publication date of the release published = datetime.datetime.strptime( newest_stbl['published_at'], "%Y-%m-%dT%H:%M:%SZ").strftime('%c') self.log.info(lead + "##### RELEASE #####") self.log.info(lead + "## Stable release %(name)s" % newest_stbl) self.log.info(lead + "## Published on %s" % published) self.log.info(lead + "## Available on %(html_url)s" % newest_stbl) # We only show the latest release if it's not # also the latest stable release if newest_V > newest_stbl_V: # We reformat the publication date of the release published = datetime.datetime.strptime( newest['published_at'], "%Y-%m-%dT%H:%M:%SZ").strftime('%c') self.log.info(lead + "##### RELEASE #####") self.log.info(lead + "## Prerelease %(name)s" % newest) self.log.info(lead + "## Published on %s" % published) self.log.info(lead + "## Available on %(html_url)s" % newest) self.log.info(lead + "###################") def __init__(self, datadir, configfile, daemon=False): # Verify if the log directory exists or create it logdir = os.path.join(datadir, 'logs') if not os.path.exists(logdir): os.mkdir(logdir) # Process log file name if daemon: if LOG_LEVEL is logging.DEBUG: logfile = os.path.join(logdir, "TraktForVLC-DEBUG.log") # Remove existing DEBUG file if os.path.isfile(logfile): os.remove(logfile) else: logfile = os.path.join( logdir, "TraktForVLC-" + DATETIME.strftime("%Y%m%d-%H%M") + ".log") logging.basicConfig( format="%(asctime)s::%(name)s::%(levelname)s::%(message)s", level=LOG_LEVEL, filename=logfile) else: logging.basicConfig( format="%(asctime)s::%(name)s::%(levelname)s::%(message)s", level=LOG_LEVEL, stream=sys.stderr) self.log = logging.getLogger("TraktForVLC") e = 'e' if (sys.platform == 'win32') else 'ë' self.log.info( "## TraktForVLC v" + __version__ + " " + __release_name__) self.log.info("## Copyright (C) 2014-2015 " + "Rapha" + e + "l Beamonte <*****@*****.**>") self.log.info("##") self.log.info("## TraktForVLC is distributed in the hope that it " + "will be useful, but") self.log.info("## with ABSOLUTELY NO WARRANTY. This is free " + "software; you are welcome") self.log.info("## to redistribute and/or modify it under the terms " + "of the GPL2.") self.log.info("") if not os.path.isfile(configfile): self.log.error("Config file " + configfile + " not found, exiting.") sys.exit(1) self.log.debug("Running on %s, with Python %s" % ( platform.platform(), platform.python_version())) self.__check_version() # Load configuration self.configfile = configfile self.__load_config() for loglvl, logstr in AVAILABLE_LOGLVL: if LOG_LEVEL <= loglvl: loglevelstr = logstr break if loglevelstr is None: loglevelstr = str(LOG_LEVEL) self.log.info("Logger level is set to %s" % loglevelstr) self.log.info("-- Will scrobble movies ? %s" % ( 'Yes' if self.DO_SCROBBLE_MOVIE else 'No')) self.log.info("-- Will scrobble tv shows ? %s" % ( 'Yes' if self.DO_SCROBBLE_TV else 'No')) self.log.info("-- Will we mark movies as being watched ? %s" % ( 'Yes' if self.DO_WATCHING_MOVIE else 'No')) self.log.info("-- Will we mark tv shows as being watched ? %s" % ( 'Yes' if self.DO_WATCHING_TV else 'No')) self.log.info("-- Videos will be scrobbled after " + str(self.SCROBBLE_PERCENT) + "% of their duration has been exceeded") self.log.info("-- Timer set to " + str(self.TIMER_INTERVAL) + " secs") self.log.info("-- Video will be marked as \"is watching\" from " + str(self.START_WATCHING_TIMER) + " secs") # VLC configuration self.vlc_ip = self.config.get("VLC", "IP") self.vlc_port = self.config.getint("VLC", "Port") self.log.info("Listening VLC to " + self.vlc_ip + ":" + str(self.vlc_port)) # Trakt app configuration trakt_id = ("0e59f99095515c228d5fbc104e342574" + "941aeeeda95946b8fa50b2b0366609bf") trakt_sc = ("3ed1d013ef80eb0bb45d8da8424b4b61" + "3713abb057ed505683caf0baf1b5c650") # Trakt user information trakt = { "PIN": None, "access_token": None, "refresh_token": None, "Username": None, # Now deprecated in Trakt v2 API "Password": None, # Now deprecated in Trakt v2 API } for opt in trakt.keys(): if self.config.has_option("Trakt", opt): trakt[opt] = self.config.get("Trakt", opt) # Initialize Trakt client modifiedTime = time.strftime( '%Y-%m-%d', time.gmtime(os.path.getmtime(get_file()))) self.trakt_client = TraktClient.TraktClient({ 'username': trakt['Username'], 'password': trakt['Password'], 'client_id': trakt_id, 'client_secret': trakt_sc, 'app_version': __version__, 'app_date': modifiedTime, 'pin': trakt['PIN'], 'access_token': trakt['access_token'], 'refresh_token': trakt['refresh_token'], 'callback_token': self.__callback_token_change, }) # Initialize TraktForVLC's cache self.resetCache() # Initialize tvdb api self.tvdb = tvdb_api.Tvdb(cache=False, language='en') self.watching_now = "" self.vlcTime = 0 self.vlc_connected = True def __load_config(self): self.config = RawConfigParser() self.config.optionxform = str self.config.read(self.configfile) # Initialize timers if SMALL_TIMERS: self.TIMER_INTERVAL = 5 self.START_WATCHING_TIMER = 5 else: self.TIMER_INTERVAL = self.config.getint("TraktForVLC", "Timer") self.START_WATCHING_TIMER = self.config.getint( "TraktForVLC", "StartWatching") # For the use of filenames instead of VLC window title self.USE_FILENAME = ( True if self.config.get("TraktForVLC", "UseFilenames") == 'Yes' else False) # Do we have to scrobble ? self.DO_SCROBBLE_MOVIE = ( True if self.config.get("TraktForVLC", "ScrobbleMovie") == 'Yes' else False) self.DO_SCROBBLE_TV = ( True if self.config.get("TraktForVLC", "ScrobbleTV") == 'Yes' else False) # Do we have to mark as watching ? self.DO_WATCHING_MOVIE = ( True if self.config.get("TraktForVLC", "WatchingMovie") == 'Yes' else False) self.DO_WATCHING_TV = ( True if self.config.get("TraktForVLC", "WatchingTV") == 'Yes' else False) # What percent should we use to scrobble videos ? self.SCROBBLE_PERCENT = self.config.getint( "TraktForVLC", "ScrobblePercent") def __callback_token_change(self, access_token, refresh_token): if self.config.has_option('Trakt', 'PIN'): self.config.remove_option('Trakt', 'PIN') self.config.set('Trakt', 'access_token', access_token) self.config.set('Trakt', 'refresh_token', refresh_token) if not self.__save_config(): self.log.debug("Error while saving tokens in configuration file!") def __save_config(self): saved = False with open(self.configfile, 'w') as configfile: self.config.write(configfile) saved = True return saved def resetCache(self, filepath=None, filename=None, filelength=None): self.log.debug("reset cache (%s, %s)" % (filename, filelength)) self.cache = { "vlc_file_path": filepath, "vlc_file_name": filename, "vlc_file_length": filelength, "scrobbled": False, "movie_info": None, "series_info": None, "series_current_ep": -1, "started_watching": None, "watching": -1, "video": {}, } def resetCacheView(self, episode=None): self.log.debug('reset cache view status (%s)' % episode) self.cache['watching'] = -1 self.cache['scrobbled'] = False self.cache['started_watching'] = None if episode is not None: self.cache['series_current_ep'] = episode def close(self, signal, frame): self.log.info("Program closed by SIGINT") sys.exit(0) def run(self): signal.signal(signal.SIGINT, self.close) while (True): try: self.main() except Exception as e: self.log.error( "An unknown error occurred", exc_info=sys.exc_info()) time.sleep(self.TIMER_INTERVAL) self.main() def main(self): try: vlc = VLCRemote(self.vlc_ip, self.vlc_port) self.vlc_connected = True except: if self.vlc_connected: self.log.info('Could not find VLC running at ' + str(self.vlc_ip) + ':' + str(self.vlc_port)) self.log.debug('Make sure your VLC player is running with ' + '--extraintf=rc --rc-host=' + str(self.vlc_ip) + ':' + str(self.vlc_port) + ' --rc-quiet', exc_info=sys.exc_info()) self.vlc_connected = False # If we were watching a video but we didn't finish it, we # have to cancel the watching status if self.cache["watching"] > -1 and not self.cache["scrobbled"]: self.trakt_client.cancelWatching( self.cache["video"]["imdbid"], self.get_episode(self.cache["video"])) # If there is something in the cache, we can purge the watching # and scrobbled information, so if the video is opened again we # will consider it's a new watch if self.cache['vlc_file_name'] is not None: self.resetCacheView() return vlcStatus = vlc.is_playing() if not vlcStatus: vlc.close() return currentFileLength = vlc.get_length() if not int(currentFileLength) > 0: self.log.debug("main::File length is 0, can't do anything") vlc.close() return currentFilePath = vlc.get_filename() if self.USE_FILENAME: currentFileName = currentFilePath.decode('utf-8') else: currentFileName = vlc.get_title().decode('utf-8') self.vlcTime = int(vlc.get_time()) # Parse the filename to verify if it comes from a stream parsed = urlparse(currentFileName) if parsed.netloc: # Set the filename using only the basename of the parsed path currentFileName = os.path.basename(parsed.path) elif self.USE_FILENAME: # Even if it's not from a stream, if it's a filename we're using # we need to keep only the basename of the parsed path currentFileName = os.path.basename(currentFileName) # Use urllib's unquote to bring back special chars currentFileName = unquote(currentFileName) if (currentFileName == self.cache["vlc_file_name"] and currentFileLength == self.cache['vlc_file_length']): if (self.cache["series_info"] is None and self.cache["movie_info"] is None): video = None elif self.cache["series_info"] is not None: video = self.get_TV(vlc, self.cache["series_info"]) else: video = self.get_Movie(vlc, self.cache["movie_info"]) else: self.log.debug("main::New file: %s (length: %s)" % (currentFileName, currentFileLength)) # If we were watching a video but we didn't finish it, we # have to cancel the watching status if self.cache["watching"] > -1 and not self.cache["scrobbled"]: self.trakt_client.cancelWatching( self.cache["video"]["imdbid"], self.get_episode(self.cache["video"])) self.resetCache(currentFilePath, currentFileName, currentFileLength) self.cache['started_watching'] = (time.time(), self.vlcTime) video = self.get_TV(vlc) if video is None: video = self.get_Movie(vlc) if video is None: self.log.info( "No tv show nor movie found for the current playing video") vlc.close() return # We cache the updated video information self.cache["video"] = video logtitle = video["title"] if video["tv"]: logtitle += (" - %01dx%02d" % (int(video["season"]), int(video["episode"]))) # If we changed episode, we have to reset the view status if (self.cache['watching'] > -1 and self.cache['series_current_ep'] != video['episode']): self.resetCacheView(video['episode']) self.cache['started_watching'] = ( time.time(), self.vlcTime % video['duration']) self.log.info(logtitle + " state : " + str(video["percentage"]) + "%") self.log.debug("main::Video: %s" % str(video)) self.log.debug("main::This video is scrobbled : " + str(self.cache["scrobbled"])) if (((video['tv'] and self.DO_SCROBBLE_TV) or (not video['tv'] and self.DO_SCROBBLE_MOVIE)) and video["percentage"] >= self.SCROBBLE_PERCENT and not self.cache["scrobbled"] and self.cache['started_watching'] is not None and ((time.time() - self.cache['started_watching'][0]) > (float(video['duration']) / 3.0)) and ((self.vlcTime - self.cache['started_watching'][1]) > (float(video['duration']) / 4.0))): self.log.info("Scrobbling " + logtitle + " to Trakt...") try: self.trakt_client.stopWatching(video["imdbid"], video["percentage"], self.get_episode(video)) self.cache["scrobbled"] = True self.log.info(logtitle + " scrobbled to Trakt !") except TraktClient.TraktError as e: self.log.error("An error occurred while trying to scrobble", exc_info=sys.exc_info()) elif (((video['tv'] and self.DO_WATCHING_TV) or (not video['tv'] and self.DO_WATCHING_MOVIE)) and video["percentage"] < self.SCROBBLE_PERCENT and not self.cache["scrobbled"] and ((float(video["duration"]) * float(video["percentage"]) / 100.0) >= self.START_WATCHING_TIMER)): self.log.debug("main::Trying to mark " + logtitle + " watching on Trakt...") try: self.trakt_client.startWatching(video["imdbid"], video["percentage"], self.get_episode(video)) self.log.info(logtitle + " is currently watching on Trakt...") self.cache["watching"] = video["percentage"] except TraktClient.TraktError as e: self.log.error("An error occurred while trying to mark as " + "watching " + logtitle, exc_info=sys.exc_info()) vlc.close() def get_episode(self, video): episode = video["tv"] if episode: episode = (video["show_imdbid"], video["season"], video["episode"]) return episode def get_TV(self, vlc, series_info=(None, None, None)): try: series, seasonNumber, episodeNumber = series_info if series is None: now_playing = parse_tv(self.cache['vlc_file_name']) if not now_playing: self.log.info( "Not able to parse a tvshow from the file title") return seriesName = now_playing['show'] seasonNumber = now_playing['season'] episodeNumber = now_playing['episodes'] if self.valid_TV(seriesName): series = self.tvdb[seriesName] self.cache["series_info"] = ( deepcopy(series), seasonNumber, episodeNumber) if series is not None: duration = int(self.cache['vlc_file_length']) time = int(self.vlcTime) # Calculate the relative time and duration depending on # the number of episodes duration = int(float(duration) / float(len(episodeNumber))) currentEpisode = episodeNumber[int(time / duration)] time = time % duration # Calculate the given percentage for the current episode percentage = time * 100 / duration try: episode = series[int(seasonNumber)][int(currentEpisode)] return self.set_video( True, series['seriesname'], series['firstaired'], episode['imdb_id'], duration, percentage, episode['seasonnumber'], episode['episodenumber'], series['imdb_id']) except: self.log.warning("Episode : No valid episode found !") self.log.debug("get_TV::Here's to help debug", exc_info=sys.exc_info()) self.cache["series_info"] = None return except: self.log.info("No matching tv show found for video playing") self.log.debug("get_TV::Here's to help debug", exc_info=sys.exc_info()) return def valid_TV(self, seriesName): try: series = self.tvdb.search(seriesName) if (len(series) == 0): self.log.debug("valid_TV::no series found with the name '%s'" % seriesName) return False return True except: self.log.debug("valid_TV::no valid title found.", exc_info=sys.exc_info()) return False def get_Movie(self, vlc, movie=None): try: duration = int(self.cache['vlc_file_length']) if movie is None: now_playing = parse_movie(self.cache['vlc_file_name']) title = now_playing['title'] year = now_playing['year'] self.log.debug("get_Movie::Now playing: %s" % str(now_playing)) if self.valid_Movie(self.cache['vlc_file_path'], title, year, duration): movie = self.cache["movie_info"] self.log.debug("get_Movie::Valid movie found: %s" % str(movie)) if movie is not None: playtime = int(self.vlcTime) percentage = playtime * 100 / duration return self.set_video( False, movie['Title'], movie['Year'], movie['imdbID'], duration, percentage) return except: self.log.info("No matching movie found for video playing") self.log.debug("get_Movie::Here's to help debug", exc_info=sys.exc_info()) return def valid_Movie(self, vlcFilePath, vlcTitle, vlcYear, vlcDuration): try: # Get Movie info movie = movie_info.get_movie_info( vlcFilePath, vlcTitle, vlcYear, vlcDuration) # Compare Movie runtime against VLC runtime time = movie['Runtime'] # Verify that the VLC duration is within 5 minutes of the # official duration if (vlcDuration >= time - 300) and (vlcDuration <= time + 300): self.cache["movie_info"] = deepcopy(movie) return True else: self.log.debug("valid_Movie::time range not respected " + "(%d +-300 != %d)" % (time, vlcDuration)) except: self.log.debug("valid_Movie::no valid title found", exc_info=sys.exc_info()) return False return False def set_video(self, tv, title, year, imdbid, duration, percentage, season=-1, episode=-1, show_imdbid=None): video = { 'tv': tv, 'title': title, 'year': year, 'imdbid': imdbid, 'duration': duration, 'percentage': percentage, 'season': season, 'episode': episode, 'show_imdbid': show_imdbid, } return video
class Config(object): """A wrapper around RawConfigParser""" def __init__(self, version=None): """Use read() to read in an existing config file. version should be an int starting with 0 that gets incremented if you want to register a new upgrade function. If None, upgrade is disabled. """ self._config = ConfigParser(dict_type=_sorted_dict) self._version = version self._loaded_version = None self._upgrade_funcs = [] self._initial = {} def _do_upgrade(self, func): assert self._loaded_version is not None assert self._version is not None old_version = self._loaded_version new_version = self._version if old_version != new_version: print_d("Config upgrade: %d->%d (%r)" % ( old_version, new_version, func)) func(self, old_version, new_version) def get_version(self): """Get the version of the loaded config file (for testing only) Raises Error if no file was loaded or versioning is disabled. """ if self._version is None: raise Error("Versioning disabled") if self._loaded_version is None: raise Error("No file loaded") return self._loaded_version def register_upgrade_function(self, function): """Register an upgrade function that gets called at each read() if the current config version and the loaded version don't match. Can also be registered after read was called. function(config, old_version: int, new_version: int) -> None """ if self._version is None: raise Error("Versioning disabled") self._upgrade_funcs.append(function) # after read(), so upgrade now if self._loaded_version is not None: self._do_upgrade(function) return function def set_inital(self, section, option, value): """Set an initial value for an option. The section must be added with add_section() first. Adds the value to the config and calling reset() will reset the value to it. """ self.set(section, option, value) self._initial.setdefault(section, {}) self._initial[section].setdefault(option, {}) self._initial[section][option] = value def reset(self, section, option): """Reset the value to the initial state""" value = self._initial[section][option] self.set(section, option, value) def options(self, section): """Returns a list of options available in the specified section.""" return self._config.options(section) def get(self, section, option, default=_DEFAULT): """get(section, option[, default]) -> str If default is not given, raises Error in case of an error """ try: return self._config.get(section, option) except Error: if default is _DEFAULT: raise return default def getboolean(self, section, option, default=_DEFAULT): """getboolean(section, option[, default]) -> bool If default is not given, raises Error in case of an error """ try: return self._config.getboolean(section, option) except Error: if default is _DEFAULT: raise if not isinstance(default, bool): raise ValueError return default def getint(self, section, option, default=_DEFAULT): """getint(section, option[, default]) -> int If default is not give, raises Error in case of an error """ try: return self._config.getint(section, option) except Error: if default is _DEFAULT: raise if not isinstance(default, int): raise ValueError return default def getfloat(self, section, option, default=_DEFAULT): """getfloat(section, option[, default]) -> float If default is not give, raises Error in case of an error """ try: return self._config.getfloat(section, option) except Error: if default is _DEFAULT: raise if not isinstance(default, float): raise ValueError return default def getstringlist(self, section, option, default=_DEFAULT): """getstringlist(section, option[, default]) -> list If default is not given, raises Error in case of an error. Gets a list of strings, using CSV to parse and delimit. """ try: value = self._config.get(section, option) except Error: if default is _DEFAULT: raise if not isinstance(default, list): raise ValueError return default parser = csv.reader( [value], lineterminator='\n', quoting=csv.QUOTE_MINIMAL) try: vals = [v.decode('utf-8') for v in parser.next()] except (csv.Error, ValueError) as e: raise Error(e) return vals def setstringlist(self, section, option, values): """Saves a list of unicode strings using the csv module""" sw = StringIO() values = [unicode(v).encode('utf-8') for v in values] writer = csv.writer(sw, lineterminator='\n', quoting=csv.QUOTE_MINIMAL) writer.writerow(values) self._config.set(section, option, sw.getvalue()) def setlist(self, section, option, values, sep=","): """Saves a list of str using ',' as a separator and \\ for escaping""" values = map(str, values) joined = join_escape(values, sep) self._config.set(section, option, joined) def getlist(self, section, option, default=_DEFAULT, sep=","): """Returns a str list saved with setlist()""" try: value = self._config.get(section, option) except Error: if default is _DEFAULT: raise if not isinstance(default, list): raise ValueError return default return split_escape(value, sep) def set(self, section, option, value): """Saves the string representation for the passed value Don't pass unicode, encode first. """ # RawConfigParser only allows string values but doesn't # scream if they are not (and it only fails before the # first config save..) if not isinstance(value, str): value = str(value) self._config.set(section, option, value) def setdefault(self, section, option, default): """Like set but only sets the new value if the option isn't set before. """ if not self._config.has_option(section, option): self._config.set(section, option, default) def write(self, filename): """Write config to filename. Can raise EnvironmentError """ assert is_fsnative(filename) mkdir(os.path.dirname(filename)) # temporary set the new version for saving if self._version is not None: self.add_section("__config__") self.set("__config__", "version", self._version) try: with atomic_save(filename, ".tmp", "wb") as fileobj: self._config.write(fileobj) finally: if self._loaded_version is not None: self.set("__config__", "version", self._loaded_version) def clear(self): """Remove all sections and initial values""" for section in self._config.sections(): self._config.remove_section(section) self._initial.clear() def is_empty(self): """Whether the config has any sections""" return not self._config.sections() def read(self, filename): """Reads the config from `filename` if the file exists, otherwise does nothing Can raise EnvironmentError, Error. """ parsed_filenames = self._config.read(filename) # don't upgrade if we just created a new config if parsed_filenames and self._version is not None: self._loaded_version = self.getint("__config__", "version", -1) for func in self._upgrade_funcs: self._do_upgrade(func) def sections(self): """Return a list of the sections available""" return self._config.sections() def has_option(self, section, option): """If the given section exists, and contains the given option""" return self._config.has_option(section, option) def remove_option(self, section, option): """Remove the specified option from the specified section Can raise Error. """ return self._config.remove_option(section, option) def add_section(self, section): """Add a section named section to the instance if it not already exists.""" if not self._config.has_section(section): self._config.add_section(section)
class MrxsFile(object): def __init__(self, filename): # Split filename dirname, ext = os.path.splitext(filename) if ext != '.mrxs': raise UnrecognizedFile # Parse slidedat self._slidedatfile = os.path.join(dirname, 'Slidedat.ini') self._dat = RawConfigParser() self._dat.optionxform = str try: with open(self._slidedatfile, 'rb') as fh: self._have_bom = (fh.read(len(UTF8_BOM)) == UTF8_BOM) if not self._have_bom: fh.seek(0) self._dat.readfp(fh) except IOError: raise UnrecognizedFile # Get file paths self._indexfile = os.path.join(dirname, self._dat.get(MRXS_HIERARCHICAL, 'INDEXFILE')) self._datafiles = [os.path.join(dirname, self._dat.get('DATAFILE', 'FILE_%d' % i)) for i in range(self._dat.getint('DATAFILE', 'FILE_COUNT'))] # Build levels self._make_levels() def _make_levels(self): self._levels = {} self._level_list = [] layer_count = self._dat.getint(MRXS_HIERARCHICAL, 'NONHIER_COUNT') for layer_id in range(layer_count): level_count = self._dat.getint(MRXS_HIERARCHICAL, 'NONHIER_%d_COUNT' % layer_id) for level_id in range(level_count): level = MrxsNonHierLevel(self._dat, layer_id, level_id, len(self._level_list)) self._levels[(level.layer_name, level.name)] = level self._level_list.append(level) @classmethod def _read_int32(cls, f): buf = f.read(4) if len(buf) != 4: raise IOError('Short read') return struct.unpack('<i', buf)[0] @classmethod def _assert_int32(cls, f, value): v = cls._read_int32(f) if v != value: raise ValueError('%d != %d' % (v, value)) def _get_data_location(self, record): with open(self._indexfile, 'rb') as fh: fh.seek(MRXS_NONHIER_ROOT_OFFSET) # seek to record table_base = self._read_int32(fh) fh.seek(table_base + record * 4) # seek to list head list_head = self._read_int32(fh) fh.seek(list_head) # seek to data page self._assert_int32(fh, 0) page = self._read_int32(fh) fh.seek(page) # check pagesize self._assert_int32(fh, 1) # read rest of prologue self._read_int32(fh) self._assert_int32(fh, 0) self._assert_int32(fh, 0) # read values position = self._read_int32(fh) size = self._read_int32(fh) fileno = self._read_int32(fh) return (self._datafiles[fileno], position, size) def _zero_record(self, record): path, offset, length = self._get_data_location(record) with open(path, 'r+b') as fh: fh.seek(0, 2) do_truncate = (fh.tell() == offset + length) if DEBUG: if do_truncate: print 'Truncating', path, 'to', offset else: print 'Zeroing', path, 'at', offset, 'for', length fh.seek(offset) buf = fh.read(len(JPEG_SOI)) if buf != JPEG_SOI: raise IOError('Unexpected data in nonhier image') if do_truncate: fh.truncate(offset) else: fh.seek(offset) fh.write('\0' * length) def _delete_index_record(self, record): if DEBUG: print 'Deleting record', record with open(self._indexfile, 'r+b') as fh: entries_to_move = len(self._level_list) - record - 1 if entries_to_move == 0: return # get base of table fh.seek(MRXS_NONHIER_ROOT_OFFSET) table_base = self._read_int32(fh) # read tail of table fh.seek(table_base + (record + 1) * 4) buf = fh.read(entries_to_move * 4) if len(buf) != entries_to_move * 4: raise IOError('Short read') # overwrite the target record fh.seek(table_base + record * 4) fh.write(buf) def _hier_keys_for_level(self, level): ret = [] for k, _ in self._dat.items(MRXS_HIERARCHICAL): if k == level.key_prefix or k.startswith(level.key_prefix + '_'): ret.append(k) return ret def _rename_section(self, old, new): if self._dat.has_section(old): if DEBUG: print '[%s] -> [%s]' % (old, new) self._dat.add_section(new) for k, v in self._dat.items(old): self._dat.set(new, k, v) self._dat.remove_section(old) elif DEBUG: print '[%s] does not exist' % old def _delete_section(self, section): if DEBUG: print 'Deleting [%s]' % section self._dat.remove_section(section) def _set_key(self, section, key, value): if DEBUG: prev = self._dat.get(section, key) print '[%s] %s: %s -> %s' % (section, key, prev, value) self._dat.set(section, key, value) def _rename_key(self, section, old, new): if DEBUG: print '[%s] %s -> %s' % (section, old, new) v = self._dat.get(section, old) self._dat.remove_option(section, old) self._dat.set(section, new, v) def _delete_key(self, section, key): if DEBUG: print 'Deleting [%s] %s' % (section, key) self._dat.remove_option(section, key) def _write(self): buf = StringIO() self._dat.write(buf) with open(self._slidedatfile, 'wb') as fh: if self._have_bom: fh.write(UTF8_BOM) fh.write(buf.getvalue().replace('\n', '\r\n')) def delete_level(self, layer_name, level_name): level = self._levels[(layer_name, level_name)] record = level.record # Zero image data self._zero_record(record) # Delete pointer from nonhier table in index self._delete_index_record(record) # Remove slidedat keys for k in self._hier_keys_for_level(level): self._delete_key(MRXS_HIERARCHICAL, k) # Remove slidedat section self._delete_section(level.section) # Rename section and keys for subsequent levels in the layer prev_level = level for cur_level in self._level_list[record + 1:]: if cur_level.layer_id != prev_level.layer_id: break for k in self._hier_keys_for_level(cur_level): new_k = k.replace(cur_level.key_prefix, prev_level.key_prefix, 1) self._rename_key(MRXS_HIERARCHICAL, k, new_k) self._set_key(MRXS_HIERARCHICAL, prev_level.section_key, prev_level.section) self._rename_section(cur_level.section, prev_level.section) prev_level = cur_level # Update level count within layer count_k = 'NONHIER_%d_COUNT' % level.layer_id count_v = self._dat.getint(MRXS_HIERARCHICAL, count_k) self._set_key(MRXS_HIERARCHICAL, count_k, count_v - 1) # Write slidedat self._write() # Refresh metadata self._make_levels()
class TraktForVLC(object): # Check if there's a newer version of TraktForVLC on the project's github, # and print informations on that subject in the logs def __check_version(self): # The leading information for lines printed by this method in logs lead = "VERSION:" # Request the github API to get the releases information github_releases = requests.get( url="https://api.github.com/repos/XaF/TraktForVLC/releases") # If there was a problem getting the releases if not github_releases.ok: self.log.error(lead + "Unable to verify new releases of TraktForVLC") return # Else, we get the json answer releases = github_releases.json() # If we didn't find any release if not releases: self.log.warning(lead + "No releases found on github") return # We get the latest release, all included newest = sorted( releases, key=lambda x: x['tag_name'], reverse=True)[0] if newest['tag_name'][:1] == "v": newest['tag_name'] = newest['tag_name'][1:] newest_V = parse_version(newest['tag_name']) # We get the latest _stable_ release newest_stbl = sorted( [r for r in releases if not r['prerelease']], key=lambda x: x['tag_name'], reverse=True)[0] if newest_stbl['tag_name'][:1] == "v": newest_stbl['tag_name'] = newest_stbl['tag_name'][1:] newest_stbl_V = parse_version(newest_stbl['tag_name']) # We parse the current version current_V = parse_version(__version__) if newest_V <= current_V: self.log.info(lead + "TraktForVLC is up to date") return # We only show the latest stable release if # it's newer than our current release if newest_stbl_V > current_V: # We reformat the publication date of the release published = datetime.datetime.strptime( newest_stbl['published_at'], "%Y-%m-%dT%H:%M:%SZ").strftime('%c') self.log.info(lead + "##### RELEASE #####") self.log.info(lead + "## Stable release %(name)s" % newest_stbl) self.log.info(lead + "## Published on %s" % published) self.log.info(lead + "## Available on %(html_url)s" % newest_stbl) # We only show the latest release if it's not # also the latest stable release if newest_V > newest_stbl_V: # We reformat the publication date of the release published = datetime.datetime.strptime( newest['published_at'], "%Y-%m-%dT%H:%M:%SZ").strftime('%c') self.log.info(lead + "##### RELEASE #####") self.log.info(lead + "## Prerelease %(name)s" % newest) self.log.info(lead + "## Published on %s" % published) self.log.info(lead + "## Available on %(html_url)s" % newest) self.log.info(lead + "###################") def __init__(self, datadir, configfile, daemon=False): # Verify if the log directory exists or create it logdir = os.path.join(datadir, 'logs') if not os.path.exists(logdir): os.mkdir(logdir) # Process log file name if daemon: if LOG_LEVEL is logging.DEBUG: logfile = os.path.join(logdir, "TraktForVLC-DEBUG.log") # Remove existing DEBUG file if os.path.isfile(logfile): os.remove(logfile) else: logfile = os.path.join( logdir, "TraktForVLC-" + DATETIME.strftime("%Y%m%d-%H%M") + ".log") logging.basicConfig( format="%(asctime)s::%(name)s::%(levelname)s::%(message)s", level=LOG_LEVEL, filename=logfile) else: logging.basicConfig( format="%(asctime)s::%(name)s::%(levelname)s::%(message)s", level=LOG_LEVEL, stream=sys.stderr) self.log = logging.getLogger("TraktForVLC") self.log.info( "## TraktForVLC v" + __version__ + " " + __release_name__) self.log.info("## Copyright (C) 2014-2015 " + "Raphaël Beamonte <*****@*****.**>") self.log.info("##") self.log.info("## TraktForVLC is distributed in the hope that it " + "will be useful, but") self.log.info("## with ABSOLUTELY NO WARRANTY. This is free " + "software; you are welcome") self.log.info("## to redistribute and/or modify it under the terms " + "of the GPL2.") self.log.info("") if not os.path.isfile(configfile): self.log.error("Config file " + configfile + " not found, exiting.") exit() self.log.debug("Running on %s, with Python %s" % ( platform.platform(), platform.python_version())) self.__check_version() # Load configuration self.configfile = configfile self.__load_config() for loglvl, logstr in AVAILABLE_LOGLVL: if LOG_LEVEL <= loglvl: loglevelstr = logstr break if loglevelstr is None: loglevelstr = str(LOG_LEVEL) self.log.info("Logger level is set to %s" % loglevelstr) self.log.info("-- Will scrobble movies ? %s" % ( 'Yes' if self.DO_SCROBBLE_MOVIE else 'No')) self.log.info("-- Will scrobble tv shows ? %s" % ( 'Yes' if self.DO_SCROBBLE_TV else 'No')) self.log.info("-- Will we mark movies as being watched ? %s" % ( 'Yes' if self.DO_WATCHING_MOVIE else 'No')) self.log.info("-- Will we mark tv shows as being watched ? %s" % ( 'Yes' if self.DO_WATCHING_TV else 'No')) self.log.info("-- Videos will be scrobbled after " + str(self.SCROBBLE_PERCENT) + "% of their duration has been exceeded") self.log.info("-- Timer set to " + str(self.TIMER_INTERVAL) + " secs") self.log.info("-- Video will be marked as \"is watching\" from " + str(self.START_WATCHING_TIMER) + " secs") # VLC configuration self.vlc_ip = self.config.get("VLC", "IP") self.vlc_port = self.config.getint("VLC", "Port") self.log.info("Listening VLC to " + self.vlc_ip + ":" + str(self.vlc_port)) # Trakt app configuration trakt_id = ("0e59f99095515c228d5fbc104e342574" + "941aeeeda95946b8fa50b2b0366609bf") trakt_sc = ("3ed1d013ef80eb0bb45d8da8424b4b61" + "3713abb057ed505683caf0baf1b5c650") # Trakt user information trakt = { "PIN": None, "access_token": None, "refresh_token": None, "Username": None, # Now deprecated in Trakt v2 API "Password": None, # Now deprecated in Trakt v2 API } for opt in trakt.keys(): if self.config.has_option("Trakt", opt): trakt[opt] = self.config.get("Trakt", opt) # Initialize Trakt client modifiedTime = time.strftime( '%Y-%m-%d', time.gmtime(os.path.getmtime(__file__))) self.trakt_client = TraktClient.TraktClient({ 'username': trakt['Username'], 'password': trakt['Password'], 'client_id': trakt_id, 'client_secret': trakt_sc, 'app_version': __version__, 'app_date': modifiedTime, 'pin': trakt['PIN'], 'access_token': trakt['access_token'], 'refresh_token': trakt['refresh_token'], 'callback_token': self.__callback_token_change, }) # Initialize TraktForVLC's cache self.resetCache() # Initialize tvdb api self.tvdb = tvdb_api.Tvdb(cache=False, language='en') self.watching_now = "" self.vlcTime = 0 self.vlc_connected = True def __load_config(self): self.config = RawConfigParser() self.config.optionxform = str self.config.read(self.configfile) # Initialize timers if SMALL_TIMERS: self.TIMER_INTERVAL = 5 self.START_WATCHING_TIMER = 5 else: self.TIMER_INTERVAL = self.config.getint("TraktForVLC", "Timer") self.START_WATCHING_TIMER = self.config.getint( "TraktForVLC", "StartWatching") # For the use of filenames instead of VLC window title self.USE_FILENAME = ( True if self.config.get("TraktForVLC", "UseFilenames") == 'Yes' else False) # Do we have to scrobble ? self.DO_SCROBBLE_MOVIE = ( True if self.config.get("TraktForVLC", "ScrobbleMovie") == 'Yes' else False) self.DO_SCROBBLE_TV = ( True if self.config.get("TraktForVLC", "ScrobbleTV") == 'Yes' else False) # Do we have to mark as watching ? self.DO_WATCHING_MOVIE = ( True if self.config.get("TraktForVLC", "WatchingMovie") == 'Yes' else False) self.DO_WATCHING_TV = ( True if self.config.get("TraktForVLC", "WatchingTV") == 'Yes' else False) # What percent should we use to scrobble videos ? self.SCROBBLE_PERCENT = self.config.getint( "TraktForVLC", "ScrobblePercent") def __callback_token_change(self, access_token, refresh_token): if self.config.has_option('Trakt', 'PIN'): self.config.remove_option('Trakt', 'PIN') self.config.set('Trakt', 'access_token', access_token) self.config.set('Trakt', 'refresh_token', refresh_token) if not self.__save_config(): self.log.debug("Error while saving tokens in configuration file!") def __save_config(self): saved = False with open(self.configfile, 'wb') as configfile: self.config.write(configfile) saved = True return saved def resetCache(self, filename=None, filelength=None): self.log.debug("reset cache (%s, %s)" % (filename, filelength)) self.cache = { "vlc_file_name": filename, "vlc_file_length": filelength, "scrobbled": False, "movie_info": None, "series_info": None, "series_current_ep": -1, "started_watching": None, "watching": -1, "video": {}, } def resetCacheView(self, episode=None): self.log.debug('reset cache view status (%s)' % episode) self.cache['watching'] = -1 self.cache['scrobbled'] = False self.cache['started_watching'] = None if episode is not None: self.cache['series_current_ep'] = episode def close(self, signal, frame): self.log.info("Program closed by SIGINT") sys.exit(0) def run(self): signal.signal(signal.SIGINT, self.close) while (True): try: self.main() except Exception as e: self.log.error( "An unknown error occurred", exc_info=sys.exc_info()) time.sleep(self.TIMER_INTERVAL) self.main() def main(self): try: vlc = VLCRemote(self.vlc_ip, self.vlc_port) self.vlc_connected = True except: if self.vlc_connected: self.log.info('Could not find VLC running at ' + str(self.vlc_ip) + ':' + str(self.vlc_port)) self.log.debug('Make sure your VLC player is running with ' + '--extraintf=rc --rc-host=' + str(self.vlc_ip) + ':' + str(self.vlc_port) + ' --rc-quiet', exc_info=sys.exc_info()) self.vlc_connected = False # If we were watching a video but we didn't finish it, we # have to cancel the watching status if self.cache["watching"] > -1 and not self.cache["scrobbled"]: self.trakt_client.cancelWatching( self.cache["video"]["imdbid"], self.get_episode(self.cache["video"])) # If there is something in the cache, we can purge the watching # and scrobbled information, so if the video is opened again we # will consider it's a new watch if self.cache['vlc_file_name'] is not None: self.resetCacheView() return vlcStatus = vlc.is_playing() if not vlcStatus: vlc.close() return currentFileLength = vlc.get_length() if not int(currentFileLength) > 0: self.log.debug("main::File length is 0, can't do anything") vlc.close() return if self.USE_FILENAME: currentFileName = vlc.get_filename() else: currentFileName = vlc.get_title() self.vlcTime = int(vlc.get_time()) # Parse the filename to verify if it comes from a stream parsed = urlparse(currentFileName) if parsed.netloc: # Set the filename using only the basename of the parsed path currentFileName = os.path.basename(parsed.path) # And use urllib's unquote to bring back special chars currentFileName = unquote(currentFileName) elif self.USE_FILENAME: # Even if it's not from a stream, if it's a filename we're using # we need to keep only the basename of the parsed path currentFileName = os.path.basename(currentFileName) if (currentFileName == self.cache["vlc_file_name"] and currentFileLength == self.cache['vlc_file_length']): if (self.cache["series_info"] is None and self.cache["movie_info"] is None): video = None elif self.cache["series_info"] is not None: video = self.get_TV(vlc, self.cache["series_info"]) else: video = self.get_Movie(vlc, self.cache["movie_info"]) else: self.log.debug("main::New file: %s (%s)" % (currentFileName, currentFileLength)) # If we were watching a video but we didn't finish it, we # have to cancel the watching status if self.cache["watching"] > -1 and not self.cache["scrobbled"]: self.trakt_client.cancelWatching( self.cache["video"]["imdbid"], self.get_episode(self.cache["video"])) self.resetCache(currentFileName, currentFileLength) self.cache['started_watching'] = (time.time(), self.vlcTime) video = self.get_TV(vlc) if video is None: video = self.get_Movie(vlc) if video is None: self.log.info( "No tv show nor movie found for the current playing video") vlc.close() return # We cache the updated video information self.cache["video"] = video logtitle = video["title"] if video["tv"]: logtitle += (" - %01dx%02d" % (int(video["season"]), int(video["episode"]))) # If we changed episode, we have to reset the view status if (self.cache['watching'] > -1 and self.cache['series_current_ep'] != video['episode']): self.resetCacheView(video['episode']) self.cache['started_watching'] = ( time.time(), self.vlcTime % video['duration']) self.log.info(logtitle + " state : " + str(video["percentage"]) + "%") self.log.debug("main::Video: %s" % str(video)) self.log.debug("main::This video is scrobbled : " + str(self.cache["scrobbled"])) if (((video['tv'] and self.DO_SCROBBLE_TV) or (not video['tv'] and self.DO_SCROBBLE_MOVIE)) and video["percentage"] >= self.SCROBBLE_PERCENT and not self.cache["scrobbled"] and self.cache['started_watching'] is not None and ((time.time() - self.cache['started_watching'][0]) > (float(video['duration']) / 3.0)) and ((self.vlcTime - self.cache['started_watching'][1]) > (float(video['duration']) / 4.0))): self.log.info("Scrobbling " + logtitle + " to Trakt...") try: self.trakt_client.stopWatching(video["imdbid"], video["percentage"], self.get_episode(video)) self.cache["scrobbled"] = True self.log.info(logtitle + " scrobbled to Trakt !") except TraktClient.TraktError as e: self.log.error("An error occurred while trying to scrobble", exc_info=sys.exc_info()) elif (((video['tv'] and self.DO_WATCHING_TV) or (not video['tv'] and self.DO_WATCHING_MOVIE)) and video["percentage"] < self.SCROBBLE_PERCENT and not self.cache["scrobbled"] and ((float(video["duration"]) * float(video["percentage"]) / 100.0) >= self.START_WATCHING_TIMER)): self.log.debug("main::Trying to mark " + logtitle + " watching on Trakt...") try: self.trakt_client.startWatching(video["imdbid"], video["percentage"], self.get_episode(video)) self.log.info(logtitle + " is currently watching on Trakt...") self.cache["watching"] = video["percentage"] except TraktClient.TraktError as e: self.log.error("An error occurred while trying to mark as " + "watching " + logtitle, exc_info=sys.exc_info()) vlc.close() def get_episode(self, video): episode = video["tv"] if episode: episode = (video["show_imdbid"], video["season"], video["episode"]) return episode def get_TV(self, vlc, series_info=(None, None, None)): try: series, seasonNumber, episodeNumber = series_info if series is None: now_playing = parse_tv(self.cache['vlc_file_name']) if not now_playing: self.log.info( "Not able to parse a tvshow from the file title") return seriesName = now_playing['show'] seasonNumber = now_playing['season'] episodeNumber = now_playing['episodes'] if self.valid_TV(seriesName): series = self.tvdb[seriesName] self.cache["series_info"] = ( deepcopy(series), seasonNumber, episodeNumber) if series is not None: duration = int(self.cache['vlc_file_length']) time = int(self.vlcTime) # Calculate the relative time and duration depending on # the number of episodes duration = int(float(duration) / float(len(episodeNumber))) currentEpisode = episodeNumber[int(time / duration)] time = time % duration # Calculate the given percentage for the current episode percentage = time * 100 / duration try: episode = series[int(seasonNumber)][int(currentEpisode)] return self.set_video( True, series['seriesname'], series['firstaired'], episode['imdb_id'], duration, percentage, episode['seasonnumber'], episode['episodenumber'], series['imdb_id']) except: self.log.warning("Episode : No valid episode found !") self.log.debug("get_TV::Here's to help debug", exc_info=sys.exc_info()) self.cache["series_info"] = None return except: self.log.info("No matching tv show found for video playing") self.log.debug("get_TV::Here's to help debug", exc_info=sys.exc_info()) return def valid_TV(self, seriesName): try: series = self.tvdb.search(seriesName) if (len(series) == 0): self.log.debug("valid_TV::no series found with the name '%s'" % seriesName) return False return True except: self.log.debug("valid_TV::no valid title found.", exc_info=sys.exc_info()) return False def get_Movie(self, vlc, movie=None): try: duration = int(self.cache['vlc_file_length']) if movie is None: now_playing = parse_movie(self.cache['vlc_file_name']) title = now_playing['title'] year = now_playing['year'] self.log.debug("get_Movie::Now playing: %s" % str(now_playing)) if self.valid_Movie(title, year, duration): movie = self.cache["movie_info"] self.log.debug("get_Movie::Valid movie found: %s" % str(movie)) if movie is not None: playtime = int(self.vlcTime) percentage = playtime * 100 / duration return self.set_video( False, movie['Title'], movie['Year'], movie['imdbID'], duration, percentage) return except: self.log.info("No matching movie found for video playing") self.log.debug("get_Movie::Here's to help debug", exc_info=sys.exc_info()) return def valid_Movie(self, vlcTitle, vlcYear, vlcDuration): try: # Get Movie info movie = movie_info.get_movie_info(vlcTitle, vlcYear) # Compare Movie runtime against VLC runtime regex = re.compile('^((?P<hour>[0-9]{1,2}).*?h)' + '?.*?(?P<min>[0-9]+).*?min?', re.IGNORECASE | re.MULTILINE) r = regex.search(movie['Runtime']) try: timeh = 0 if r.group('hour') is None else int(r.group('hour')) timem = 0 if r.group('min') is None else int(r.group('min')) time = timeh * 60 * 60 + timem * 60 except: self.log.debug("valid_Movie::unable to compute the duration", exc_info=sys.exc_info()) return False # Verify that the VLC duration is within 5 minutes of the # official duration if (vlcDuration >= time - 300) and (vlcDuration <= time + 300): self.cache["movie_info"] = deepcopy(movie) return True else: self.log.debug("valid_Movie::time range not respected " + "(%d +-300 != %d)" % (time, vlcDuration)) except: self.log.debug("valid_Movie::no valid title found", exc_info=sys.exc_info()) return False return False def set_video(self, tv, title, year, imdbid, duration, percentage, season=-1, episode=-1, show_imdbid=None): video = { 'tv': tv, 'title': title, 'year': year, 'imdbid': imdbid, 'duration': duration, 'percentage': percentage, 'season': season, 'episode': episode, 'show_imdbid': show_imdbid, } return video
def remove_option(self, section, option): if self.has_option(section, option): RawConfigParser.remove_option(self, section, option)
class Configuration(object): def __init__(self, bus, target_name, file_name): self.bus = bus self.target_name = target_name self.file_name = file_name self.config = RawConfigParser() self.config.read(file_name) if target_name is not None: self.service = bus.create_service({"type": "autoconfigure", "target": target_name}, active=False, use_py_object=_AutoconfigureService(self)) self.sections_object = self.service.create_object("sections", self._compute_sections(), doc=sections_object_doc) self.options_object = self.service.create_object("options", self._compute_options(), doc=options_object_doc) @synchronized(_lock) def _save(self): with open(self.file_name, "w") as file: self.config.write(file) self.sections_object.set_value(self._compute_sections()) self.options_object.set_value(self._compute_options()) @synchronized(_lock) def _compute_sections(self): return self.config.sections() @synchronized(_lock) def _compute_options(self): return dict((section, dict(self.config.items(section))) for section in self.config.sections()) @synchronized(_lock) def sections(self): return self.config.sections() @synchronized(_lock) def add_section(self, name): self.config.add_section(name) self._save() @synchronized(_lock) def has_section(self, name): return self.config.has_section(name) @synchronized(_lock) def options(self, section): return self.config.options(section) @synchronized(_lock) def has_option(self, section, option): return self.config.has_option(section, option) @synchronized(_lock) def get(self, section, option): return self.config.get(section, option) @synchronized(_lock) def getint(self, section, option): return self.config.getint(section, option) @synchronized(_lock) def getfloat(self, section, option): return self.config.getfloat(section, option) @synchronized(_lock) def getboolean(self, section, option): return self.config.getboolean(section, option) @synchronized(_lock) def items(self, section): return self.config.items(section) @synchronized(_lock) def set(self, section, option, value): self.config.set(section, option, value) self._save() @synchronized(_lock) def remove_option(self, section, option): self.config.remove_option(section, option) self._save() @synchronized(_lock) def remove_section(self, section): self.config.remove_section(section) self._save()