def setUpClass(cls):
os.environ['PYSEMANTIC_CONFIG'] = "test.conf"
pr.CONF_FILE_NAME = "test.conf"
cls.testenv = os.environ
cls.test_config_path = op.join(os.getcwd(), "test.conf")
shutil.copy(TEST_CONFIG_FILE_PATH, cls.test_config_path)
# Change the relative paths in the config file to absolute paths
parser = RawConfigParser()
parser.read(cls.test_config_path)
for section in parser.sections():
schema_path = parser.get(section, "specfile")
parser.remove_option(section, "specfile")
parser.set(section, "specfile",
op.join(op.abspath(op.dirname(__file__)), schema_path))
with open(cls.test_config_path, "w") as fileobj:
parser.write(fileobj)
# change the relative paths in the test dictionary to absolute paths
with open(TEST_DATA_DICT, "r") as fileobj:
cls.org_specs = yaml.load(fileobj, Loader=yaml.CLoader)
new_specs = deepcopy(cls.org_specs)
for _, specs in new_specs.iteritems():
path = specs['path']
specs['path'] = op.join(op.abspath(op.dirname(__file__)), path)
# Rewrite this to the file
with open(TEST_DATA_DICT, "w") as fileobj:
yaml.dump(new_specs, fileobj, Dumper=yaml.CDumper,
default_flow_style=False)
def iter_backends(self):
"""
Iterate on backends.
:returns: each tuple contains the backend name, module name and module options
:rtype: :class:`tuple`
"""
config = RawConfigParser()
config.read(self.confpath)
changed = False
for backend_name in config.sections():
params = dict(config.items(backend_name))
try:
module_name = params.pop('_module')
except KeyError:
try:
module_name = params.pop('_backend')
config.set(backend_name, '_module', module_name)
config.remove_option(backend_name, '_backend')
changed = True
except KeyError:
warning('Missing field "_module" for configured backend "%s"', backend_name)
continue
yield backend_name, module_name, params
if changed:
with open(self.confpath, 'wb') as f:
config.write(f)
def tearDownClass(cls):
try:
# modify the testdata back
with open(TEST_DATA_DICT, "r") as fileobj:
test_data = yaml.load(fileobj, Loader=yaml.CLoader)
test_data['iris']['path'] = op.join("testdata", "iris.csv")
test_data['person_activity']['path'] = op.join(
"testdata", "person_activity.tsv")
del test_data['multi_iris']
with open(TEST_DATA_DICT, "w") as fileobj:
test_data = yaml.dump(test_data,
fileobj,
Dumper=yaml.CDumper,
default_flow_style=False)
# Change the config files back
parser = RawConfigParser()
parser.read(cls.test_conf_file)
parser.remove_option("pysemantic", "specfile")
parser.set("pysemantic", "specfile",
op.join("testdata", "test_dictionary.yaml"))
with open(TEST_CONFIG_FILE_PATH, 'w') as fileobj:
parser.write(fileobj)
finally:
os.unlink(cls.test_conf_file)
os.unlink(cls.copied_iris_path)
def tearDownClass(cls):
try:
# modify the testdata back
with open(TEST_DATA_DICT, "r") as fileobj:
test_data = yaml.load(fileobj, Loader=yaml.CLoader)
test_data['iris']['path'] = op.join("testdata", "iris.csv")
test_data['person_activity']['path'] = op.join("testdata",
"person_activity.tsv")
del test_data['multi_iris']
with open(TEST_DATA_DICT, "w") as fileobj:
test_data = yaml.dump(test_data, fileobj, Dumper=yaml.CDumper,
default_flow_style=False)
# Change the config files back
parser = RawConfigParser()
parser.read(cls.test_conf_file)
parser.remove_option("pysemantic", "specfile")
parser.set("pysemantic", "specfile",
op.join("testdata", "test_dictionary.yaml"))
with open(TEST_CONFIG_FILE_PATH, 'w') as fileobj:
parser.write(fileobj)
finally:
os.unlink(cls.test_conf_file)
os.unlink(cls.copied_iris_path)
def rc_to_ini(rc_info, default_section=SECTION):
"""Convert a .rc file to a ConfigParser (.ini-like object)
Items are assumed to be in app:assembl section,
unless prefixed by "{section}__" .
Keys prefixed with an underscore are not passed on.
Keys prefixed with a star are put in the global (DEFAULT) section.
Value of '__delete_key__' is eliminated if existing.
"""
p = Parser()
for key, val in rc_info.iteritems():
if not key or key.startswith('_'):
continue
if key[0] == '*':
# Global keys
section = "DEFAULT"
key = key[1:]
elif '__' in key:
section, key = key.split('__', 1)
else:
section = default_section
ensureSection(p, section)
if val == '__delete_key__':
# Allow to remove a variable from rc
# so we can fall back to underlying ini
p.remove_option(section, key)
else:
p.set(section, key, val)
return p
def yaml_to_ini(yaml_conf, default_section='app:assembl'):
"""Convert a .yaml file to a ConfigParser (.ini-like object)
Items are assumed to be in app:assembl section,
unless prefixed by "{section}__" .
Keys prefixed with an underscore are not passed on.
Keys prefixed with a star are put in the global (DEFAULT) section.
Value of '__delete_key__' is eliminated if existing.
"""
p = RawConfigParser()
ensureSection(p, default_section)
for key, val in yaml_conf.iteritems():
if key.startswith('_'):
continue
if isinstance(val, dict):
if key in _known_invoke_sections:
continue
ensureSection(p, key)
for subk, subv in val.iteritems():
p.set(key, subk, val_to_ini(subv))
else:
if val == '__delete_key__':
# Allow to remove a variable from rc
# so we can fall back to underlying ini
p.remove_option(default_section, key)
else:
p.set(default_section, key, val_to_ini(val))
return p
def remove_option(self, section, option, also_remove_default=False):
try:
if also_remove_default:
DEFAULTS[section].pop(option)
RawConfigParser.remove_option(self, section, option)
except (NoSectionError, KeyError):
pass
def setUpClass(cls):
os.environ['PYSEMANTIC_CONFIG'] = "test.conf"
pr.CONF_FILE_NAME = "test.conf"
cls.testenv = os.environ
cls.test_config_path = op.join(os.getcwd(), "test.conf")
shutil.copy(TEST_CONFIG_FILE_PATH, cls.test_config_path)
# Change the relative paths in the config file to absolute paths
parser = RawConfigParser()
parser.read(cls.test_config_path)
for section in parser.sections():
schema_path = parser.get(section, "specfile")
parser.remove_option(section, "specfile")
parser.set(section, "specfile",
op.join(op.abspath(op.dirname(__file__)), schema_path))
with open(cls.test_config_path, "w") as fileobj:
parser.write(fileobj)
# change the relative paths in the test dictionary to absolute paths
with open(TEST_DATA_DICT, "r") as fileobj:
cls.org_specs = yaml.load(fileobj, Loader=Loader)
new_specs = deepcopy(cls.org_specs)
for _, specs in new_specs.iteritems():
path = specs['path']
specs['path'] = op.join(op.abspath(op.dirname(__file__)), path)
# Rewrite this to the file
with open(TEST_DATA_DICT, "w") as fileobj:
yaml.dump(new_specs, fileobj, Dumper=Dumper,
default_flow_style=False)
def _path_fixer(filepath, root=None):
"""Change all the relative paths in `filepath` to absolute ones.
:param filepath: File to be changed
:param root: Root path with which the relative paths are prefixed. If None
(default), the directory with this file is the root.
"""
if root is None:
root = op.join(op.abspath(op.dirname(__file__)))
if filepath.endswith((".yaml", ".yml")):
with open(filepath, "r") as fileobj:
data = yaml.load(fileobj, Loader=Loader)
for specs in data.itervalues():
specs['path'] = op.join(root, specs['path'])
with open(filepath, "w") as fileobj:
yaml.dump(data, fileobj, Dumper=Dumper, default_flow_style=False)
elif filepath.endswith(".conf"):
parser = RawConfigParser()
parser.read(filepath)
for section in parser.sections():
path = parser.get(section, "specfile")
parser.remove_option(section, "specfile")
parser.set(section, "specfile", op.join(root, path))
with open(filepath, "w") as fileobj:
parser.write(fileobj)
def remove_option(self, section, option, also_remove_default=False):
try:
if also_remove_default:
DEFAULTS[section].pop(option)
RawConfigParser.remove_option(self, section, option)
except (NoSectionError, KeyError):
pass
def iter_backends(self):
"""
Iterate on backends.
:returns: each tuple contains the backend name, module name and module options
:rtype: :class:`tuple`
"""
config = RawConfigParser()
config.read(self.confpath)
changed = False
for backend_name in config.sections():
params = dict(config.items(backend_name))
try:
module_name = params.pop('_module')
except KeyError:
try:
module_name = params.pop('_backend')
config.set(backend_name, '_module', module_name)
config.remove_option(backend_name, '_backend')
changed = True
except KeyError:
warning(
'Missing field "_module" for configured backend "%s"',
backend_name)
continue
yield backend_name, module_name, params
if changed:
with open(self.confpath, 'wb') as f:
config.write(f)
def _path_fixer(filepath, root=None):
"""Change all the relative paths in `filepath` to absolute ones.
:param filepath: File to be changed
:param root: Root path with which the relative paths are prefixed. If None
(default), the directory with this file is the root.
"""
if root is None:
root = op.join(op.abspath(op.dirname(__file__)))
if filepath.endswith((".yaml", ".yml")):
with open(filepath, "r") as fileobj:
data = yaml.load(fileobj, Loader=Loader)
for specs in data.itervalues():
specs['path'] = op.join(root, specs['path'])
with open(filepath, "w") as fileobj:
yaml.dump(data, fileobj, Dumper=Dumper,
default_flow_style=False)
elif filepath.endswith(".conf"):
parser = RawConfigParser()
parser.read(filepath)
for section in parser.sections():
path = parser.get(section, "specfile")
parser.remove_option(section, "specfile")
parser.set(section, "specfile", op.join(root, path))
with open(filepath, "w") as fileobj:
parser.write(fileobj)
class ParamStore(object):
def __init__(self, root_dir, file_name):
if not os.path.isdir(root_dir):
os.makedirs(root_dir)
self._path = os.path.join(root_dir, file_name)
self._dirty = False
# open config file
self._config = RawConfigParser()
self._config.read(self._path)
def __del__(self):
self.flush()
def flush(self):
if not self._dirty:
return
self._dirty = False
of = open(self._path, "w")
self._config.write(of)
of.close()
def get(self, section, option, default=None):
"""Get a parameter value and return a string.
If default is specified and section or option are not defined
in the file, they are created and set to default, which is
then the return value.
"""
if not self._config.has_option(section, option):
if default is not None:
self.set(section, option, default)
return default
return self._config.get(section, option)
def get_datetime(self, section, option, default=None):
result = self.get(section, option, default)
if result:
return safestrptime(result)
return result
def set(self, section, option, value):
"""Set option in section to string value."""
if not self._config.has_section(section):
self._config.add_section(section)
elif self._config.has_option(section, option) and self._config.get(section, option) == value:
return
self._config.set(section, option, value)
self._dirty = True
def unset(self, section, option):
"""Remove option from section."""
if not self._config.has_section(section):
return
if self._config.has_option(section, option):
self._config.remove_option(section, option)
self._dirty = True
if not self._config.options(section):
self._config.remove_section(section)
self._dirty = True
class ConfigStore(object):
def __init__(self, name):
self.config = RawConfigParser()
self.file_opts = {}
if sys.version_info[0] >= 3:
self.file_opts['encoding'] = 'utf-8'
if hasattr(appdirs, 'user_config_dir'):
data_dir = appdirs.user_config_dir('photini')
else:
data_dir = appdirs.user_data_dir('photini')
if not os.path.isdir(data_dir):
os.makedirs(data_dir, mode=0700)
self.file_name = os.path.join(data_dir, '%s.ini' % name)
if name == 'editor':
for old_file_name in (os.path.expanduser('~/photini.ini'),
os.path.join(data_dir, 'photini.ini')):
if os.path.exists(old_file_name):
self.config.read(old_file_name, **self.file_opts)
self.save()
os.unlink(old_file_name)
self.config.read(self.file_name, **self.file_opts)
self.timer = QtCore.QTimer()
self.timer.setSingleShot(True)
self.timer.setInterval(3000)
self.timer.timeout.connect(self.save)
self.has_section = self.config.has_section
def get(self, section, option, default=None):
if self.config.has_option(section, option):
result = self.config.get(section, option)
if sys.version_info[0] < 3:
return result.decode('utf-8')
return result
if default is not None:
self.set(section, option, default)
return default
def set(self, section, option, value):
if not self.config.has_section(section):
self.config.add_section(section)
if (self.config.has_option(section, option) and
self.config.get(section, option) == value):
return
if sys.version_info[0] < 3:
value = value.encode('utf-8')
self.config.set(section, option, value)
self.timer.start()
def remove_section(self, section):
if not self.config.has_section(section):
return
for option in self.config.options(section):
self.config.remove_option(section, option)
self.config.remove_section(section)
self.timer.start()
def save(self):
self.config.write(open(self.file_name, 'w', **self.file_opts))
os.chmod(self.file_name, 0600)
class FileStorage(object):
"""docstring for FileStorage"""
def __init__(self, filepath):
super(FileStorage, self).__init__()
self.filepath = filepath
self.filetype = 'ini'
self.__cp = RawConfigParser()
self.__cp.read(self.filepath)
self.__section = 'data'
if not os.path.isfile(self.filepath):
with open(self.filepath, 'w') as f:
self.__cp.add_section(self.__section)
self.__cp.write(f)
def __flush(self):
with open(self.filepath, 'w') as f:
self.__cp.write(f)
def insert(self, key, data):
if self.__cp.get(self.__section, key):
raise KeyExistsError()
if isinstance(data, dict):
data = json.dumps(data)
self.__cp.set(self.__section, key, data)
self.__flush()
def delete(self, key):
self.__cp.remove_option(self.__section, key)
self.__flush()
def find(self, key, to_json=False):
try:
data = self.__cp.get(self.__section, key)
except:
data = None
if not data:
return {}
if to_json:
try:
return json.loads(data)
except ValueError:
print('cannot decode data')
print(data)
return data
def update(self, key, data):
if isinstance(data, dict):
data = json.dumps(data)
self.__cp.set(self.__section, key, data)
self.__flush()
def upsert(self, key, data):
if isinstance(data, dict):
data = json.dumps(data)
self.__cp.set(self.__section, key, data)
self.__flush()
def remove_item_from_section(self, ini_file, section_name, key_value):
"""
Removes option from *.ini file
:parameter ini_file, ini file where the removing should be done
:parameter section_name, section where the option wants to be removed
:parameter key_value, value which needs to be removed
"""
confp = RawConfigParser()
confp.read(ini_file)
confp.remove_option(section_name, key_value)
with open(ini_file, 'w') as configfile:
confp.write(configfile)
def remove_option(self, option):
"""
Removes an option (in ``section/key`` syntax),
thus will not be saved anymore
:param option: the option path
:type option: string
"""
splitvals = option.split('/')
section, key = "/".join(splitvals[:-1]), splitvals[-1]
RawConfigParser.remove_option(self, section, key)
def remove_item_from_section(self, ini_file, section_name, key_value):
"""
Removes option from *.ini file
:parameter ini_file, ini file where the removing should be done
:parameter section_name, section where the option wants to be removed
:parameter key_value, value which needs to be removed
"""
confp = RawConfigParser()
confp.read(ini_file)
confp.remove_option(section_name, key_value)
with open(ini_file, 'w') as configfile:
confp.write(configfile)
def merge(master, updates, outfile, **kwargs):
if not master:
master = _find_master()
if not isinstance(updates, (list, tuple)):
updates = [updates]
print "Merging files: %s and %s" % (master, ', '.join(updates))
parser = RawConfigParser()
parser.read(updates)
with open(master, 'r') as orig:
with open(outfile, 'w') as new:
current_section = None
for line in orig:
sec_m = re.match("^\[([\w\d_\-\s]+)\]\s*", line, re.IGNORECASE)
if sec_m:
current_section = sec_m.group(1)
new.write(line)
else:
if not parser.has_section(current_section):
new.write(line)
continue
var_m = re.match("^(?:;)?([\w\d\_\-\.]+)\s*=\s*(.*)\n$",
line, re.IGNORECASE)
if var_m:
key, value = var_m.groups()
if parser.has_option(current_section, key):
new_value = parser.get(current_section, key)
# print "REPLACING: %s = %s with value %s" % (key, value, new_value)
new.write("%s = %s\n" % (key, new_value % kwargs))
parser.remove_option(current_section, key)
if not parser.items(current_section):
parser.remove_section(current_section)
else:
new.write(line)
else:
new.write(line)
if parser.sections():
#print "The following values were not set:"
for s in parser.sections():
new.write("")
new.write("[%s]\n" % s)
for t in parser.items(s):
new.write("%s = %s\n" % t)
def merge(master, updates, outfile, **kwargs):
if not master:
master = _find_master()
if not isinstance(updates, (list, tuple)):
updates = [updates]
print "Merging files: %s and %s" % (master, ', '.join(updates))
parser = RawConfigParser()
parser.read(updates)
with open(master, 'r') as orig:
with open(outfile, 'w') as new:
current_section=None
for line in orig:
sec_m = re.match("^\[([\w\d_\-\s]+)\]\s*", line, re.IGNORECASE)
if sec_m:
current_section=sec_m.group(1)
new.write(line)
else:
if not parser.has_section(current_section):
new.write(line)
continue
var_m = re.match("^(?:;)?([\w\d\_\-\.]+)\s*=\s*(.*)\n$", line, re.IGNORECASE)
if var_m:
key, value = var_m.groups()
if parser.has_option(current_section, key):
new_value = parser.get(current_section, key)
# print "REPLACING: %s = %s with value %s" % (key, value, new_value)
new.write("%s = %s\n" % (key, new_value % kwargs))
parser.remove_option(current_section, key)
if not parser.items(current_section):
parser.remove_section(current_section)
else:
new.write(line)
else:
new.write(line)
if parser.sections():
#print "The following values were not set:"
for s in parser.sections():
new.write("")
new.write("[%s]\n" % s)
for t in parser.items(s):
new.write("%s = %s\n" % t)
def ConvertRanks():
print "Converting ranks..."
start = time.time()
ConfigReader = RawConfigParser()
ConfigReader.read("../../ranks.ini")
Items = ConfigReader.items("ranks")
for Item in Items:
Username,Rank = Item
if OldToNew.has_key(Rank):
ConfigReader.remove_option("ranks",Username)
ConfigReader.set("ranks",Username,OldToNew[Rank])
fHandle = open("../../ranks.ini","w")
ConfigReader.write(fHandle)
fHandle.close()
print "Converted ranks in %f seconds" %(time.time()-start)
def remove_options(self, section, data):
"""Retrieve options from a section
"""
#print self.data['conf_file']
if 'conf_file' in self.data and self.data['conf_file'] is not None:
config = RawConfigParser()
config.read([self.data['conf_file']])
for key in data:
config.remove_option(section, key)
if section in self._cache and key in self._cache[section]:
del self._cache[section][key]
with open(self.data['conf_file'], 'wb') as configfile:
config.write(configfile)
return True
return False
class SystemProvider(object):
def __init__(self, config):
self.config = config
self.log = kaizen.logging.getLogger(self)
self.configparser = RawConfigParser()
def load(self, filename=None):
self.configparser = RawConfigParser()
if not filename:
filename = self.config.get("system")
if not filename or not os.path.isfile(filename):
self.log.debug("no config file found for system povided "\
"dependencies. Config file %r will be created" % \
filename)
else:
self.log.debug("Reading system provide file %s" % filename)
self.configparser.read(filename)
if not self.configparser.has_section("provides"):
self.log.debug("system config file '%s' has to provides section. "\
"Section will be created." % filename)
self.configparser.add_section("provides")
def provides(self, name):
return self.configparser and self.configparser.has_option("provides",
name)
def get(self, name):
if not self.provides(name):
return None
version = self.configparser.get("provides", name)
return SystemDependency(name, version)
def add(self, name, version):
if not self.configparser:
return
if version is None:
version = ''
self.configparser.set("provides", name, version)
def remove(self, name):
if not self.configparser:
return False
return self.configparser.remove_option("provides", name)
def save(self, filename=None):
if not filename:
filename = self.config.get("system")
f = open(filename, "w")
self.configparser.write(f)
f.close()
def is_empty(self):
return self.configparser is None and \
len(self.configparser.items("provides")) == 0
def list(self):
if not self.configparser:
return []
return self.configparser.items("provides")
def set_schema_fpath(project_name, schema_fpath):
"""Set the schema path for a given project.
:param project_name: Name of the project
:param schema_fpath: path to the yaml file to be used as the schema for \
the project.
:type project_name: str
:type schema_fpath: str
:return: True, if setting the schema path was successful.
:Example:
>>> set_schema_fpath('skynet', '/path/to/new/schema.yaml')
True
"""
path = locate_config_file()
parser = RawConfigParser()
parser.read(path)
if project_name in parser.sections():
if not parser.remove_option(project_name, "specfile"):
raise MissingProject
else:
parser.set(project_name, "specfile", schema_fpath)
with open(path, "w") as f:
parser.write(f)
return True
raise MissingProject
def set_schema_fpath(project_name, schema_fpath):
"""Set the schema path for a given project.
:param project_name: Name of the project
:param schema_fpath: path to the yaml file to be used as the schema for \
the project.
:type project_name: str
:type schema_fpath: str
:return: True, if setting the schema path was successful.
:Example:
>>> set_schema_fpath('skynet', '/path/to/new/schema.yaml')
True
"""
path = locate_config_file()
parser = RawConfigParser()
parser.read(path)
if project_name in parser.sections():
if not parser.remove_option(project_name, "specfile"):
raise MissingProject
else:
parser.set(project_name, "specfile", schema_fpath)
with open(path, "w") as f:
parser.write(f)
return True
raise MissingProject
def test_set_schema_fpath(self):
"""Test if programmatically setting a schema file to an existing
project works."""
old_schempath = pr.get_default_specfile("pysemantic")
try:
self.assertTrue(pr.set_schema_fpath("pysemantic", "/foo/bar"))
self.assertEqual(pr.get_default_specfile("pysemantic"), "/foo/bar")
self.assertRaises(MissingProject, pr.set_schema_fpath, "foobar",
"/foo/bar")
finally:
conf_path = pr.locate_config_file()
parser = RawConfigParser()
parser.read(conf_path)
parser.remove_option("pysemantic", "specfile")
parser.set("pysemantic", "specfile", old_schempath)
with open(TEST_CONFIG_FILE_PATH, "w") as fileobj:
parser.write(fileobj)
def WriteSignleConfigValue(file, section, entry, value, remove = False):
try:
config = RawConfigParser()
config.read(file)
if remove:
if config.has_option(section, entry):
config.remove_option(section, entry)
else:
config.set(section, entry, str(value))
# Writing our configuration file disk
with open(file, 'wb') as configfile:
config.write(configfile)
except Exception as e1:
log.error("Error Writing Config File (WriteSignleConfigValue): " + str(e1))
def test_set_schema_fpath(self):
"""Test if programmatically setting a schema file to an existing
project works."""
old_schempath = pr.get_default_specfile("pysemantic")
try:
self.assertTrue(pr.set_schema_fpath("pysemantic", "/foo/bar"))
self.assertEqual(pr.get_default_specfile("pysemantic"),
"/foo/bar")
self.assertRaises(MissingProject, pr.set_schema_fpath,
"foobar", "/foo/bar")
finally:
conf_path = pr.locate_config_file()
parser = RawConfigParser()
parser.read(conf_path)
parser.remove_option("pysemantic", "specfile")
parser.set("pysemantic", "specfile", old_schempath)
with open(TEST_CONFIG_FILE_PATH, "w") as fileobj:
parser.write(fileobj)
def set_options(self, kwargs):
config = RawConfigParser()
fp = StringIO(self.data)
config.readfp(fp)
for key, val in kwargs.items():
if key == "filter_by":
if val is None:
config.remove_option("options", "filter_by")
else:
config.set("options", key, " ".join(val))
else:
config.set("options", key, val)
fp = StringIO()
config.write(fp)
fp.seek(0)
self.data = fp.read()
self.save()
def setUpClass(cls):
cls.maxDiff = None
# modify the testdata dict to have absolute paths
with open(TEST_DATA_DICT, "r") as fileobj:
test_data = yaml.load(fileobj, Loader=Loader)
for _, specs in test_data.iteritems():
path = op.join(op.abspath(op.dirname(__file__)), specs['path'])
specs['path'] = path
# Put in the multifile specs
cls.copied_iris_path = test_data['iris']['path'].replace(
"iris", "iris2")
dframe = pd.read_csv(test_data['iris']['path'])
dframe.to_csv(cls.copied_iris_path, index=False)
copied_iris_specs = deepcopy(test_data['iris'])
copied_iris_specs['path'] = [
copied_iris_specs['path'], cls.copied_iris_path
]
copied_iris_specs['nrows'] = [150, 150]
test_data['multi_iris'] = copied_iris_specs
with open(TEST_DATA_DICT, "w") as fileobj:
yaml.dump(test_data,
fileobj,
Dumper=Dumper,
default_flow_style=False)
cls.data_specs = test_data
_path_fixer(TEST_XL_DICT)
# Fix config file to have absolute paths
config_fname = op.basename(TEST_CONFIG_FILE_PATH)
cls.test_conf_file = op.join(os.getcwd(), config_fname)
parser = RawConfigParser()
parser.read(TEST_CONFIG_FILE_PATH)
for project in ("pysemantic", "test_excel"):
specfile = parser.get(project, 'specfile')
specfile = op.join(op.abspath(op.dirname(__file__)), specfile)
parser.remove_option(project, "specfile")
parser.set(project, "specfile", specfile)
with open(cls.test_conf_file, 'w') as fileobj:
parser.write(fileobj)
pr.CONF_FILE_NAME = config_fname
def remove_option(self, section, option):
"""Removes an option from a section.
:param section: the section from which to remove the option
:type section: string
:param option: the option to be removed
:type option: string
:rtype: boolean
:returns: True if the option existed
"""
return RCP.remove_option(self, section, option)
def parse_config(self, filename):
from ConfigParser import RawConfigParser
import io
config = RawConfigParser()
config.readfp(io.open(filename, 'r', encoding='utf_8_sig'))
for s in config.sections():
port = int(config.get(s, 'port'))
config.remove_option(s, 'port')
xsize, ysize = [int(d) for d in config.get(s, 'size').split(",")]
config.remove_option(s, 'size')
x_off, y_off = [int(d) for d in config.get(s, 'offset').split(",")]
config.remove_option(s, 'offset')
self.offsets[s] = (x_off, y_off)
for device, offset in config.items(s):
x_off, y_off = [int(d) for d in offset.split(",")]
if self.offsets.has_key(device):
if (x_off, y_off) != self.offsets[device]:
raise RuntimeError("conflicting offsets for device %s" % device)
self.offsets[device] = (x_off, y_off)
if s in self.transtbl: self.transtbl[s].append(device)
else: self.transtbl[s] = [device]
if device in self.transtbl: self.transtbl[device].append(s)
else: self.transtbl[device] = [s]
self.add_virtual(s, xsize, ysize, port)
def edit_config(filename, settings, dry_run=False):
"""Edit a configuration file to include `settings`
`settings` is a dictionary of dictionaries or ``None`` values, keyed by
command/section name. A ``None`` value means to delete the entire section,
while a dictionary lists settings to be changed or deleted in that section.
A setting of ``None`` means to delete that setting.
"""
from ConfigParser import RawConfigParser
log.debug("Reading configuration from %s", filename)
opts = RawConfigParser()
opts.read([filename])
for section, options in settings.items():
if options is None:
log.info("Deleting section [%s] from %s", section, filename)
opts.remove_section(section)
else:
if not opts.has_section(section):
log.debug("Adding new section [%s] to %s", section, filename)
opts.add_section(section)
for option,value in options.items():
if value is None:
log.debug("Deleting %s.%s from %s",
section, option, filename
)
opts.remove_option(section,option)
if not opts.options(section):
log.info("Deleting empty [%s] section from %s",
section, filename)
opts.remove_section(section)
else:
log.debug(
"Setting %s.%s to %r in %s",
section, option, value, filename
)
opts.set(section,option,value)
log.info("Writing %s", filename)
if not dry_run:
f = open(filename,'w'); opts.write(f); f.close()
def setUpClass(cls):
cls.maxDiff = None
# modify the testdata dict to have absolute paths
with open(TEST_DATA_DICT, "r") as fileobj:
test_data = yaml.load(fileobj, Loader=Loader)
for _, specs in test_data.iteritems():
path = op.join(op.abspath(op.dirname(__file__)), specs['path'])
specs['path'] = path
# Put in the multifile specs
cls.copied_iris_path = test_data['iris']['path'].replace("iris",
"iris2")
dframe = pd.read_csv(test_data['iris']['path'])
dframe.to_csv(cls.copied_iris_path, index=False)
copied_iris_specs = deepcopy(test_data['iris'])
copied_iris_specs['path'] = [copied_iris_specs['path'],
cls.copied_iris_path]
copied_iris_specs['nrows'] = [150, 150]
test_data['multi_iris'] = copied_iris_specs
with open(TEST_DATA_DICT, "w") as fileobj:
yaml.dump(test_data, fileobj, Dumper=Dumper,
default_flow_style=False)
cls.data_specs = test_data
_path_fixer(TEST_XL_DICT)
# Fix config file to have absolute paths
config_fname = op.basename(TEST_CONFIG_FILE_PATH)
cls.test_conf_file = op.join(os.getcwd(), config_fname)
parser = RawConfigParser()
parser.read(TEST_CONFIG_FILE_PATH)
for project in ("pysemantic", "test_excel"):
specfile = parser.get(project, 'specfile')
specfile = op.join(op.abspath(op.dirname(__file__)), specfile)
parser.remove_option(project, "specfile")
parser.set(project, "specfile", specfile)
with open(cls.test_conf_file, 'w') as fileobj:
parser.write(fileobj)
pr.CONF_FILE_NAME = config_fname
def edit_config(filename, settings, dry_run=False):
"""Edit a configuration file to include `settings`
`settings` is a dictionary of dictionaries or ``None`` values, keyed by
command/section name. A ``None`` value means to delete the entire section,
while a dictionary lists settings to be changed or deleted in that section.
A setting of ``None`` means to delete that setting.
"""
from ConfigParser import RawConfigParser
log.debug("Reading configuration from %s", filename)
opts = RawConfigParser()
opts.read([filename])
for section, options in settings.items():
if options is None:
log.info("Deleting section [%s] from %s", section, filename)
opts.remove_section(section)
else:
if not opts.has_section(section):
log.debug("Adding new section [%s] to %s", section, filename)
opts.add_section(section)
for option, value in options.items():
if value is None:
log.debug("Deleting %s.%s from %s", section, option,
filename)
opts.remove_option(section, option)
if not opts.options(section):
log.info("Deleting empty [%s] section from %s",
section, filename)
opts.remove_section(section)
else:
log.debug("Setting %s.%s to %r in %s", section, option,
value, filename)
opts.set(section, option, value)
log.info("Writing %s", filename)
if not dry_run:
f = open(filename, 'w')
opts.write(f)
f.close()
def parse_config(self, filename):
from ConfigParser import RawConfigParser
import io
config = RawConfigParser()
with io.open(filename, 'r', encoding='utf_8_sig') as fp:
config.readfp(fp)
for s in config.sections():
port = int(config.get(s, 'port'))
config.remove_option(s, 'port')
if sys.argv[1] == '-s':
self.snoopy_port = int(config.get(s, 'snoopy_port'))
config.remove_option(s, 'snoopy_port')
self.snoopy_addr = config.get(s, 'snoopy_addr')
config.remove_option(s, 'snoopy_addr')
xsize, ysize = [int(d) for d in config.get(s, 'size').split(",")]
config.remove_option(s, 'size')
x_off, y_off = [int(d) for d in config.get(s, 'offset').split(",")]
config.remove_option(s, 'offset')
self.offsets[s] = (x_off, y_off)
for device, offset in config.items(s):
x_off, y_off = [int(d) for d in offset.split(",")]
if self.offsets.has_key(device):
if (x_off, y_off) != self.offsets[device]:
raise RuntimeError(
"conflicting offsets for device %s" % device)
self.offsets[device] = (x_off, y_off)
if s in self.transtbl: self.transtbl[s].append(device)
else: self.transtbl[s] = [device]
if device in self.transtbl: self.transtbl[device].append(s)
else: self.transtbl[device] = [s]
self.add_virtual(s, xsize, ysize, port)
def iter_backends(self):
config = RawConfigParser()
config.read(self.confpath)
changed = False
for backend_name in config.sections():
params = dict(config.items(backend_name))
try:
module_name = params.pop('_module')
except KeyError:
try:
module_name = params.pop('_backend')
config.set(backend_name, '_module', module_name)
config.remove_option(backend_name, '_backend')
changed = True
except KeyError:
warning('Missing field "_module" for configured backend "%s"', backend_name)
continue
yield backend_name, module_name, params
if changed:
with open(self.confpath, 'wb') as f:
config.write(f)
def unsetFilenameInConfigFile(self, root_path, name):
"""Remove filename field association in config file
@root_path: Path where config file is stored
@param name: Field name
"""
path = os.path.join(root_path, self.cfg_filename)
# Config file doesn't exists
if not os.path.exists(path):
return
# Initialize config file
config = RawConfigParser()
# Read old config file
fd = open(path, 'r')
try:
# Read file
config.readfp(fd)
finally:
fd.close()
# Update config file
if config.has_section(self.cfg_filename_section):
config.remove_option(self.cfg_filename_section, name)
if not config.options(self.cfg_filename_section):
# Section is empty so remove config file
rm_file(path)
else:
fd = open(path, 'w')
try:
# Write file
config.write(fd)
finally:
fd.close()
def unsetFilenameInConfigFile(self, root_path, name):
"""Remove filename field association in config file
@root_path: Path where config file is stored
@param name: Field name
"""
path = os.path.join(root_path, self.cfg_filename)
# Config file doesn't exists
if not os.path.exists(path):
return
# Initialize config file
config = RawConfigParser()
# Read old config file
fd = open(path, 'r')
try:
# Read file
config.readfp(fd)
finally:
fd.close()
# Update config file
if config.has_section(self.cfg_filename_section):
config.remove_option(self.cfg_filename_section, name)
if not config.options(self.cfg_filename_section):
# Section is empty so remove config file
rm_file(path)
else:
fd = open(path, 'w')
try:
# Write file
config.write(fd)
finally:
fd.close()
def reset(self, name):
# If custom conf file is not set then it is only read only configs
if self.custom_conf_file is None:
raise GconfNotConfigurable()
# If a config can not be modified
if name != "all" and not self._is_configurable(name):
raise GconfNotConfigurable()
cnf = RawConfigParser()
with open(self.custom_conf_file) as f:
cnf.readfp(f)
# Nothing to Reset, Not configured
if name != "all":
if not cnf.has_option("vars", name):
return True
# Remove option from custom conf file
cnf.remove_option("vars", name)
else:
# Remove and add empty section, do not disturb if config file
# already has any other section
try:
cnf.remove_section("vars")
except NoSectionError:
pass
cnf.add_section("vars")
with open(self.tmp_conf_file, "w") as fw:
cnf.write(fw)
os.rename(self.tmp_conf_file, self.custom_conf_file)
self.reload()
return True
class Config(object):
"""A wrapper around RawConfigParser.
Provides a ``defaults`` attribute of the same type which can be used
to set default values.
"""
def __init__(self, version=None, _defaults=True):
"""Use read() to read in an existing config file.
version should be an int starting with 0 that gets incremented if you
want to register a new upgrade function. If None, upgrade is disabled.
"""
self._config = ConfigParser(dict_type=_sorted_dict)
self.defaults = None
if _defaults:
self.defaults = Config(_defaults=False)
self._version = version
self._loaded_version = None
self._upgrade_funcs = []
def _do_upgrade(self, func):
assert self._loaded_version is not None
assert self._version is not None
old_version = self._loaded_version
new_version = self._version
if old_version != new_version:
print_d("Config upgrade: %d->%d (%r)" % (
old_version, new_version, func))
func(self, old_version, new_version)
def get_version(self):
"""Get the version of the loaded config file (for testing only)
Raises Error if no file was loaded or versioning is disabled.
"""
if self._version is None:
raise Error("Versioning disabled")
if self._loaded_version is None:
raise Error("No file loaded")
return self._loaded_version
def register_upgrade_function(self, function):
"""Register an upgrade function that gets called at each read()
if the current config version and the loaded version don't match.
Can also be registered after read was called.
function(config, old_version: int, new_version: int) -> None
"""
if self._version is None:
raise Error("Versioning disabled")
self._upgrade_funcs.append(function)
# after read(), so upgrade now
if self._loaded_version is not None:
self._do_upgrade(function)
return function
def reset(self, section, option):
"""Reset the value to the default state"""
assert self.defaults is not None
self.set(section, option, self.defaults.get(section, option))
def options(self, section):
"""Returns a list of options available in the specified section."""
try:
options = self._config.options(section)
except NoSectionError:
if self.defaults:
return self.defaults.options(section)
raise
else:
if self.defaults:
try:
options.extend(self.defaults.options(section))
options = list_unique(options)
except NoSectionError:
pass
return options
def get(self, section, option, default=_DEFAULT):
"""get(section, option[, default]) -> str
If default is not given or set, raises Error in case of an error
"""
try:
return self._config.get(section, option)
except Error:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.get(section, option)
except Error:
pass
raise
return default
def gettext(self, *args, **kwargs):
value = self.get(*args, **kwargs)
if PY2:
value = value.decode("utf-8")
else:
# make sure there are no surrogates
value.encode("utf-8")
return value
def getbytes(self, section, option, default=_DEFAULT):
try:
value = self._config.get(section, option)
if PY3:
value = value.encode("utf-8", "surrogateescape")
return value
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getbytes(section, option)
except Error:
pass
raise Error(e)
return default
def getboolean(self, section, option, default=_DEFAULT):
"""getboolean(section, option[, default]) -> bool
If default is not given or set, raises Error in case of an error
"""
try:
return self._config.getboolean(section, option)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getboolean(section, option)
except Error:
pass
raise Error(e)
return default
def getint(self, section, option, default=_DEFAULT):
"""getint(section, option[, default]) -> int
If default is not give or set, raises Error in case of an error
"""
try:
return int(self._config.getfloat(section, option))
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getint(section, option)
except Error:
pass
raise Error(e)
return default
def getfloat(self, section, option, default=_DEFAULT):
"""getfloat(section, option[, default]) -> float
If default is not give or set, raises Error in case of an error
"""
try:
return self._config.getfloat(section, option)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getfloat(section, option)
except Error:
pass
raise Error(e)
return default
def getstringlist(self, section, option, default=_DEFAULT):
"""getstringlist(section, option[, default]) -> list
If default is not given or set, raises Error in case of an error.
Gets a list of strings, using CSV to parse and delimit.
"""
try:
value = self._config.get(section, option)
parser = csv.reader(
[value], lineterminator='\n', quoting=csv.QUOTE_MINIMAL)
try:
if PY2:
vals = [v.decode('utf-8') for v in next(parser)]
else:
vals = next(parser)
except (csv.Error, ValueError) as e:
raise Error(e)
return vals
except Error as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getstringlist(section, option)
except Error:
pass
raise Error(e)
return default
def setstringlist(self, section, option, values):
"""Saves a list of unicode strings using the csv module"""
if PY2:
sw = cBytesIO()
values = [text_type(v).encode('utf-8') for v in values]
else:
sw = StringIO()
values = [text_type(v) for v in values]
writer = csv.writer(sw, lineterminator='\n', quoting=csv.QUOTE_MINIMAL)
writer.writerow(values)
self.set(section, option, sw.getvalue())
def setlist(self, section, option, values, sep=","):
"""Saves a list of str using ',' as a separator and \\ for escaping"""
values = [str(v) for v in values]
joined = join_escape(values, sep)
self.set(section, option, joined)
def getlist(self, section, option, default=_DEFAULT, sep=","):
"""Returns a str list saved with setlist()"""
try:
value = self._config.get(section, option)
return split_escape(value, sep)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getlist(section, option, sep=sep)
except Error:
pass
raise Error(e)
return default
def set(self, section, option, value):
"""Saves the string representation for the passed value
Don't pass unicode, encode first.
"""
if PY3 and isinstance(value, bytes):
raise TypeError("use setbytes")
# RawConfigParser only allows string values but doesn't
# scream if they are not (and it only fails before the
# first config save..)
if not isinstance(value, str):
value = str(value)
try:
self._config.set(section, option, value)
except NoSectionError:
if self.defaults and self.defaults.has_section(section):
self._config.add_section(section)
self._config.set(section, option, value)
else:
raise
def settext(self, section, option, value):
value = text_type(value)
if PY2:
value = value.encode("utf-8")
else:
# make sure there are no surrogates
value.encode("utf-8")
self.set(section, option, value)
def setbytes(self, section, option, value):
assert isinstance(value, bytes)
if PY3:
value = value.decode("utf-8", "surrogateescape")
self.set(section, option, value)
def write(self, filename):
"""Write config to filename.
Can raise EnvironmentError
"""
assert isinstance(filename, fsnative)
mkdir(os.path.dirname(filename))
# temporary set the new version for saving
if self._version is not None:
self.add_section("__config__")
self.set("__config__", "version", self._version)
try:
with atomic_save(filename, "wb") as fileobj:
if PY2:
self._config.write(fileobj)
else:
temp = StringIO()
self._config.write(temp)
data = temp.getvalue().encode("utf-8", "surrogateescape")
fileobj.write(data)
finally:
if self._loaded_version is not None:
self.set("__config__", "version", self._loaded_version)
def clear(self):
"""Remove all sections."""
for section in self._config.sections():
self._config.remove_section(section)
def is_empty(self):
"""Whether the config has any sections"""
return not self._config.sections()
def read(self, filename):
"""Reads the config from `filename` if the file exists,
otherwise does nothing
Can raise EnvironmentError, Error.
"""
try:
with open(filename, "rb") as fileobj:
if PY3:
fileobj = StringIO(
fileobj.read().decode("utf-8", "surrogateescape"))
self._config.readfp(fileobj, filename)
except (IOError, OSError):
return
# don't upgrade if we just created a new config
if self._version is not None:
self._loaded_version = self.getint("__config__", "version", -1)
for func in self._upgrade_funcs:
self._do_upgrade(func)
def has_option(self, section, option):
"""If the given section exists, and contains the given option"""
return self._config.has_option(section, option) or (
self.defaults and self.defaults.has_option(section, option))
def has_section(self, section):
"""If the given section exists"""
return self._config.has_section(section) or (
self.defaults and self.defaults.has_section(section))
def remove_option(self, section, option):
"""Remove the specified option from the specified section
Can raise Error.
"""
return self._config.remove_option(section, option)
def add_section(self, section):
"""Add a section named section to the instance if it not already
exists."""
if not self._config.has_section(section):
self._config.add_section(section)
class AwsCredentials(object):
"""Wraps a RawConfigParser to treat a section named 'default' as a nomral section."""
__REAL_DEFAULT_SECTION_NAME = 'default'
__TEMP_DEFAULT_SECTION_NAME = '__default__'
__REAL_DEFAULT_SECTION_HEADING = '[' + __REAL_DEFAULT_SECTION_NAME + ']'
__TEMP_DEFAULT_SECTION_HEADING = '[' + __TEMP_DEFAULT_SECTION_NAME + ']'
def __init__(self):
self.__config = RawConfigParser()
def read(self, path):
with open(path, 'r') as file:
content = file.read()
content = content.replace(self.__REAL_DEFAULT_SECTION_HEADING,
self.__TEMP_DEFAULT_SECTION_HEADING)
content_io = StringIO(content)
self.__config.readfp(content_io)
def write(self, path):
content_io = StringIO()
self.__config.write(content_io)
content = content_io.getvalue()
content = content.replace(self.__TEMP_DEFAULT_SECTION_HEADING,
self.__REAL_DEFAULT_SECTION_HEADING)
with open(path, 'w') as file:
file.write(content)
def __to_temp_name(self, name):
if name == self.__REAL_DEFAULT_SECTION_NAME:
name = self.__TEMP_DEFAULT_SECTION_NAME
return name
def __to_real_name(self, name):
if name == self.__TEMP_DEFAULT_SECTION_NAME:
name = self.__REAL_DEFAULT_SECTION_NAME
return name
def sections(self):
sections = self.__config.sections()
sections = [self.__to_real_name(section) for section in sections]
return sections
def add_section(self, section):
section = self.__to_temp_name(section)
self.__config.add_section(section)
def has_section(self, section):
section = self.__to_temp_name(section)
return self.__config.has_section(section)
def options(self, section):
section = self.__to_temp_name(section)
return self.__config.options(section)
def has_option(self, section, option):
section = self.__to_temp_name(section)
return self.__config.has_option(section, option)
def get(self, section, option):
section = self.__to_temp_name(section)
return self.__config.get(section, option)
def items(self, section):
section = self.__to_temp_name(section)
return self.__config.items(section)
def set(self, section, option, value):
section = self.__to_temp_name(section)
self.__config.set(section, option, value)
def remove_option(self, section, option):
section = self.__to_temp_name(section)
return self.__config.remove_option(section, section, option)
def remove_section(self, section):
section = self.__to_temp_name(section)
return self.__config.remove_section(section)
def remove_option(self, option):
return _RawConfigParser.remove_option(self, self._default_section,
option)
def remove_option(self, section, option):
if self.has_option(section, option):
RawConfigParser.remove_option(self, section, option)
class TestPolicyUid(unittest.TestCase):
def remove_option(self, section, option):
self.config.remove_option(section, option)
fp = open(conf.config_file, "w")
self.config.write(fp)
fp.close()
def set(self, section, option, value):
self.config.set(section, option, value)
fp = open(conf.config_file, "w")
self.config.write(fp)
fp.close()
@classmethod
def setup_class(self, *args, **kw):
self.config = RawConfigParser()
self.config.read(conf.config_file)
from tests.functional.purge_users import purge_users
purge_users()
self.user = {
'local': 'john.doe',
'domain': 'example.org'
}
self.login = conf.get('ldap', 'bind_dn')
self.password = conf.get('ldap', 'bind_pw')
self.domain = conf.get('kolab', 'primary_domain')
result = wap_client.authenticate(self.login, self.password, self.domain)
@classmethod
def teardown_class(self, *args, **kw):
self.config.remove_option('example.org', 'policy_uid')
fp = open(conf.config_file, "w")
self.config.write(fp)
fp.close()
from tests.functional.purge_users import purge_users
purge_users()
def test_001_default(self):
from tests.functional.user_add import user_add
user_add("John", "Doe")
from tests.functional.synchronize import synchronize_once
synchronize_once()
auth = Auth()
auth.connect()
user = auth.find_recipient('*****@*****.**')
user_info = wap_client.user_info(user)
self.assertEqual(user_info['uid'], "doe")
from tests.functional.purge_users import purge_users
purge_users()
def test_002_givenname_dot_surname(self):
self.set('example.org', 'policy_uid', '%(givenname)s.%(surname)s')
from tests.functional.user_add import user_add
user_add("John", "Doe")
from tests.functional.synchronize import synchronize_once
synchronize_once()
auth = Auth()
auth.connect()
user = auth.find_recipient('*****@*****.**')
user_info = wap_client.user_info(user)
self.assertEqual(user_info['uid'], "John.Doe")
from tests.functional.purge_users import purge_users
purge_users()
def test_003_givenname_fc_dot_surname(self):
self.set('example.org', 'policy_uid', "'%(givenname)s'[0:1].%(surname)s")
from tests.functional.user_add import user_add
user_add("John", "Doe")
from tests.functional.synchronize import synchronize_once
synchronize_once()
auth = Auth()
auth.connect()
user = auth.find_recipient('*****@*****.**')
user_info = wap_client.user_info(user)
self.assertEqual(user_info['uid'], "J.Doe")
from tests.functional.purge_users import purge_users
purge_users()
def test_004_givenname(self):
self.set('example.org', 'policy_uid', '%(givenname)s')
from tests.functional.user_add import user_add
user_add("John", "Doe")
from tests.functional.synchronize import synchronize_once
synchronize_once()
auth = Auth()
auth.connect()
user = auth.find_recipient('*****@*****.**')
user_info = wap_client.user_info(user)
self.assertEqual(user_info['uid'], "John")
from tests.functional.purge_users import purge_users
purge_users()
def test_005_lowercase_givenname(self):
self.set('example.org', 'policy_uid', '%(givenname)s.lower()')
from tests.functional.user_add import user_add
user_add("John", "Doe")
from tests.functional.synchronize import synchronize_once
synchronize_once()
auth = Auth()
auth.connect()
user = auth.find_recipient('*****@*****.**')
user_info = wap_client.user_info(user)
self.assertEqual(user_info['uid'], "john")
from tests.functional.purge_users import purge_users
purge_users()
def test_006_lowercase_givenname_surname(self):
self.set('example.org', 'policy_uid', "%(givenname)s.lower().%(surname)s.lower()")
from tests.functional.user_add import user_add
user_add("John", "Doe")
from tests.functional.synchronize import synchronize_once
synchronize_once()
auth = Auth()
auth.connect()
user = auth.find_recipient('*****@*****.**')
user_info = wap_client.user_info(user)
self.assertEqual(user_info['uid'], "john.doe")
from tests.functional.purge_users import purge_users
purge_users()
def main(args):
config = ConfigParser({"htrc": False, "sentences": "False"})
config.read(args.config_file)
if config.getboolean("main", "sentences"):
from vsm.extensions.ldasentences import CorpusSent as Corpus
else:
from vsm.corpus import Corpus
if args.lang is None:
args.lang = []
args.corpus_path = config.get("main", "corpus_file")
c = Corpus.load(args.corpus_path)
# check for htrc metadata
if args.htrc or config.get("main", "htrc"):
htrc_langs = get_htrc_langs(args)
if htrc_langs:
args.lang.extend(new_langs)
# auto-guess a language
"""
new_langs = [lang for lang in detect_langs(c) if lang in langs and lang not in args.lang]
if new_langs:
args.lang.extend(new_langs)
"""
# add default locale if no other languages are specified
# do not add if in quiet mode -- make everything explicit
if not args.lang and not args.quiet:
import locale
locale = locale.getdefaultlocale()[0].split('_')[0].lower()
if locale in langs.keys():
args.lang.append(locale)
# check for any new candidates
args.lang = [lang for lang in args.lang if stop_language(c, langs[lang])]
if args.lang and not args.quiet:
args.lang = lang_prompt(args.lang)
stoplist = set()
# Apply stop words
print " "
for lang in args.lang:
print "Applying", langs[lang], "stopwords"
candidates = stop_language(c, langs[lang])
if len(candidates):
stoplist.update(candidates)
# Apply custom stopwords file
if args.stopword_file:
with open(args.stopword_file, encoding='utf8') as swf:
candidates = [unidecode(word.strip()) for word in swf]
if len(candidates):
print "Applying custom stopword file to remove {} word{}.".format(
len(candidates), 's' if len(candidates) > 1 else '')
stoplist.update(candidates)
if args.min_word_len:
candidates = get_small_words(c, args.min_word_len)
if len(candidates):
print "Filtering {} small word{} with less than {} characters.".format(
len(candidates), 's' if len(candidates) > 1 else '',
args.min_word_len)
stoplist.update(candidates)
if not args.special_chars:
candidates = get_special_chars(c)
if len(candidates):
print "Filtering {} word{} with special characters.".format(
len(candidates), 's' if len(candidates) > 1 else '')
stoplist.update(candidates)
if args.high_filter is None and not args.quiet:
args.high_filter, candidates = get_high_filter(args, c, words=stoplist)
if len(candidates):
print "Filtering {} high frequency word{}.".format(
len(candidates), 's' if len(candidates) > 1 else '')
stoplist.update(candidates)
elif args.high_filter > 0:
candidates = get_candidate_words(c, args.high_filter, sort=False)
if len(candidates):
print "Filtering {} high frequency word{}.".format(
len(candidates), 's' if len(candidates) > 1 else '')
stoplist.update(candidates)
if args.low_filter is None and not args.quiet:
args.low_filter, candidates = get_low_filter(args, c, words=stoplist)
if len(candidates):
print "Filtering {} low frequency word{}.".format(
len(candidates), 's' if len(candidates) > 1 else '')
stoplist.update(candidates)
elif args.low_filter > 0:
candidates = get_candidate_words(c, -1 * args.low_filter, sort=False)
if len(candidates):
print "Filtering {} low frequency words.".format(len(candidates))
stoplist.update(candidates)
if not stoplist:
print "No stopwords applied.\n\n"
sys.exit(0)
else:
print "\n\nApplying {} stopword{}".format(
len(stoplist), 's' if len(stoplist) > 1 else '')
c.in_place_stoplist(stoplist)
print "\n"
def name_corpus(dirname, languages, lowfreq=None, highfreq=None):
items, counts = get_items_counts(c.corpus)
corpus_name = [dirname]
if args.lang:
corpus_name.append('nltk')
corpus_name.append(''.join(args.lang))
if lowfreq > 0:
corpus_name.append('freq%s' % lowfreq)
else:
corpus_name.append('freq%s' % min(counts))
if highfreq > 0:
corpus_name.append('N%s' % highfreq)
else:
corpus_name.append('freq%s' % max(counts))
corpus_name = '-'.join(corpus_name)
corpus_name += '.npz'
return corpus_name
dirname = os.path.basename(args.corpus_path).split('-nltk-')[0].replace(
'.npz', '')
corpus_name = name_corpus(dirname, ['en'], args.low_filter,
args.high_filter)
model_path = os.path.dirname(args.corpus_path)
args.corpus_path = os.path.join(model_path, corpus_name)
c.save(args.corpus_path)
config.set("main", "corpus_file", args.corpus_path)
config.remove_option("main", "model_pattern")
with open(args.config_file, 'wb') as configfh:
config.write(configfh)
class Config(object):
"""A wrapper around RawConfigParser"""
def __init__(self, version=None):
"""Use read() to read in an existing config file.
version should be an int starting with 0 that gets incremented if you
want to register a new upgrade function. If None, upgrade is disabled.
"""
self._config = ConfigParser(dict_type=_sorted_dict)
self._version = version
self._loaded_version = None
self._upgrade_funcs = []
self._initial = {}
def _do_upgrade(self, func):
assert self._loaded_version is not None
assert self._version is not None
old_version = self._loaded_version
new_version = self._version
if old_version != new_version:
print_d("Config upgrade: %d->%d (%r)" %
(old_version, new_version, func))
func(self, old_version, new_version)
def get_version(self):
"""Get the version of the loaded config file (for testing only)
Raises Error if no file was loaded or versioning is disabled.
"""
if self._version is None:
raise Error("Versioning disabled")
if self._loaded_version is None:
raise Error("No file loaded")
return self._loaded_version
def register_upgrade_function(self, function):
"""Register an upgrade function that gets called at each read()
if the current config version and the loaded version don't match.
Can also be registered after read was called.
function(config, old_version: int, new_version: int) -> None
"""
if self._version is None:
raise Error("Versioning disabled")
self._upgrade_funcs.append(function)
# after read(), so upgrade now
if self._loaded_version is not None:
self._do_upgrade(function)
return function
def set_inital(self, section, option, value):
"""Set an initial value for an option.
The section must be added with add_section() first.
Adds the value to the config and calling reset()
will reset the value to it.
"""
self.set(section, option, value)
self._initial.setdefault(section, {})
self._initial[section].setdefault(option, {})
self._initial[section][option] = value
def reset(self, section, option):
"""Reset the value to the initial state"""
value = self._initial[section][option]
self.set(section, option, value)
def options(self, section):
"""Returns a list of options available in the specified section."""
return self._config.options(section)
def get(self, *args):
"""get(section, option[, default]) -> str
If default is not given, raises Error in case of an error
"""
if len(args) == 3:
try:
return self._config.get(*args[:2])
except Error:
return args[-1]
return self._config.get(*args)
def getboolean(self, *args):
"""getboolean(section, option[, default]) -> bool
If default is not given, raises Error in case of an error
"""
if len(args) == 3:
if not isinstance(args[-1], bool):
raise ValueError
try:
return self._config.getboolean(*args[:2])
# ValueError if the value found in the config file
# does not match any string representation -> so catch it too
except (ValueError, Error):
return args[-1]
return self._config.getboolean(*args)
def getint(self, *args):
"""getint(section, option[, default]) -> int
If default is not give, raises Error in case of an error
"""
if len(args) == 3:
if not isinstance(args[-1], int):
raise ValueError
try:
return self._config.getint(*args[:2])
except Error:
return args[-1]
return self._config.getint(*args)
def getfloat(self, *args):
"""getfloat(section, option[, default]) -> float
If default is not give, raises Error in case of an error
"""
if len(args) == 3:
if not isinstance(args[-1], float):
raise ValueError
try:
return self._config.getfloat(*args[:2])
except Error:
return args[-1]
return self._config.getfloat(*args)
def getstringlist(self, *args):
"""getstringlist(section, option[, default]) -> list
If default is not given, raises Error in case of an error.
Gets a list of strings, using CSV to parse and delimit.
"""
if len(args) == 3:
if not isinstance(args[-1], list):
raise ValueError
try:
value = self._config.get(*args[:2])
except Error:
return args[-1]
else:
value = self._config.get(*args)
parser = csv.reader([value],
lineterminator='\n',
quoting=csv.QUOTE_MINIMAL)
try:
vals = [v.decode('utf-8') for v in parser.next()]
except (csv.Error, ValueError) as e:
raise Error(e)
return vals
def setstringlist(self, section, option, values):
"""Saves a list of unicode strings using the csv module"""
sw = StringIO()
values = [unicode(v).encode('utf-8') for v in values]
writer = csv.writer(sw, lineterminator='\n', quoting=csv.QUOTE_MINIMAL)
writer.writerow(values)
self._config.set(section, option, sw.getvalue())
def set(self, section, option, value):
"""Saves the string representation for the passed value
Don't pass unicode, encode first.
"""
# RawConfigParser only allows string values but doesn't
# scream if they are not (and it only fails before the
# first config save..)
if not isinstance(value, str):
value = str(value)
self._config.set(section, option, value)
def setdefault(self, section, option, default):
"""Like set but only sets the new value if the option
isn't set before.
"""
if not self._config.has_option(section, option):
self._config.set(section, option, default)
def write(self, filename):
"""Write config to filename.
Can raise EnvironmentError
"""
assert is_fsnative(filename)
mkdir(os.path.dirname(filename))
# temporary set the new version for saving
if self._version is not None:
self.add_section("__config__")
self.set("__config__", "version", self._version)
try:
with atomic_save(filename, ".tmp", "wb") as fileobj:
self._config.write(fileobj)
finally:
if self._loaded_version is not None:
self.set("__config__", "version", self._loaded_version)
def clear(self):
"""Remove all sections and initial values"""
for section in self._config.sections():
self._config.remove_section(section)
self._initial.clear()
def is_empty(self):
"""Whether the config has any sections"""
return not self._config.sections()
def read(self, filename):
"""Reads the config from `filename` if the file exists,
otherwise does nothing
Can raise EnvironmentError, Error.
"""
parsed_filenames = self._config.read(filename)
# don't upgrade if we just created a new config
if parsed_filenames and self._version is not None:
self._loaded_version = self.getint("__config__", "version", -1)
for func in self._upgrade_funcs:
self._do_upgrade(func)
def sections(self):
"""Return a list of the sections available"""
return self._config.sections()
def has_option(self, section, option):
"""If the given section exists, and contains the given option"""
return self._config.has_option(section, option)
def remove_option(self, section, option):
"""Remove the specified option from the specified section
Can raise Error.
"""
return self._config.remove_option(section, option)
def add_section(self, section):
"""Add a section named section to the instance if it not already
exists."""
if not self._config.has_section(section):
self._config.add_section(section)
def load_config(path): # type: (str) -> RawConfigParser
"""Load the config file from the given path, applying fixes as needed.
@todo: Refactor all this
"""
first_run = not os.path.exists(path)
config = RawConfigParser()
# Make keys case-sensitive because keysyms must be
config.optionxform = str # type: ignore # (Cannot assign to a method)
# TODO: Maybe switch to two config files so I can have only the keys in the
# keymap case-sensitive?
config.read(path)
dirty = False
if not config.has_section('general'):
config.add_section('general')
# Change this if you make backwards-incompatible changes to the
# section and key naming in the config file.
config.set('general', 'cfg_schema', 1)
dirty = True
for key, val in DEFAULTS['general'].items():
if not config.has_option('general', key):
config.set('general', key, str(val))
dirty = True
mk_raw = modkeys = config.get('general', 'ModMask')
if ' ' in modkeys.strip() and '<' not in modkeys:
modkeys = '<%s>' % '><'.join(modkeys.strip().split())
logging.info("Updating modkeys format:\n %r --> %r", mk_raw, modkeys)
config.set('general', 'ModMask', modkeys)
dirty = True
# Either load the keybindings or use and save the defaults
if config.has_section('keys'):
keymap = dict(config.items('keys'))
else:
keymap = DEFAULTS['keys']
config.add_section('keys')
for row in keymap.items():
config.set('keys', row[0], row[1])
dirty = True
# Migrate from the deprecated syntax for punctuation keysyms
for key in keymap:
# Look up unrecognized shortkeys in a hardcoded dict and
# replace with valid names like ',' -> 'comma'
if key in KEYLOOKUP:
logging.warn("Updating config file from deprecated keybind syntax:"
"\n\t%r --> %r", key, KEYLOOKUP[key])
config.remove_option('keys', key)
config.set('keys', KEYLOOKUP[key], keymap[key])
dirty = True
if dirty:
cfg_file = open(path, 'wb')
# TODO: REPORT: Argument 1 to "write" of "RawConfigParser" has
# incompatible type BinaryIO"; expected "file"
config.write(cfg_file) # type: ignore
cfg_file.close()
if first_run:
logging.info("Wrote default config file to %s", path)
return config
class TestPolicyUid(unittest.TestCase):
def remove_option(self, section, option):
self.config.remove_option(section, option)
fp = open(conf.config_file, "w")
self.config.write(fp)
fp.close()
def set(self, section, option, value):
self.config.set(section, option, value)
fp = open(conf.config_file, "w")
self.config.write(fp)
fp.close()
@classmethod
def setup_class(self, *args, **kw):
self.config = RawConfigParser()
self.config.read(conf.config_file)
from tests.functional.purge_users import purge_users
purge_users()
self.user = {
'local': 'john.doe',
'domain': 'example.org'
}
self.login = conf.get('ldap', 'bind_dn')
self.password = conf.get('ldap', 'bind_pw')
self.domain = conf.get('kolab', 'primary_domain')
result = wap_client.authenticate(self.login, self.password, self.domain)
@classmethod
def teardown_class(self, *args, **kw):
self.config.remove_option('example.org', 'policy_uid')
fp = open(conf.config_file, "w")
self.config.write(fp)
fp.close()
from tests.functional.purge_users import purge_users
purge_users()
def test_001_default(self):
from tests.functional.user_add import user_add
user_add("John", "Doe")
from tests.functional.synchronize import synchronize_once
synchronize_once()
auth = Auth()
auth.connect()
user = auth.find_recipient('*****@*****.**')
user_info = wap_client.user_info(user)
self.assertEqual(user_info['uid'], "doe")
from tests.functional.purge_users import purge_users
purge_users()
def test_002_givenname_dot_surname(self):
self.set('example.org', 'policy_uid', '%(givenname)s.%(surname)s')
from tests.functional.user_add import user_add
user_add("John", "Doe")
from tests.functional.synchronize import synchronize_once
synchronize_once()
auth = Auth()
auth.connect()
user = auth.find_recipient('*****@*****.**')
user_info = wap_client.user_info(user)
self.assertEqual(user_info['uid'], "John.Doe")
from tests.functional.purge_users import purge_users
purge_users()
def test_003_givenname_fc_dot_surname(self):
self.set('example.org', 'policy_uid', "'%(givenname)s'[0:1].%(surname)s")
from tests.functional.user_add import user_add
user_add("John", "Doe")
from tests.functional.synchronize import synchronize_once
synchronize_once()
auth = Auth()
auth.connect()
user = auth.find_recipient('*****@*****.**')
user_info = wap_client.user_info(user)
self.assertEqual(user_info['uid'], "J.Doe")
from tests.functional.purge_users import purge_users
purge_users()
def test_004_givenname(self):
self.set('example.org', 'policy_uid', '%(givenname)s')
from tests.functional.user_add import user_add
user_add("John", "Doe")
from tests.functional.synchronize import synchronize_once
synchronize_once()
auth = Auth()
auth.connect()
user = auth.find_recipient('*****@*****.**')
user_info = wap_client.user_info(user)
self.assertEqual(user_info['uid'], "John")
from tests.functional.purge_users import purge_users
purge_users()
def test_005_lowercase_givenname(self):
self.set('example.org', 'policy_uid', '%(givenname)s.lower()')
from tests.functional.user_add import user_add
user_add("John", "Doe")
from tests.functional.synchronize import synchronize_once
synchronize_once()
auth = Auth()
auth.connect()
user = auth.find_recipient('*****@*****.**')
user_info = wap_client.user_info(user)
self.assertEqual(user_info['uid'], "john")
from tests.functional.purge_users import purge_users
purge_users()
def test_006_lowercase_givenname_surname(self):
self.set('example.org', 'policy_uid', "%(givenname)s.lower().%(surname)s.lower()")
from tests.functional.user_add import user_add
user_add("John", "Doe")
from tests.functional.synchronize import synchronize_once
synchronize_once()
auth = Auth()
auth.connect()
user = auth.find_recipient('*****@*****.**')
user_info = wap_client.user_info(user)
self.assertEqual(user_info['uid'], "john.doe")
from tests.functional.purge_users import purge_users
purge_users()
class ParamStore(object):
def __init__(self, root_dir, file_name):
self._lock = Lock()
with self._lock:
if not os.path.isdir(root_dir):
raise RuntimeError(
'Directory "' + root_dir + '" does not exist.')
self._path = os.path.join(root_dir, file_name)
self._dirty = False
# open config file
self._config = RawConfigParser()
self._config.read(self._path)
def __del__(self):
self.flush()
def flush(self):
if not self._dirty:
return
with self._lock:
self._dirty = False
of = open(self._path, 'w')
self._config.write(of)
of.close()
def get(self, section, option, default=None):
"""Get a parameter value and return a string.
If default is specified and section or option are not defined
in the file, they are created and set to default, which is
then the return value.
"""
with self._lock:
if not self._config.has_option(section, option):
if default is not None:
self._set(section, option, default)
return default
return self._config.get(section, option)
def get_datetime(self, section, option, default=None):
result = self.get(section, option, default)
if result:
return safestrptime(result)
return result
def set(self, section, option, value):
"""Set option in section to string value."""
with self._lock:
self._set(section, option, value)
def _set(self, section, option, value):
if not self._config.has_section(section):
self._config.add_section(section)
elif (self._config.has_option(section, option) and
self._config.get(section, option) == value):
return
self._config.set(section, option, value)
self._dirty = True
def unset(self, section, option):
"""Remove option from section."""
with self._lock:
if not self._config.has_section(section):
return
if self._config.has_option(section, option):
self._config.remove_option(section, option)
self._dirty = True
if not self._config.options(section):
self._config.remove_section(section)
self._dirty = True
class TraktForVLC(object):
# Check if there's a newer version of TraktForVLC on the project's github,
# and print informations on that subject in the logs
def __check_version(self):
# The leading information for lines printed by this method in logs
lead = "VERSION:"
# Request the github API to get the releases information
github_releases = requests.get(
url="https://api.github.com/repos/XaF/TraktForVLC/releases")
# If there was a problem getting the releases
if not github_releases.ok:
self.log.error(lead +
"Unable to verify new releases of TraktForVLC")
return
# Else, we get the json answer
releases = github_releases.json()
# If we didn't find any release
if not releases:
self.log.warning(lead +
"No releases found on github")
return
# We get the latest release, all included
newest = sorted(
releases,
key=lambda x: x['tag_name'],
reverse=True)[0]
if newest['tag_name'][:1] == "v":
newest['tag_name'] = newest['tag_name'][1:]
newest_V = parse_version(newest['tag_name'])
# We get the latest _stable_ release
newest_stbl = sorted(
[r for r in releases if not r['prerelease']],
key=lambda x: x['tag_name'],
reverse=True)[0]
if newest_stbl['tag_name'][:1] == "v":
newest_stbl['tag_name'] = newest_stbl['tag_name'][1:]
newest_stbl_V = parse_version(newest_stbl['tag_name'])
# We parse the current version
current_V = parse_version(__version__)
if newest_V <= current_V:
self.log.info(lead + "TraktForVLC is up to date")
return
# We only show the latest stable release if
# it's newer than our current release
if newest_stbl_V > current_V:
# We reformat the publication date of the release
published = datetime.datetime.strptime(
newest_stbl['published_at'],
"%Y-%m-%dT%H:%M:%SZ").strftime('%c')
self.log.info(lead + "##### RELEASE #####")
self.log.info(lead + "## Stable release %(name)s" % newest_stbl)
self.log.info(lead + "## Published on %s" % published)
self.log.info(lead + "## Available on %(html_url)s" % newest_stbl)
# We only show the latest release if it's not
# also the latest stable release
if newest_V > newest_stbl_V:
# We reformat the publication date of the release
published = datetime.datetime.strptime(
newest['published_at'],
"%Y-%m-%dT%H:%M:%SZ").strftime('%c')
self.log.info(lead + "##### RELEASE #####")
self.log.info(lead + "## Prerelease %(name)s" % newest)
self.log.info(lead + "## Published on %s" % published)
self.log.info(lead + "## Available on %(html_url)s" % newest)
self.log.info(lead + "###################")
def __init__(self, datadir, configfile, daemon=False):
# Verify if the log directory exists or create it
logdir = os.path.join(datadir, 'logs')
if not os.path.exists(logdir):
os.mkdir(logdir)
# Process log file name
if daemon:
if LOG_LEVEL is logging.DEBUG:
logfile = os.path.join(logdir, "TraktForVLC-DEBUG.log")
# Remove existing DEBUG file
if os.path.isfile(logfile):
os.remove(logfile)
else:
logfile = os.path.join(
logdir,
"TraktForVLC-" + DATETIME.strftime("%Y%m%d-%H%M") + ".log")
logging.basicConfig(
format="%(asctime)s::%(name)s::%(levelname)s::%(message)s",
level=LOG_LEVEL,
filename=logfile)
else:
logging.basicConfig(
format="%(asctime)s::%(name)s::%(levelname)s::%(message)s",
level=LOG_LEVEL,
stream=sys.stderr)
self.log = logging.getLogger("TraktForVLC")
e = 'e' if (sys.platform == 'win32') else 'ë'
self.log.info(
"## TraktForVLC v" + __version__ + " " + __release_name__)
self.log.info("## Copyright (C) 2014-2015 " +
"Rapha" + e + "l Beamonte <*****@*****.**>")
self.log.info("##")
self.log.info("## TraktForVLC is distributed in the hope that it " +
"will be useful, but")
self.log.info("## with ABSOLUTELY NO WARRANTY. This is free " +
"software; you are welcome")
self.log.info("## to redistribute and/or modify it under the terms " +
"of the GPL2.")
self.log.info("")
if not os.path.isfile(configfile):
self.log.error("Config file " +
configfile +
" not found, exiting.")
sys.exit(1)
self.log.debug("Running on %s, with Python %s" % (
platform.platform(), platform.python_version()))
self.__check_version()
# Load configuration
self.configfile = configfile
self.__load_config()
for loglvl, logstr in AVAILABLE_LOGLVL:
if LOG_LEVEL <= loglvl:
loglevelstr = logstr
break
if loglevelstr is None:
loglevelstr = str(LOG_LEVEL)
self.log.info("Logger level is set to %s" % loglevelstr)
self.log.info("-- Will scrobble movies ? %s" % (
'Yes' if self.DO_SCROBBLE_MOVIE else 'No'))
self.log.info("-- Will scrobble tv shows ? %s" % (
'Yes' if self.DO_SCROBBLE_TV else 'No'))
self.log.info("-- Will we mark movies as being watched ? %s" % (
'Yes' if self.DO_WATCHING_MOVIE else 'No'))
self.log.info("-- Will we mark tv shows as being watched ? %s" % (
'Yes' if self.DO_WATCHING_TV else 'No'))
self.log.info("-- Videos will be scrobbled after " +
str(self.SCROBBLE_PERCENT) +
"% of their duration has been exceeded")
self.log.info("-- Timer set to " +
str(self.TIMER_INTERVAL) +
" secs")
self.log.info("-- Video will be marked as \"is watching\" from " +
str(self.START_WATCHING_TIMER) +
" secs")
# VLC configuration
self.vlc_ip = self.config.get("VLC", "IP")
self.vlc_port = self.config.getint("VLC", "Port")
self.log.info("Listening VLC to " +
self.vlc_ip + ":" + str(self.vlc_port))
# Trakt app configuration
trakt_id = ("0e59f99095515c228d5fbc104e342574" +
"941aeeeda95946b8fa50b2b0366609bf")
trakt_sc = ("3ed1d013ef80eb0bb45d8da8424b4b61" +
"3713abb057ed505683caf0baf1b5c650")
# Trakt user information
trakt = {
"PIN": None,
"access_token": None,
"refresh_token": None,
"Username": None, # Now deprecated in Trakt v2 API
"Password": None, # Now deprecated in Trakt v2 API
}
for opt in trakt.keys():
if self.config.has_option("Trakt", opt):
trakt[opt] = self.config.get("Trakt", opt)
# Initialize Trakt client
modifiedTime = time.strftime(
'%Y-%m-%d',
time.gmtime(os.path.getmtime(get_file())))
self.trakt_client = TraktClient.TraktClient({
'username': trakt['Username'],
'password': trakt['Password'],
'client_id': trakt_id,
'client_secret': trakt_sc,
'app_version': __version__,
'app_date': modifiedTime,
'pin': trakt['PIN'],
'access_token': trakt['access_token'],
'refresh_token': trakt['refresh_token'],
'callback_token': self.__callback_token_change,
})
# Initialize TraktForVLC's cache
self.resetCache()
# Initialize tvdb api
self.tvdb = tvdb_api.Tvdb(cache=False, language='en')
self.watching_now = ""
self.vlcTime = 0
self.vlc_connected = True
def __load_config(self):
self.config = RawConfigParser()
self.config.optionxform = str
self.config.read(self.configfile)
# Initialize timers
if SMALL_TIMERS:
self.TIMER_INTERVAL = 5
self.START_WATCHING_TIMER = 5
else:
self.TIMER_INTERVAL = self.config.getint("TraktForVLC", "Timer")
self.START_WATCHING_TIMER = self.config.getint(
"TraktForVLC", "StartWatching")
# For the use of filenames instead of VLC window title
self.USE_FILENAME = (
True if self.config.get("TraktForVLC", "UseFilenames") == 'Yes'
else False)
# Do we have to scrobble ?
self.DO_SCROBBLE_MOVIE = (
True if self.config.get("TraktForVLC", "ScrobbleMovie") == 'Yes'
else False)
self.DO_SCROBBLE_TV = (
True if self.config.get("TraktForVLC", "ScrobbleTV") == 'Yes'
else False)
# Do we have to mark as watching ?
self.DO_WATCHING_MOVIE = (
True if self.config.get("TraktForVLC", "WatchingMovie") == 'Yes'
else False)
self.DO_WATCHING_TV = (
True if self.config.get("TraktForVLC", "WatchingTV") == 'Yes'
else False)
# What percent should we use to scrobble videos ?
self.SCROBBLE_PERCENT = self.config.getint(
"TraktForVLC", "ScrobblePercent")
def __callback_token_change(self, access_token, refresh_token):
if self.config.has_option('Trakt', 'PIN'):
self.config.remove_option('Trakt', 'PIN')
self.config.set('Trakt', 'access_token', access_token)
self.config.set('Trakt', 'refresh_token', refresh_token)
if not self.__save_config():
self.log.debug("Error while saving tokens in configuration file!")
def __save_config(self):
saved = False
with open(self.configfile, 'w') as configfile:
self.config.write(configfile)
saved = True
return saved
def resetCache(self, filepath=None, filename=None, filelength=None):
self.log.debug("reset cache (%s, %s)" % (filename, filelength))
self.cache = {
"vlc_file_path": filepath,
"vlc_file_name": filename,
"vlc_file_length": filelength,
"scrobbled": False,
"movie_info": None,
"series_info": None,
"series_current_ep": -1,
"started_watching": None,
"watching": -1,
"video": {},
}
def resetCacheView(self, episode=None):
self.log.debug('reset cache view status (%s)' % episode)
self.cache['watching'] = -1
self.cache['scrobbled'] = False
self.cache['started_watching'] = None
if episode is not None:
self.cache['series_current_ep'] = episode
def close(self, signal, frame):
self.log.info("Program closed by SIGINT")
sys.exit(0)
def run(self):
signal.signal(signal.SIGINT, self.close)
while (True):
try:
self.main()
except Exception as e:
self.log.error(
"An unknown error occurred",
exc_info=sys.exc_info())
time.sleep(self.TIMER_INTERVAL)
self.main()
def main(self):
try:
vlc = VLCRemote(self.vlc_ip, self.vlc_port)
self.vlc_connected = True
except:
if self.vlc_connected:
self.log.info('Could not find VLC running at ' +
str(self.vlc_ip) + ':' + str(self.vlc_port))
self.log.debug('Make sure your VLC player is running with ' +
'--extraintf=rc --rc-host=' +
str(self.vlc_ip) + ':' + str(self.vlc_port) +
' --rc-quiet', exc_info=sys.exc_info())
self.vlc_connected = False
# If we were watching a video but we didn't finish it, we
# have to cancel the watching status
if self.cache["watching"] > -1 and not self.cache["scrobbled"]:
self.trakt_client.cancelWatching(
self.cache["video"]["imdbid"],
self.get_episode(self.cache["video"]))
# If there is something in the cache, we can purge the watching
# and scrobbled information, so if the video is opened again we
# will consider it's a new watch
if self.cache['vlc_file_name'] is not None:
self.resetCacheView()
return
vlcStatus = vlc.is_playing()
if not vlcStatus:
vlc.close()
return
currentFileLength = vlc.get_length()
if not int(currentFileLength) > 0:
self.log.debug("main::File length is 0, can't do anything")
vlc.close()
return
currentFilePath = vlc.get_filename()
if self.USE_FILENAME:
currentFileName = currentFilePath.decode('utf-8')
else:
currentFileName = vlc.get_title().decode('utf-8')
self.vlcTime = int(vlc.get_time())
# Parse the filename to verify if it comes from a stream
parsed = urlparse(currentFileName)
if parsed.netloc:
# Set the filename using only the basename of the parsed path
currentFileName = os.path.basename(parsed.path)
elif self.USE_FILENAME:
# Even if it's not from a stream, if it's a filename we're using
# we need to keep only the basename of the parsed path
currentFileName = os.path.basename(currentFileName)
# Use urllib's unquote to bring back special chars
currentFileName = unquote(currentFileName)
if (currentFileName == self.cache["vlc_file_name"]
and currentFileLength == self.cache['vlc_file_length']):
if (self.cache["series_info"] is None
and self.cache["movie_info"] is None):
video = None
elif self.cache["series_info"] is not None:
video = self.get_TV(vlc, self.cache["series_info"])
else:
video = self.get_Movie(vlc, self.cache["movie_info"])
else:
self.log.debug("main::New file: %s (length: %s)"
% (currentFileName, currentFileLength))
# If we were watching a video but we didn't finish it, we
# have to cancel the watching status
if self.cache["watching"] > -1 and not self.cache["scrobbled"]:
self.trakt_client.cancelWatching(
self.cache["video"]["imdbid"],
self.get_episode(self.cache["video"]))
self.resetCache(currentFilePath, currentFileName,
currentFileLength)
self.cache['started_watching'] = (time.time(), self.vlcTime)
video = self.get_TV(vlc)
if video is None:
video = self.get_Movie(vlc)
if video is None:
self.log.info(
"No tv show nor movie found for the current playing video")
vlc.close()
return
# We cache the updated video information
self.cache["video"] = video
logtitle = video["title"]
if video["tv"]:
logtitle += (" - %01dx%02d"
% (int(video["season"]), int(video["episode"])))
# If we changed episode, we have to reset the view status
if (self.cache['watching'] > -1
and self.cache['series_current_ep'] != video['episode']):
self.resetCacheView(video['episode'])
self.cache['started_watching'] = (
time.time(),
self.vlcTime % video['duration'])
self.log.info(logtitle + " state : " + str(video["percentage"]) + "%")
self.log.debug("main::Video: %s" % str(video))
self.log.debug("main::This video is scrobbled : " +
str(self.cache["scrobbled"]))
if (((video['tv'] and self.DO_SCROBBLE_TV)
or (not video['tv'] and self.DO_SCROBBLE_MOVIE))
and video["percentage"] >= self.SCROBBLE_PERCENT
and not self.cache["scrobbled"]
and self.cache['started_watching'] is not None
and ((time.time() - self.cache['started_watching'][0])
> (float(video['duration']) / 3.0))
and ((self.vlcTime - self.cache['started_watching'][1])
> (float(video['duration']) / 4.0))):
self.log.info("Scrobbling " + logtitle + " to Trakt...")
try:
self.trakt_client.stopWatching(video["imdbid"],
video["percentage"],
self.get_episode(video))
self.cache["scrobbled"] = True
self.log.info(logtitle + " scrobbled to Trakt !")
except TraktClient.TraktError as e:
self.log.error("An error occurred while trying to scrobble",
exc_info=sys.exc_info())
elif (((video['tv'] and self.DO_WATCHING_TV)
or (not video['tv'] and self.DO_WATCHING_MOVIE))
and video["percentage"] < self.SCROBBLE_PERCENT
and not self.cache["scrobbled"]
and ((float(video["duration"]) * float(video["percentage"])
/ 100.0) >= self.START_WATCHING_TIMER)):
self.log.debug("main::Trying to mark " +
logtitle +
" watching on Trakt...")
try:
self.trakt_client.startWatching(video["imdbid"],
video["percentage"],
self.get_episode(video))
self.log.info(logtitle + " is currently watching on Trakt...")
self.cache["watching"] = video["percentage"]
except TraktClient.TraktError as e:
self.log.error("An error occurred while trying to mark as " +
"watching " + logtitle,
exc_info=sys.exc_info())
vlc.close()
def get_episode(self, video):
episode = video["tv"]
if episode:
episode = (video["show_imdbid"],
video["season"],
video["episode"])
return episode
def get_TV(self, vlc, series_info=(None, None, None)):
try:
series, seasonNumber, episodeNumber = series_info
if series is None:
now_playing = parse_tv(self.cache['vlc_file_name'])
if not now_playing:
self.log.info(
"Not able to parse a tvshow from the file title")
return
seriesName = now_playing['show']
seasonNumber = now_playing['season']
episodeNumber = now_playing['episodes']
if self.valid_TV(seriesName):
series = self.tvdb[seriesName]
self.cache["series_info"] = (
deepcopy(series),
seasonNumber,
episodeNumber)
if series is not None:
duration = int(self.cache['vlc_file_length'])
time = int(self.vlcTime)
# Calculate the relative time and duration depending on
# the number of episodes
duration = int(float(duration) / float(len(episodeNumber)))
currentEpisode = episodeNumber[int(time / duration)]
time = time % duration
# Calculate the given percentage for the current episode
percentage = time * 100 / duration
try:
episode = series[int(seasonNumber)][int(currentEpisode)]
return self.set_video(
True,
series['seriesname'],
series['firstaired'],
episode['imdb_id'],
duration,
percentage,
episode['seasonnumber'],
episode['episodenumber'],
series['imdb_id'])
except:
self.log.warning("Episode : No valid episode found !")
self.log.debug("get_TV::Here's to help debug",
exc_info=sys.exc_info())
self.cache["series_info"] = None
return
except:
self.log.info("No matching tv show found for video playing")
self.log.debug("get_TV::Here's to help debug",
exc_info=sys.exc_info())
return
def valid_TV(self, seriesName):
try:
series = self.tvdb.search(seriesName)
if (len(series) == 0):
self.log.debug("valid_TV::no series found with the name '%s'"
% seriesName)
return False
return True
except:
self.log.debug("valid_TV::no valid title found.",
exc_info=sys.exc_info())
return False
def get_Movie(self, vlc, movie=None):
try:
duration = int(self.cache['vlc_file_length'])
if movie is None:
now_playing = parse_movie(self.cache['vlc_file_name'])
title = now_playing['title']
year = now_playing['year']
self.log.debug("get_Movie::Now playing: %s" % str(now_playing))
if self.valid_Movie(self.cache['vlc_file_path'], title,
year, duration):
movie = self.cache["movie_info"]
self.log.debug("get_Movie::Valid movie found: %s"
% str(movie))
if movie is not None:
playtime = int(self.vlcTime)
percentage = playtime * 100 / duration
return self.set_video(
False,
movie['Title'],
movie['Year'],
movie['imdbID'],
duration,
percentage)
return
except:
self.log.info("No matching movie found for video playing")
self.log.debug("get_Movie::Here's to help debug",
exc_info=sys.exc_info())
return
def valid_Movie(self, vlcFilePath, vlcTitle, vlcYear, vlcDuration):
try:
# Get Movie info
movie = movie_info.get_movie_info(
vlcFilePath, vlcTitle, vlcYear, vlcDuration)
# Compare Movie runtime against VLC runtime
time = movie['Runtime']
# Verify that the VLC duration is within 5 minutes of the
# official duration
if (vlcDuration >= time - 300) and (vlcDuration <= time + 300):
self.cache["movie_info"] = deepcopy(movie)
return True
else:
self.log.debug("valid_Movie::time range not respected " +
"(%d +-300 != %d)" % (time, vlcDuration))
except:
self.log.debug("valid_Movie::no valid title found",
exc_info=sys.exc_info())
return False
return False
def set_video(self, tv, title, year, imdbid, duration, percentage,
season=-1, episode=-1, show_imdbid=None):
video = {
'tv': tv,
'title': title,
'year': year,
'imdbid': imdbid,
'duration': duration,
'percentage': percentage,
'season': season,
'episode': episode,
'show_imdbid': show_imdbid,
}
return video
class Config(object):
"""A wrapper around RawConfigParser"""
def __init__(self, version=None):
"""Use read() to read in an existing config file.
version should be an int starting with 0 that gets incremented if you
want to register a new upgrade function. If None, upgrade is disabled.
"""
self._config = ConfigParser(dict_type=_sorted_dict)
self._version = version
self._loaded_version = None
self._upgrade_funcs = []
self._initial = {}
def _do_upgrade(self, func):
assert self._loaded_version is not None
assert self._version is not None
old_version = self._loaded_version
new_version = self._version
if old_version != new_version:
print_d("Config upgrade: %d->%d (%r)" % (
old_version, new_version, func))
func(self, old_version, new_version)
def get_version(self):
"""Get the version of the loaded config file (for testing only)
Raises Error if no file was loaded or versioning is disabled.
"""
if self._version is None:
raise Error("Versioning disabled")
if self._loaded_version is None:
raise Error("No file loaded")
return self._loaded_version
def register_upgrade_function(self, function):
"""Register an upgrade function that gets called at each read()
if the current config version and the loaded version don't match.
Can also be registered after read was called.
function(config, old_version: int, new_version: int) -> None
"""
if self._version is None:
raise Error("Versioning disabled")
self._upgrade_funcs.append(function)
# after read(), so upgrade now
if self._loaded_version is not None:
self._do_upgrade(function)
return function
def set_inital(self, section, option, value):
"""Set an initial value for an option.
The section must be added with add_section() first.
Adds the value to the config and calling reset()
will reset the value to it.
"""
self.set(section, option, value)
self._initial.setdefault(section, {})
self._initial[section].setdefault(option, {})
self._initial[section][option] = value
def reset(self, section, option):
"""Reset the value to the initial state"""
value = self._initial[section][option]
self.set(section, option, value)
def options(self, section):
"""Returns a list of options available in the specified section."""
return self._config.options(section)
def get(self, section, option, default=_DEFAULT):
"""get(section, option[, default]) -> str
If default is not given, raises Error in case of an error
"""
try:
return self._config.get(section, option)
except Error:
if default is _DEFAULT:
raise
return default
def getboolean(self, section, option, default=_DEFAULT):
"""getboolean(section, option[, default]) -> bool
If default is not given, raises Error in case of an error
"""
try:
return self._config.getboolean(section, option)
except Error:
if default is _DEFAULT:
raise
if not isinstance(default, bool):
raise ValueError
return default
def getint(self, section, option, default=_DEFAULT):
"""getint(section, option[, default]) -> int
If default is not give, raises Error in case of an error
"""
try:
return self._config.getint(section, option)
except Error:
if default is _DEFAULT:
raise
if not isinstance(default, int):
raise ValueError
return default
def getfloat(self, section, option, default=_DEFAULT):
"""getfloat(section, option[, default]) -> float
If default is not give, raises Error in case of an error
"""
try:
return self._config.getfloat(section, option)
except Error:
if default is _DEFAULT:
raise
if not isinstance(default, float):
raise ValueError
return default
def getstringlist(self, section, option, default=_DEFAULT):
"""getstringlist(section, option[, default]) -> list
If default is not given, raises Error in case of an error.
Gets a list of strings, using CSV to parse and delimit.
"""
try:
value = self._config.get(section, option)
except Error:
if default is _DEFAULT:
raise
if not isinstance(default, list):
raise ValueError
return default
parser = csv.reader(
[value], lineterminator='\n', quoting=csv.QUOTE_MINIMAL)
try:
vals = [v.decode('utf-8') for v in parser.next()]
except (csv.Error, ValueError) as e:
raise Error(e)
return vals
def setstringlist(self, section, option, values):
"""Saves a list of unicode strings using the csv module"""
sw = StringIO()
values = [unicode(v).encode('utf-8') for v in values]
writer = csv.writer(sw, lineterminator='\n', quoting=csv.QUOTE_MINIMAL)
writer.writerow(values)
self._config.set(section, option, sw.getvalue())
def setlist(self, section, option, values, sep=","):
"""Saves a list of str using ',' as a separator and \\ for escaping"""
values = map(str, values)
joined = join_escape(values, sep)
self._config.set(section, option, joined)
def getlist(self, section, option, default=_DEFAULT, sep=","):
"""Returns a str list saved with setlist()"""
try:
value = self._config.get(section, option)
except Error:
if default is _DEFAULT:
raise
if not isinstance(default, list):
raise ValueError
return default
return split_escape(value, sep)
def set(self, section, option, value):
"""Saves the string representation for the passed value
Don't pass unicode, encode first.
"""
# RawConfigParser only allows string values but doesn't
# scream if they are not (and it only fails before the
# first config save..)
if not isinstance(value, str):
value = str(value)
self._config.set(section, option, value)
def setdefault(self, section, option, default):
"""Like set but only sets the new value if the option
isn't set before.
"""
if not self._config.has_option(section, option):
self._config.set(section, option, default)
def write(self, filename):
"""Write config to filename.
Can raise EnvironmentError
"""
assert is_fsnative(filename)
mkdir(os.path.dirname(filename))
# temporary set the new version for saving
if self._version is not None:
self.add_section("__config__")
self.set("__config__", "version", self._version)
try:
with atomic_save(filename, ".tmp", "wb") as fileobj:
self._config.write(fileobj)
finally:
if self._loaded_version is not None:
self.set("__config__", "version", self._loaded_version)
def clear(self):
"""Remove all sections and initial values"""
for section in self._config.sections():
self._config.remove_section(section)
self._initial.clear()
def is_empty(self):
"""Whether the config has any sections"""
return not self._config.sections()
def read(self, filename):
"""Reads the config from `filename` if the file exists,
otherwise does nothing
Can raise EnvironmentError, Error.
"""
parsed_filenames = self._config.read(filename)
# don't upgrade if we just created a new config
if parsed_filenames and self._version is not None:
self._loaded_version = self.getint("__config__", "version", -1)
for func in self._upgrade_funcs:
self._do_upgrade(func)
def sections(self):
"""Return a list of the sections available"""
return self._config.sections()
def has_option(self, section, option):
"""If the given section exists, and contains the given option"""
return self._config.has_option(section, option)
def remove_option(self, section, option):
"""Remove the specified option from the specified section
Can raise Error.
"""
return self._config.remove_option(section, option)
def add_section(self, section):
"""Add a section named section to the instance if it not already
exists."""
if not self._config.has_section(section):
self._config.add_section(section)
class MrxsFile(object):
def __init__(self, filename):
# Split filename
dirname, ext = os.path.splitext(filename)
if ext != '.mrxs':
raise UnrecognizedFile
# Parse slidedat
self._slidedatfile = os.path.join(dirname, 'Slidedat.ini')
self._dat = RawConfigParser()
self._dat.optionxform = str
try:
with open(self._slidedatfile, 'rb') as fh:
self._have_bom = (fh.read(len(UTF8_BOM)) == UTF8_BOM)
if not self._have_bom:
fh.seek(0)
self._dat.readfp(fh)
except IOError:
raise UnrecognizedFile
# Get file paths
self._indexfile = os.path.join(dirname,
self._dat.get(MRXS_HIERARCHICAL, 'INDEXFILE'))
self._datafiles = [os.path.join(dirname,
self._dat.get('DATAFILE', 'FILE_%d' % i))
for i in range(self._dat.getint('DATAFILE', 'FILE_COUNT'))]
# Build levels
self._make_levels()
def _make_levels(self):
self._levels = {}
self._level_list = []
layer_count = self._dat.getint(MRXS_HIERARCHICAL, 'NONHIER_COUNT')
for layer_id in range(layer_count):
level_count = self._dat.getint(MRXS_HIERARCHICAL,
'NONHIER_%d_COUNT' % layer_id)
for level_id in range(level_count):
level = MrxsNonHierLevel(self._dat, layer_id, level_id,
len(self._level_list))
self._levels[(level.layer_name, level.name)] = level
self._level_list.append(level)
@classmethod
def _read_int32(cls, f):
buf = f.read(4)
if len(buf) != 4:
raise IOError('Short read')
return struct.unpack('<i', buf)[0]
@classmethod
def _assert_int32(cls, f, value):
v = cls._read_int32(f)
if v != value:
raise ValueError('%d != %d' % (v, value))
def _get_data_location(self, record):
with open(self._indexfile, 'rb') as fh:
fh.seek(MRXS_NONHIER_ROOT_OFFSET)
# seek to record
table_base = self._read_int32(fh)
fh.seek(table_base + record * 4)
# seek to list head
list_head = self._read_int32(fh)
fh.seek(list_head)
# seek to data page
self._assert_int32(fh, 0)
page = self._read_int32(fh)
fh.seek(page)
# check pagesize
self._assert_int32(fh, 1)
# read rest of prologue
self._read_int32(fh)
self._assert_int32(fh, 0)
self._assert_int32(fh, 0)
# read values
position = self._read_int32(fh)
size = self._read_int32(fh)
fileno = self._read_int32(fh)
return (self._datafiles[fileno], position, size)
def _zero_record(self, record):
path, offset, length = self._get_data_location(record)
with open(path, 'r+b') as fh:
fh.seek(0, 2)
do_truncate = (fh.tell() == offset + length)
if DEBUG:
if do_truncate:
print 'Truncating', path, 'to', offset
else:
print 'Zeroing', path, 'at', offset, 'for', length
fh.seek(offset)
buf = fh.read(len(JPEG_SOI))
if buf != JPEG_SOI:
raise IOError('Unexpected data in nonhier image')
if do_truncate:
fh.truncate(offset)
else:
fh.seek(offset)
fh.write('\0' * length)
def _delete_index_record(self, record):
if DEBUG:
print 'Deleting record', record
with open(self._indexfile, 'r+b') as fh:
entries_to_move = len(self._level_list) - record - 1
if entries_to_move == 0:
return
# get base of table
fh.seek(MRXS_NONHIER_ROOT_OFFSET)
table_base = self._read_int32(fh)
# read tail of table
fh.seek(table_base + (record + 1) * 4)
buf = fh.read(entries_to_move * 4)
if len(buf) != entries_to_move * 4:
raise IOError('Short read')
# overwrite the target record
fh.seek(table_base + record * 4)
fh.write(buf)
def _hier_keys_for_level(self, level):
ret = []
for k, _ in self._dat.items(MRXS_HIERARCHICAL):
if k == level.key_prefix or k.startswith(level.key_prefix + '_'):
ret.append(k)
return ret
def _rename_section(self, old, new):
if self._dat.has_section(old):
if DEBUG:
print '[%s] -> [%s]' % (old, new)
self._dat.add_section(new)
for k, v in self._dat.items(old):
self._dat.set(new, k, v)
self._dat.remove_section(old)
elif DEBUG:
print '[%s] does not exist' % old
def _delete_section(self, section):
if DEBUG:
print 'Deleting [%s]' % section
self._dat.remove_section(section)
def _set_key(self, section, key, value):
if DEBUG:
prev = self._dat.get(section, key)
print '[%s] %s: %s -> %s' % (section, key, prev, value)
self._dat.set(section, key, value)
def _rename_key(self, section, old, new):
if DEBUG:
print '[%s] %s -> %s' % (section, old, new)
v = self._dat.get(section, old)
self._dat.remove_option(section, old)
self._dat.set(section, new, v)
def _delete_key(self, section, key):
if DEBUG:
print 'Deleting [%s] %s' % (section, key)
self._dat.remove_option(section, key)
def _write(self):
buf = StringIO()
self._dat.write(buf)
with open(self._slidedatfile, 'wb') as fh:
if self._have_bom:
fh.write(UTF8_BOM)
fh.write(buf.getvalue().replace('\n', '\r\n'))
def delete_level(self, layer_name, level_name):
level = self._levels[(layer_name, level_name)]
record = level.record
# Zero image data
self._zero_record(record)
# Delete pointer from nonhier table in index
self._delete_index_record(record)
# Remove slidedat keys
for k in self._hier_keys_for_level(level):
self._delete_key(MRXS_HIERARCHICAL, k)
# Remove slidedat section
self._delete_section(level.section)
# Rename section and keys for subsequent levels in the layer
prev_level = level
for cur_level in self._level_list[record + 1:]:
if cur_level.layer_id != prev_level.layer_id:
break
for k in self._hier_keys_for_level(cur_level):
new_k = k.replace(cur_level.key_prefix,
prev_level.key_prefix, 1)
self._rename_key(MRXS_HIERARCHICAL, k, new_k)
self._set_key(MRXS_HIERARCHICAL, prev_level.section_key,
prev_level.section)
self._rename_section(cur_level.section, prev_level.section)
prev_level = cur_level
# Update level count within layer
count_k = 'NONHIER_%d_COUNT' % level.layer_id
count_v = self._dat.getint(MRXS_HIERARCHICAL, count_k)
self._set_key(MRXS_HIERARCHICAL, count_k, count_v - 1)
# Write slidedat
self._write()
# Refresh metadata
self._make_levels()
class TraktForVLC(object):
# Check if there's a newer version of TraktForVLC on the project's github,
# and print informations on that subject in the logs
def __check_version(self):
# The leading information for lines printed by this method in logs
lead = "VERSION:"
# Request the github API to get the releases information
github_releases = requests.get(
url="https://api.github.com/repos/XaF/TraktForVLC/releases")
# If there was a problem getting the releases
if not github_releases.ok:
self.log.error(lead +
"Unable to verify new releases of TraktForVLC")
return
# Else, we get the json answer
releases = github_releases.json()
# If we didn't find any release
if not releases:
self.log.warning(lead +
"No releases found on github")
return
# We get the latest release, all included
newest = sorted(
releases,
key=lambda x: x['tag_name'],
reverse=True)[0]
if newest['tag_name'][:1] == "v":
newest['tag_name'] = newest['tag_name'][1:]
newest_V = parse_version(newest['tag_name'])
# We get the latest _stable_ release
newest_stbl = sorted(
[r for r in releases if not r['prerelease']],
key=lambda x: x['tag_name'],
reverse=True)[0]
if newest_stbl['tag_name'][:1] == "v":
newest_stbl['tag_name'] = newest_stbl['tag_name'][1:]
newest_stbl_V = parse_version(newest_stbl['tag_name'])
# We parse the current version
current_V = parse_version(__version__)
if newest_V <= current_V:
self.log.info(lead + "TraktForVLC is up to date")
return
# We only show the latest stable release if
# it's newer than our current release
if newest_stbl_V > current_V:
# We reformat the publication date of the release
published = datetime.datetime.strptime(
newest_stbl['published_at'],
"%Y-%m-%dT%H:%M:%SZ").strftime('%c')
self.log.info(lead + "##### RELEASE #####")
self.log.info(lead + "## Stable release %(name)s" % newest_stbl)
self.log.info(lead + "## Published on %s" % published)
self.log.info(lead + "## Available on %(html_url)s" % newest_stbl)
# We only show the latest release if it's not
# also the latest stable release
if newest_V > newest_stbl_V:
# We reformat the publication date of the release
published = datetime.datetime.strptime(
newest['published_at'],
"%Y-%m-%dT%H:%M:%SZ").strftime('%c')
self.log.info(lead + "##### RELEASE #####")
self.log.info(lead + "## Prerelease %(name)s" % newest)
self.log.info(lead + "## Published on %s" % published)
self.log.info(lead + "## Available on %(html_url)s" % newest)
self.log.info(lead + "###################")
def __init__(self, datadir, configfile, daemon=False):
# Verify if the log directory exists or create it
logdir = os.path.join(datadir, 'logs')
if not os.path.exists(logdir):
os.mkdir(logdir)
# Process log file name
if daemon:
if LOG_LEVEL is logging.DEBUG:
logfile = os.path.join(logdir, "TraktForVLC-DEBUG.log")
# Remove existing DEBUG file
if os.path.isfile(logfile):
os.remove(logfile)
else:
logfile = os.path.join(
logdir,
"TraktForVLC-" + DATETIME.strftime("%Y%m%d-%H%M") + ".log")
logging.basicConfig(
format="%(asctime)s::%(name)s::%(levelname)s::%(message)s",
level=LOG_LEVEL,
filename=logfile)
else:
logging.basicConfig(
format="%(asctime)s::%(name)s::%(levelname)s::%(message)s",
level=LOG_LEVEL,
stream=sys.stderr)
self.log = logging.getLogger("TraktForVLC")
self.log.info(
"## TraktForVLC v" + __version__ + " " + __release_name__)
self.log.info("## Copyright (C) 2014-2015 " +
"Raphaël Beamonte <*****@*****.**>")
self.log.info("##")
self.log.info("## TraktForVLC is distributed in the hope that it " +
"will be useful, but")
self.log.info("## with ABSOLUTELY NO WARRANTY. This is free " +
"software; you are welcome")
self.log.info("## to redistribute and/or modify it under the terms " +
"of the GPL2.")
self.log.info("")
if not os.path.isfile(configfile):
self.log.error("Config file " +
configfile +
" not found, exiting.")
exit()
self.log.debug("Running on %s, with Python %s" % (
platform.platform(), platform.python_version()))
self.__check_version()
# Load configuration
self.configfile = configfile
self.__load_config()
for loglvl, logstr in AVAILABLE_LOGLVL:
if LOG_LEVEL <= loglvl:
loglevelstr = logstr
break
if loglevelstr is None:
loglevelstr = str(LOG_LEVEL)
self.log.info("Logger level is set to %s" % loglevelstr)
self.log.info("-- Will scrobble movies ? %s" % (
'Yes' if self.DO_SCROBBLE_MOVIE else 'No'))
self.log.info("-- Will scrobble tv shows ? %s" % (
'Yes' if self.DO_SCROBBLE_TV else 'No'))
self.log.info("-- Will we mark movies as being watched ? %s" % (
'Yes' if self.DO_WATCHING_MOVIE else 'No'))
self.log.info("-- Will we mark tv shows as being watched ? %s" % (
'Yes' if self.DO_WATCHING_TV else 'No'))
self.log.info("-- Videos will be scrobbled after " +
str(self.SCROBBLE_PERCENT) +
"% of their duration has been exceeded")
self.log.info("-- Timer set to " +
str(self.TIMER_INTERVAL) +
" secs")
self.log.info("-- Video will be marked as \"is watching\" from " +
str(self.START_WATCHING_TIMER) +
" secs")
# VLC configuration
self.vlc_ip = self.config.get("VLC", "IP")
self.vlc_port = self.config.getint("VLC", "Port")
self.log.info("Listening VLC to " +
self.vlc_ip + ":" + str(self.vlc_port))
# Trakt app configuration
trakt_id = ("0e59f99095515c228d5fbc104e342574" +
"941aeeeda95946b8fa50b2b0366609bf")
trakt_sc = ("3ed1d013ef80eb0bb45d8da8424b4b61" +
"3713abb057ed505683caf0baf1b5c650")
# Trakt user information
trakt = {
"PIN": None,
"access_token": None,
"refresh_token": None,
"Username": None, # Now deprecated in Trakt v2 API
"Password": None, # Now deprecated in Trakt v2 API
}
for opt in trakt.keys():
if self.config.has_option("Trakt", opt):
trakt[opt] = self.config.get("Trakt", opt)
# Initialize Trakt client
modifiedTime = time.strftime(
'%Y-%m-%d',
time.gmtime(os.path.getmtime(__file__)))
self.trakt_client = TraktClient.TraktClient({
'username': trakt['Username'],
'password': trakt['Password'],
'client_id': trakt_id,
'client_secret': trakt_sc,
'app_version': __version__,
'app_date': modifiedTime,
'pin': trakt['PIN'],
'access_token': trakt['access_token'],
'refresh_token': trakt['refresh_token'],
'callback_token': self.__callback_token_change,
})
# Initialize TraktForVLC's cache
self.resetCache()
# Initialize tvdb api
self.tvdb = tvdb_api.Tvdb(cache=False, language='en')
self.watching_now = ""
self.vlcTime = 0
self.vlc_connected = True
def __load_config(self):
self.config = RawConfigParser()
self.config.optionxform = str
self.config.read(self.configfile)
# Initialize timers
if SMALL_TIMERS:
self.TIMER_INTERVAL = 5
self.START_WATCHING_TIMER = 5
else:
self.TIMER_INTERVAL = self.config.getint("TraktForVLC", "Timer")
self.START_WATCHING_TIMER = self.config.getint(
"TraktForVLC", "StartWatching")
# For the use of filenames instead of VLC window title
self.USE_FILENAME = (
True if self.config.get("TraktForVLC", "UseFilenames") == 'Yes'
else False)
# Do we have to scrobble ?
self.DO_SCROBBLE_MOVIE = (
True if self.config.get("TraktForVLC", "ScrobbleMovie") == 'Yes'
else False)
self.DO_SCROBBLE_TV = (
True if self.config.get("TraktForVLC", "ScrobbleTV") == 'Yes'
else False)
# Do we have to mark as watching ?
self.DO_WATCHING_MOVIE = (
True if self.config.get("TraktForVLC", "WatchingMovie") == 'Yes'
else False)
self.DO_WATCHING_TV = (
True if self.config.get("TraktForVLC", "WatchingTV") == 'Yes'
else False)
# What percent should we use to scrobble videos ?
self.SCROBBLE_PERCENT = self.config.getint(
"TraktForVLC", "ScrobblePercent")
def __callback_token_change(self, access_token, refresh_token):
if self.config.has_option('Trakt', 'PIN'):
self.config.remove_option('Trakt', 'PIN')
self.config.set('Trakt', 'access_token', access_token)
self.config.set('Trakt', 'refresh_token', refresh_token)
if not self.__save_config():
self.log.debug("Error while saving tokens in configuration file!")
def __save_config(self):
saved = False
with open(self.configfile, 'wb') as configfile:
self.config.write(configfile)
saved = True
return saved
def resetCache(self, filename=None, filelength=None):
self.log.debug("reset cache (%s, %s)" % (filename, filelength))
self.cache = {
"vlc_file_name": filename,
"vlc_file_length": filelength,
"scrobbled": False,
"movie_info": None,
"series_info": None,
"series_current_ep": -1,
"started_watching": None,
"watching": -1,
"video": {},
}
def resetCacheView(self, episode=None):
self.log.debug('reset cache view status (%s)' % episode)
self.cache['watching'] = -1
self.cache['scrobbled'] = False
self.cache['started_watching'] = None
if episode is not None:
self.cache['series_current_ep'] = episode
def close(self, signal, frame):
self.log.info("Program closed by SIGINT")
sys.exit(0)
def run(self):
signal.signal(signal.SIGINT, self.close)
while (True):
try:
self.main()
except Exception as e:
self.log.error(
"An unknown error occurred",
exc_info=sys.exc_info())
time.sleep(self.TIMER_INTERVAL)
self.main()
def main(self):
try:
vlc = VLCRemote(self.vlc_ip, self.vlc_port)
self.vlc_connected = True
except:
if self.vlc_connected:
self.log.info('Could not find VLC running at ' +
str(self.vlc_ip) + ':' + str(self.vlc_port))
self.log.debug('Make sure your VLC player is running with ' +
'--extraintf=rc --rc-host=' +
str(self.vlc_ip) + ':' + str(self.vlc_port) +
' --rc-quiet', exc_info=sys.exc_info())
self.vlc_connected = False
# If we were watching a video but we didn't finish it, we
# have to cancel the watching status
if self.cache["watching"] > -1 and not self.cache["scrobbled"]:
self.trakt_client.cancelWatching(
self.cache["video"]["imdbid"],
self.get_episode(self.cache["video"]))
# If there is something in the cache, we can purge the watching
# and scrobbled information, so if the video is opened again we
# will consider it's a new watch
if self.cache['vlc_file_name'] is not None:
self.resetCacheView()
return
vlcStatus = vlc.is_playing()
if not vlcStatus:
vlc.close()
return
currentFileLength = vlc.get_length()
if not int(currentFileLength) > 0:
self.log.debug("main::File length is 0, can't do anything")
vlc.close()
return
if self.USE_FILENAME:
currentFileName = vlc.get_filename()
else:
currentFileName = vlc.get_title()
self.vlcTime = int(vlc.get_time())
# Parse the filename to verify if it comes from a stream
parsed = urlparse(currentFileName)
if parsed.netloc:
# Set the filename using only the basename of the parsed path
currentFileName = os.path.basename(parsed.path)
# And use urllib's unquote to bring back special chars
currentFileName = unquote(currentFileName)
elif self.USE_FILENAME:
# Even if it's not from a stream, if it's a filename we're using
# we need to keep only the basename of the parsed path
currentFileName = os.path.basename(currentFileName)
if (currentFileName == self.cache["vlc_file_name"]
and currentFileLength == self.cache['vlc_file_length']):
if (self.cache["series_info"] is None
and self.cache["movie_info"] is None):
video = None
elif self.cache["series_info"] is not None:
video = self.get_TV(vlc, self.cache["series_info"])
else:
video = self.get_Movie(vlc, self.cache["movie_info"])
else:
self.log.debug("main::New file: %s (%s)"
% (currentFileName, currentFileLength))
# If we were watching a video but we didn't finish it, we
# have to cancel the watching status
if self.cache["watching"] > -1 and not self.cache["scrobbled"]:
self.trakt_client.cancelWatching(
self.cache["video"]["imdbid"],
self.get_episode(self.cache["video"]))
self.resetCache(currentFileName, currentFileLength)
self.cache['started_watching'] = (time.time(), self.vlcTime)
video = self.get_TV(vlc)
if video is None:
video = self.get_Movie(vlc)
if video is None:
self.log.info(
"No tv show nor movie found for the current playing video")
vlc.close()
return
# We cache the updated video information
self.cache["video"] = video
logtitle = video["title"]
if video["tv"]:
logtitle += (" - %01dx%02d"
% (int(video["season"]), int(video["episode"])))
# If we changed episode, we have to reset the view status
if (self.cache['watching'] > -1
and self.cache['series_current_ep'] != video['episode']):
self.resetCacheView(video['episode'])
self.cache['started_watching'] = (
time.time(),
self.vlcTime % video['duration'])
self.log.info(logtitle + " state : " + str(video["percentage"]) + "%")
self.log.debug("main::Video: %s" % str(video))
self.log.debug("main::This video is scrobbled : " +
str(self.cache["scrobbled"]))
if (((video['tv'] and self.DO_SCROBBLE_TV)
or (not video['tv'] and self.DO_SCROBBLE_MOVIE))
and video["percentage"] >= self.SCROBBLE_PERCENT
and not self.cache["scrobbled"]
and self.cache['started_watching'] is not None
and ((time.time() - self.cache['started_watching'][0])
> (float(video['duration']) / 3.0))
and ((self.vlcTime - self.cache['started_watching'][1])
> (float(video['duration']) / 4.0))):
self.log.info("Scrobbling " + logtitle + " to Trakt...")
try:
self.trakt_client.stopWatching(video["imdbid"],
video["percentage"],
self.get_episode(video))
self.cache["scrobbled"] = True
self.log.info(logtitle + " scrobbled to Trakt !")
except TraktClient.TraktError as e:
self.log.error("An error occurred while trying to scrobble",
exc_info=sys.exc_info())
elif (((video['tv'] and self.DO_WATCHING_TV)
or (not video['tv'] and self.DO_WATCHING_MOVIE))
and video["percentage"] < self.SCROBBLE_PERCENT
and not self.cache["scrobbled"]
and ((float(video["duration"]) * float(video["percentage"])
/ 100.0) >= self.START_WATCHING_TIMER)):
self.log.debug("main::Trying to mark " +
logtitle +
" watching on Trakt...")
try:
self.trakt_client.startWatching(video["imdbid"],
video["percentage"],
self.get_episode(video))
self.log.info(logtitle + " is currently watching on Trakt...")
self.cache["watching"] = video["percentage"]
except TraktClient.TraktError as e:
self.log.error("An error occurred while trying to mark as " +
"watching " + logtitle,
exc_info=sys.exc_info())
vlc.close()
def get_episode(self, video):
episode = video["tv"]
if episode:
episode = (video["show_imdbid"],
video["season"],
video["episode"])
return episode
def get_TV(self, vlc, series_info=(None, None, None)):
try:
series, seasonNumber, episodeNumber = series_info
if series is None:
now_playing = parse_tv(self.cache['vlc_file_name'])
if not now_playing:
self.log.info(
"Not able to parse a tvshow from the file title")
return
seriesName = now_playing['show']
seasonNumber = now_playing['season']
episodeNumber = now_playing['episodes']
if self.valid_TV(seriesName):
series = self.tvdb[seriesName]
self.cache["series_info"] = (
deepcopy(series),
seasonNumber,
episodeNumber)
if series is not None:
duration = int(self.cache['vlc_file_length'])
time = int(self.vlcTime)
# Calculate the relative time and duration depending on
# the number of episodes
duration = int(float(duration) / float(len(episodeNumber)))
currentEpisode = episodeNumber[int(time / duration)]
time = time % duration
# Calculate the given percentage for the current episode
percentage = time * 100 / duration
try:
episode = series[int(seasonNumber)][int(currentEpisode)]
return self.set_video(
True,
series['seriesname'],
series['firstaired'],
episode['imdb_id'],
duration,
percentage,
episode['seasonnumber'],
episode['episodenumber'],
series['imdb_id'])
except:
self.log.warning("Episode : No valid episode found !")
self.log.debug("get_TV::Here's to help debug",
exc_info=sys.exc_info())
self.cache["series_info"] = None
return
except:
self.log.info("No matching tv show found for video playing")
self.log.debug("get_TV::Here's to help debug",
exc_info=sys.exc_info())
return
def valid_TV(self, seriesName):
try:
series = self.tvdb.search(seriesName)
if (len(series) == 0):
self.log.debug("valid_TV::no series found with the name '%s'"
% seriesName)
return False
return True
except:
self.log.debug("valid_TV::no valid title found.",
exc_info=sys.exc_info())
return False
def get_Movie(self, vlc, movie=None):
try:
duration = int(self.cache['vlc_file_length'])
if movie is None:
now_playing = parse_movie(self.cache['vlc_file_name'])
title = now_playing['title']
year = now_playing['year']
self.log.debug("get_Movie::Now playing: %s" % str(now_playing))
if self.valid_Movie(title, year, duration):
movie = self.cache["movie_info"]
self.log.debug("get_Movie::Valid movie found: %s"
% str(movie))
if movie is not None:
playtime = int(self.vlcTime)
percentage = playtime * 100 / duration
return self.set_video(
False,
movie['Title'],
movie['Year'],
movie['imdbID'],
duration,
percentage)
return
except:
self.log.info("No matching movie found for video playing")
self.log.debug("get_Movie::Here's to help debug",
exc_info=sys.exc_info())
return
def valid_Movie(self, vlcTitle, vlcYear, vlcDuration):
try:
# Get Movie info
movie = movie_info.get_movie_info(vlcTitle, vlcYear)
# Compare Movie runtime against VLC runtime
regex = re.compile('^((?P<hour>[0-9]{1,2}).*?h)' +
'?.*?(?P<min>[0-9]+).*?min?',
re.IGNORECASE | re.MULTILINE)
r = regex.search(movie['Runtime'])
try:
timeh = 0 if r.group('hour') is None else int(r.group('hour'))
timem = 0 if r.group('min') is None else int(r.group('min'))
time = timeh * 60 * 60 + timem * 60
except:
self.log.debug("valid_Movie::unable to compute the duration",
exc_info=sys.exc_info())
return False
# Verify that the VLC duration is within 5 minutes of the
# official duration
if (vlcDuration >= time - 300) and (vlcDuration <= time + 300):
self.cache["movie_info"] = deepcopy(movie)
return True
else:
self.log.debug("valid_Movie::time range not respected " +
"(%d +-300 != %d)" % (time, vlcDuration))
except:
self.log.debug("valid_Movie::no valid title found",
exc_info=sys.exc_info())
return False
return False
def set_video(self, tv, title, year, imdbid, duration, percentage,
season=-1, episode=-1, show_imdbid=None):
video = {
'tv': tv,
'title': title,
'year': year,
'imdbid': imdbid,
'duration': duration,
'percentage': percentage,
'season': season,
'episode': episode,
'show_imdbid': show_imdbid,
}
return video
def remove_option(self, section, option):
if self.has_option(section, option):
RawConfigParser.remove_option(self, section, option)
class Configuration(object):
def __init__(self, bus, target_name, file_name):
self.bus = bus
self.target_name = target_name
self.file_name = file_name
self.config = RawConfigParser()
self.config.read(file_name)
if target_name is not None:
self.service = bus.create_service({"type": "autoconfigure", "target": target_name},
active=False, use_py_object=_AutoconfigureService(self))
self.sections_object = self.service.create_object("sections",
self._compute_sections(), doc=sections_object_doc)
self.options_object = self.service.create_object("options",
self._compute_options(), doc=options_object_doc)
@synchronized(_lock)
def _save(self):
with open(self.file_name, "w") as file:
self.config.write(file)
self.sections_object.set_value(self._compute_sections())
self.options_object.set_value(self._compute_options())
@synchronized(_lock)
def _compute_sections(self):
return self.config.sections()
@synchronized(_lock)
def _compute_options(self):
return dict((section, dict(self.config.items(section)))
for section in self.config.sections())
@synchronized(_lock)
def sections(self):
return self.config.sections()
@synchronized(_lock)
def add_section(self, name):
self.config.add_section(name)
self._save()
@synchronized(_lock)
def has_section(self, name):
return self.config.has_section(name)
@synchronized(_lock)
def options(self, section):
return self.config.options(section)
@synchronized(_lock)
def has_option(self, section, option):
return self.config.has_option(section, option)
@synchronized(_lock)
def get(self, section, option):
return self.config.get(section, option)
@synchronized(_lock)
def getint(self, section, option):
return self.config.getint(section, option)
@synchronized(_lock)
def getfloat(self, section, option):
return self.config.getfloat(section, option)
@synchronized(_lock)
def getboolean(self, section, option):
return self.config.getboolean(section, option)
@synchronized(_lock)
def items(self, section):
return self.config.items(section)
@synchronized(_lock)
def set(self, section, option, value):
self.config.set(section, option, value)
self._save()
@synchronized(_lock)
def remove_option(self, section, option):
self.config.remove_option(section, option)
self._save()
@synchronized(_lock)
def remove_section(self, section):
self.config.remove_section(section)
self._save()
