def run(self):
try:
config_file = DEFAULT_CONF_FILE
if not os.path.isfile(config_file):
raise ValueError('Configuration file not found: {0}'.format(config_file))
config = RawConfigParser()
config.read(config_file)
if 'security' in config.sections():
return 'The configuration file is already migrated to the new version'
config.add_section('security')
config.add_section('database')
for item in config.items('paths'):
if item[0] == 'database_file':
config.set('database', item[0], item[1])
else:
config.set('security', item[0], item[1])
config.remove_section('paths')
config.set('security', 'crl_file_url', 'None')
config.set('logging', 'log_level', 'INFO')
with open(config_file, 'w') as file:
config.write(file)
except Exception as exc:
return exc
return 'Configuration file migrated'
def remove_section(self, section):
""" Wrapper around the ConfigParser.remove_section() method.
This method only calls the ConfigParser.remove_section() method
if the section actually exists.
"""
if self.has_section(section):
RawConfigParser.remove_section(self, section)
def push_database_data(email, password):
resource = 'nprstuff'
filename = '%s.conf' % resource
baseConfDir = os.path.expanduser('~/.local/config/%s' % resource)
absPath = os.path.join(baseConfDir, filename)
if not os.path.isfile(absPath):
cparser = RawConfigParser()
cparser.set('DATABASE_DATA', 'email', dbEmail)
cparser.set('DATABASE_DATA', 'password', dbPasswd)
else:
cparser = ConfigParser.ConfigParser()
cparser.read(absPath)
if cparser.has_section('DATABASE_DATA'):
cparser.remove_section('DATABASE_DATA')
cparser.add_section('DATABASE_DATA')
cparser.set('DATABASE_DATA', 'email', email)
cparser.set('DATABASE_DATA', 'password', password)
with open(absPath, 'wb') as openfile:
cparser.write(openfile)
os.chmod(absPath, 0o600)
def edit_config(filename, settings, dry_run=False):
"""Edit a configuration file to include `settings`
`settings` is a dictionary of dictionaries or ``None`` values, keyed by
command/section name. A ``None`` value means to delete the entire section,
while a dictionary lists settings to be changed or deleted in that section.
A setting of ``None`` means to delete that setting.
"""
from configparser import RawConfigParser
log.debug("Reading configuration from %s", filename)
opts = RawConfigParser()
opts.read([filename])
for section, options in list(settings.items()):
if options is None:
log.info("Deleting section [%s] from %s", section, filename)
opts.remove_section(section)
else:
if not opts.has_section(section):
log.debug("Adding new section [%s] to %s", section, filename)
opts.add_section(section)
for option,value in list(options.items()):
if value is None:
log.debug("Deleting %s.%s from %s",
section, option, filename
)
opts.remove_option(section,option)
if not opts.options(section):
log.info("Deleting empty [%s] section from %s",
section, filename)
opts.remove_section(section)
else:
log.debug(
"Setting %s.%s to %r in %s",
section, option, value, filename
)
opts.set(section,option,value)
log.info("Writing %s", filename)
if not dry_run:
f = open(filename,'w'); opts.write(f); f.close()
class ConfigFileBackingStore(backing_store.BackingStore):
def __init__(self, path):
super(ConfigFileBackingStore, self).__init__(path)
self.configParser = RawConfigParser()
self.configParser.read(self.path)
def identifiers(self):
return self.configParser.sections()
def add_identifier(self, ident):
try:
self.configParser.add_section(ident)
except DuplicateSectionError:
raise ValueError("The identifier `%s` already exists" % str(ident))
def remove_identifier(self, ident):
self.configParser.remove_section(ident)
def keys(self, ident):
try:
return self.configParser.options(ident)
except NoSectionError:
raise ValueError("No identifier named `%s` exists" % str(ident))
def get(self, ident, key, default=None):
try:
val = self.configParser.get(ident, key.lower())
return val
except (NoSectionError, NoOptionError):
return default
def set(self, ident, key, value):
self.configParser.set(ident, key.lower(), value)
def save(self):
try:
with open(self.path, "w") as configFile:
self.configParser.write(configFile)
except IOError:
raise IOError("Cannot save data to `%s`. Permission Denied")
def edit_config(filename, settings, dry_run=False):
"""Edit a configuration file to include `settings`
`settings` is a dictionary of dictionaries or ``None`` values, keyed by
command/section name. A ``None`` value means to delete the entire section,
while a dictionary lists settings to be changed or deleted in that section.
A setting of ``None`` means to delete that setting.
"""
from configparser import RawConfigParser
log.debug("Reading configuration from %s", filename)
opts = RawConfigParser()
opts.read([filename])
for section, options in list(settings.items()):
if options is None:
log.info("Deleting section [%s] from %s", section, filename)
opts.remove_section(section)
else:
if not opts.has_section(section):
log.debug("Adding new section [%s] to %s", section, filename)
opts.add_section(section)
for option, value in list(options.items()):
if value is None:
log.debug("Deleting %s.%s from %s", section, option,
filename)
opts.remove_option(section, option)
if not opts.options(section):
log.info("Deleting empty [%s] section from %s",
section, filename)
opts.remove_section(section)
else:
log.debug("Setting %s.%s to %r in %s", section, option,
value, filename)
opts.set(section, option, value)
log.info("Writing %s", filename)
if not dry_run:
f = open(filename, 'w')
opts.write(f)
f.close()
def main(args):
"""docstring for main"""
configparser = RawConfigParser()
configparser.read(args.projects)
if args.project in configparser.sections():
yn = input('Do you really want to remove {0}? (y/N) '.format(
args.project))
if not yn or yn.capitalize().startswith('N'):
return
path = configparser.get(args.project, 'path')
try:
shutil.rmtree(path)
except FileNotFoundError:
logging.debug('{0} does not exists'.format(path))
configparser.remove_section(args.project)
with open(args.projects, 'w') as f:
configparser.write(f)
logging.debug('Project {0} is removed'.format(args.project))
else:
raise Exception('Project {0} does not exists'.format(args.project))
class Config(object):
"""A wrapper around RawConfigParser.
Provides a ``defaults`` attribute of the same type which can be used
to set default values.
"""
def __init__(self, version=None, _defaults=True):
"""Use read() to read in an existing config file.
version should be an int starting with 0 that gets incremented if you
want to register a new upgrade function. If None, upgrade is disabled.
"""
self._config = ConfigParser(dict_type=_sorted_dict)
self.defaults = None
if _defaults:
self.defaults = Config(_defaults=False)
self._version = version
self._loaded_version = None
self._upgrade_funcs = []
def _do_upgrade(self, func):
assert self._loaded_version is not None
assert self._version is not None
old_version = self._loaded_version
new_version = self._version
if old_version != new_version:
print_d("Config upgrade: %d->%d (%r)" % (
old_version, new_version, func))
func(self, old_version, new_version)
def get_version(self):
"""Get the version of the loaded config file (for testing only)
Raises Error if no file was loaded or versioning is disabled.
"""
if self._version is None:
raise Error("Versioning disabled")
if self._loaded_version is None:
raise Error("No file loaded")
return self._loaded_version
def register_upgrade_function(self, function):
"""Register an upgrade function that gets called at each read()
if the current config version and the loaded version don't match.
Can also be registered after read was called.
function(config, old_version: int, new_version: int) -> None
"""
if self._version is None:
raise Error("Versioning disabled")
self._upgrade_funcs.append(function)
# after read(), so upgrade now
if self._loaded_version is not None:
self._do_upgrade(function)
return function
def reset(self, section, option):
"""Reset the value to the default state"""
assert self.defaults is not None
try:
self._config.remove_option(section, option)
except NoSectionError:
pass
def options(self, section):
"""Returns a list of options available in the specified section."""
try:
options = self._config.options(section)
except NoSectionError:
if self.defaults:
return self.defaults.options(section)
raise
else:
if self.defaults:
try:
options.extend(self.defaults.options(section))
options = list_unique(options)
except NoSectionError:
pass
return options
def get(self, section, option, default=_DEFAULT):
"""get(section, option[, default]) -> str
If default is not given or set, raises Error in case of an error
"""
try:
return self._config.get(section, option)
except Error:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.get(section, option)
except Error:
pass
raise
return default
def gettext(self, *args, **kwargs):
value = self.get(*args, **kwargs)
# make sure there are no surrogates
value.encode("utf-8")
return value
def getbytes(self, section, option, default=_DEFAULT):
try:
value = self._config.get(section, option)
value = value.encode("utf-8", "surrogateescape")
return value
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getbytes(section, option)
except Error:
pass
raise Error(e)
return default
def getboolean(self, section, option, default=_DEFAULT):
"""getboolean(section, option[, default]) -> bool
If default is not given or set, raises Error in case of an error
"""
try:
return self._config.getboolean(section, option)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getboolean(section, option)
except Error:
pass
raise Error(e)
return default
def getint(self, section, option, default=_DEFAULT):
"""getint(section, option[, default]) -> int
If default is not give or set, raises Error in case of an error
"""
try:
return int(self._config.getfloat(section, option))
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getint(section, option)
except Error:
pass
raise Error(e)
return default
def getfloat(self, section, option, default=_DEFAULT):
"""getfloat(section, option[, default]) -> float
If default is not give or set, raises Error in case of an error
"""
try:
return self._config.getfloat(section, option)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getfloat(section, option)
except Error:
pass
raise Error(e)
return default
def getstringlist(self, section, option, default=_DEFAULT):
"""getstringlist(section, option[, default]) -> list
If default is not given or set, raises Error in case of an error.
Gets a list of strings, using CSV to parse and delimit.
"""
try:
value = self._config.get(section, option)
parser = csv.reader(
[value], lineterminator='\n', quoting=csv.QUOTE_MINIMAL)
try:
vals = next(parser)
except (csv.Error, ValueError) as e:
raise Error(e)
return vals
except Error as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getstringlist(section, option)
except Error:
pass
raise Error(e)
return default
def setstringlist(self, section, option, values):
"""Saves a list of unicode strings using the csv module"""
sw = StringIO()
values = [str(v) for v in values]
writer = csv.writer(sw, lineterminator='\n', quoting=csv.QUOTE_MINIMAL)
writer.writerow(values)
self.set(section, option, sw.getvalue())
def setlist(self, section, option, values, sep=","):
"""Saves a list of str using ',' as a separator and \\ for escaping"""
values = [str(v) for v in values]
joined = join_escape(values, sep)
self.set(section, option, joined)
def getlist(self, section, option, default=_DEFAULT, sep=","):
"""Returns a str list saved with setlist()"""
try:
value = self._config.get(section, option)
return split_escape(value, sep)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getlist(section, option, sep=sep)
except Error:
pass
raise Error(e)
return default
def set(self, section, option, value):
"""Saves the string representation for the passed value
Don't pass unicode, encode first.
"""
if isinstance(value, bytes):
raise TypeError("use setbytes")
# RawConfigParser only allows string values but doesn't
# scream if they are not (and it only fails before the
# first config save..)
if not isinstance(value, str):
value = str(value)
try:
self._config.set(section, option, value)
except NoSectionError:
if self.defaults and self.defaults.has_section(section):
self._config.add_section(section)
self._config.set(section, option, value)
else:
raise
def settext(self, section, option, value):
value = str(value)
# make sure there are no surrogates
value.encode("utf-8")
self.set(section, option, value)
def setbytes(self, section, option, value):
assert isinstance(value, bytes)
value = value.decode("utf-8", "surrogateescape")
self.set(section, option, value)
def write(self, filename):
"""Write config to filename.
Can raise EnvironmentError
"""
assert isinstance(filename, fsnative)
mkdir(os.path.dirname(filename))
# temporary set the new version for saving
if self._version is not None:
self.add_section("__config__")
self.set("__config__", "version", self._version)
try:
with atomic_save(filename, "wb") as fileobj:
temp = StringIO()
self._config.write(temp)
data = temp.getvalue().encode("utf-8", "surrogateescape")
fileobj.write(data)
finally:
if self._loaded_version is not None:
self.set("__config__", "version", self._loaded_version)
def clear(self):
"""Remove all sections."""
for section in self._config.sections():
self._config.remove_section(section)
def is_empty(self):
"""Whether the config has any sections"""
return not self._config.sections()
def read(self, filename):
"""Reads the config from `filename` if the file exists,
otherwise does nothing
Can raise EnvironmentError, Error.
"""
try:
with open(filename, "rb") as fileobj:
fileobj = StringIO(
fileobj.read().decode("utf-8", "surrogateescape"))
self._config.readfp(fileobj, filename)
except (IOError, OSError):
return
# don't upgrade if we just created a new config
if self._version is not None:
self._loaded_version = self.getint("__config__", "version", -1)
for func in self._upgrade_funcs:
self._do_upgrade(func)
def has_option(self, section, option):
"""If the given section exists, and contains the given option"""
return self._config.has_option(section, option) or (
self.defaults and self.defaults.has_option(section, option))
def has_section(self, section):
"""If the given section exists"""
return self._config.has_section(section) or (
self.defaults and self.defaults.has_section(section))
def remove_option(self, section, option):
"""Remove the specified option from the specified section
Can raise Error.
"""
return self._config.remove_option(section, option)
def add_section(self, section):
"""Add a section named section to the instance if it not already
exists."""
if not self._config.has_section(section):
self._config.add_section(section)
def get_merged_config(**options):
"""Get the final merged configuration for supvervisord, as a string.
This is the top-level function exported by this module. It combines
the config file from the main project with default settings and those
specified in the command-line, processes various special section names,
and returns the resulting configuration as a string.
"""
# Find and load the containing project module.
# This can be specified explicity using the --project-dir option.
# Otherwise, we attempt to guess by looking for the manage.py file.
project_dir = options.get("project_dir")
if project_dir is None:
project_dir = guess_project_dir()
# Find the config file to load.
# Default to <project-dir>/supervisord.conf.
config_file = options.get("config_file")
if config_file is None:
config_file = os.path.join(project_dir, CONFIG_FILE)
# Build the default template context variables.
# This is mostly useful information about the project and environment.
ctx = {
"PROJECT_DIR": project_dir,
"PYTHON": os.path.realpath(os.path.abspath(sys.executable)),
"SUPERVISOR_OPTIONS": rerender_options(options),
"settings": settings,
"environ": os.environ,
}
# Initialise the ConfigParser.
# Fortunately for us, ConfigParser has merge-multiple-config-files
# functionality built into it. You just read each file in turn, and
# values from later files overwrite values from former.
cfg = RawConfigParser()
# Start from the default configuration options.
data = render_config(DEFAULT_CONFIG, ctx).decode("ascii")
cfg.readfp(StringIO(data))
# Add in the project-specific config file.
with open(config_file, "r") as f:
data = render_config(f.read(), ctx)
cfg.readfp(StringIO(data.decode("utf-8")))
# Add in the options specified on the command-line.
cfg.readfp(StringIO(get_config_from_options(**options)))
# Add options from [program:__defaults__] to each program section
# if it happens to be missing that option.
PROG_DEFAULTS = "program:__defaults__"
if cfg.has_section(PROG_DEFAULTS):
for option in cfg.options(PROG_DEFAULTS):
default = cfg.get(PROG_DEFAULTS, option)
for section in cfg.sections():
if section.startswith("program:"):
if not cfg.has_option(section, option):
cfg.set(section, option, default)
cfg.remove_section(PROG_DEFAULTS)
# Add options from [program:__overrides__] to each program section
# regardless of whether they already have that option.
PROG_OVERRIDES = "program:__overrides__"
if cfg.has_section(PROG_OVERRIDES):
for option in cfg.options(PROG_OVERRIDES):
override = cfg.get(PROG_OVERRIDES, option)
for section in cfg.sections():
if section.startswith("program:"):
cfg.set(section, option, override)
cfg.remove_section(PROG_OVERRIDES)
# Make sure we've got a port configured for supervisorctl to
# talk to supervisord. It's passworded based on secret key.
# If they have configured a unix socket then use that, otherwise
# use an inet server on localhost at fixed-but-randomish port.
username = hashlib.md5(settings.SECRET_KEY.encode("utf-8")).hexdigest()[:7]
password = hashlib.md5(username.encode("utf-8")).hexdigest()
if cfg.has_section("unix_http_server"):
set_if_missing(cfg, "unix_http_server", "username", username)
set_if_missing(cfg, "unix_http_server", "password", password)
serverurl = "unix://" + cfg.get("unix_http_server", "file")
else:
# This picks a "random" port in the 9000 range to listen on.
# It's derived from the secret key, so it's stable for a given
# project but multiple projects are unlikely to collide.
port = int(hashlib.md5(password.encode("utf-8")).hexdigest()[:3],
16) % 1000
addr = "127.0.0.1:9%03d" % (port, )
set_if_missing(cfg, "inet_http_server", "port", addr)
set_if_missing(cfg, "inet_http_server", "username", username)
set_if_missing(cfg, "inet_http_server", "password", password)
serverurl = "http://" + cfg.get("inet_http_server", "port")
set_if_missing(cfg, "supervisorctl", "serverurl", serverurl)
set_if_missing(cfg, "supervisorctl", "username", username)
set_if_missing(cfg, "supervisorctl", "password", password)
set_if_missing(cfg, "rpcinterface:supervisor",
"supervisor.rpcinterface_factory",
"supervisor.rpcinterface:make_main_rpcinterface")
# Remove any [program:] sections with exclude=true
for section in cfg.sections():
try:
if cfg.getboolean(section, "exclude"):
cfg.remove_section(section)
except NoOptionError:
pass
# Sanity-check to give better error messages.
for section in cfg.sections():
if section.startswith("program:"):
if not cfg.has_option(section, "command"):
msg = "Process name '%s' has no command configured"
raise ValueError(msg % (section.split(":", 1)[-1]))
# Write it out to a StringIO and return the data
s = StringIO()
cfg.write(s)
return s.getvalue()
class Config:
"""A wrapper around RawConfigParser.
Provides a ``defaults`` attribute of the same type which can be used
to set default values.
"""
def __init__(self, version=None, _defaults=True):
"""Use read() to read in an existing config file.
version should be an int starting with 0 that gets incremented if you
want to register a new upgrade function. If None, upgrade is disabled.
"""
self._config = ConfigParser(dict_type=_sorted_dict)
self.defaults = None
if _defaults:
self.defaults = Config(_defaults=False)
self._version = version
self._loaded_version = None
self._upgrade_funcs = []
def _do_upgrade(self, func):
assert self._loaded_version is not None
assert self._version is not None
old_version = self._loaded_version
new_version = self._version
if old_version != new_version:
print_d("Config upgrade: %d->%d (%r)" %
(old_version, new_version, func))
func(self, old_version, new_version)
def get_version(self):
"""Get the version of the loaded config file (for testing only)
Raises Error if no file was loaded or versioning is disabled.
"""
if self._version is None:
raise Error("Versioning disabled")
if self._loaded_version is None:
raise Error("No file loaded")
return self._loaded_version
def register_upgrade_function(self, function):
"""Register an upgrade function that gets called at each read()
if the current config version and the loaded version don't match.
Can also be registered after read was called.
function(config, old_version: int, new_version: int) -> None
"""
if self._version is None:
raise Error("Versioning disabled")
self._upgrade_funcs.append(function)
# after read(), so upgrade now
if self._loaded_version is not None:
self._do_upgrade(function)
return function
def reset(self, section, option):
"""Reset the value to the default state"""
assert self.defaults is not None
try:
self._config.remove_option(section, option)
except NoSectionError:
pass
def options(self, section):
"""Returns a list of options available in the specified section."""
try:
options = self._config.options(section)
except NoSectionError:
if self.defaults:
return self.defaults.options(section)
raise
else:
if self.defaults:
try:
options.extend(self.defaults.options(section))
options = list_unique(options)
except NoSectionError:
pass
return options
def get(self, section, option, default=_DEFAULT):
"""get(section, option[, default]) -> str
If default is not given or set, raises Error in case of an error
"""
try:
return self._config.get(section, option)
except Error:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.get(section, option)
except Error:
pass
raise
return default
def gettext(self, *args, **kwargs):
value = self.get(*args, **kwargs)
# make sure there are no surrogates
value.encode("utf-8")
return value
def getbytes(self, section, option, default=_DEFAULT):
try:
value = self._config.get(section, option)
value = value.encode("utf-8", "surrogateescape")
return value
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getbytes(section, option)
except Error:
pass
raise Error(e)
return default
def getboolean(self, section, option, default=_DEFAULT):
"""getboolean(section, option[, default]) -> bool
If default is not given or set, raises Error in case of an error
"""
try:
return self._config.getboolean(section, option)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getboolean(section, option)
except Error:
pass
raise Error(e)
return default
def getint(self, section, option, default=_DEFAULT):
"""getint(section, option[, default]) -> int
If default is not give or set, raises Error in case of an error
"""
try:
return int(self._config.getfloat(section, option))
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getint(section, option)
except Error:
pass
raise Error(e)
return default
def getfloat(self, section, option, default=_DEFAULT):
"""getfloat(section, option[, default]) -> float
If default is not give or set, raises Error in case of an error
"""
try:
return self._config.getfloat(section, option)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getfloat(section, option)
except Error:
pass
raise Error(e)
return default
def getstringlist(self, section, option, default=_DEFAULT):
"""getstringlist(section, option[, default]) -> list
If default is not given or set, raises Error in case of an error.
Gets a list of strings, using CSV to parse and delimit.
"""
try:
value = self._config.get(section, option)
parser = csv.reader([value],
lineterminator='\n',
quoting=csv.QUOTE_MINIMAL)
try:
vals = next(parser)
except (csv.Error, ValueError) as e:
raise Error(e)
return vals
except Error as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getstringlist(section, option)
except Error:
pass
raise Error(e)
return default
def setstringlist(self, section, option, values):
"""Saves a list of unicode strings using the csv module"""
sw = StringIO()
values = [str(v) for v in values]
writer = csv.writer(sw, lineterminator='\n', quoting=csv.QUOTE_MINIMAL)
writer.writerow(values)
self.set(section, option, sw.getvalue())
def setlist(self, section, option, values, sep=","):
"""Saves a list of str using ',' as a separator and \\ for escaping"""
values = [str(v) for v in values]
joined = join_escape(values, sep)
self.set(section, option, joined)
def getlist(self, section, option, default=_DEFAULT, sep=","):
"""Returns a str list saved with setlist()"""
try:
value = self._config.get(section, option)
return split_escape(value, sep)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getlist(section, option, sep=sep)
except Error:
pass
raise Error(e)
return default
def set(self, section, option, value):
"""Saves the string representation for the passed value
Don't pass unicode, encode first.
"""
if isinstance(value, bytes):
raise TypeError("use setbytes")
# RawConfigParser only allows string values but doesn't
# scream if they are not (and it only fails before the
# first config save..)
if not isinstance(value, str):
value = str(value)
try:
self._config.set(section, option, value)
except NoSectionError:
if self.defaults and self.defaults.has_section(section):
self._config.add_section(section)
self._config.set(section, option, value)
else:
raise
def settext(self, section, option, value):
value = str(value)
# make sure there are no surrogates
value.encode("utf-8")
self.set(section, option, value)
def setbytes(self, section, option, value):
assert isinstance(value, bytes)
value = value.decode("utf-8", "surrogateescape")
self.set(section, option, value)
def write(self, filename):
"""Write config to filename.
Can raise EnvironmentError
"""
assert isinstance(filename, fsnative)
mkdir(os.path.dirname(filename))
# temporary set the new version for saving
if self._version is not None:
self.add_section("__config__")
self.set("__config__", "version", self._version)
try:
with atomic_save(filename, "wb") as fileobj:
temp = StringIO()
self._config.write(temp)
data = temp.getvalue().encode("utf-8", "surrogateescape")
fileobj.write(data)
finally:
if self._loaded_version is not None:
self.set("__config__", "version", self._loaded_version)
def clear(self):
"""Remove all sections."""
for section in self._config.sections():
self._config.remove_section(section)
def is_empty(self):
"""Whether the config has any sections"""
return not self._config.sections()
def read(self, filename):
"""Reads the config from `filename` if the file exists,
otherwise does nothing
Can raise EnvironmentError, Error.
"""
try:
with open(filename, "rb") as fileobj:
fileobj = StringIO(fileobj.read().decode(
"utf-8", "surrogateescape"))
self._config.readfp(fileobj, filename)
except (IOError, OSError):
return
# don't upgrade if we just created a new config
if self._version is not None:
self._loaded_version = self.getint("__config__", "version", -1)
for func in self._upgrade_funcs:
self._do_upgrade(func)
def has_option(self, section, option):
"""If the given section exists, and contains the given option"""
return self._config.has_option(
section, option) or (self.defaults
and self.defaults.has_option(section, option))
def has_section(self, section):
"""If the given section exists"""
return self._config.has_section(section) or (
self.defaults and self.defaults.has_section(section))
def remove_option(self, section, option):
"""Remove the specified option from the specified section
Can raise Error.
"""
return self._config.remove_option(section, option)
def add_section(self, section):
"""Add a section named section to the instance if it not already
exists."""
if not self._config.has_section(section):
self._config.add_section(section)
class ParamStore(object):
def __init__(self, root_dir, file_name):
self._lock = threading.Lock()
with self._lock:
if not os.path.isdir(root_dir):
raise RuntimeError('Directory "' + root_dir +
'" does not exist.')
self._path = os.path.join(root_dir, file_name)
self._dirty = False
# open config file
self._config = RawConfigParser()
self._config.read(self._path)
def flush(self):
if not self._dirty:
return
with self._lock:
self._dirty = False
with open(self._path, 'w') as of:
self._config.write(of)
def get(self, section, option, default=None):
"""Get a parameter value and return a string.
If default is specified and section or option are not defined
in the file, they are created and set to default, which is
then the return value.
"""
with self._lock:
if not self._config.has_option(section, option):
if default is not None:
self._set(section, option, default)
return default
return self._config.get(section, option)
def get_datetime(self, section, option, default=None):
result = self.get(section, option, default)
if result:
return WSDateTime.from_csv(result)
return result
def set(self, section, option, value):
"""Set option in section to string value."""
with self._lock:
self._set(section, option, value)
def _set(self, section, option, value):
if not self._config.has_section(section):
self._config.add_section(section)
elif (self._config.has_option(section, option)
and self._config.get(section, option) == value):
return
self._config.set(section, option, value)
self._dirty = True
def unset(self, section, option):
"""Remove option from section."""
with self._lock:
if not self._config.has_section(section):
return
if self._config.has_option(section, option):
self._config.remove_option(section, option)
self._dirty = True
if not self._config.options(section):
self._config.remove_section(section)
self._dirty = True
suite1 = sys.argv[1]
suite2 = sys.argv[2]
metaval1, cp1 = read_rose_cp('{}/app/um/rose-app.conf'.format(suite1))
metaval2, cp2 = read_rose_cp('{}/app/um/rose-app.conf'.format(suite2))
if metaval1 != metaval2:
print('WARNING: comparing different metavals')
print(metaval1)
print(metaval2)
print()
cpdiff = RawConfigParser()
secs1 = set(cp1.sections())
secs2 = set(cp2.sections())
# Common sections.
sboth = secs1 & secs2
for sec in sboth:
if sec[:2] == '!!':
continue
compare_sections(cp1[sec], cp2[sec], cpdiff)
for sec in cpdiff.sections():
if len(cpdiff[sec]) == 0:
cpdiff.remove_section(sec)
output_fn = 'rose-app.um.{}_{}.diff.conf'.format(suite1, suite2)
print('Writing to {}'.format(output_fn))
with open(output_fn, 'w') as f:
cpdiff.write(f)
count_file = r"../count.ini"
busy_file = r"../is_busy.txt"
needed_count = 6
to_delete = []
config = RawConfigParser(allow_no_value=True)
config.read_file(open(count_file))
for each_section in config.sections():
if each_section == "global" or each_section == "failed":
continue
if each_section == "waifu" or each_section == "husbando":
continue
count = 0
for (each_key, each_val) in config.items(each_section):
if each_val is None:
if each_section not in to_delete:
to_delete.append(each_section)
else:
count += int(each_val)
if count <= needed_count:
to_delete.append(each_section)
for sec in to_delete:
config.remove_section(sec)
while True:
if not os.path.exists(busy_file):
with open(count_file, 'w') as configfile:
config.write(configfile)
sys.exit(0)
time.sleep(1)
class ParamStore(object):
def __init__(self, root_dir, file_name):
self._lock = threading.Lock()
with self._lock:
if not os.path.isdir(root_dir):
raise RuntimeError(
'Directory "' + root_dir + '" does not exist.')
self._path = os.path.join(root_dir, file_name)
self._dirty = False
# open config file
self._config = RawConfigParser()
self._config.read(self._path)
def flush(self):
if not self._dirty:
return
with self._lock:
self._dirty = False
with open(self._path, 'w') as of:
self._config.write(of)
def get(self, section, option, default=None):
"""Get a parameter value and return a string.
If default is specified and section or option are not defined
in the file, they are created and set to default, which is
then the return value.
"""
with self._lock:
if not self._config.has_option(section, option):
if default is not None:
self._set(section, option, default)
return default
return self._config.get(section, option)
def get_datetime(self, section, option, default=None):
result = self.get(section, option, default)
if result:
return WSDateTime.from_csv(result)
return result
def set(self, section, option, value):
"""Set option in section to string value."""
with self._lock:
self._set(section, option, value)
def _set(self, section, option, value):
if not self._config.has_section(section):
self._config.add_section(section)
elif (self._config.has_option(section, option) and
self._config.get(section, option) == value):
return
self._config.set(section, option, value)
self._dirty = True
def unset(self, section, option):
"""Remove option from section."""
with self._lock:
if not self._config.has_section(section):
return
if self._config.has_option(section, option):
self._config.remove_option(section, option)
self._dirty = True
if not self._config.options(section):
self._config.remove_section(section)
self._dirty = True
def edit_config_info(config):
if os.name == 'posix':
default_editor = 'vi'
elif os.name == 'nt':
default_editor = 'notepad'
else:
raise Exception("unsupported OS type %s" % os.name)
EDITOR = os.environ.get('EDITOR', default_editor)
opts = [
'title',
'tabstop',
'svg_width',
'table_width',
'desc',
]
# ----
# Create a ConfigParser that contains relevant sections of the config.
# ----
name = config['name']
tmp_config = RawConfigParser()
tmp_config.add_section(name)
for opt in opts:
tmp_config.set(name, opt, str(config[opt]))
# ----
# We need the temp file to edit to have the correct, OS specific
# line endings. So we create a StringIO buffer first to get the
# file content from the ConfigParser, then change '\n' into
# os.linesep when creating the temp file.
# ----
buf = StringIO()
tmp_config.write(buf)
tf = tempfile.NamedTemporaryFile(suffix=".tmp.conf", delete=False)
tf_name = tf.name
tf.write(buf.getvalue().replace('\n', os.linesep).encode('utf-8'))
tf.close()
# ----
# Call the editor.
# ----
subprocess.call([EDITOR, tf_name])
# ----
# Remove all sections from the ConfigParser object, read back
# the temp file and extract the one expected section.
# ----
for s in tmp_config.sections():
tmp_config.remove_section(s)
tf = open(tf_name, 'r')
tmp_config.readfp(tf)
tf.close()
os.remove(tf_name)
if len(tmp_config.sections()) != 1:
raise Exception("config must have exactly one section")
name = tmp_config.sections()[0]
config['name'] = name
for opt in opts:
if tmp_config.has_option(name, opt):
config[opt] = str(tmp_config.get(name, opt))
def get_merged_config(self, **options):
"""Get the final merged configuration for supvervisord, as a string.
This is the top-level function exported by this module. It combines
the config file from the main project with default settings and those
specified in the command-line, processes various special section names,
and returns the resulting configuration as a string.
"""
config_file = "supervisord.conf"
# Initialise the ConfigParser.
# Fortunately for us, ConfigParser has merge-multiple-config-files
# functionality built into it. You just read each file in turn, and
# values from later files overwrite values from former.
cfg = RawConfigParser()
# Start from the default configuration options.
cfg.readfp(StringIO(self.DEFAULT_CONFIG))
"""
# Add in the project-specific config file.
with open(config_file,"r") as f:
data = f.read()
cfg.readfp(StringIO(data))
"""
# Add in the options from the self.services
cfg.readfp(StringIO(self.get_config_from_services()))
# Add in the options specified on the command-line.
cfg.readfp(StringIO(self.get_config_from_options(**options)))
# Add options from [program:__defaults__] to each program section
# if it happens to be missing that option.
PROG_DEFAULTS = "program:__defaults__"
if cfg.has_section(PROG_DEFAULTS):
for option in cfg.options(PROG_DEFAULTS):
default = cfg.get(PROG_DEFAULTS,option)
for section in cfg.sections():
if section.startswith("program:"):
if not cfg.has_option(section,option):
cfg.set(section,option,default)
cfg.remove_section(PROG_DEFAULTS)
# Add options from [program:__overrides__] to each program section
# regardless of whether they already have that option.
PROG_OVERRIDES = "program:__overrides__"
if cfg.has_section(PROG_OVERRIDES):
for option in cfg.options(PROG_OVERRIDES):
override = cfg.get(PROG_OVERRIDES,option)
for section in cfg.sections():
if section.startswith("program:"):
cfg.set(section,option,override)
cfg.remove_section(PROG_OVERRIDES)
# Make sure we've got a port configured for supervisorctl to
# talk to supervisord. It's passworded based on secret key.
# If they have configured a unix socket then use that, otherwise
# use an inet server on localhost at fixed-but-randomish port.
username = hashlib.md5("angelo".encode('utf-8')).hexdigest()[:7]
password = hashlib.md5(username.encode('utf-8')).hexdigest()
if cfg.has_section("unix_http_server"):
self.set_if_missing(cfg,"unix_http_server","username",username)
self.set_if_missing(cfg,"unix_http_server","password",password)
serverurl = "unix://" + cfg.get("unix_http_server","file")
else:
# This picks a "random" port in the 9000 range to listen on.
# It's derived from the secret key, so it's stable for a given
# project but multiple projects are unlikely to collide.
port = int(hashlib.md5(password.encode('utf-8')).hexdigest()[:3],16) % 1000
addr = "127.0.0.1:9%03d" % (port,)
self.set_if_missing(cfg,"inet_http_server","port",addr)
self.set_if_missing(cfg,"inet_http_server","username",username)
self.set_if_missing(cfg,"inet_http_server","password",password)
serverurl = "http://" + cfg.get("inet_http_server","port")
self.set_if_missing(cfg,"supervisorctl","serverurl",serverurl)
self.set_if_missing(cfg,"supervisorctl","username",username)
self.set_if_missing(cfg,"supervisorctl","password",password)
self.set_if_missing(cfg,"rpcinterface:supervisor",
"supervisor.rpcinterface_factory",
"supervisor.rpcinterface:make_main_rpcinterface")
# Remove any [program:] sections with exclude=true
for section in cfg.sections():
try:
if cfg.getboolean(section,"exclude"):
cfg.remove_section(section)
except NoOptionError:
pass
# Sanity-check to give better error messages.
for section in cfg.sections():
if section.startswith("program:"):
if not cfg.has_option(section,"command"):
msg = "Process name '%s' has no command configured"
raise ValueError(msg % (section.split(":",1)[-1]))
# Write it out to a StringIO and return the data
s = StringIO()
cfg.write(s)
return s.getvalue()
class BaseConfigStore(object):
# the actual config store functionality
def __init__(self, name, *arg, **kw):
super(BaseConfigStore, self).__init__(*arg, **kw)
self.dirty = False
self.config = RawConfigParser()
config_dir = os.environ.get('PHOTINI_CONFIG')
if config_dir:
config_dir = os.path.expanduser(config_dir)
elif hasattr(appdirs, 'user_config_dir'):
config_dir = appdirs.user_config_dir('photini')
else:
config_dir = appdirs.user_data_dir('photini')
if not os.path.isdir(config_dir):
os.makedirs(config_dir, mode=stat.S_IRWXU)
self.file_name = os.path.join(config_dir, name + '.ini')
if os.path.isfile(self.file_name):
kwds = {'encoding': 'utf-8'}
with open(self.file_name, 'r', **kwds) as fp:
self.config.read_file(fp)
self.has_section = self.config.has_section
def get(self, section, option, default=None):
if self.config.has_option(section, option):
return self.config.get(section, option)
if default is not None:
self.set(section, option, default)
return default
def set(self, section, option, value):
if not self.config.has_section(section):
self.config.add_section(section)
elif (self.config.has_option(section, option)
and self.config.get(section, option) == value):
return
self.config.set(section, option, value)
self.dirty = True
def delete(self, section, option):
if not self.config.has_section(section):
return
if self.config.has_option(section, option):
self.config.remove_option(section, option)
if not self.config.options(section):
self.config.remove_section(section)
self.dirty = True
def remove_section(self, section):
if not self.config.has_section(section):
return
for option in self.config.options(section):
self.config.remove_option(section, option)
self.config.remove_section(section)
self.dirty = True
def save(self):
if not self.dirty:
return
kwds = {'encoding': 'utf-8'}
with open(self.file_name, 'w', **kwds) as fp:
self.config.write(fp)
os.chmod(self.file_name, stat.S_IRUSR | stat.S_IWUSR)
self.dirty = False
class ConfigEditor:
def __init__(self):
self.config_parser = RawConfigParser()
self.config_file = None
def open(self, config_file):
"""
Open and read a config file
:param config_file: config file path
"""
logger.debug('Opening {}'.format(config_file))
self.config_file = config_file
if os.access(config_file, os.R_OK):
logger.debug('Parsing config file')
self.config_parser.read(config_file)
def read(self, section, option, fallback=None):
"""
Returns a config option value from config file
:param section: section where the option is stored
:param option: option name
:param fallback: (optional) fallback value
:return: a config option value
:rtype: string
"""
if self.config_file == None:
raise InvalidOperation('read')
if fallback is None:
return self.config_parser.get(section, option)
else:
return self.config_parser.get(section, option, fallback=fallback)
def readboolean(self, section, option, fallback=False):
"""
Returns a boolean config option value from config file
:param section: section where the option is stored
:param option: option name
:param fallback: (optional) fallback value
:return: a config option value
:rtype: boolean
"""
if self.config_file == None:
raise InvalidOperation('readboolean')
return self.config_parser.getboolean(section,
option,
fallback=fallback)
def write(self, section, option, value):
"""
Write a config option value in config object
:param section: section where the option is stored
:param option: option name
:param value: option value
"""
if self.config_file == None:
raise InvalidOperation('write')
if section != 'DEFAULT' and not self.config_parser.has_section(
section):
logger.debug('Adding new section {}'.format(section))
self.config_parser.add_section(section)
logger.debug('Adding {}.{} with value {}'.format(
section, option, value))
self.config_parser.set(section, option, value)
def remove(self, section, option):
"""
Remove a config option in config object
:param section: section where the option is stored
:param option: option name
:return: True if option is removed, False if not exist
:rtype: boolean
"""
if self.config_file == None:
raise InvalidOperation('remove')
logger.debug('Removing {}.{}'.format(section, option))
option_removed = self.config_parser.remove_option(section, option)
if section != 'DEFAULT' and option_removed:
if self.config_parser.items(section) == self.config_parser.items(
'DEFAULT'):
logger.debug('Removing empty section {}'.format(section))
self.config_parser.remove_section(section)
return option_removed
def remove_project(self, project):
"""
Remove a project (config section in config object)
:param project: section name
:return: True if section is removed, False if not exist
:rtype: boolean
"""
if self.config_file == None:
raise InvalidOperation('remove')
logger.debug('Removing {}'.format(project))
return self.config_parser.remove_section(project)
def list(self):
"""
List config sections
:return: list of projects (sections in config)
:rtype: list
"""
if self.config_file == None:
raise InvalidOperation('list')
return self.config_parser.sections()
def list_enabled_projects(self):
"""
Get the list of enabled projects
:return: list of enabled projects
:rtype: list
"""
if self.config_file == None:
raise InvalidOperation('list_enabled_projects')
try:
return self.config_parser.get('DEFAULT', 'sync_projects').split()
except NoOptionError:
return []
def enable_project(self, project):
"""
Enable a project adding it to sync_projects
:param project: project name
"""
if self.config_file == None:
raise InvalidOperation('enable_project')
logger.debug('Enabling project {}'.format(project))
enabled_projects = self.list_enabled_projects()
enabled_projects.append(project)
enabled_projects.sort()
self.config_parser.set('DEFAULT', 'sync_projects',
' '.join(enabled_projects))
def disable_project(self, project):
"""
Disable a project removing it from sync_projects
:param project: project name
:return: True if project is disabled, False if not
:rtype: boolean
"""
if self.config_file == None:
raise InvalidOperation('disable_project')
logger.debug('Disabling project {}'.format(project))
enabled_projects = self.list_enabled_projects()
try:
enabled_projects.remove(project)
self.config_parser.set('DEFAULT', 'sync_projects',
' '.join(enabled_projects))
return True
except ValueError:
logger.debug(
'Nothing to do, {} is not in enabled projects'.format(project))
return False
def has_project(self, project):
"""
Check if a project (a section in config) is present
:param project: section name
:return: True if section exists, False if not
:rtype: boolean
"""
if self.config_file == None:
raise InvalidOperation('has_project')
return self.config_parser.has_section(project)
def has_project_enabled(self, project):
"""
Check if a project is enabled
:param project: project name
:return: True if project is enabled, False if not
:rtype: boolean
"""
if self.config_file == None:
raise InvalidOperation('has_project_enabled')
return True if project in self.list_enabled_projects() else False
def save(self):
"""
Save the config object in config file
"""
if self.config_file == None:
raise InvalidOperation('save')
logger.debug('Saving config in config file')
with open(self.config_file, 'w') as configfile:
self.config_parser.write(configfile)
self.config_file = None
def clean(self):
"""
Cleans the config editor
"""
logger.debug('Cleaning config editor')
self.config_parser = RawConfigParser()
self.config_file = None
class MrxsFile(object):
def __init__(self, filename):
# Split filename
dirname, ext = os.path.splitext(filename)
if ext != '.mrxs':
raise UnrecognizedFile
# Parse slidedat
self._slidedatfile = os.path.join(dirname, 'Slidedat.ini')
self._slidedatfile_upperdir = os.path.join(dirname, '../Slidedat.ini')
self._dat = RawConfigParser()
self._dat.optionxform = str
self._anonymize_meta(dirname)
try:
with open(self._slidedatfile, 'r', encoding="utf-8-sig") as fh:
self._have_bom = (fh.read(len(UTF8_BOM)) == UTF8_BOM)
if not self._have_bom:
fh.seek(0)
self._dat.read_file(fh)
except IOError:
raise UnrecognizedFile
# Get file paths
self._indexfile = os.path.join(
dirname, self._dat.get(MRXS_HIERARCHICAL, 'INDEXFILE'))
self._datafiles = [
os.path.join(dirname, self._dat.get('DATAFILE', 'FILE_%d' % i))
for i in range(self._dat.getint('DATAFILE', 'FILE_COUNT'))
]
# Build levels
self._make_levels()
def _anonymize_meta(self, dirname):
for filename in [
os.path.join(dirname, 'Slidedat.ini'),
os.path.join(dirname, '../Slidedat.ini')
]:
filedata = ''
if os.path.exists(filename):
with open(filename, 'r') as fh:
filedata = fh.read()
filedata = re.sub(r"SLIDE_NAME.*", "SLIDE_NAME = None",
filedata)
filedata = re.sub(r"PROJECT_NAME.*", "PROJECT_NAME = None",
filedata)
filedata = re.sub(r"SLIDE_CREATIONDATETIME.*",
"SLIDE_CREATIONDATETIME = None",
filedata)
fh.close
with open(filename, 'w') as fh:
fh.write(filedata)
fh.close()
def _make_levels(self):
self._levels = {}
self._level_list = []
layer_count = self._dat.getint(MRXS_HIERARCHICAL, 'NONHIER_COUNT')
for layer_id in range(layer_count):
level_count = self._dat.getint(MRXS_HIERARCHICAL,
'NONHIER_%d_COUNT' % layer_id)
for level_id in range(level_count):
level = MrxsNonHierLevel(self._dat, layer_id, level_id,
len(self._level_list))
self._levels[(level.layer_name, level.name)] = level
self._level_list.append(level)
@classmethod
def _read_int32(cls, f):
buf = f.read(4)
if len(buf) != 4:
raise IOError('Short read')
return struct.unpack('<i', buf)[0]
@classmethod
def _assert_int32(cls, f, value):
v = cls._read_int32(f)
if v != value:
raise ValueError('%d != %d' % (v, value))
def _get_data_location(self, record):
with open(self._indexfile, 'rb') as fh:
fh.seek(MRXS_NONHIER_ROOT_OFFSET)
# seek to record
table_base = self._read_int32(fh)
fh.seek(table_base + record * 4)
# seek to list head
list_head = self._read_int32(fh)
fh.seek(list_head)
# seek to data page
self._assert_int32(fh, 0)
page = self._read_int32(fh)
fh.seek(page)
# check pagesize
self._assert_int32(fh, 1)
# read rest of prologue
self._read_int32(fh)
self._assert_int32(fh, 0)
self._assert_int32(fh, 0)
# read values
position = self._read_int32(fh)
size = self._read_int32(fh)
fileno = self._read_int32(fh)
return (self._datafiles[fileno], position, size)
def _zero_record(self, record):
path, offset, length = self._get_data_location(record)
with open(path, 'r+b') as fh:
fh.seek(0, 2)
do_truncate = (fh.tell() == offset + length)
if DEBUG:
if do_truncate:
print('Truncating', path, 'to', offset)
else:
print('Zeroing', path, 'at', offset, 'for', length)
fh.seek(offset)
buf = fh.read(len(JPEG_SOI))
# print(buf)
# exit()
if buf != JPEG_SOI:
raise IOError('Unexpected data in nonhier image')
if do_truncate:
fh.truncate(offset)
else:
fh.seek(offset)
fh.write('\0' * length)
def _delete_index_record(self, record):
if DEBUG:
print('Deleting record', record)
with open(self._indexfile, 'r+b') as fh:
entries_to_move = len(self._level_list) - record - 1
if entries_to_move == 0:
return
# get base of table
fh.seek(MRXS_NONHIER_ROOT_OFFSET)
table_base = self._read_int32(fh)
# read tail of table
fh.seek(table_base + (record + 1) * 4)
buf = fh.read(entries_to_move * 4)
if len(buf) != entries_to_move * 4:
raise IOError('Short read')
# overwrite the target record
fh.seek(table_base + record * 4)
fh.write(buf)
def _hier_keys_for_level(self, level):
ret = []
for k, _ in self._dat.items(MRXS_HIERARCHICAL):
if k == level.key_prefix or k.startswith(level.key_prefix + '_'):
ret.append(k)
return ret
def _rename_section(self, old, new):
if self._dat.has_section(old):
if DEBUG:
print('[%s] -> [%s]' % (old, new))
self._dat.add_section(new)
for k, v in self._dat.items(old):
self._dat.set(new, k, v)
self._dat.remove_section(old)
elif DEBUG:
print('[%s] does not exist' % old)
def _delete_section(self, section):
if DEBUG:
print('Deleting [%s]' % section)
self._dat.remove_section(section)
def _set_key(self, section, key, value):
if DEBUG:
prev = self._dat.get(section, key)
print('[%s] %s: %s -> %s' % (section, key, prev, value))
self._dat.set(section, key, value)
def _rename_key(self, section, old, new):
if DEBUG:
print('[%s] %s -> %s' % (section, old, new))
v = self._dat.get(section, old)
self._dat.remove_option(section, old)
self._dat.set(section, new, v)
def _delete_key(self, section, key):
if DEBUG:
print('Deleting [%s] %s' % (section, key))
self._dat.remove_option(section, key)
def _write(self):
buf = StringIO()
self._dat.write(buf)
with open(self._slidedatfile, 'wb') as fh:
if self._have_bom:
fh.write(UTF8_BOM.encode())
fh.write(buf.getvalue().replace('\n', '\r\n').encode())
def delete_level(self, layer_name, level_name):
level = self._levels[(layer_name, level_name)]
record = level.record
# Zero image data
self._zero_record(record)
# Delete pointer from nonhier table in index
self._delete_index_record(record)
# Remove slidedat keys
for k in self._hier_keys_for_level(level):
self._delete_key(MRXS_HIERARCHICAL, k)
# Remove slidedat section
self._delete_section(level.section)
# Rename section and keys for subsequent levels in the layer
prev_level = level
for cur_level in self._level_list[record + 1:]:
if cur_level.layer_id != prev_level.layer_id:
break
for k in self._hier_keys_for_level(cur_level):
new_k = k.replace(cur_level.key_prefix, prev_level.key_prefix,
1)
self._rename_key(MRXS_HIERARCHICAL, k, new_k)
self._set_key(MRXS_HIERARCHICAL, prev_level.section_key,
prev_level.section)
self._rename_section(cur_level.section, prev_level.section)
prev_level = cur_level
# Update level count within layer
count_k = 'NONHIER_%d_COUNT' % level.layer_id
count_v = self._dat.getint(MRXS_HIERARCHICAL, count_k)
self._set_key(MRXS_HIERARCHICAL, count_k, count_v - 1)
# Write slidedat
self._write()
# Refresh metadata
self._make_levels()
class MrxsFile(object):
def __init__(self, filename):
# Split filename
dirname, ext = os.path.splitext(filename)
if ext != ".mrxs":
raise UnrecognizedFile
# Parse slidedat
self._slidedatfile = os.path.join(dirname, "Slidedat.ini")
self._dat = RawConfigParser()
self._dat.optionxform = str
try:
with open(self._slidedatfile, "rb") as fh:
self._have_bom = fh.read(len(UTF8_BOM)) == UTF8_BOM
if not self._have_bom:
fh.seek(0)
self._dat.readfp(fh)
except IOError:
raise UnrecognizedFile
# Get file paths
self._indexfile = os.path.join(
dirname, self._dat.get(MRXS_HIERARCHICAL, "INDEXFILE"))
self._datafiles = [
os.path.join(dirname, self._dat.get("DATAFILE", "FILE_%d" % i))
for i in range(self._dat.getint("DATAFILE", "FILE_COUNT"))
]
# Build levels
self._make_levels()
def _make_levels(self):
self._levels = {}
self._level_list = []
layer_count = self._dat.getint(MRXS_HIERARCHICAL, "NONHIER_COUNT")
for layer_id in range(layer_count):
level_count = self._dat.getint(MRXS_HIERARCHICAL,
"NONHIER_%d_COUNT" % layer_id)
for level_id in range(level_count):
level = MrxsNonHierLevel(self._dat, layer_id, level_id,
len(self._level_list))
self._levels[(level.layer_name, level.name)] = level
self._level_list.append(level)
@classmethod
def _read_int32(cls, f):
buf = f.read(4)
if len(buf) != 4:
raise IOError("Short read")
return struct.unpack("<i", buf)[0]
@classmethod
def _assert_int32(cls, f, value):
v = cls._read_int32(f)
if v != value:
raise ValueError("%d != %d" % (v, value))
def _get_data_location(self, record):
with open(self._indexfile, "rb") as fh:
fh.seek(MRXS_NONHIER_ROOT_OFFSET)
# seek to record
table_base = self._read_int32(fh)
fh.seek(table_base + record * 4)
# seek to list head
list_head = self._read_int32(fh)
fh.seek(list_head)
# seek to data page
self._assert_int32(fh, 0)
page = self._read_int32(fh)
fh.seek(page)
# check pagesize
self._assert_int32(fh, 1)
# read rest of prologue
self._read_int32(fh)
self._assert_int32(fh, 0)
self._assert_int32(fh, 0)
# read values
position = self._read_int32(fh)
size = self._read_int32(fh)
fileno = self._read_int32(fh)
return (self._datafiles[fileno], position, size)
def _zero_record(self, record):
path, offset, length = self._get_data_location(record)
with open(path, "r+b") as fh:
fh.seek(0, 2)
do_truncate = fh.tell() == offset + length
if DEBUG:
if do_truncate:
print("Truncating", path, "to", offset)
else:
print("Zeroing", path, "at", offset, "for", length)
fh.seek(offset)
buf = fh.read(len(JPEG_SOI))
if buf != JPEG_SOI:
raise IOError("Unexpected data in nonhier image")
if do_truncate:
fh.truncate(offset)
else:
fh.seek(offset)
fh.write("\0" * length)
def _delete_index_record(self, record):
if DEBUG:
print("Deleting record", record)
with open(self._indexfile, "r+b") as fh:
entries_to_move = len(self._level_list) - record - 1
if entries_to_move == 0:
return
# get base of table
fh.seek(MRXS_NONHIER_ROOT_OFFSET)
table_base = self._read_int32(fh)
# read tail of table
fh.seek(table_base + (record + 1) * 4)
buf = fh.read(entries_to_move * 4)
if len(buf) != entries_to_move * 4:
raise IOError("Short read")
# overwrite the target record
fh.seek(table_base + record * 4)
fh.write(buf)
def _hier_keys_for_level(self, level):
ret = []
for k, _ in self._dat.items(MRXS_HIERARCHICAL):
if k == level.key_prefix or k.startswith(level.key_prefix + "_"):
ret.append(k)
return ret
def _rename_section(self, old, new):
if self._dat.has_section(old):
if DEBUG:
print("[%s] -> [%s]" % (old, new))
self._dat.add_section(new)
for k, v in self._dat.items(old):
self._dat.set(new, k, v)
self._dat.remove_section(old)
elif DEBUG:
print("[%s] does not exist" % old)
def _delete_section(self, section):
if DEBUG:
print("Deleting [%s]" % section)
self._dat.remove_section(section)
def _set_key(self, section, key, value):
if DEBUG:
prev = self._dat.get(section, key)
print("[%s] %s: %s -> %s" % (section, key, prev, value))
self._dat.set(section, key, value)
def _rename_key(self, section, old, new):
if DEBUG:
print("[%s] %s -> %s" % (section, old, new))
v = self._dat.get(section, old)
self._dat.remove_option(section, old)
self._dat.set(section, new, v)
def _delete_key(self, section, key):
if DEBUG:
print("Deleting [%s] %s" % (section, key))
self._dat.remove_option(section, key)
def _write(self):
buf = io.StringIO()
self._dat.write(buf)
with open(self._slidedatfile, "wb") as fh:
if self._have_bom:
fh.write(bytearray(UTF8_BOM))
fh.write(bytearray(buf.getvalue().replace("\n", "\r\n")))
def delete_level(self, layer_name, level_name):
level = self._levels[(layer_name, level_name)]
record = level.record
# Zero image data
self._zero_record(record)
# Delete pointer from nonhier table in index
self._delete_index_record(record)
# Remove slidedat keys
for k in self._hier_keys_for_level(level):
self._delete_key(MRXS_HIERARCHICAL, k)
# Remove slidedat section
self._delete_section(level.section)
# Rename section and keys for subsequent levels in the layer
prev_level = level
for cur_level in self._level_list[record + 1:]:
if cur_level.layer_id != prev_level.layer_id:
break
for k in self._hier_keys_for_level(cur_level):
new_k = k.replace(cur_level.key_prefix, prev_level.key_prefix,
1)
self._rename_key(MRXS_HIERARCHICAL, k, new_k)
self._set_key(MRXS_HIERARCHICAL, prev_level.section_key,
prev_level.section)
self._rename_section(cur_level.section, prev_level.section)
prev_level = cur_level
# Update level count within layer
count_k = "NONHIER_%d_COUNT" % level.layer_id
count_v = self._dat.getint(MRXS_HIERARCHICAL, count_k)
self._set_key(MRXS_HIERARCHICAL, count_k, count_v - 1)
# Write slidedat
self._write()
# Refresh metadata
self._make_levels()
