def create_new_conf_from_modifications(self):
"""
Return a new RawConfigParser instance that has been created from the
non-default modifications returned by the `modifications` property
above.
"""
# This is a bit hacky as the underlying config classes don't really
# support the notion of "only write out sections/options that have
# changed since we loaded the defaults".
if not self.repo_path:
raise RepositoryNotSet()
mods = self.modifications
if not mods:
raise NoModificationsMade()
filename = self.writable_repo_override_conf_filename
conf = RawConfigParser()
conf.read(filename)
for (section, options) in mods.items():
conf.add_section(section)
for (option, value) in options.items():
conf.set(section, option, value)
return conf
def set(self, section, option, new_value):
with self.lock:
if self.callback and self.has_section(section) and self.has_option(section, option):
old_value = self.get(section, option)
if not self.callback(section, option, new_value, old_value):
raise OperationNotPossibleAtRuntimeException
RawConfigParser.set(self, section, option, new_value)
def save():
"""Saves FileDirectives into ConfigFile."""
section = "*"
module = sys.modules[__name__]
parser = RawConfigParser()
parser.optionxform = str # Force case-sensitivity on names
parser.add_section(section)
try:
f, fname = open(ConfigFile, "wb"), util.longpath(ConfigFile)
f.write("# %s configuration autowritten on %s.\n" %
(fname, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
for name in FileDirectives:
try:
value = getattr(module, name)
parser.set(section, name, json.dumps(value))
except Exception:
pass
for name in OptionalFileDirectives:
try:
value = getattr(module, name, None)
if OptionalFileDirectiveDefaults.get(name) != value:
parser.set(section, name, json.dumps(value))
except Exception:
pass
parser.write(f)
f.close()
except Exception:
pass # Fail silently
def install_mercurial_hook():
"""
Installs the mercurial precommit hook by adding a hook to the hgrc
file in the .hg directory of the repository.
"""
repo_dir = get_repo_dir()
config_file = os.path.join(repo_dir, '.hg', 'hgrc')
config_parser = RawConfigParser()
config_parser.read(config_file)
precommit_abs_file = os.path.join(repo_dir, 'scripts',
'codestyleprecommit.py')
section = 'hooks'
key = 'pretxncommit.precommit'
value = 'python:%s:mercurial_hook' % precommit_abs_file
if not config_parser.has_section(section):
config_parser.add_section(section)
config_parser.set(section, key, value)
with open(config_file, 'w') as config:
config_parser.write(config)
def _save_config_file(self, config):
"""Write config to disk.
"""
# Generate a sanitized version of our running configuration
sane_config = RawConfigParser()
for section_name, section in config._config.items():
sane_config.add_section(section_name)
for option_name, value in section.items():
if section_name == 'general':
if option_name in ['config_file']:
continue
if value is not None:
sane_config.set(section_name, option_name, str(value))
config_dir = self.config_file.parent
if not config_dir.exists():
config_dir.mkdir(parents=True, exist_ok=True)
# Write the config file atomically.
self.acquire_lock()
with NamedTemporaryFile(mode='w', dir=str(config_dir),
delete=False) as tmpfile:
sane_config.write(tmpfile)
if os.path.getsize(tmpfile.name) > 0:
os.replace(tmpfile.name, str(self.config_file))
else:
self.log.warning(
'Config file saving failed, not replacing %s with %s.',
str(self.config_file), tmpfile.name)
self.release_lock()
def configure(self, settings=None):
"""Parses arg `settings` or settings file if None passed and sets class
attributes accordingly
"""
config = ConfigParser()
# using str instead of optionxform not to .lower() options
config.optionxform = str
if settings is not None:
for section in settings:
config.add_section(section)
for key, value in settings[section].items():
config.set(section, key, str(value))
else:
settings_path = os.path.join(get_project_root(),
SETTINGS_FILE_NAME)
config.read(settings_path)
for section in config.sections():
# legacy format. use following after dropping py2 support:
# for key, value in config[section].items():
# setattr(getattr(self, section), key, value)
for key in config.options(section):
setattr(getattr(self, section), key, config.get(section, key))
self._configure_logging()
self._configure_thirdparty_logging()
self.configured = True
def test_init_yes_simple():
cfg = RawConfigParser()
cfg.add_section('group fooers')
cfg.set('group fooers', 'members', 'jdoe')
cfg.set('group fooers', 'init', 'foo/bar')
eq(access.haveAccess(config=cfg, user='******', mode='init', path='foo/bar'),
('repositories', 'foo/bar', 'init'))
def add_config(self, options):
"""Creates analysis.conf file from current analysis options.
@param options: current configuration options, dict format.
@return: operation status.
"""
global ERROR_MESSAGE
if type(options) != dict:
return False
config = RawConfigParser()
config.add_section("analysis")
try:
for key, value in options.items():
# Options can be UTF encoded.
if isinstance(value, basestring):
try:
value = value.encode("utf-8")
except UnicodeEncodeError:
pass
config.set("analysis", key, value)
config_path = os.path.join(ANALYZER_FOLDER, "analysis.conf")
with open(config_path, "wb") as config_file:
config.write(config_file)
except Exception as e:
ERROR_MESSAGE = str(e)
return False
return True
def expand_input(ifile, parameters):
""" Expands an input file by expanding some of its arguments.
"""
d = load_input(ifile, parameters, d={}, upper=False, raw=True)
base = os.path.splitext(ifile)[0]
config = RawConfigParser()
config.read(ifile)
l = []
for i, d0 in enumerate(expand_dict(d)):
fname = '%s_%.4d.ini' % (base, i)
l.append(fname)
with open(fname, 'w') as fp:
for k, v in d0.iteritems():
for sect in config.sections():
try:
old = config.get(sect, k)
config.set(sect, k, v)
except NoOptionError:
pass
config.write(fp)
return l
class NotebookInfo(object):
def __init__(self, folder):
if not isinstance(folder, unicode):
raise ValueError("folder argument must be unicode")
self.folder = folder
self.__load()
def __load(self):
self.__parser = RawConfigParser()
# Fallback with the modtime of the folder as "last_modified"
st = os.stat(self.folder)
self.__parser.add_section('Notebook')
self.__parser.set('Notebook', 'last_modified', str(st.st_mtime))
index_file = os.path.join(self.folder, "index.rnb")
try:
f = open(index_file, "r")
except IOError, e:
# If not readable, just ignore
return
try:
self.__parser.readfp(f)
except ParsingError:
# If not readable, just ignore
return
finally:
f.close()
def set(self, name, value):
if self.custom_conf_file is None:
raise GconfNotConfigurable()
if not self._is_configurable(name):
raise GconfNotConfigurable()
if not self._is_valid_value(name, value):
raise GconfInvalidValue()
curr_val = self.gconf.get(name, None)
if curr_val == value:
return True
cnf = RawConfigParser()
with open(self.custom_conf_file) as f:
cnf.readfp(f)
if not cnf.has_section("vars"):
cnf.add_section("vars")
cnf.set("vars", name, value)
with open(self.tmp_conf_file, "w") as fw:
cnf.write(fw)
os.rename(self.tmp_conf_file, self.custom_conf_file)
self.reload()
return True
def _path_fixer(filepath, root=None):
"""Change all the relative paths in `filepath` to absolute ones.
:param filepath: File to be changed
:param root: Root path with which the relative paths are prefixed. If None
(default), the directory with this file is the root.
"""
if root is None:
root = op.join(op.abspath(op.dirname(__file__)))
if filepath.endswith((".yaml", ".yml")):
with open(filepath, "r") as fileobj:
data = yaml.load(fileobj, Loader=yaml.CLoader)
for specs in data.itervalues():
specs['path'] = op.join(root, specs['path'])
with open(filepath, "w") as fileobj:
yaml.dump(data,
fileobj,
Dumper=yaml.CDumper,
default_flow_style=False)
elif filepath.endswith(".conf"):
parser = RawConfigParser()
parser.read(filepath)
for section in parser.sections():
path = parser.get(section, "specfile")
parser.remove_option(section, "specfile")
parser.set(section, "specfile", op.join(root, path))
with open(filepath, "w") as fileobj:
parser.write(fileobj)
def tearDownClass(cls):
try:
# modify the testdata back
with open(TEST_DATA_DICT, "r") as fileobj:
test_data = yaml.load(fileobj, Loader=yaml.CLoader)
test_data['iris']['path'] = op.join("testdata", "iris.csv")
test_data['person_activity']['path'] = op.join(
"testdata", "person_activity.tsv")
del test_data['multi_iris']
with open(TEST_DATA_DICT, "w") as fileobj:
test_data = yaml.dump(test_data,
fileobj,
Dumper=yaml.CDumper,
default_flow_style=False)
# Change the config files back
parser = RawConfigParser()
parser.read(cls.test_conf_file)
parser.remove_option("pysemantic", "specfile")
parser.set("pysemantic", "specfile",
op.join("testdata", "test_dictionary.yaml"))
with open(TEST_CONFIG_FILE_PATH, 'w') as fileobj:
parser.write(fileobj)
finally:
os.unlink(cls.test_conf_file)
os.unlink(cls.copied_iris_path)
def save(self, filename=None):
if filename is None:
filename = self.write_path
self.before_save()
config_parser = RawConfigParser()
#config_parser.readfp(
for section_name, section in self.sections.items():
config_parser.add_section(section_name)
for item in section.items:
type_process = {
str: str,
bool: str,
int: str,
float: str,
"pickle": do_pickling
}[section.item_types[item]]
# look it up now. If this is a lazily evaluated item, find its
# value before we close
# TODO: is this what we really want to do?
value = section[item]
config_parser.set(section_name, item, type_process(value))
directory = os.path.dirname(filename)
if not os.path.exists(directory):
os.makedirs(directory)
config_parser.write(open(filename, "w"))
def save(self, filename=None):
if filename is None:
filename = self.write_path
self.before_save()
config_parser = RawConfigParser()
#config_parser.readfp(
for section_name, section in self.sections.items():
config_parser.add_section(section_name)
for item in section.items:
type_process = {
str: str,
bool: str,
int: str,
float: str,
"pickle": do_pickling
}[section.item_types[item]]
# look it up now. If this is a lazily evaluated item, find its
# value before we close
# TODO: is this what we really want to do?
value = section[item]
config_parser.set(section_name, item, type_process(value))
directory = os.path.dirname(filename)
if not os.path.exists(directory):
os.makedirs(directory)
config_parser.write(open(filename, "w"))
def test_read_yes_all():
cfg = RawConfigParser()
cfg.add_section('group fooers')
cfg.set('group fooers', 'members', '@all')
cfg.set('group fooers', 'readonly', 'foo/bar')
eq(access.haveAccess(config=cfg, user='******', mode='readonly', path='foo/bar'),
('repositories', 'foo/bar'))
def test_clean_no_extension(self):
repo = self.make_clone("clone to clean", 'default')
rc_path = os.path.join(repo.target_dir, '.hg', 'hgrc')
# make sure that 'purge' extension is NOT activated
# we expect it to be installed on the system this test runs
# note that that's the case with Debian and Red Hat families of
# distributions
parser = RawConfigParser()
parser.read(rc_path)
parser.add_section('extensions')
parser.set('extensions', 'hgext.purge', '!')
with open(rc_path, 'w') as rc:
parser.write(rc)
dirty_dir = os.path.join(repo.target_dir, 'dirty')
os.mkdir(dirty_dir)
dirty_files = (os.path.join(repo.target_dir, 'untracked.pyc'),
os.path.join(dirty_dir, 'untracked2.pyo'))
for path in dirty_files:
with open(path, 'w') as f:
f.write('content')
repo.clean()
for path in dirty_files:
self.failIf(os.path.exists(path),
"Untracked file %r should have been removed" % path)
self.failIf(os.path.exists(dirty_dir),
"Untracked dir should have been removed")
def save(self):
config = RawConfigParser()
for name, version in self.versions.items():
config.set(DEFAULTSECT, name, version)
with open_for_config(os.path.join(self.path, self.VERSIONS_LIST)) as fp:
config.write(fp)
def make_vmu(in_path, out_base):
path_conf = out_base + '.vmu'
path_0 = out_base + '.l0'
path_1 = out_base + '.l1'
with OpenSlide(in_path) as osr:
l0 = VmuLevel(osr, 0)
l1 = VmuLevel(osr, osr.get_best_level_for_downsample(32))
for i, l in enumerate([l0, l1]):
print 'Level %d: %d pixels/column' % (i, l.column_width)
l0.save(path_0)
l1.save(path_1)
section = 'Uncompressed Virtual Microscope Specimen'
conf = {
'NoLayers': '1',
'ImageFile': path_0,
'MapFile': path_1,
'BitsPerPixel': '36',
'PixelOrder': 'RGB',
}
c = RawConfigParser()
c.optionxform = str
c.add_section(section)
for k, v in conf.iteritems():
c.set(section, k, v)
with open(path_conf, 'w') as fh:
c.write(fh)
def create_config_file(config_file, random_music_home):
"""
Create a configuration file.
:param config_file: path to config file we are creating
:type config_file: str
:param random_music_home: home of random_music application (i.e. where
index files are stored
:type random_music_home: str
"""
sys.stdout.write(
"You do not appear to have a config file, lets create one!\n")
sys.stdout.write("Creating config file at %s\n" % config_file)
config = RawConfigParser()
config.add_section('config')
config.set('config', 'loop_songs', 'true')
config.set('config', 'randomise', 'true')
config.set('config', 'index_dir',
os.path.join(random_music_home, "indicies"))
music_client = DEFAULT_MUSIC_CLIENT
while not which(music_client):
music_client = raw_input("The music player '%s' could not be found "
"on your path. Please input a different "
"music player:" % music_client)
config.set('config', 'music_client', music_client)
user_music_dirs = ""
while not all([os.path.isdir(d) for d in user_music_dirs.split(",")]):
user_music_dirs = raw_input("Input a csv list of full paths to "
"your music dirs:")
config.set('config', 'music_dirs', user_music_dirs)
with open(config_file, 'wb') as fh:
config.write(fh)
def write(self, config_data, filepath=None):
"""
Create a dotfile from keyword arguments.
:param config_data:
Dict of config settings
:param filepath:
(Optional) Path to write
"""
if filepath is None:
filepath = self.filepath
config = RawConfigParser()
section = SECTION_NAME
config.add_section(section)
# Set the config settings
for key, val in config_data.iteritems():
config.set(section, key, val)
with open(filepath, 'wb') as dotfile:
config.write(dotfile)
self.enforce_perms()
log.debug('wrote %s' % filepath)
def setUpClass(cls):
os.environ['PYSEMANTIC_CONFIG'] = "test.conf"
pr.CONF_FILE_NAME = "test.conf"
cls.testenv = os.environ
cls.test_config_path = op.join(os.getcwd(), "test.conf")
shutil.copy(TEST_CONFIG_FILE_PATH, cls.test_config_path)
# Change the relative paths in the config file to absolute paths
parser = RawConfigParser()
parser.read(cls.test_config_path)
for section in parser.sections():
schema_path = parser.get(section, "specfile")
parser.remove_option(section, "specfile")
parser.set(section, "specfile",
op.join(op.abspath(op.dirname(__file__)), schema_path))
with open(cls.test_config_path, "w") as fileobj:
parser.write(fileobj)
# change the relative paths in the test dictionary to absolute paths
with open(TEST_DATA_DICT, "r") as fileobj:
cls.org_specs = yaml.load(fileobj, Loader=Loader)
new_specs = deepcopy(cls.org_specs)
for _, specs in new_specs.iteritems():
path = specs['path']
specs['path'] = op.join(op.abspath(op.dirname(__file__)), path)
# Rewrite this to the file
with open(TEST_DATA_DICT, "w") as fileobj:
yaml.dump(new_specs,
fileobj,
Dumper=Dumper,
default_flow_style=False)
def save_config(self):
"""Save the current configuration to the config file.
"""
self.log.debug("Saving config file to '%s'", self.config_file)
if not self.config_file:
self.log.warning('%s.config_file file not set, not saving config!', self.__class__.__name__)
return
self.acquire_lock()
config = RawConfigParser()
for section_name, section in self.config._config.items():
config.add_section(section_name)
for option_name, value in section.items():
if section_name == 'general':
if option_name in ['save_config']:
continue
config.set(section_name, option_name, str(value))
with NamedTemporaryFile(mode='w', dir=os.path.dirname(self.config_file), delete=False) as tmpfile:
config.write(tmpfile)
# Move the new config file into place atomically
if os.path.getsize(tmpfile.name) > 0:
os.rename(tmpfile.name, self.config_file)
else:
self.log.warning('Config file saving failed, not replacing %s with %s.', self.config_file, tmpfile.name)
self.release_lock()
class ApplicationConfig(object):
"""A thin wrapper around ConfigParser that remembers what we read.
The remembered settings can then be written out to a minimal config file
when building the Elastic Beanstalk zipfile.
"""
def __init__(self):
self.input = RawConfigParser()
config_filename = os.environ.get("CONFIG", "production.ini")
with open(config_filename) as f:
self.input.readfp(f)
self.output = RawConfigParser()
def get(self, section, key):
value = self.input.get(section, key)
# remember that we needed this configuration value
if (section.upper() != "DEFAULT"
and not self.output.has_section(section)):
self.output.add_section(section)
self.output.set(section, key, value)
return value
def to_config(self):
io = cStringIO.StringIO()
self.output.write(io)
return io.getvalue()
def rc_to_ini(rc_info, default_section=SECTION):
"""Convert a .rc file to a ConfigParser (.ini-like object)
Items are assumed to be in app:assembl section,
unless prefixed by "{section}__" .
Keys prefixed with an underscore are not passed on.
Keys prefixed with a star are put in the global (DEFAULT) section.
Value of '__delete_key__' is eliminated if existing.
"""
p = Parser()
for key, val in rc_info.iteritems():
if not key or key.startswith('_'):
continue
if key[0] == '*':
# Global keys
section = "DEFAULT"
key = key[1:]
elif '__' in key:
section, key = key.split('__', 1)
else:
section = default_section
ensureSection(p, section)
if val == '__delete_key__':
# Allow to remove a variable from rc
# so we can fall back to underlying ini
p.remove_option(section, key)
else:
p.set(section, key, val)
return p
def setFilenameInConfigFile(self, root_path, name, filename):
"""Add new filename association in config file
@root_path: Path where config file is stored
@param name: Field name
@param filename: Filename of value stored in field
"""
path = os.path.join(root_path, self.cfg_filename)
# Update file
config = RawConfigParser()
if os.path.exists(path):
# Read file
fd = open(path, 'r')
try:
config.readfp(fd)
finally:
fd.close()
# Create section if it doesn't exist
if not config.has_section(self.cfg_filename_section):
config.add_section(self.cfg_filename_section)
config.set(self.cfg_filename_section, name, filename)
fd = open(path, 'w')
try:
# Write file
config.write(fd)
finally:
fd.close()
def test_write_yes_map():
cfg = RawConfigParser()
cfg.add_section('group fooers')
cfg.set('group fooers', 'members', 'jdoe')
cfg.set('group fooers', 'map writable foo/bar', 'quux/thud')
eq(access.haveAccess(config=cfg, user='******', mode='writable', path='foo/bar'),
('repositories', 'quux/thud'))
def parse_probe(probename, probe, parentname, parent):
probe_path = parentname + probename
probename = probename[1:]
probe_type = probe.pop("probe type", None)
options = RawConfigParser()
# probe metadata
if "metadata" in probe:
probe_metadata = dict()
for k,v in probe.pop("metadata").items():
probe_metadata[str(k)] = str(v)
else:
probe_metadata = dict()
# probe policy
options = RawConfigParser()
options.add_section("policy")
for k,v in probe.pop("policy", dict()).items():
options.set("policy", str(k), str(v))
probe_policy = Section("policy", options, os.path.dirname(systemfile.path))
# probe settings
options = RawConfigParser()
options.add_section("settings")
for k in filter(lambda k: not k.startswith("/"), probe.keys()):
options.set("settings", str(k), str(probe.pop(k)))
probe_settings = Section("settings", options, os.path.dirname(systemfile.path))
# probe children
children = dict()
for childname,childprobe in probe.items():
parse_probe(childname, childprobe, probe_path, children)
parent[probename] = ProbeSpec(probe_path, probe_type, probe_settings, probe_metadata, probe_policy, children)
def save():
"""Saves FileDirectives into ConfigFile."""
section = "*"
module = sys.modules[__name__]
parser = RawConfigParser()
parser.optionxform = str # Force case-sensitivity on names
parser.add_section(section)
try:
f, fname = open(ConfigFile, "wb"), util.longpath(ConfigFile)
f.write("# %s configuration autowritten on %s.\n" %
(fname, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
for name in FileDirectives:
try:
parser.set(section, name, json.dumps(getattr(module, name)))
except Exception: pass
for name in OptionalFileDirectives:
try:
value = getattr(module, name, None)
if OptionalFileDirectiveDefaults.get(name) != value:
parser.set(section, name, json.dumps(value))
except Exception: pass
parser.write(f)
f.close()
except Exception:
pass # Fail silently
def test_read_yes_map_wouldHaveWritable():
cfg = RawConfigParser()
cfg.add_section('group fooers')
cfg.set('group fooers', 'members', 'jdoe')
cfg.set('group fooers', 'map writable foo/bar', 'quux/thud')
eq(access.haveAccess(config=cfg, user='******', mode='readonly', path='foo/bar'),
None)
def init_admin_repository(
git_dir,
pubkey,
user,
):
repository.init(path=git_dir,
template=resource_filename('gitosis.templates', 'admin'))
repository.init(path=git_dir, )
if not repository.has_initial_commit(git_dir):
log.info('Making initial commit...')
# ConfigParser does not guarantee order, so jump through hoops
# to make sure [gitosis] is first
cfg_file = StringIO()
print >> cfg_file, '[gitosis]'
print >> cfg_file
cfg = RawConfigParser()
cfg.add_section('group gitosis-admin')
cfg.set('group gitosis-admin', 'members', user)
cfg.set('group gitosis-admin', 'writable', 'gitosis-admin')
cfg.write(cfg_file)
initial_commit(
git_dir=git_dir,
cfg=cfg_file.getvalue(),
pubkey=pubkey,
user=user,
)
def install_mercurial_hook():
"""
Installs the mercurial precommit hook by adding a hook to the hgrc
file in the .hg directory of the repository.
"""
repo_dir = get_repo_dir()
config_file = os.path.join(repo_dir, ".hg", "hgrc")
config_parser = RawConfigParser()
config_parser.read(config_file)
precommit_abs_file = os.path.join(repo_dir, "scripts", "codestyleprecommit.py")
section = "hooks"
key = "pretxncommit.precommit"
value = "python:%s:mercurial_hook" % precommit_abs_file
if not config_parser.has_section(section):
config_parser.add_section(section)
config_parser.set(section, key, value)
with open(config_file, "w") as config:
config_parser.write(config)
def write(self, config_data, filepath=None):
"""
Create a dotfile from keyword arguments.
:param config_data:
Dict of config settings
:param filepath:
(Optional) Path to write
"""
if filepath is None:
filepath = self.filepath
config = RawConfigParser()
section = constants.SECTION_NAME
config.add_section(section)
# Set the config settings
for key, val in config_data.iteritems():
config.set(section, key, val)
with open(filepath, 'wb') as dotfile:
config.write(dotfile)
self.enforce_perms()
log.debug('wrote %s' % filepath)
def getAccount(account_key):
from ConfigParser import RawConfigParser, NoOptionError, NoSectionError
account_file = '.account'
config = RawConfigParser()
with open(account_file, 'r') as fp:
config.readfp(fp)
account = config.get('account', account_key)
password = None
password_section = 'password'
try:
password = config.get(password_section, account_key)
except NoSectionError:
config.add_section(password_section)
except NoOptionError:
pass
aes = AESCipher(account)
if password:
return account, aes.decrypt(password).encode('UTF-8')
from getpass import getpass
password = getpass(account_key + ' of ' +account + "'s password: ")
config.set(password_section, account_key, aes.encrypt(password))
with open(account_file, 'w') as fp:
config.write(fp)
return account, password
def load_configs(shutit):
"""Responsible for loading config files into ShutIt.
Recurses down from configured shutit module paths.
"""
cfg = shutit.cfg
# Get root default config.
configs = [('defaults', StringIO.StringIO(_default_cnf))]
# Add the shutit global host- and user-specific config file.
configs.append(os.path.join(shutit.shutit_main_dir,
'configs/' + socket.gethostname() + '_' + cfg['host']['real_user'] + '.cnf'))
configs.append(os.path.join(cfg['shutit_home'], 'config'))
# Add the local build.cnf
configs.append('configs/build.cnf')
# Get passed-in config(s)
for config_file_name in cfg['build']['extra_configs']:
run_config_file = os.path.expanduser(config_file_name)
if not os.path.isfile(run_config_file):
print('Did not recognise ' + run_config_file +
' as a file - do you need to touch ' + run_config_file + '?')
sys.exit()
configs.append(run_config_file)
# Image to use to start off. The script should be idempotent, so running it
# on an already built image should be ok, and is advised to reduce diff space required.
if cfg['build']['interactive'] >= 3 or cfg['action']['show_config']:
msg = ''
for c in configs:
if type(c) is tuple:
c = c[0]
msg = msg + '\t\n' + c
shutit.log('\t' + c)
if cfg['build']['interactive'] >= 3:
print textwrap.dedent("""\n""") + msg + textwrap.dedent("""
Looking at config files in the above order (even if they
do not exist - you may want to create them).
If you get a "Port already in use:" error,
run:
docker ps -a | grep -w <port> | awk '{print $1}' | xargs docker kill
or
sudo docker ps -a | grep -w <port> | awk '{print $1}' | xargs sudo docker kill
""" + colour('31','[Hit return to continue]'))
raw_input('')
# Interpret any config overrides, write to a file and add them to the
# list of configs to be interpreted
if cfg['build']['config_overrides']:
# We don't need layers, this is a temporary configparser
override_cp = RawConfigParser()
for o_sec, o_key, o_val in cfg['build']['config_overrides']:
if not override_cp.has_section(o_sec):
override_cp.add_section(o_sec)
override_cp.set(o_sec, o_key, o_val)
override_fd = StringIO.StringIO()
override_cp.write(override_fd)
override_fd.seek(0)
configs.append(('overrides', override_fd))
cfg_parser = get_configs(shutit,configs)
get_base_config(cfg, cfg_parser)
def install_mercurial_hook():
"""
Installs the mercurial precommit hook by adding a hook to the hgrc
file in the .hg directory of the repository.
"""
repo_dir = get_repo_dir()
config_file = os.path.join(repo_dir, '.hg', 'hgrc')
config_parser = RawConfigParser()
config_parser.read(config_file)
precommit_abs_file = os.path.join(repo_dir, 'scripts',
'codestyleprecommit.py')
section = 'hooks'
key = 'pretxncommit.precommit'
value = 'python:%s:mercurial_hook' % precommit_abs_file
if not config_parser.has_section(section):
config_parser.add_section(section)
config_parser.set(section, key, value)
with open(config_file, 'w') as config:
config_parser.write(config)
def set_schema_fpath(project_name, schema_fpath):
"""Set the schema path for a given project.
:param project_name: Name of the project
:param schema_fpath: path to the yaml file to be used as the schema for \
the project.
:type project_name: str
:type schema_fpath: str
:return: True, if setting the schema path was successful.
:Example:
>>> set_schema_fpath('skynet', '/path/to/new/schema.yaml')
True
"""
path = locate_config_file()
parser = RawConfigParser()
parser.read(path)
if project_name in parser.sections():
if not parser.remove_option(project_name, "specfile"):
raise MissingProject
else:
parser.set(project_name, "specfile", schema_fpath)
with open(path, "w") as f:
parser.write(f)
return True
raise MissingProject
def test_init_yes_simple():
cfg = RawConfigParser()
cfg.add_section('group fooers')
cfg.set('group fooers', 'members', 'jdoe')
cfg.set('group fooers', 'init', 'foo/bar')
eq(access.haveAccess(config=cfg, user='******', mode='init', path='foo/bar'),
('repositories', 'foo/bar', 'init'))
def convertDownloadCheckpoints(checkpoint_dir):
# Convert tribler <= 6.2 download checkpoints to tribler 6.3
if os.path.exists(checkpoint_dir):
for old_filename in glob.glob(os.path.join(checkpoint_dir, '*.pickle')):
old_checkpoint = None
try:
with open(old_filename, "rb") as old_file:
old_checkpoint = pickle.load(old_file)
except:
pass
if old_checkpoint:
new_checkpoint = RawConfigParser()
new_checkpoint.add_section('downloadconfig')
new_checkpoint.add_section('state')
for key, value in old_checkpoint['dlconfig'].iteritems():
if key in ['saveas', 'max_upload_rate', 'max_download_rate', 'super_seeder', 'mode',
'selected_files', 'correctedfilename']:
new_checkpoint.set('downloadconfig', key, value)
new_checkpoint.set('state', 'version', PERSISTENTSTATE_CURRENTVERSION)
new_checkpoint.set('state', 'engineresumedata', old_checkpoint['engineresumedata'])
new_checkpoint.set('state', 'dlstate', old_checkpoint['dlstate'])
new_checkpoint.set('state', 'metainfo', old_checkpoint['metainfo'])
with open(old_filename.replace('.pickle', '.state'), "wb") as new_file:
new_checkpoint.write(new_file)
os.remove(old_filename)
def init_admin_repository(
git_dir,
pubkey,
user,
):
repository.init(
path=git_dir,
template=resource_filename('gitosis.templates', 'admin')
)
repository.init(
path=git_dir,
)
hook = os.path.join(git_dir, 'hooks', 'post-update')
os.chmod(hook, 0755)
if not repository.has_initial_commit(git_dir):
log.info('Making initial commit...')
# ConfigParser does not guarantee order, so jump through hoops
# to make sure [gitosis] is first
cfg_file = StringIO()
print >>cfg_file, '[gitosis]'
print >>cfg_file
cfg = RawConfigParser()
cfg.add_section('group gitosis-admin')
cfg.set('group gitosis-admin', 'members', user)
cfg.set('group gitosis-admin', 'writable', 'gitosis-admin')
cfg.write(cfg_file)
initial_commit(
git_dir=git_dir,
cfg=cfg_file.getvalue(),
pubkey=pubkey,
user=user,
)
def tearDownClass(cls):
try:
# modify the testdata back
with open(TEST_DATA_DICT, "r") as fileobj:
test_data = yaml.load(fileobj, Loader=yaml.CLoader)
test_data['iris']['path'] = op.join("testdata", "iris.csv")
test_data['person_activity']['path'] = op.join("testdata",
"person_activity.tsv")
del test_data['multi_iris']
with open(TEST_DATA_DICT, "w") as fileobj:
test_data = yaml.dump(test_data, fileobj, Dumper=yaml.CDumper,
default_flow_style=False)
# Change the config files back
parser = RawConfigParser()
parser.read(cls.test_conf_file)
parser.remove_option("pysemantic", "specfile")
parser.set("pysemantic", "specfile",
op.join("testdata", "test_dictionary.yaml"))
with open(TEST_CONFIG_FILE_PATH, 'w') as fileobj:
parser.write(fileobj)
finally:
os.unlink(cls.test_conf_file)
os.unlink(cls.copied_iris_path)
def next_serial(serial_file=CA_SERIALNO):
"""
Get the next serial number if we're using an NSS-based self-signed CA.
The file is an ini-like file with following properties:
lastvalue = the last serial number handed out
nextreplica = the serial number the next replica should start with
replicainterval = the number to add to nextreplica the next time a
replica is created
File locking is attempted so we have unique serial numbers.
"""
fp = None
parser = RawConfigParser()
if ipautil.file_exists(serial_file):
try:
fp = open(serial_file, "r+")
fcntl.flock(fp.fileno(), fcntl.LOCK_EX)
parser.readfp(fp)
serial = parser.getint('selfsign', 'lastvalue')
cur_serial = serial + 1
except IOError, e:
raise RuntimeError("Unable to determine serial number: %s" % str(e))
except MissingSectionHeaderError:
fcntl.flock(fp.fileno(), fcntl.LOCK_UN)
fp.close()
f=open(serial_file,"r")
r = f.readline()
f.close()
cur_serial = int(r) + 1
fp = open(serial_file, "w")
fcntl.flock(fp.fileno(), fcntl.LOCK_EX)
parser.add_section('selfsign')
parser.set('selfsign', 'nextreplica', 500000)
parser.set('selfsign', 'replicainterval', 500000)
def iter_backends(self):
"""
Iterate on backends.
:returns: each tuple contains the backend name, module name and module options
:rtype: :class:`tuple`
"""
config = RawConfigParser()
config.read(self.confpath)
changed = False
for backend_name in config.sections():
params = dict(config.items(backend_name))
try:
module_name = params.pop('_module')
except KeyError:
try:
module_name = params.pop('_backend')
config.set(backend_name, '_module', module_name)
config.remove_option(backend_name, '_backend')
changed = True
except KeyError:
warning('Missing field "_module" for configured backend "%s"', backend_name)
continue
yield backend_name, module_name, params
if changed:
with open(self.confpath, 'wb') as f:
config.write(f)
def add_backend(self, backend_name, module_name, params, edit=False):
"""
Add a backend to config.
:param backend_name: name of the backend in config
:param module_name: name of the Python submodule to run
:param params: params to pass to the module
:type params: :class:`dict`
"""
if not backend_name:
raise ValueError(u'Please give a name to the configured backend.')
config = RawConfigParser()
config.read(self.confpath)
if not edit:
try:
config.add_section(backend_name)
except DuplicateSectionError:
raise BackendAlreadyExists(backend_name)
config.set(backend_name, '_module', module_name)
for key, value in params.iteritems():
if isinstance(value, unicode):
value = value.encode('utf-8')
config.set(backend_name, key, value)
with open(self.confpath, 'wb') as f:
config.write(f)
def test_user():
cfg = RawConfigParser()
cfg.add_section('user jdoe')
cfg.set('user jdoe', 'readonly', 'foo xyzzy bar')
eq(access.haveAccess(
config=cfg, user='******', mode='readonly', path='xyzzy'),
('repositories', 'xyzzy'))
def test_write_no_simple_wouldHaveReadonly():
cfg = RawConfigParser()
cfg.add_section('group fooers')
cfg.set('group fooers', 'members', 'jdoe')
cfg.set('group fooers', 'readonly', 'foo/bar')
eq(access.haveAccess(config=cfg, user='******', mode='writable', path='foo/bar'),
None)
class ApplicationConfig(object):
"""A thin wrapper around ConfigParser that remembers what we read.
The remembered settings can then be written out to a minimal config file
when building the Elastic Beanstalk zipfile.
"""
def __init__(self):
self.input = RawConfigParser()
with open("production.ini") as f:
self.input.readfp(f)
self.output = RawConfigParser()
def get(self, section, key):
value = self.input.get(section, key)
# remember that we needed this configuration value
if (section.upper() != "DEFAULT" and
not self.output.has_section(section)):
self.output.add_section(section)
self.output.set(section, key, value)
return value
def to_config(self):
io = cStringIO.StringIO()
self.output.write(io)
return io.getvalue()
def create_config_file(config_file, random_music_home):
"""
Create a configuration file.
:param config_file: path to config file we are creating
:type config_file: str
:param random_music_home: home of random_music application (i.e. where
index files are stored
:type random_music_home: str
"""
sys.stdout.write("You do not appear to have a config file, lets create one!\n")
sys.stdout.write("Creating config file at %s\n" % config_file)
config = RawConfigParser()
config.add_section('config')
config.set('config', 'loop_songs', 'true')
config.set('config', 'randomise', 'true')
config.set('config', 'index_dir', os.path.join(random_music_home,
"indicies"))
music_client = DEFAULT_MUSIC_CLIENT
while not which(music_client):
music_client = raw_input("The music player '%s' could not be found "
"on your path. Please input a different "
"music player:" % music_client)
config.set('config', 'music_client', music_client)
user_music_dirs = ""
while not all([os.path.isdir(d) for d in user_music_dirs.split(",")]):
user_music_dirs = raw_input("Input a csv list of full paths to "
"your music dirs:")
config.set('config', 'music_dirs', user_music_dirs)
with open(config_file, 'wb') as fh:
config.write(fh)
def test_push_inits_sets_export_ok():
tmp = util.maketemp()
cfg = RawConfigParser()
cfg.add_section('gitosis')
repositories = path.join(tmp, 'repositories')
os.mkdir(repositories)
cfg.set('gitosis', 'repositories', repositories)
generated = path.join(tmp, 'generated')
os.mkdir(generated)
cfg.set('gitosis', 'generate-files-in', generated)
cfg.add_section('group foo')
cfg.set('group foo', 'members', 'jdoe')
cfg.set('group foo', 'writable', 'foo')
cfg.add_section('repo foo')
cfg.set('repo foo', 'daemon', 'yes')
cfg.add_section('rsp')
cfg.set('rsp', 'haveAccessURL', 'example.org')
serve.serve(
cfg=cfg,
user='******',
command="git-receive-pack 'foo'",
)
eq(os.listdir(repositories), ['foo.git'])
export = path.join(repositories, 'foo.git', 'git-daemon-export-ok')
assert path.exists(export)
def _path_fixer(filepath, root=None):
"""Change all the relative paths in `filepath` to absolute ones.
:param filepath: File to be changed
:param root: Root path with which the relative paths are prefixed. If None
(default), the directory with this file is the root.
"""
if root is None:
root = op.join(op.abspath(op.dirname(__file__)))
if filepath.endswith((".yaml", ".yml")):
with open(filepath, "r") as fileobj:
data = yaml.load(fileobj, Loader=Loader)
for specs in data.itervalues():
specs['path'] = op.join(root, specs['path'])
with open(filepath, "w") as fileobj:
yaml.dump(data, fileobj, Dumper=Dumper,
default_flow_style=False)
elif filepath.endswith(".conf"):
parser = RawConfigParser()
parser.read(filepath)
for section in parser.sections():
path = parser.get(section, "specfile")
parser.remove_option(section, "specfile")
parser.set(section, "specfile", op.join(root, path))
with open(filepath, "w") as fileobj:
parser.write(fileobj)
def test_push_inits_subdir_parent_exists():
tmp = util.maketemp()
cfg = RawConfigParser()
cfg.add_section('gitosis')
repositories = path.join(tmp, 'repositories')
os.mkdir(repositories)
foo = path.join(repositories, 'foo')
# silly mode on purpose; not to be touched
os.mkdir(foo, 0751)
cfg.set('gitosis', 'repositories', repositories)
generated = path.join(tmp, 'generated')
os.mkdir(generated)
cfg.set('gitosis', 'generate-files-in', generated)
cfg.add_section('group foo')
cfg.set('group foo', 'members', 'jdoe')
cfg.set('group foo', 'writable', 'foo/bar')
cfg.add_section('rsp')
cfg.set('rsp', 'haveAccessURL', 'example.org')
serve.serve(
cfg=cfg,
user='******',
command="git-receive-pack 'foo/bar.git'",
)
eq(os.listdir(repositories), ['foo'])
util.check_mode(foo, 0751, is_dir=True)
eq(os.listdir(foo), ['bar.git'])
assert path.isfile(path.join(repositories, 'foo', 'bar.git', 'HEAD'))
def test_no_notListed():
cfg = RawConfigParser()
cfg.add_section('group hackers')
cfg.set('group hackers', 'members', 'wsmith')
gen = group.getMembership(config=cfg, user='******')
eq(gen.next(), 'all')
assert_raises(StopIteration, gen.next)
def write(path, section, key, value): conf = RawConfigParser() conf.read(path) conf.set( section, key, value ) configfile = open(path, 'w') conf.write(configfile) configfile.close()
def create_new_conf_from_modifications(self):
"""
Return a new RawConfigParser instance that has been created from the
non-default modifications returned by the `modifications` property
above.
"""
# This is a bit hacky as the underlying config classes don't really
# support the notion of "only write out sections/options that have
# changed since we loaded the defaults".
if not self.repo_path:
raise RepositoryNotSet()
mods = self.modifications
if not mods:
raise NoModificationsMade()
filename = self.writable_repo_override_conf_filename
conf = RawConfigParser()
conf.read(filename)
for (section, options) in mods.items():
conf.add_section(section)
for (option, value) in options.items():
conf.set(section, option, value)
return conf
class OriginAuthStore(object):
def __init__(self, config_file):
self.config_file = config_file
self.config = RawConfigParser()
self.config.read(config_file)
def origin(self, name):
return OriginAuth(self, name)
def __getitem__(self, origin_name):
try:
return dict(self.config.items(origin_name))
except NoSectionError:
return {}
def __setitem__(self, origin_name, auth):
try:
self.config.remove_section(origin_name)
except NoSectionError:
pass
if auth:
self.config.add_section(origin_name)
for key, val in auth.iteritems():
self.config.set(origin_name, key, val)
with open(self.config_file, 'w') as f:
self.config.write(f)
try:
os.chmod(self.config_file, stat.S_IRUSR | stat.S_IWUSR)
except OSError:
print 'Unable to chmod 600 %s' % self.config_file # TODO: Test
def savePackageVersion(packageName, version):
settings = RawConfigParser()
settings.read(PackageConfiguration.getSettingsFile())
for section in settings.sections():
if settings.get(section, "PackageName") == packageName:
settings.set(section, "PackageVersion", version)
with open(PackageConfiguration.getSettingsFile(), 'wb') as configfile:
settings.write(configfile)
def test_yes_middle():
cfg = RawConfigParser()
cfg.add_section('group hackers')
cfg.set('group hackers', 'members', 'wsmith jdoe danny')
gen = group.getMembership(config=cfg, user='******')
eq(gen.next(), 'hackers')
eq(gen.next(), 'all')
assert_raises(StopIteration, gen.next)
def set(self, section, option, value=None):
if not self.has_section(section):
self.add_section(section)
if isinstance(value, unicode):
value = self.encode(value)
elif not isinstance(value, str):
value = str(value)
RawConfigParser.set(self, section, option, value)
def save(self):
config = RawConfigParser()
for name, version in self.versions.items():
config.set(DEFAULTSECT, name, version)
with open_for_config(os.path.join(self.path,
self.VERSIONS_LIST)) as fp:
config.write(fp)
def write_submodule_config(f, name, path, url):
config = RawConfigParser()
section = 'submodule "%s"' % (name)
config.add_section(section)
config.set(section, 'path', path)
config.set(section, 'url', url)
config.write(f)
