def test_clean_no_extension(self):
repo = self.make_clone("clone to clean", 'default')
rc_path = os.path.join(repo.target_dir, '.hg', 'hgrc')
# make sure that 'purge' extension is NOT activated
# we expect it to be installed on the system this test runs
# note that that's the case with Debian and Red Hat families of
# distributions
parser = RawConfigParser()
parser.read(rc_path)
parser.add_section('extensions')
parser.set('extensions', 'hgext.purge', '!')
with open(rc_path, 'w') as rc:
parser.write(rc)
dirty_dir = os.path.join(repo.target_dir, 'dirty')
os.mkdir(dirty_dir)
dirty_files = (os.path.join(repo.target_dir, 'untracked.pyc'),
os.path.join(dirty_dir, 'untracked2.pyo'))
for path in dirty_files:
with open(path, 'w') as f:
f.write('content')
repo.clean()
for path in dirty_files:
self.failIf(os.path.exists(path),
"Untracked file %r should have been removed" % path)
self.failIf(os.path.exists(dirty_dir),
"Untracked dir should have been removed")
class ApplicationConfig(object):
"""A thin wrapper around ConfigParser that remembers what we read.
The remembered settings can then be written out to a minimal config file
when building the Elastic Beanstalk zipfile.
"""
def __init__(self):
self.input = RawConfigParser()
with open("production.ini") as f:
self.input.readfp(f)
self.output = RawConfigParser()
def get(self, section, key):
value = self.input.get(section, key)
# remember that we needed this configuration value
if (section.upper() != "DEFAULT" and
not self.output.has_section(section)):
self.output.add_section(section)
self.output.set(section, key, value)
return value
def to_config(self):
io = cStringIO.StringIO()
self.output.write(io)
return io.getvalue()
def _save_config_file(self, config):
"""Write config to disk.
"""
# Generate a sanitized version of our running configuration
sane_config = RawConfigParser()
for section_name, section in config.items():
sane_config.add_section(section_name)
for option_name, value in section.items():
if section_name == 'general':
if option_name in ['config_file']:
continue
if value is not None:
sane_config.set(section_name, option_name, str(value))
config_dir = self.config_file.parent
if not config_dir.exists():
config_dir.mkdir(parents=True, exist_ok=True)
# Write the config file atomically.
self.acquire_lock()
with NamedTemporaryFile(mode='w', dir=str(config_dir),
delete=False) as tmpfile:
sane_config.write(tmpfile)
if os.path.getsize(tmpfile.name) > 0:
os.replace(tmpfile.name, str(self.config_file))
else:
self.log.warning(
'Config file saving failed, not replacing %s with %s.',
str(self.config_file), tmpfile.name)
self.release_lock()
def write(path, section, key, value): conf = RawConfigParser() conf.read(path) conf.set( section, key, value ) configfile = open(path, 'w') conf.write(configfile) configfile.close()
def _path_fixer(filepath, root=None):
"""Change all the relative paths in `filepath` to absolute ones.
:param filepath: File to be changed
:param root: Root path with which the relative paths are prefixed. If None
(default), the directory with this file is the root.
"""
if root is None:
root = op.join(op.abspath(op.dirname(__file__)))
if filepath.endswith((".yaml", ".yml")):
with open(filepath, "r") as fileobj:
data = yaml.load(fileobj, Loader=Loader)
for specs in data.itervalues():
specs['path'] = op.join(root, specs['path'])
with open(filepath, "w") as fileobj:
yaml.dump(data, fileobj, Dumper=Dumper,
default_flow_style=False)
elif filepath.endswith(".conf"):
parser = RawConfigParser()
parser.read(filepath)
for section in parser.sections():
path = parser.get(section, "specfile")
parser.remove_option(section, "specfile")
parser.set(section, "specfile", op.join(root, path))
with open(filepath, "w") as fileobj:
parser.write(fileobj)
def init_admin_repository(
git_dir,
pubkey,
user,
):
repository.init(
path=git_dir,
template=resource_filename('gitosis.templates', 'admin')
)
repository.init(
path=git_dir,
)
# can't rely on setuptools and all kinds of distro packaging to
# have kept our templates executable, it seems
os.chmod(os.path.join(git_dir, 'hooks', 'post-update'), 0755)
if not repository.has_initial_commit(git_dir):
log.info('Making initial commit...')
# ConfigParser does not guarantee order, so jump through hoops
# to make sure [gitosis] is first
cfg_file = StringIO()
print >>cfg_file, '[gitosis]'
print >>cfg_file
cfg = RawConfigParser()
cfg.add_section('group gitosis-admin')
cfg.set('group gitosis-admin', 'members', user)
cfg.set('group gitosis-admin', 'writable', 'gitosis-admin')
cfg.write(cfg_file)
initial_commit(
git_dir=git_dir,
cfg=cfg_file.getvalue(),
pubkey=pubkey,
user=user,
)
class BaseSettings:
"""
Class for core settings functionality.
"""
def __init__(self):
self.saveData = None
self.loadData = None
self.settingsFile = get_path() + r'\toolbox.ini'
self.config = RawConfigParser()
def Exists(self):
if os.path.isfile(self.settingsFile): return True
else: return False
def SetValue(self, section, key, value):
try:
self.config.set(section, str(key), value)
except Exception:
self.config.add_section(section)
self.config.set(section, str(key), value)
self.Write()
def Write(self):
with open(self.settingsFile, 'wb') as cfgFile:
self.config.write(cfgFile)
def Read(self):
if self.Exists(): self.config.read(self.settingsFile)
else: raise ValueError("File does not exist")
def GetValue(self, section, key):
return self.config.get(section, str(key))
def create_local_ini(c):
"""Compose local.ini from the given .yaml file"""
from assembl.scripts.ini_files import extract_saml_info, populate_random, find_ini_file, combine_ini
assert running_locally(c)
c.config.DEFAULT = c.config.get('DEFAULT', {})
c.config.DEFAULT['code_root'] = c.config.code_root # Is this used?
# Special case: uwsgi does not do internal computations.
if 'uwsgi' not in c.config:
c.config.uwsgi = {}
c.config.uwsgi.virtualenv = c.config.projectpath + '/venv'
ini_sequence = c.config.get('ini_files', None)
assert ini_sequence, "Define ini_files"
ini_sequence = ini_sequence.split()
base = RawConfigParser()
random_file = c.config.get('random_file', None)
for overlay in ini_sequence:
if overlay == 'RC_DATA':
overlay = yaml_to_ini(c.config)
elif overlay.startswith('RANDOM'):
templates = overlay.split(':')[1:]
overlay = populate_random(random_file, templates,
extract_saml_info(c.config))
else:
overlay = find_ini_file(
overlay, os.path.join(c.config.code_root, 'assembl',
'configs'))
assert overlay, "Cannot find " + overlay
combine_ini(base, overlay)
ini_file_name = c.config.get('_internal', {}).get('ini_file', 'local.ini')
local_ini_path = os.path.join(c.config.projectpath, ini_file_name)
if exists(c, local_ini_path):
c.run('cp %s %s.bak' % (local_ini_path, local_ini_path))
with open(local_ini_path, 'w') as f:
base.write(f)
def saveUserPackages(packagesConfiguration):
settings = RawConfigParser()
for i in range(len(packagesConfiguration)):
settings.add_section(CONFIGSECTIONAME)
settings.set(CONFIGSECTIONAME, "PackageName",
packagesConfiguration[i].package_name)
settings.set(CONFIGSECTIONAME, "Location",
packagesConfiguration[i].network_path)
settings.set(CONFIGSECTIONAME, "PackageVersion",
packagesConfiguration[i].getCurrentVersion())
settings.set(CONFIGSECTIONAME, "Connection Type", "Local")
settings.set(CONFIGSECTIONAME, "UseVCS",
packagesConfiguration[i].use_vcs)
settings.set(CONFIGSECTIONAME, "VCSType",
packagesConfiguration[i].vcs_type)
if packagesConfiguration[i].connection_type == ConnectionType.FTP:
settings.set(CONFIGSECTIONAME, "Host",
packagesConfiguration[i].host)
settings.set(CONFIGSECTIONAME, "Username",
packagesConfiguration[i].username)
settings.set(CONFIGSECTIONAME, "Password",
packagesConfiguration[i].password)
settings.set(CONFIGSECTIONAME, "FTP Path",
packagesConfiguration[i].ftp_path)
with open(PackageConfiguration.getSettingsFile(), 'wb') as configfile:
settings.write(configfile)
def write(self, config_data, filepath=None):
"""
Create a dotfile from keyword arguments.
:param config_data:
Dict of config settings
:param filepath:
(Optional) Path to write
"""
if filepath is None:
filepath = self.filepath
config = RawConfigParser()
section = SECTION_NAME
config.add_section(section)
# Set the config settings
for key, val in config_data.iteritems():
config.set(section, key, val)
with open(filepath, 'wb') as dotfile:
config.write(dotfile)
self.enforce_perms()
log.debug('wrote %s' % filepath)
class Ini():
def __init__(self):
self.config = RawConfigParser()
def get_items(self, section='MySQL'):
self.config.read(INI_PATH)
return dict(self.config.items(section))
def get_value(self, section, option):
self.config.read(INI_PATH)
return self.config.get(section, option)
def set_updatedate(self, function):
self.config.read(INI_PATH)
self.config.set(function, 'updateddate',
time.strftime('%Y-%m-%d %H:%M:%S'))
with open(INI_PATH, 'wb') as fp:
self.config.write(fp)
def get_all_biz(self):
self.config.read(INI_PATH)
sections = self.config.sections()
ini = Ini()
result = {}
for section in sections:
if section not in ('MySQL', 'API'):
result[section] = ini.get_items(section)
return result
def create_config_file(config_file, random_music_home):
"""
Create a configuration file.
:param config_file: path to config file we are creating
:type config_file: str
:param random_music_home: home of random_music application (i.e. where
index files are stored
:type random_music_home: str
"""
sys.stdout.write(
"You do not appear to have a config file, lets create one!\n")
sys.stdout.write("Creating config file at %s\n" % config_file)
config = RawConfigParser()
config.add_section('config')
config.set('config', 'loop_songs', 'true')
config.set('config', 'randomise', 'true')
config.set('config', 'index_dir',
os.path.join(random_music_home, "indicies"))
music_client = DEFAULT_MUSIC_CLIENT
while not which(music_client):
music_client = raw_input("The music player '%s' could not be found "
"on your path. Please input a different "
"music player:" % music_client)
config.set('config', 'music_client', music_client)
user_music_dirs = ""
while not all([os.path.isdir(d) for d in user_music_dirs.split(",")]):
user_music_dirs = raw_input("Input a csv list of full paths to "
"your music dirs:")
config.set('config', 'music_dirs', user_music_dirs)
with open(config_file, 'wb') as fh:
config.write(fh)
def _save(widget, chooser):
config = RawConfigParser()
horizontals = self.area_clue.get_horizontals()
verticals = self.area_clue.get_verticals()
path = chooser.get_filename()
path += '.cwg' if not path.endswith('.cwg') else ''
game = self.game_area.get_actual_game()
config.add_section('Horizontals')
for x in horizontals:
config.set('Horizontals', str(x), horizontals[x])
config.add_section('Verticals')
for x in verticals:
config.set('Verticals', str(x), horizontals[x])
config.add_section('Game')
for x in range(1, 11):
config.set('Game', str(x), game.split('\n')[x - 1])
with open(path, 'wb') as configfile:
config.write(configfile)
chooser._destroy()
def set(path, section, option, value):
conf = RawConfigParser()
conf.read(path)
conf.set(section, option, value)
configfile = open(path, "w")
conf.write(configfile)
configfile.close()
def save(self, filename, private=False):
"""
Save repository into a file (modules.list for example).
:param filename: path to file to save repository.
:type filename: str
:param private: if enabled, save URL of repository.
:type private: bool
"""
config = RawConfigParser()
config.set(DEFAULTSECT, 'name', self.name)
config.set(DEFAULTSECT, 'update', self.update)
config.set(DEFAULTSECT, 'maintainer', self.maintainer)
config.set(DEFAULTSECT, 'signed', int(self.signed))
config.set(DEFAULTSECT, 'key_update', self.key_update)
if private:
config.set(DEFAULTSECT, 'url', self.url)
for module in self.modules.values():
config.add_section(module.name)
for key, value in module.dump():
if sys.version_info.major == 2:
# python2's configparser enforces bytes coercion with str(value)...
config.set(module.name, key, to_unicode(value).encode('utf-8'))
else:
config.set(module.name, key, value)
with open_for_config(filename) as f:
config.write(f)
def add_backend(self, backend_name, module_name, params, edit=False):
"""
Add a backend to config.
:param backend_name: name of the backend in config
:param module_name: name of the Python submodule to run
:param params: params to pass to the module
:type params: :class:`dict`
"""
if not backend_name:
raise ValueError(u'Please give a name to the configured backend.')
config = RawConfigParser()
config.read(self.confpath)
if not edit:
try:
config.add_section(backend_name)
except DuplicateSectionError:
raise BackendAlreadyExists(backend_name)
config.set(backend_name, '_module', module_name)
for key, value in params.iteritems():
if isinstance(value, unicode):
value = value.encode('utf-8')
config.set(backend_name, key, value)
with open(self.confpath, 'wb') as f:
config.write(f)
def make_vmu(in_path, out_base):
path_conf = out_base + '.vmu'
path_0 = out_base + '.l0'
path_1 = out_base + '.l1'
with OpenSlide(in_path) as osr:
l0 = VmuLevel(osr, 0)
l1 = VmuLevel(osr, osr.get_best_level_for_downsample(32))
for i, l in enumerate([l0, l1]):
print 'Level %d: %d pixels/column' % (i, l.column_width)
l0.save(path_0)
l1.save(path_1)
section = 'Uncompressed Virtual Microscope Specimen'
conf = {
'NoLayers': '1',
'ImageFile': path_0,
'MapFile': path_1,
'BitsPerPixel': '36',
'PixelOrder': 'RGB',
}
c = RawConfigParser()
c.optionxform = str
c.add_section(section)
for k, v in conf.iteritems():
c.set(section, k, v)
with open(path_conf, 'w') as fh:
c.write(fh)
def iter_backends(self):
"""
Iterate on backends.
:returns: each tuple contains the backend name, module name and module options
:rtype: :class:`tuple`
"""
config = RawConfigParser()
config.read(self.confpath)
changed = False
for backend_name in config.sections():
params = dict(config.items(backend_name))
try:
module_name = params.pop('_module')
except KeyError:
try:
module_name = params.pop('_backend')
config.set(backend_name, '_module', module_name)
config.remove_option(backend_name, '_backend')
changed = True
except KeyError:
warning(
'Missing field "_module" for configured backend "%s"',
backend_name)
continue
yield backend_name, module_name, params
if changed:
with open(self.confpath, 'wb') as f:
config.write(f)
def add_backend(self, backend_name, module_name, params, edit=False):
"""
Add a backend to config.
:param backend_name: name of the backend in config
:param module_name: name of the Python submodule to run
:param params: params to pass to the module
:type params: :class:`dict`
"""
if not backend_name:
raise ValueError(u'Please give a name to the configured backend.')
config = RawConfigParser()
config.read(self.confpath)
if not edit:
try:
config.add_section(backend_name)
except DuplicateSectionError:
raise BackendAlreadyExists(backend_name)
config.set(backend_name, '_module', module_name)
for key, value in params.iteritems():
if isinstance(value, unicode):
value = value.encode('utf-8')
config.set(backend_name, key, value)
with open(self.confpath, 'wb') as f:
config.write(f)
def _set(key, value):
filename, section, item = key.split(".", 2)
config = RawConfigParser()
config.read("/opt/OpenvStorage/config/{0}.cfg".format(filename))
config.set(section, item, value)
with open("/opt/OpenvStorage/config/{0}.cfg".format(filename), "w") as config_file:
config.write(config_file)
def getversioncfg():
import re
from ConfigParser import RawConfigParser
vd0 = dict(version=FALLBACK_VERSION, commit='', date='', timestamp=0)
# first fetch data from gitarchivecfgfile, ignore if it is unexpanded
g = vd0.copy()
cp0 = RawConfigParser(vd0)
cp0.read(gitarchivecfgfile)
if '$Format:' not in cp0.get('DEFAULT', 'commit'):
g = cp0.defaults()
mx = re.search(r'\btag: v(\d[^,]*)', g.pop('refnames'))
if mx:
g['version'] = mx.group(1)
# then try to obtain version data from git.
gitdir = os.path.join(MYDIR, '.git')
if os.path.exists(gitdir) or 'GIT_DIR' in os.environ:
try:
g = gitinfo()
except OSError:
pass
# finally, check and update the active version file
cp = RawConfigParser()
cp.read(versioncfgfile)
d = cp.defaults()
rewrite = not d or (g['commit'] and (g['version'] != d.get('version')
or g['commit'] != d.get('commit')))
if rewrite:
cp.set('DEFAULT', 'version', g['version'])
cp.set('DEFAULT', 'commit', g['commit'])
cp.set('DEFAULT', 'date', g['date'])
cp.set('DEFAULT', 'timestamp', g['timestamp'])
cp.write(open(versioncfgfile, 'w'))
return cp
def my_cnf_inspect(self):
"""
Fix nonexistent paths to log-error and pid-file
"""
self.my_cnf_manager('backup')
track = {
'files': ('log-error', ),
'paths': ('pid-file', )
}
default_log = '/var/lib/mysql/mysqld.error.log'
default_pid = '/var/lib/mysql/mysqld.pid'
conf = RawConfigParser(allow_no_value=True)
conf.read('/etc/my.cnf.prev')
# try to find non-existent paths, defined in /etc/my.cnf
for s in conf.sections():
for opt, val in conf.items(s):
if opt in track['files']:
# inspect whole path
if not os.path.exists(val):
print 'NO LOG for {opt} --> {v}'.format(opt=opt, v=val)
conf.set(s, opt, default_log)
elif opt in track['paths']:
# inspect dir path
if not os.path.exists(os.path.dirname(val)):
print 'NO PATH for {opt} --> {v}'.format(opt=opt, v=val)
conf.set(s, opt, default_pid)
with open('/etc/my.cnf', 'wb') as configfile:
conf.write(configfile)
def save():
"""Saves FileDirectives into ConfigFile."""
section = "*"
module = sys.modules[__name__]
parser = RawConfigParser()
parser.optionxform = str # Force case-sensitivity on names
parser.add_section(section)
try:
f, fname = open(ConfigFile, "wb"), util.longpath(ConfigFile)
f.write("# %s %s configuration written on %s.\n" %
(Title, Version,
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
for name in FileDirectives:
try:
parser.set(section, name, json.dumps(getattr(module, name)))
except Exception:
pass
for name in OptionalFileDirectives:
try:
value = getattr(module, name, None)
if OptionalFileDirectiveDefaults.get(name) != value:
parser.set(section, name, json.dumps(value))
except Exception:
pass
parser.write(f)
f.close()
except Exception:
pass # Fail silently
def save(self, filename, private=False):
"""
Save repository into a file (modules.list for example).
:param filename: path to file to save repository.
:type filename: str
:param private: if enabled, save URL of repository.
:type private: bool
"""
config = RawConfigParser()
config.set(DEFAULTSECT, 'name', self.name)
config.set(DEFAULTSECT, 'update', self.update)
config.set(DEFAULTSECT, 'maintainer', self.maintainer)
config.set(DEFAULTSECT, 'signed', int(self.signed))
config.set(DEFAULTSECT, 'key_update', self.key_update)
if private:
config.set(DEFAULTSECT, 'url', self.url)
for module in self.modules.values():
config.add_section(module.name)
for key, value in module.dump():
if sys.version_info.major == 2:
# python2's configparser enforces bytes coercion with str(value)...
config.set(module.name, key, to_unicode(value).encode('utf-8'))
else:
config.set(module.name, key, value)
with open_for_config(filename) as f:
config.write(f)
class ApplicationConfig(object):
"""A thin wrapper around ConfigParser that remembers what we read.
The remembered settings can then be written out to a minimal config file
when building the Elastic Beanstalk zipfile.
"""
def __init__(self):
self.input = RawConfigParser()
config_filename = os.environ.get("CONFIG", "production.ini")
with open(config_filename) as f:
self.input.readfp(f)
self.output = RawConfigParser()
def get(self, section, key):
value = self.input.get(section, key)
# remember that we needed this configuration value
if (section.upper() != "DEFAULT"
and not self.output.has_section(section)):
self.output.add_section(section)
self.output.set(section, key, value)
return value
def to_config(self):
io = cStringIO.StringIO()
self.output.write(io)
return io.getvalue()
def save(self):
config = RawConfigParser()
for name, version in self.versions.items():
config.set(DEFAULTSECT, name, version)
with open_for_config(os.path.join(self.path, self.VERSIONS_LIST)) as fp:
config.write(fp)
def save(self):
config = RawConfigParser()
config.add_section('GENERAL')
config.set('GENERAL', 'number_of_devices', self.number_of_devices)
config.set('GENERAL', 'device_code_name', self.device_code_name)
config.set('GENERAL', 'device_type', self.device_type)
config.add_section('LOGGING')
config.set('LOGGING', 'log_level', self.log_level)
config.set('LOGGING', 'log_folder', self.log_folder)
config.add_section('PATHS')
config.set('PATHS', 'srfpc_path', self.srfpc_path)
config.set('PATHS', 'eac_path', self.eac_path)
config.set('PATHS', 'cc2541_firmware_path', self.cc2541_firmware_path)
config.set('PATHS', 'efm32_firmware_path', self.efm32_firmware_path)
config.set('PATHS', 'efm32_firmware_path_final',
self.efm32_firmware_path_final)
config.add_section('EB IDs and J-LINK S/Ns')
config.set('EB IDs and J-LINK S/Ns', 'eb_ids',
','.join(self.eb_id_list))
config.set('EB IDs and J-LINK S/Ns', 'jlink_sn',
','.join(self.jlink_sn_list))
with open('settings.cfg', 'wb') as configfile:
config.write(configfile)
def dump(cls, data, fh):
cp = ConfigParser()
if isinstance(cp.__module__, bytes): # pragma: nocover
_fh = fh
encode = lambda x: x.encode('utf8')
else: # pragma: nocover
_fh = codecs.getwriter('utf8')(fh)
encode = lambda x: x
def _set(cp, section, key, value):
cp.set(encode(section), encode(key), encode(value))
for i, _item in enumerate(data):
item = map_keys(_item, cls.fields, reverse=True)
section = _str(i)
cp.add_section(encode(section))
for key in item:
if key == 'bday':
dt = item.first(key)
if dt.year == 1900:
value = dt.strftime('--%m-%d')
else:
value = dt.strftime('%Y-%m-%d')
_set(cp, section, key, value)
elif key in cls.fields:
_set(cp, section, key, item.join(key))
cp.write(_fh)
def create_config_file(config_file, random_music_home):
"""
Create a configuration file.
:param config_file: path to config file we are creating
:type config_file: str
:param random_music_home: home of random_music application (i.e. where
index files are stored
:type random_music_home: str
"""
sys.stdout.write("You do not appear to have a config file, lets create one!\n")
sys.stdout.write("Creating config file at %s\n" % config_file)
config = RawConfigParser()
config.add_section('config')
config.set('config', 'loop_songs', 'true')
config.set('config', 'randomise', 'true')
config.set('config', 'index_dir', os.path.join(random_music_home,
"indicies"))
music_client = DEFAULT_MUSIC_CLIENT
while not which(music_client):
music_client = raw_input("The music player '%s' could not be found "
"on your path. Please input a different "
"music player:" % music_client)
config.set('config', 'music_client', music_client)
user_music_dirs = ""
while not all([os.path.isdir(d) for d in user_music_dirs.split(",")]):
user_music_dirs = raw_input("Input a csv list of full paths to "
"your music dirs:")
config.set('config', 'music_dirs', user_music_dirs)
with open(config_file, 'wb') as fh:
config.write(fh)
def setFilenameInConfigFile(self, root_path, name, filename):
"""Add new filename association in config file
@root_path: Path where config file is stored
@param name: Field name
@param filename: Filename of value stored in field
"""
path = os.path.join(root_path, self.cfg_filename)
# Update file
config = RawConfigParser()
if os.path.exists(path):
# Read file
fd = open(path, 'r')
try:
config.readfp(fd)
finally:
fd.close()
# Create section if it doesn't exist
if not config.has_section(self.cfg_filename_section):
config.add_section(self.cfg_filename_section)
config.set(self.cfg_filename_section, name, filename)
fd = open(path, 'w')
try:
# Write file
config.write(fd)
finally:
fd.close()
def write(self, config_data, filepath=None):
"""
Create a dotfile from keyword arguments.
:param config_data:
Dict of config settings
:param filepath:
(Optional) Path to write
"""
if filepath is None:
filepath = self.filepath
config = RawConfigParser()
section = constants.SECTION_NAME
config.add_section(section)
# Set the config settings
for key, val in config_data.iteritems():
config.set(section, key, val)
with open(filepath, 'wb') as dotfile:
config.write(dotfile)
self.enforce_perms()
log.debug('wrote %s' % filepath)
def add_config(self, options):
"""Creates analysis.conf file from current analysis options.
@param options: current configuration options, dict format.
@return: operation status.
"""
global ERROR_MESSAGE
if type(options) != dict:
return False
config = RawConfigParser()
config.add_section("analysis")
try:
for key, value in options.items():
# Options can be UTF encoded.
if isinstance(value, basestring):
try:
value = value.encode("utf-8")
except UnicodeEncodeError:
pass
config.set("analysis", key, value)
config_path = os.path.join(ANALYZER_FOLDER, "analysis.conf")
with open(config_path, "wb") as config_file:
config.write(config_file)
except Exception as e:
ERROR_MESSAGE = str(e)
return False
return True
def save(self):
config = RawConfigParser()
for name, version in self.versions.items():
config.set(DEFAULTSECT, name, version)
with open_for_config(os.path.join(self.path, self.VERSIONS_LIST)) as fp:
config.write(fp)
def save_config(self):
"""Save the current configuration to the config file.
"""
self.log.debug("Saving config file to '%s'", self.config_file)
if not self.config_file:
self.log.warning('%s.config_file file not set, not saving config!',
self.__class__.__name__)
return
self.acquire_lock()
config = RawConfigParser()
for section_name, section in self.config._config.items():
config.add_section(section_name)
for option_name, value in section.items():
if section_name == 'general':
if option_name in ['save_config']:
continue
config.set(section_name, option_name, str(value))
with NamedTemporaryFile(mode='w',
dir=os.path.dirname(self.config_file),
delete=False) as tmpfile:
config.write(tmpfile)
# Move the new config file into place atomically
if os.path.getsize(tmpfile.name) > 0:
os.rename(tmpfile.name, self.config_file)
else:
self.log.warning(
'Config file saving failed, not replacing %s with %s.',
self.config_file, tmpfile.name)
self.release_lock()
def _writefp(cls, fp, data):
config = RawConfigParser()
for section_name, section in data.items():
config.add_section(section_name)
for k, v in section.items():
config.set(section_name, k, v)
config.write(fp)
def tearDownClass(cls):
try:
# modify the testdata back
with open(TEST_DATA_DICT, "r") as fileobj:
test_data = yaml.load(fileobj, Loader=yaml.CLoader)
test_data['iris']['path'] = op.join("testdata", "iris.csv")
test_data['person_activity']['path'] = op.join("testdata",
"person_activity.tsv")
del test_data['multi_iris']
with open(TEST_DATA_DICT, "w") as fileobj:
test_data = yaml.dump(test_data, fileobj, Dumper=yaml.CDumper,
default_flow_style=False)
# Change the config files back
parser = RawConfigParser()
parser.read(cls.test_conf_file)
parser.remove_option("pysemantic", "specfile")
parser.set("pysemantic", "specfile",
op.join("testdata", "test_dictionary.yaml"))
with open(TEST_CONFIG_FILE_PATH, 'w') as fileobj:
parser.write(fileobj)
finally:
os.unlink(cls.test_conf_file)
os.unlink(cls.copied_iris_path)
def save():
"""Saves FileDirectives into ConfigFile."""
section = "*"
module = sys.modules[__name__]
parser = RawConfigParser()
parser.optionxform = str # Force case-sensitivity on names
parser.add_section(section)
try:
f, fname = open(ConfigFile, "wb"), util.longpath(ConfigFile)
f.write("# %s configuration autowritten on %s.\n" %
(fname, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
for name in FileDirectives:
try:
parser.set(section, name, json.dumps(getattr(module, name)))
except Exception: pass
for name in OptionalFileDirectives:
try:
value = getattr(module, name, None)
if OptionalFileDirectiveDefaults.get(name) != value:
parser.set(section, name, json.dumps(value))
except Exception: pass
parser.write(f)
f.close()
except Exception:
pass # Fail silently
def _path_fixer(filepath, root=None):
"""Change all the relative paths in `filepath` to absolute ones.
:param filepath: File to be changed
:param root: Root path with which the relative paths are prefixed. If None
(default), the directory with this file is the root.
"""
if root is None:
root = op.join(op.abspath(op.dirname(__file__)))
if filepath.endswith((".yaml", ".yml")):
with open(filepath, "r") as fileobj:
data = yaml.load(fileobj, Loader=Loader)
for specs in data.itervalues():
specs['path'] = op.join(root, specs['path'])
with open(filepath, "w") as fileobj:
yaml.dump(data, fileobj, Dumper=Dumper, default_flow_style=False)
elif filepath.endswith(".conf"):
parser = RawConfigParser()
parser.read(filepath)
for section in parser.sections():
path = parser.get(section, "specfile")
parser.remove_option(section, "specfile")
parser.set(section, "specfile", op.join(root, path))
with open(filepath, "w") as fileobj:
parser.write(fileobj)
def iter_backends(self):
"""
Iterate on backends.
:returns: each tuple contains the backend name, module name and module options
:rtype: :class:`tuple`
"""
config = RawConfigParser()
config.read(self.confpath)
changed = False
for backend_name in config.sections():
params = dict(config.items(backend_name))
try:
module_name = params.pop('_module')
except KeyError:
try:
module_name = params.pop('_backend')
config.set(backend_name, '_module', module_name)
config.remove_option(backend_name, '_backend')
changed = True
except KeyError:
warning('Missing field "_module" for configured backend "%s"', backend_name)
continue
yield backend_name, module_name, params
if changed:
with open(self.confpath, 'wb') as f:
config.write(f)
def init_admin_repository(
git_dir,
pubkey,
user,
):
repository.init(
path=git_dir,
template=resource_filename('gitosis.templates', 'admin')
)
repository.init(
path=git_dir,
)
hook = os.path.join(git_dir, 'hooks', 'post-update')
os.chmod(hook, 0755)
if not repository.has_initial_commit(git_dir):
log.info('Making initial commit...')
# ConfigParser does not guarantee order, so jump through hoops
# to make sure [gitosis] is first
cfg_file = StringIO()
print >>cfg_file, '[gitosis]'
print >>cfg_file
cfg = RawConfigParser()
cfg.add_section('group gitosis-admin')
cfg.set('group gitosis-admin', 'members', user)
cfg.set('group gitosis-admin', 'writable', 'gitosis-admin')
cfg.write(cfg_file)
initial_commit(
git_dir=git_dir,
cfg=cfg_file.getvalue(),
pubkey=pubkey,
user=user,
)
def load_configs(shutit):
"""Responsible for loading config files into ShutIt.
Recurses down from configured shutit module paths.
"""
cfg = shutit.cfg
# Get root default config.
configs = [('defaults', StringIO.StringIO(_default_cnf))]
# Add the shutit global host- and user-specific config file.
configs.append(os.path.join(shutit.shutit_main_dir,
'configs/' + socket.gethostname() + '_' + cfg['host']['real_user'] + '.cnf'))
configs.append(os.path.join(cfg['shutit_home'], 'config'))
# Add the local build.cnf
configs.append('configs/build.cnf')
# Get passed-in config(s)
for config_file_name in cfg['build']['extra_configs']:
run_config_file = os.path.expanduser(config_file_name)
if not os.path.isfile(run_config_file):
print('Did not recognise ' + run_config_file +
' as a file - do you need to touch ' + run_config_file + '?')
sys.exit()
configs.append(run_config_file)
# Image to use to start off. The script should be idempotent, so running it
# on an already built image should be ok, and is advised to reduce diff space required.
if cfg['build']['interactive'] >= 3 or cfg['action']['show_config']:
msg = ''
for c in configs:
if type(c) is tuple:
c = c[0]
msg = msg + '\t\n' + c
shutit.log('\t' + c)
if cfg['build']['interactive'] >= 3:
print textwrap.dedent("""\n""") + msg + textwrap.dedent("""
Looking at config files in the above order (even if they
do not exist - you may want to create them).
If you get a "Port already in use:" error,
run:
docker ps -a | grep -w <port> | awk '{print $1}' | xargs docker kill
or
sudo docker ps -a | grep -w <port> | awk '{print $1}' | xargs sudo docker kill
""" + colour('31','[Hit return to continue]'))
raw_input('')
# Interpret any config overrides, write to a file and add them to the
# list of configs to be interpreted
if cfg['build']['config_overrides']:
# We don't need layers, this is a temporary configparser
override_cp = RawConfigParser()
for o_sec, o_key, o_val in cfg['build']['config_overrides']:
if not override_cp.has_section(o_sec):
override_cp.add_section(o_sec)
override_cp.set(o_sec, o_key, o_val)
override_fd = StringIO.StringIO()
override_cp.write(override_fd)
override_fd.seek(0)
configs.append(('overrides', override_fd))
cfg_parser = get_configs(shutit,configs)
get_base_config(cfg, cfg_parser)
def save(self, filename=None):
if filename is None:
filename = self.write_path
self.before_save()
config_parser = RawConfigParser()
#config_parser.readfp(
for section_name, section in self.sections.items():
config_parser.add_section(section_name)
for item in section.items:
type_process = {
str: str,
bool: str,
int: str,
float: str,
"pickle": do_pickling
}[section.item_types[item]]
# look it up now. If this is a lazily evaluated item, find its
# value before we close
# TODO: is this what we really want to do?
value = section[item]
config_parser.set(section_name, item, type_process(value))
directory = os.path.dirname(filename)
if not os.path.exists(directory):
os.makedirs(directory)
config_parser.write(open(filename, "w"))
class OriginAuthStore(object):
def __init__(self, config_file):
self.config_file = config_file
self.config = RawConfigParser()
self.config.read(config_file)
def origin(self, name):
return OriginAuth(self, name)
def __getitem__(self, origin_name):
try:
return dict(self.config.items(origin_name))
except NoSectionError:
return {}
def __setitem__(self, origin_name, auth):
try:
self.config.remove_section(origin_name)
except NoSectionError:
pass
if auth:
self.config.add_section(origin_name)
for key, val in auth.iteritems():
self.config.set(origin_name, key, val)
with open(self.config_file, 'w') as f:
self.config.write(f)
try:
os.chmod(self.config_file, stat.S_IRUSR | stat.S_IWUSR)
except OSError:
print 'Unable to chmod 600 %s' % self.config_file # TODO: Test
def save(self, filename, private=False):
"""
Save repository into a file (modules.list for example).
:param filename: path to file to save repository.
:type filename: str
:param private: if enabled, save URL of repository.
:type private: bool
"""
config = RawConfigParser()
config.set(DEFAULTSECT, 'name', self.name)
config.set(DEFAULTSECT, 'update', self.update)
config.set(DEFAULTSECT, 'maintainer', self.maintainer)
config.set(DEFAULTSECT, 'signed', int(self.signed))
config.set(DEFAULTSECT, 'key_update', self.key_update)
if private:
config.set(DEFAULTSECT, 'url', self.url)
for module in self.modules.itervalues():
config.add_section(module.name)
for key, value in module.dump():
config.set(module.name, key, to_unicode(value).encode('utf-8'))
with open(filename, 'wb') as f:
config.write(f)
class Configuration(object):
def __init__(self, filename='.jira.cfg'):
self.parser = RawConfigParser()
filepath = os.path.expanduser(filename)
if os.path.exists(filepath): self.parser.read(filepath)
else:
# Set up config file
self.parser.add_section('jira_default')
self.parser.set('jira_default','username',raw_input('username: '******'password: '******'confirm: ')
if passwd != passwd2: print 'passwords do not match.'
else: passwd_confirmed = True
self.parser.set('jira_default','password', passwd)
self.parser.set('jira_default','host',raw_input('host (e.g jira.atlassian.com): '))
self.parser.set('jira_default','path','/rest/api/latest')
# Color-coded statuses
self.parser.add_section('colors')
self.parser.set('colors','Closed','blue')
self.parser.set('colors','Resolved','green')
self.parser.set('colors','In Progress','magenta')
f = open(filepath, 'w')
self.parser.write(f)
os.chmod(filepath, 0600) #Only user can read this
def write_sts_token(self, profile, access_key_id, secret_access_key,
session_token):
""" Writes STS auth information to credentials file """
region = 'us-east-1'
output = 'json'
if not os.path.exists(self.creds_dir):
os.makedirs(self.creds_dir)
config = RawConfigParser()
if os.path.isfile(self.creds_file):
config.read(self.creds_file)
if not config.has_section(profile):
config.add_section(profile)
config.set(profile, 'output', output)
config.set(profile, 'region', region)
config.set(profile, 'aws_access_key_id', access_key_id)
config.set(profile, 'aws_secret_access_key', secret_access_key)
config.set(profile, 'aws_session_token', session_token)
with open(self.creds_file, 'w+') as configfile:
config.write(configfile)
print("Temporary credentials written to profile: %s" % profile)
print("Invoke using: aws --profile %s <service> <command>" % profile)
class Configuration(object):
def __init__(self, filename):
self.parser = RawConfigParser()
filepath = os.path.expanduser(filename)
if os.path.exists(filepath): self.parser.read(filepath)
else:
# Set up config file
self.parser.add_section('jira_default')
self.parser.set('jira_default', 'username', raw_input('username: '******'password: '******'confirm: ')
if passwd != passwd2: print 'passwords do not match.'
else: passwd_confirmed = True
self.parser.set('jira_default', 'password', passwd)
self.parser.set('jira_default', 'host', raw_input('host (e.g jira.atlassian.com): '))
self.parser.set('jira_default', 'path', '/rest/api/latest')
# Color-coded statuses
#self.parser.set('colors', 'Resolved', 'green')
#self.parser.set('colors', 'In Progress', 'magenta')
f = open(filepath, 'w')
self.parser.write(f)
os.chmod(filepath, 0600) #Only user can read this
def install_mercurial_hook():
"""
Installs the mercurial precommit hook by adding a hook to the hgrc
file in the .hg directory of the repository.
"""
repo_dir = get_repo_dir()
config_file = os.path.join(repo_dir, '.hg', 'hgrc')
config_parser = RawConfigParser()
config_parser.read(config_file)
precommit_abs_file = os.path.join(repo_dir, 'scripts',
'codestyleprecommit.py')
section = 'hooks'
key = 'pretxncommit.precommit'
value = 'python:%s:mercurial_hook' % precommit_abs_file
if not config_parser.has_section(section):
config_parser.add_section(section)
config_parser.set(section, key, value)
with open(config_file, 'w') as config:
config_parser.write(config)
def set_schema_fpath(project_name, schema_fpath):
"""Set the schema path for a given project.
:param project_name: Name of the project
:param schema_fpath: path to the yaml file to be used as the schema for \
the project.
:type project_name: str
:type schema_fpath: str
:return: True, if setting the schema path was successful.
:Example:
>>> set_schema_fpath('skynet', '/path/to/new/schema.yaml')
True
"""
path = locate_config_file()
parser = RawConfigParser()
parser.read(path)
if project_name in parser.sections():
if not parser.remove_option(project_name, "specfile"):
raise MissingProject
else:
parser.set(project_name, "specfile", schema_fpath)
with open(path, "w") as f:
parser.write(f)
return True
raise MissingProject
def install_mercurial_hook():
"""
Installs the mercurial precommit hook by adding a hook to the hgrc
file in the .hg directory of the repository.
"""
repo_dir = get_repo_dir()
config_file = os.path.join(repo_dir, '.hg', 'hgrc')
config_parser = RawConfigParser()
config_parser.read(config_file)
precommit_abs_file = os.path.join(repo_dir, 'scripts',
'codestyleprecommit.py')
section = 'hooks'
key = 'pretxncommit.precommit'
value = 'python:%s:mercurial_hook' % precommit_abs_file
if not config_parser.has_section(section):
config_parser.add_section(section)
config_parser.set(section, key, value)
with open(config_file, 'w') as config:
config_parser.write(config)
def save(self, filename, private=False):
"""
Save repository into a file (modules.list for example).
:param filename: path to file to save repository.
:type filename: str
:param private: if enabled, save URL of repository.
:type private: bool
"""
config = RawConfigParser()
config.set(DEFAULTSECT, 'name', self.name)
config.set(DEFAULTSECT, 'update', self.update)
config.set(DEFAULTSECT, 'maintainer', self.maintainer)
config.set(DEFAULTSECT, 'signed', int(self.signed))
config.set(DEFAULTSECT, 'key_update', self.key_update)
if private:
config.set(DEFAULTSECT, 'url', self.url)
for module in self.modules.itervalues():
config.add_section(module.name)
for key, value in module.dump():
config.set(module.name, key, to_unicode(value).encode('utf-8'))
with open(filename, 'wb') as f:
config.write(f)
def modify_node_value(self, master_node=None, child_node=None, value=''):
#todo 节点信息和配置文件路径不能为空
if not master_node or not child_node:
raise UserWarning(
'Please fill in the complete node parameter information!!!')
if not self.config_file_path:
raise UserWarning('Please specify the profile path!!!')
conf = RawConfigParser()
conf_status = conf.read(self.config_file_path)
#todo 验证配置文件路径是否正确
if conf_status:
try:
conf.set(section=master_node, option=child_node, value=value)
conf.write(open(self.config_file_path, 'w'))
except NoSectionError as e:
raise UserWarning(
'Please check the machine.conf file,section[ %s ] '
'is not exist!' % (e.section))
except AttributeError:
raise UserWarning(
'The profile node information is not found!!!')
else:
raise UserWarning(
'Please check that the configuration file path is correct!!!')
def getversioncfg():
import re
from ConfigParser import RawConfigParser
vd0 = dict(version=FALLBACK_VERSION, commit='', date='', timestamp=0)
# first fetch data from gitarchivecfgfile, ignore if it is unexpanded
g = vd0.copy()
cp0 = RawConfigParser(vd0)
cp0.read(gitarchivecfgfile)
if '$Format:' not in cp0.get('DEFAULT', 'commit'):
g = cp0.defaults()
mx = re.search(r'\btag: v(\d[^,]*)', g.pop('refnames'))
if mx:
g['version'] = mx.group(1)
# then try to obtain version data from git.
gitdir = os.path.join(MYDIR, '.git')
if os.path.exists(gitdir) or 'GIT_DIR' in os.environ:
try:
g = gitinfo()
except OSError:
pass
# finally, check and update the active version file
cp = RawConfigParser()
cp.read(versioncfgfile)
d = cp.defaults()
rewrite = not d or (g['commit'] and (
g['version'] != d.get('version') or g['commit'] != d.get('commit')))
if rewrite:
cp.set('DEFAULT', 'version', g['version'])
cp.set('DEFAULT', 'commit', g['commit'])
cp.set('DEFAULT', 'date', g['date'])
cp.set('DEFAULT', 'timestamp', g['timestamp'])
cp.write(open(versioncfgfile, 'w'))
return cp
def set_schema_fpath(project_name, schema_fpath):
"""Set the schema path for a given project.
:param project_name: Name of the project
:param schema_fpath: path to the yaml file to be used as the schema for \
the project.
:type project_name: str
:type schema_fpath: str
:return: True, if setting the schema path was successful.
:Example:
>>> set_schema_fpath('skynet', '/path/to/new/schema.yaml')
True
"""
path = locate_config_file()
parser = RawConfigParser()
parser.read(path)
if project_name in parser.sections():
if not parser.remove_option(project_name, "specfile"):
raise MissingProject
else:
parser.set(project_name, "specfile", schema_fpath)
with open(path, "w") as f:
parser.write(f)
return True
raise MissingProject
def my_cnf_inspect(self):
"""
Fix nonexistent paths to log-error and pid-file
"""
self.my_cnf_manager('backup')
track = {'files': ('log-error', ), 'paths': ('pid-file', )}
default_log = '/var/lib/mysql/mysqld.error.log'
default_pid = '/var/lib/mysql/mysqld.pid'
conf = RawConfigParser(allow_no_value=True)
conf.read('/etc/my.cnf.prev')
# try to find non-existent paths, defined in /etc/my.cnf
for s in conf.sections():
for opt, val in conf.items(s):
if opt in track['files']:
# inspect whole path
if not os.path.exists(val):
print 'NO LOG for {opt} --> {v}'.format(opt=opt, v=val)
conf.set(s, opt, default_log)
elif opt in track['paths']:
# inspect dir path
if not os.path.exists(os.path.dirname(val)):
print 'NO PATH for {opt} --> {v}'.format(opt=opt,
v=val)
conf.set(s, opt, default_pid)
with open('/etc/my.cnf', 'wb') as configfile:
conf.write(configfile)
def getAccount(account_key):
from ConfigParser import RawConfigParser, NoOptionError, NoSectionError
account_file = '.account'
config = RawConfigParser()
with open(account_file, 'r') as fp:
config.readfp(fp)
account = config.get('account', account_key)
password = None
password_section = 'password'
try:
password = config.get(password_section, account_key)
except NoSectionError:
config.add_section(password_section)
except NoOptionError:
pass
aes = AESCipher(account)
if password:
return account, aes.decrypt(password).encode('UTF-8')
from getpass import getpass
password = getpass(account_key + ' of ' +account + "'s password: ")
config.set(password_section, account_key, aes.encrypt(password))
with open(account_file, 'w') as fp:
config.write(fp)
return account, password
def setUpClass(cls):
os.environ['PYSEMANTIC_CONFIG'] = "test.conf"
pr.CONF_FILE_NAME = "test.conf"
cls.testenv = os.environ
cls.test_config_path = op.join(os.getcwd(), "test.conf")
shutil.copy(TEST_CONFIG_FILE_PATH, cls.test_config_path)
# Change the relative paths in the config file to absolute paths
parser = RawConfigParser()
parser.read(cls.test_config_path)
for section in parser.sections():
schema_path = parser.get(section, "specfile")
parser.remove_option(section, "specfile")
parser.set(section, "specfile",
op.join(op.abspath(op.dirname(__file__)), schema_path))
with open(cls.test_config_path, "w") as fileobj:
parser.write(fileobj)
# change the relative paths in the test dictionary to absolute paths
with open(TEST_DATA_DICT, "r") as fileobj:
cls.org_specs = yaml.load(fileobj, Loader=yaml.CLoader)
new_specs = deepcopy(cls.org_specs)
for _, specs in new_specs.iteritems():
path = specs['path']
specs['path'] = op.join(op.abspath(op.dirname(__file__)), path)
# Rewrite this to the file
with open(TEST_DATA_DICT, "w") as fileobj:
yaml.dump(new_specs, fileobj, Dumper=yaml.CDumper,
default_flow_style=False)
def install_mercurial_hook():
"""
Installs the mercurial precommit hook by adding a hook to the hgrc
file in the .hg directory of the repository.
"""
repo_dir = get_repo_dir()
config_file = os.path.join(repo_dir, ".hg", "hgrc")
config_parser = RawConfigParser()
config_parser.read(config_file)
precommit_abs_file = os.path.join(repo_dir, "scripts", "codestyleprecommit.py")
section = "hooks"
key = "pretxncommit.precommit"
value = "python:%s:mercurial_hook" % precommit_abs_file
if not config_parser.has_section(section):
config_parser.add_section(section)
config_parser.set(section, key, value)
with open(config_file, "w") as config:
config_parser.write(config)
def update_config_file(payload_filename, config_path):
# type: (str, str) -> None
"""Update the configuration file."""
original_config = ConfigParser()
original_config.read(config_path)
# new config file object
config = RawConfigParser()
# update the info section
config.add_section('info')
options = original_config.options('info')
for option in options:
if option != "payloadpath":
config.set('info', option, original_config.get('info', option))
else:
dirname = os.path.dirname(
original_config.get('info', 'payloadpath'))
filepath = os.path.join(dirname, payload_filename)
config.set('info', option, filepath)
# update the context section
config.add_section('context')
dirname = os.path.dirname(original_config.get('context',
'update_path'))
filepath = os.path.join(dirname, payload_filename)
config.set('context', 'update_path', filepath)
with open(config_path, 'wb') as configfile:
config.write(configfile)
class CredentialsFile(object):
def __init__(self, filename=path.expanduser('~/.aws/credentials')):
self._filename = filename
self._config = RawConfigParser()
with open(self._filename, 'r') as f:
self._config.readfp(f)
@property
def keyId(self):
return self._config.get('default', 'aws_access_key_id')
@property
def secretKey(self):
return self._config.get('default', 'aws_secret_access_key')
def updateCredentials(self, keyId, secretKey):
"""Write new credentials to the config file.
I'll also set the umask to 0066, but since I'm the only thing writing
files you shouldn't mind.
"""
self._config.set('default', 'aws_access_key_id', keyId)
self._config.set('default', 'aws_secret_access_key', secretKey)
os.umask(0o0066)
os.rename(self._filename, self._filename+'~')
with open(self._filename, 'w') as f:
self._config.write(f)
