def _test_option(self, option):
setattr(self.cnf, option, 'value')
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('value', c.get(option))
setattr(self.cnf, option, None)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertRaises(NoPathError, c.get, option)
def _test_numeric_option(self, option):
self.assertRaises(ValueError, setattr, self.cnf, option, 'NotNumericValue')
setattr(self.cnf, option, 113)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('113', c.get(option))
self.assertEqual(113, getattr(self.cnf, option))
setattr(self.cnf, option, None)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertRaises(NoPathError, c.get, option)
def _test_path_option(self, option):
self.assertRaises(ValueError, setattr, self.cnf, option, '/not/exists')
setattr(self.cnf, option, '/tmp')
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('/tmp', c.get(option))
self.assertEqual('/tmp', getattr(self.cnf, option))
def get_system_variables(self):
#TESTING REQUIRED!
conf = Configuration(self._config_format)
conf.read(self._config_path)
vars = {}
for section in conf.sections('./'):
vars[section] = conf.get(section)
return vars
def __init__(self, body):
config = Configuration("apache")
try:
config.reads(str(body))
except ParseError, e:
LOG.error(
"MetaConf failed to parse Apache VirtualHost body: \n%s" %
body)
e._err = body + "\n" + e._err
raise
def _https_config_exists(self):
config_dir = os.path.dirname(self.api.app_inc_path)
conf_path = os.path.join(config_dir, 'https.include')
config = None
try:
config = Configuration('nginx')
config.read(conf_path)
except (Exception, BaseException), e:
raise HandlerError('Cannot read/parse nginx main configuration file: %s' % str(e))
def _reload_backends(self):
self._logger.info('Updating mysql-proxy backends list')
self.config = Configuration('mysql')
if os.path.exists(CONFIG_FILE_PATH):
self.config.read(CONFIG_FILE_PATH)
self.config.remove('./mysql-proxy/proxy-backend-addresses')
self.config.remove(
'./mysql-proxy/proxy-read-only-backend-addresses')
try:
self.config.get('./mysql-proxy')
except NoPathError:
self.config.add('./mysql-proxy')
queryenv = bus.queryenv_service
roles = queryenv.list_roles()
master = None
slaves = []
for role in roles:
if not is_mysql_role(role.behaviour):
continue
for host in role.hosts:
ip = host.internal_ip or host.external_ip
if host.replication_master:
master = ip
else:
slaves.append(ip)
if master:
self._logger.debug(
'Adding mysql master %s to mysql-proxy defaults file', master)
self.config.add('./mysql-proxy/proxy-backend-addresses',
'%s:3306' % master)
if slaves:
self._logger.debug(
'Adding mysql slaves to mysql-proxy defaults file: %s',
', '.join(slaves))
for slave in slaves:
self.config.add(
'./mysql-proxy/proxy-read-only-backend-addresses',
'%s:3306' % slave)
self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
self.config.set('./mysql-proxy/daemon', 'true', force=True)
self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
if self.service.version > (0, 8, 0):
self.config.set('./mysql-proxy/plugins', 'proxy', force=True)
self._logger.debug('Saving new mysql-proxy defaults file')
self.config.write(CONFIG_FILE_PATH)
os.chmod(CONFIG_FILE_PATH, 0660)
self.service.restart()
def load(self, preset_type):
'''
@rtype: Preset
@raise OSError: When cannot read preset file
@raise MetaconfError: When experience problems with preset file parsing
'''
self._logger.debug('Loading %s %s preset' %
(preset_type, self.service_name))
ini = Configuration('ini')
ini.read(self._filename(preset_type))
return CnfPreset(ini.get('general/name'), dict(ini.items('settings/')))
def _manifest(self):
f_manifest = CnfController._manifest
base_manifest = f_manifest.fget(self)
path = self._manifest_path
s = {}
out = None
if not self._merged_manifest:
cmd = '%s --no-defaults --verbose SU_EXEC' % mysql_svc.MYSQLD_PATH
out = system2('%s - mysql -s %s -c "%s"' % (SU_EXEC, BASH, cmd),
shell=True, raise_exc=False, silent=True)[0]
if out:
raw = out.split(49*'-'+' '+24*'-')
if raw:
a = raw[-1].split('\n')
if len(a) > 5:
b = a[1:-5]
for item in b:
c = item.split()
if len(c) > 1:
key = c[0]
val = ' '.join(c[1:])
s[key.strip()] = val.strip()
if s:
m_config = Configuration('ini')
if os.path.exists(path):
m_config.read(path)
for variable in base_manifest:
name = variable.name
dv_path = './%s/default-value' % name
try:
old_value = m_config.get(dv_path)
if name in s:
new_value = s[name]
else:
name = name.replace('_','-')
if name in s:
new_value = self.definitions[s[name]] if s[name] in self.definitions else s[name]
if old_value != new_value and new_value != '(No default value)':
LOG.debug('Replacing %s default value %s with precompiled value %s',
name, old_value, new_value)
m_config.set(path=dv_path, value=new_value, force=True)
except NoPathError:
pass
m_config.write(path)
self._merged_manifest = _CnfManifest(path)
return self._merged_manifest
def __init__(self, manifest_path):
self._options = []
ini = Configuration('ini')
ini.read(manifest_path)
try:
self._defaults = dict(ini.items('__defaults__'))
except NoPathError:
self._defaults = dict()
for name in ini.sections("./"):
if name == '__defaults__':
continue
self._options.append(
_OptionSpec.from_ini(ini, name, self._defaults))
def download_and_restore(self, volume, snapshot, tranzit_path):
# Load manifest
clear_queue(self._writer_queue)
clear_queue(self._download_queue)
self._download_finished.clear()
transfer = self._transfer_cls()
mnf_path = transfer.download(snapshot.path, tranzit_path)
mnf = Configuration('ini')
mnf.read(mnf_path)
volume.fs_created = False
volume.mkfs(snapshot.fstype)
remote_path = os.path.dirname(snapshot.path)
# Get links with md5 sums
links = [(os.path.join(remote_path, chunk[0]), chunk[1])
for chunk in mnf.items('chunks')]
links.sort()
# Download 2 first chunks
for link in links[:2]:
transfer.download(link[0], tranzit_path)
chunk_path = os.path.join(tranzit_path, os.path.basename(link[0]))
if self._md5sum(chunk_path) != link[1]:
raise Exception("md5sum of chunk %s is not correct." %
chunk_path)
self._writer_queue.put(chunk_path)
if hasattr(snapshot,
'snap_strategy') and snapshot.snap_strategy == 'data':
restore_strategy = DataRestoreStrategy(self._logger)
else:
restore_strategy = DeviceRestoreStrategy(self._logger)
writer = threading.Thread(target=restore_strategy.restore,
name='writer',
args=(self._writer_queue, volume,
self._download_finished))
writer.start()
# Add remaining files to download queue
for link in links[2:]:
self._download_queue.put(link)
downloader = threading.Thread(name="Downloader",
target=self._downloader,
args=(tranzit_path, ))
downloader.start()
downloader.join()
writer.join()
def _test_bool_option(self, option):
self.assertRaises(ValueError, setattr, self.cnf, option, 'NotBoolValue')
setattr(self.cnf, option, True)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('true', c.get(option))
self.assertEqual(True, getattr(self.cnf, option))
setattr(self.cnf, option, False)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('false', c.get(option))
self.assertEqual(False, getattr(self.cnf, option))
setattr(self.cnf, option, None)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertRaises(NoPathError, c.get, option)
c.set(option, 'NotBool', force=True)
c.write(self.cnf_path)
self.assertRaises(ValueError, getattr, self.cnf, option)
def _write_manifest(self, snapshot, tranzit_path):
''' Make snapshot manifest '''
manifest_path = os.path.join(
tranzit_path, '%s.%s' % (snapshot.id, self.MANIFEST_NAME))
self._logger.info('Writing snapshot manifest file in %s',
manifest_path)
config = Configuration('ini')
config.add('snapshot/description', snapshot.description, force=True)
config.add('snapshot/created_at', time.strftime("%Y-%m-%d %H:%M:%S"))
config.add('snapshot/pack_method', 'pigz') # Not used yet
for chunk, md5 in self._chunks_md5.iteritems():
config.add('chunks/%s' % chunk, md5, force=True)
config.write(manifest_path)
return manifest_path
def on_reload(self):
self.queryenv = bus.queryenv_service
self.platform = bus.platform
self.private_ip = self.platform.get_private_ip()
self.zone = self.platform.get_avail_zone()
cnf = bus.cnf
self.ini = cnf.rawini
self.role_name = self.ini.get(config.SECT_GENERAL, config.OPT_ROLE_NAME)
self.storage_path = self.ini.get(CNF_SECTION, OPT_STORAGE_PATH)
self.storage_conf_path = self.ini.get(CNF_SECTION, OPT_STORAGE_CNF_PATH)
self.data_file_directory = self.storage_path + "/datafile"
self.commit_log_directory = self.storage_path + "/commitlog"
self.cassandra_conf = Configuration('xml')
try:
self.cassandra_conf.read(self.storage_conf_path)
except (OSError, MetaconfError, ParseError), e:
self._logger.error('Cassandra storage-conf.xml is broken. %s' % e)
def save(self, preset, preset_type):
'''
@type preset: CnfPreset
@type preset_type: CnfPresetStore.PresetType
@raise ValueError: When `preset` is not an instance of CnfPreset
@raise OSError: When cannot save preset file
'''
if not isinstance(preset, CnfPreset):
raise ValueError(
'argument `preset` should be a CnfPreset instance, %s is given',
type(preset))
self._logger.debug('Saving preset as %s' % preset_type)
ini = Configuration('ini')
ini.add('general')
ini.add(
'general/name', preset.name if
(hasattr(preset, 'name') and preset.name) else 'Noname')
ini.add('settings')
for k, v in preset.settings.items():
ini.add('settings/%s' % k, v)
ini.write(self._filename(preset_type))
def _manifest(self):
class HeadRequest(urllib2.Request):
def get_method(self):
return "HEAD"
manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour
path = self._manifest_path
url_handle = urllib2.urlopen(HeadRequest(manifest_url))
headers = url_handle.info()
url_last_modified = headers.getdate("Last-Modified")
file_modified = tuple(time.localtime(
os.path.getmtime(path))) if os.path.exists(path) else None
if not file_modified or url_last_modified > file_modified:
self._logger.debug('Fetching %s', manifest_url)
response = urllib2.urlopen(manifest_url)
data = response.read()
if data:
old_manifest = Configuration('ini')
if os.path.exists(path):
old_manifest.read(path)
new_manifest = Configuration('ini')
o = StringIO()
o.write(data)
o.seek(0)
new_manifest.readfp(o)
new_sections = new_manifest.sections('./')
old_sections = old_manifest.sections('./')
diff_path = os.path.join(os.path.dirname(path),
self.behaviour + '.incdiff')
diff = Configuration('ini')
if old_sections and old_sections != new_sections:
#skipping diff if no previous manifest found or it is equal to the new one
if os.path.exists(diff_path):
diff.read(diff_path)
sys_vars = self.get_system_variables()
for section in new_sections:
if section not in old_sections and sys_vars.has_key(
section):
sys_var = sys_vars[section]
if self.definitions:
if self.definitions.has_key(sys_var):
sys_var = self.definitions[sys_var]
diff.add('./%s/default-value' % section,
sys_var,
force=True)
diff.write(diff_path)
if os.path.exists(diff_path):
diff.read(diff_path)
for variable in diff.sections('./'):
sys_value = diff.get('./%s/default-value' % variable)
if sys_value and variable in new_manifest.sections('./'):
new_manifest.set('./%s/default-value' % variable,
sys_value,
force=True)
new_manifest.write(path)
return _CnfManifest(path)
def apply_preset(self, preset):
conf = Configuration(self._config_format)
conf.read(self._config_path)
self._before_apply_preset()
ver = self._software_version
for opt in self._manifest:
path = opt.name if not opt.section else '%s/%s' % (opt.section,
opt.name)
try:
value = conf.get(path)
except NoPathError:
value = ''
if opt.name in preset.settings:
new_value = preset.settings[opt.name]
# Skip unsupported
if ver and opt.supported_from and opt.supported_from > ver:
self._logger.debug(
"Skipping option '%s' supported from %s; installed %s"
% (opt.name, opt.supported_from, ver))
continue
if not opt.default_value:
self._logger.debug("Option '%s' has no default value" %
opt.name)
pass
elif new_value == opt.default_value:
if value:
self._logger.debug(
"Option '%s' equal to default. Removing." %
opt.name)
conf.remove(path)
self._after_remove_option(opt)
continue
if self.definitions and new_value in self.definitions:
manifest = Configuration('ini')
if os.path.exists(self._manifest_path):
manifest.read(self._manifest_path)
try:
if manifest.get('%s/type' % opt.name) == 'boolean':
new_value = self.definitions[new_value]
except NoPathError, e:
pass
self._logger.debug("Check that '%s' value changed:'%s'='%s'" %
(opt.name, value, new_value))
if new_value == value:
self._logger.debug("Skip option '%s'. Not changed" %
opt.name)
pass
else:
self._logger.debug("Set option '%s' = '%s'" %
(opt.name, new_value))
self._logger.debug('Set path %s = %s', path, new_value)
conf.set(path, new_value, force=True)
self._after_set_option(opt, new_value)
else:
if value:
self._logger.debug(
"Removing option '%s'. Not found in preset" % opt.name)
conf.remove(path)
self._after_remove_option(opt)
def _init_configuration(self):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
def __init__(self, path):
self._cnf = Configuration("apache")
self.path = path
