def apply_preset(self, preset):
conf = Configuration(self._config_format)
conf.read(self._config_path)
self._before_apply_preset()
ver = self._software_version
for opt in self._manifest:
path = opt.name if not opt.section else '%s/%s' % (opt.section, opt.name)
try:
value = conf.get(path)
except NoPathError:
value = ''
if opt.name in preset.settings:
new_value = preset.settings[opt.name]
# Skip unsupported
if ver and opt.supported_from and opt.supported_from > ver:
self._logger.debug("Skipping option '%s' supported from %s; installed %s" %
(opt.name, opt.supported_from, ver))
continue
if not opt.default_value:
self._logger.debug("Option '%s' has no default value" % opt.name)
pass
elif new_value == opt.default_value:
if value:
self._logger.debug("Option '%s' equal to default. Removing." % opt.name)
conf.remove(path)
self._after_remove_option(opt)
continue
if self.definitions and new_value in self.definitions:
manifest = Configuration('ini')
if os.path.exists(self._manifest_path):
manifest.read(self._manifest_path)
try:
if manifest.get('%s/type' % opt.name) == 'boolean':
new_value = self.definitions[new_value]
except NoPathError, e:
pass
self._logger.debug("Check that '%s' value changed:'%s'='%s'"%(opt.name, value, new_value))
if new_value == value:
self._logger.debug("Skip option '%s'. Not changed" % opt.name)
pass
else:
self._logger.debug("Set option '%s' = '%s'" % (opt.name, new_value))
self._logger.debug('Set path %s = %s', path, new_value)
conf.set(path, new_value, force=True)
self._after_set_option(opt, new_value)
else:
if value:
self._logger.debug("Removing option '%s'. Not found in preset" % opt.name)
conf.remove(path)
self._after_remove_option(opt)
def _manifest(self):
f_manifest = CnfController._manifest
base_manifest = f_manifest.fget(self)
path = self._manifest_path
s = {}
out = None
if not self._merged_manifest:
cmd = '%s --no-defaults --verbose SU_EXEC' % mysql_svc.MYSQLD_PATH
out = system2('%s - mysql -s %s -c "%s"' % (SU_EXEC, BASH, cmd),
shell=True, raise_exc=False, silent=True)[0]
if out:
raw = out.split(49*'-'+' '+24*'-')
if raw:
a = raw[-1].split('\n')
if len(a) > 5:
b = a[1:-5]
for item in b:
c = item.split()
if len(c) > 1:
key = c[0]
val = ' '.join(c[1:])
s[key.strip()] = val.strip()
if s:
m_config = Configuration('ini')
if os.path.exists(path):
m_config.read(path)
for variable in base_manifest:
name = variable.name
dv_path = './%s/default-value' % name
try:
old_value = m_config.get(dv_path)
if name in s:
new_value = s[name]
else:
name = name.replace('_','-')
if name in s:
new_value = self.definitions[s[name]] if s[name] in self.definitions else s[name]
if old_value != new_value and new_value != '(No default value)':
LOG.debug('Replacing %s default value %s with precompiled value %s',
name, old_value, new_value)
m_config.set(path=dv_path, value=new_value, force=True)
except NoPathError:
pass
m_config.write(path)
self._merged_manifest = _CnfManifest(path)
return self._merged_manifest
def _test_bool_option(self, option):
self.assertRaises(ValueError, setattr, self.cnf, option, 'NotBoolValue')
setattr(self.cnf, option, True)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('true', c.get(option))
self.assertEqual(True, getattr(self.cnf, option))
setattr(self.cnf, option, False)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('false', c.get(option))
self.assertEqual(False, getattr(self.cnf, option))
setattr(self.cnf, option, None)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertRaises(NoPathError, c.get, option)
c.set(option, 'NotBool', force=True)
c.write(self.cnf_path)
self.assertRaises(ValueError, getattr, self.cnf, option)
def _test_bool_option(self, option):
self.assertRaises(ValueError, setattr, self.cnf, option, 'NotBoolValue')
setattr(self.cnf, option, True)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('true', c.get(option))
self.assertEqual(True, getattr(self.cnf, option))
setattr(self.cnf, option, False)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('false', c.get(option))
self.assertEqual(False, getattr(self.cnf, option))
setattr(self.cnf, option, None)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertRaises(NoPathError, c.get, option)
c.set(option, 'NotBool', force=True)
c.write(self.cnf_path)
self.assertRaises(ValueError, getattr, self.cnf, option)
class MysqlProxyHandler(ServiceCtlHandler):
def __init__(self):
self._logger = logging.getLogger(__name__)
self.service = initdv2.lookup(BEHAVIOUR)
self._service_name = BEHAVIOUR
bus.on(init=self.on_init)
def accept(self,
message,
queue,
behaviour=None,
platform=None,
os=None,
dist=None):
return message.behaviour and is_mysql_role(
message.behaviour) and message.name in (
Messages.HOST_UP, Messages.HOST_DOWN, NEW_MASTER_UP,
DbMsrMessages.DBMSR_NEW_MASTER_UP)
def on_init(self):
bus.on(start=self.on_start,
before_host_up=self.on_before_host_up,
reload=self.on_reload)
def on_reload(self):
self._reload_backends()
def on_start(self):
cnf = bus.cnf
if cnf.state == config.ScalarizrState.RUNNING:
self._reload_backends()
def on_before_host_up(self, msg):
self._reload_backends()
def _reload_backends(self):
self._logger.info('Updating mysql-proxy backends list')
self.config = Configuration('mysql')
if os.path.exists(CONFIG_FILE_PATH):
self.config.read(CONFIG_FILE_PATH)
self.config.remove('./mysql-proxy/proxy-backend-addresses')
self.config.remove(
'./mysql-proxy/proxy-read-only-backend-addresses')
try:
self.config.get('./mysql-proxy')
except NoPathError:
self.config.add('./mysql-proxy')
queryenv = bus.queryenv_service
roles = queryenv.list_roles()
master = None
slaves = []
for role in roles:
if not is_mysql_role(role.behaviour):
continue
for host in role.hosts:
ip = host.internal_ip or host.external_ip
if host.replication_master:
master = ip
else:
slaves.append(ip)
if master:
self._logger.debug(
'Adding mysql master %s to mysql-proxy defaults file', master)
self.config.add('./mysql-proxy/proxy-backend-addresses',
'%s:3306' % master)
if slaves:
self._logger.debug(
'Adding mysql slaves to mysql-proxy defaults file: %s',
', '.join(slaves))
for slave in slaves:
self.config.add(
'./mysql-proxy/proxy-read-only-backend-addresses',
'%s:3306' % slave)
self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
self.config.set('./mysql-proxy/daemon', 'true', force=True)
self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
if self.service.version > (0, 8, 0):
self.config.set('./mysql-proxy/plugins', 'proxy', force=True)
self._logger.debug('Saving new mysql-proxy defaults file')
self.config.write(CONFIG_FILE_PATH)
os.chmod(CONFIG_FILE_PATH, 0660)
self.service.restart()
def on_HostUp(self, message):
self._reload_backends()
on_DbMsr_NewMasterUp = on_Mysql_NewMasterUp = on_HostDown = on_HostUp
def _manifest(self):
class HeadRequest(urllib2.Request):
def get_method(self):
return "HEAD"
manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour
path = self._manifest_path
url_handle = urllib2.urlopen(HeadRequest(manifest_url))
headers = url_handle.info()
url_last_modified = headers.getdate("Last-Modified")
file_modified = tuple(time.localtime(
os.path.getmtime(path))) if os.path.exists(path) else None
if not file_modified or url_last_modified > file_modified:
self._logger.debug('Fetching %s', manifest_url)
response = urllib2.urlopen(manifest_url)
data = response.read()
if data:
old_manifest = Configuration('ini')
if os.path.exists(path):
old_manifest.read(path)
new_manifest = Configuration('ini')
o = StringIO()
o.write(data)
o.seek(0)
new_manifest.readfp(o)
new_sections = new_manifest.sections('./')
old_sections = old_manifest.sections('./')
diff_path = os.path.join(os.path.dirname(path),
self.behaviour + '.incdiff')
diff = Configuration('ini')
if old_sections and old_sections != new_sections:
#skipping diff if no previous manifest found or it is equal to the new one
if os.path.exists(diff_path):
diff.read(diff_path)
sys_vars = self.get_system_variables()
for section in new_sections:
if section not in old_sections and sys_vars.has_key(
section):
sys_var = sys_vars[section]
if self.definitions:
if self.definitions.has_key(sys_var):
sys_var = self.definitions[sys_var]
diff.add('./%s/default-value' % section,
sys_var,
force=True)
diff.write(diff_path)
if os.path.exists(diff_path):
diff.read(diff_path)
for variable in diff.sections('./'):
sys_value = diff.get('./%s/default-value' % variable)
if sys_value and variable in new_manifest.sections('./'):
new_manifest.set('./%s/default-value' % variable,
sys_value,
force=True)
new_manifest.write(path)
return _CnfManifest(path)
def apply_preset(self, preset):
conf = Configuration(self._config_format)
conf.read(self._config_path)
self._before_apply_preset()
ver = self._software_version
for opt in self._manifest:
path = opt.name if not opt.section else '%s/%s' % (opt.section,
opt.name)
try:
value = conf.get(path)
except NoPathError:
value = ''
if opt.name in preset.settings:
new_value = preset.settings[opt.name]
# Skip unsupported
if ver and opt.supported_from and opt.supported_from > ver:
self._logger.debug(
"Skipping option '%s' supported from %s; installed %s"
% (opt.name, opt.supported_from, ver))
continue
if not opt.default_value:
self._logger.debug("Option '%s' has no default value" %
opt.name)
pass
elif new_value == opt.default_value:
if value:
self._logger.debug(
"Option '%s' equal to default. Removing." %
opt.name)
conf.remove(path)
self._after_remove_option(opt)
continue
if self.definitions and new_value in self.definitions:
manifest = Configuration('ini')
if os.path.exists(self._manifest_path):
manifest.read(self._manifest_path)
try:
if manifest.get('%s/type' % opt.name) == 'boolean':
new_value = self.definitions[new_value]
except NoPathError, e:
pass
self._logger.debug("Check that '%s' value changed:'%s'='%s'" %
(opt.name, value, new_value))
if new_value == value:
self._logger.debug("Skip option '%s'. Not changed" %
opt.name)
pass
else:
self._logger.debug("Set option '%s' = '%s'" %
(opt.name, new_value))
self._logger.debug('Set path %s = %s', path, new_value)
conf.set(path, new_value, force=True)
self._after_set_option(opt, new_value)
else:
if value:
self._logger.debug(
"Removing option '%s'. Not found in preset" % opt.name)
conf.remove(path)
self._after_remove_option(opt)
class MysqlProxyHandler(ServiceCtlHandler):
def __init__(self):
self._logger = logging.getLogger(__name__)
self.service = initdv2.lookup(BEHAVIOUR)
self._service_name = BEHAVIOUR
bus.on(init=self.on_init)
def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None):
return message.behaviour and is_mysql_role(message.behaviour) and message.name in (
Messages.HOST_UP,
Messages.HOST_DOWN,
NEW_MASTER_UP,
DbMsrMessages.DBMSR_NEW_MASTER_UP
)
def on_init(self):
bus.on(
start=self.on_start,
before_host_up=self.on_before_host_up,
reload=self.on_reload
)
def on_reload(self):
self._reload_backends()
def on_start(self):
cnf = bus.cnf
if cnf.state == config.ScalarizrState.RUNNING:
self._reload_backends()
def on_before_host_up(self, msg):
self._reload_backends()
def _reload_backends(self):
self._logger.info('Updating mysql-proxy backends list')
self.config = Configuration('mysql')
if os.path.exists(CONFIG_FILE_PATH):
self.config.read(CONFIG_FILE_PATH)
self.config.remove('./mysql-proxy/proxy-backend-addresses')
self.config.remove('./mysql-proxy/proxy-read-only-backend-addresses')
try:
self.config.get('./mysql-proxy')
except NoPathError:
self.config.add('./mysql-proxy')
queryenv = bus.queryenv_service
roles = queryenv.list_roles()
master = None
slaves = []
for role in roles:
if not is_mysql_role(role.behaviour):
continue
for host in role.hosts:
ip = host.internal_ip or host.external_ip
if host.replication_master:
master = ip
else:
slaves.append(ip)
if master:
self._logger.debug('Adding mysql master %s to mysql-proxy defaults file', master)
self.config.add('./mysql-proxy/proxy-backend-addresses', '%s:3306' % master)
if slaves:
self._logger.debug('Adding mysql slaves to mysql-proxy defaults file: %s', ', '.join(slaves))
for slave in slaves:
self.config.add('./mysql-proxy/proxy-read-only-backend-addresses', '%s:3306' % slave)
self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
self.config.set('./mysql-proxy/daemon', 'true', force=True)
self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
if self.service.version > (0,8,0):
self.config.set('./mysql-proxy/plugins', 'proxy', force=True)
self._logger.debug('Saving new mysql-proxy defaults file')
self.config.write(CONFIG_FILE_PATH)
os.chmod(CONFIG_FILE_PATH, 0660)
self.service.restart()
def on_HostUp(self, message):
self._reload_backends()
on_DbMsr_NewMasterUp = on_Mysql_NewMasterUp = on_HostDown = on_HostUp
class MongoDBConfig(BaseConfig):
config_type = "mongodb"
config_name = os.path.basename(CONFIG_PATH_DEFAULT)
@classmethod
def find(cls, config_dir=None):
# conf_path = UBUNTU_CONFIG_PATH if disttool.is_ubuntu() else CENTOS_CONFIG_PATH
return cls(os.path.join(config_dir, cls.config_name) if config_dir else CONFIG_PATH_DEFAULT)
def set(self, option, value):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
if value:
self.data.set(option, value, force=True)
else:
self.data.remove(option)
if self.autosave:
self.save_data()
self.data = None
def set_bool_option(self, option, value):
try:
assert value in (None, True, False)
except AssertionError:
raise ValueError("%s must be a boolean (got %s instead)" % (option, type(value)))
value_to_set = None if value is None else str(value).lower()
self.set(option, value_to_set)
def get_bool_option(self, option):
value = self.get(option)
try:
assert not value or value == "true" or value == "false"
except AssertionError:
raise ValueError("%s must be true or false (got %s instead)" % (option, type(value)))
return True if value == "true" else False
def _get_logpath(self):
return self.get("logpath")
def _set_logpath(self, path):
self.set("logpath", path)
def _get_replSet(self):
return self.get("replSet")
def _set_replSet(self, name):
self.set("replSet", name)
def _get_port(self):
return self.get_numeric_option("port")
def _set_port(self, number):
self.set_numeric_option("port", number)
def _get_logappend(self):
return self.get_bool_option("logappend")
def _set_logappend(self, on=True):
self.set_bool_option("logappend", on)
def _get_dbpath(self):
return self.get("dbpath")
def _set_dbpath(self, path):
self.set_path_type_option("dbpath", path)
def _get_nojournal(self):
return self.get_bool_option("nojournal")
def _set_nojournal(self, on=False):
self.set_bool_option("nojournal", on)
def _get_nohttpinterface(self):
return self.get_bool_option("nohttpinterface")
def _set_nohttpinterface(self, on=False):
self.set_bool_option("nohttpinterface", on)
def _get_rest(self):
return self.get_bool_option("rest")
def _set_rest(self, on=False):
self.set_bool_option("rest", on)
def _set_shardsvr(self, value):
self.set_bool_option("shardsvr", value)
def _get_shardsvr(self):
return self.get_bool_option("shardsvr")
shardsvr = property(_get_shardsvr, _set_shardsvr)
rest = property(_get_rest, _set_rest)
nohttpinterface = property(_get_nohttpinterface, _set_nohttpinterface)
nojournal = property(_get_nojournal, _set_nojournal)
dbpath = property(_get_dbpath, _set_dbpath)
logappend = property(_get_logappend, _set_logappend)
port = property(_get_port, _set_port)
replSet = property(_get_replSet, _set_replSet)
logpath = property(_get_logpath, _set_logpath)
class BaseRedisConfig(BaseConfig):
config_type = 'redis'
def set(self, option, value, append=False):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
if value:
if append:
self.data.add(option, str(value))
else:
self.data.set(option,str(value), force=True)
else:
self.data.comment(option)
if self.autosave:
self.save_data()
self.data = None
def set_sequential_option(self, option, seq):
is_typle = type(seq) is tuple
try:
assert seq is None or is_typle
except AssertionError:
raise ValueError('%s must be a sequence (got %s instead)' % (option, seq))
self.set(option, ' '.join(map(str,seq)) if is_typle else None)
def get_sequential_option(self, option):
raw = self.get(option)
return raw.split() if raw else ()
def get_list(self, option):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
try:
value = self.data.get_list(option)
except NoPathError:
try:
value = getattr(self, option+'_default')
except AttributeError:
value = ()
if self.autosave:
self.data = None
return value
def get_dict_option(self, option):
raw = self.get_list(option)
d = {}
for raw_value in raw:
k,v = raw_value.split()
if k and v:
d[k] = v
return d
def set_dict_option(self, option, d):
try:
assert d is None or type(d)==dict
#cleaning up
#TODO: make clean process smarter using indexes
for i in self.get_list(option):
self.set(option+'[0]', None)
#adding multiple entries
for k,v in d.items():
val = ' '.join(map(str,'%s %s'%(k,v)))
self.set(option, val, append=True)
except ValueError:
raise ValueError('%s must be a sequence (got %s instead)' % (option, d))
def _manifest(self):
class HeadRequest(urllib2.Request):
def get_method(self):
return "HEAD"
manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour
path = self._manifest_path
url_handle = urllib2.urlopen(HeadRequest(manifest_url))
headers = url_handle.info()
url_last_modified = headers.getdate("Last-Modified")
file_modified = tuple(time.localtime(os.path.getmtime(path))) if os.path.exists(path) else None
if not file_modified or url_last_modified > file_modified:
self._logger.debug('Fetching %s', manifest_url)
response = urllib2.urlopen(manifest_url)
data = response.read()
if data:
old_manifest = Configuration('ini')
if os.path.exists(path):
old_manifest.read(path)
new_manifest = Configuration('ini')
o = StringIO()
o.write(data)
o.seek(0)
new_manifest.readfp(o)
new_sections = new_manifest.sections('./')
old_sections = old_manifest.sections('./')
diff_path = os.path.join(os.path.dirname(path), self.behaviour + '.incdiff')
diff = Configuration('ini')
if old_sections and old_sections != new_sections:
#skipping diff if no previous manifest found or it is equal to the new one
if os.path.exists(diff_path):
diff.read(diff_path)
sys_vars = self.get_system_variables()
for section in new_sections:
if section not in old_sections and sys_vars.has_key(section):
sys_var = sys_vars[section]
if self.definitions:
if self.definitions.has_key(sys_var):
sys_var = self.definitions[sys_var]
diff.add('./%s/default-value' % section, sys_var, force=True)
diff.write(diff_path)
if os.path.exists(diff_path):
diff.read(diff_path)
for variable in diff.sections('./'):
sys_value = diff.get('./%s/default-value' % variable)
if sys_value and variable in new_manifest.sections('./'):
new_manifest.set('./%s/default-value' % variable, sys_value, force=True)
new_manifest.write(path)
return _CnfManifest(path)
class BaseConfig(object):
'''
Parent class for object representations of postgresql.conf and recovery.conf which fortunately both have similar syntax
'''
autosave = None
path = None
data = None
config_name = None
config_type = None
comment_empty = False
def __init__(self, path, autosave=True):
self._logger = logging.getLogger(__name__)
self.autosave = autosave
self.path = path
@classmethod
def find(cls, config_dir):
return cls(os.path.join(config_dir.path, cls.config_name))
def set(self, option, value):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
if value:
self.data.set(option,str(value), force=True)
elif self.comment_empty:
self.data.comment(option)
if self.autosave:
self.save_data()
self.data = None
def set_path_type_option(self, option, path):
if not os.path.exists(path):
raise ValueError('%s %s does not exist' % (option, path))
self.set(option, path)
def set_numeric_option(self, option, number):
try:
assert number is None or type(number) is int
except AssertionError:
raise ValueError('%s must be a number (got %s instead)' % (option, number))
is_numeric = type(number) is int
self.set(option, str(number) if is_numeric else None)
def get(self, option):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
try:
value = self.data.get(option)
except NoPathError:
try:
value = getattr(self, option+'_default')
except AttributeError:
value = None
if self.autosave:
self.data = None
return value
def get_numeric_option(self, option):
value = self.get(option)
try:
assert value is None or int(value)
except AssertionError:
raise ValueError('%s must be a number (got %s instead)' % (option, type(value)))
return value if value is None else int(value)
def save_data(self):
if self.data:
self.data.write(self.path)
