def _manifest(self):
f_manifest = CnfController._manifest
base_manifest = f_manifest.fget(self)
path = self._manifest_path
s = {}
out = None
if not self._merged_manifest:
cmd = '%s --no-defaults --verbose SU_EXEC' % mysql_svc.MYSQLD_PATH
out = system2('%s - mysql -s %s -c "%s"' % (SU_EXEC, BASH, cmd),
shell=True, raise_exc=False, silent=True)[0]
if out:
raw = out.split(49*'-'+' '+24*'-')
if raw:
a = raw[-1].split('\n')
if len(a) > 5:
b = a[1:-5]
for item in b:
c = item.split()
if len(c) > 1:
key = c[0]
val = ' '.join(c[1:])
s[key.strip()] = val.strip()
if s:
m_config = Configuration('ini')
if os.path.exists(path):
m_config.read(path)
for variable in base_manifest:
name = variable.name
dv_path = './%s/default-value' % name
try:
old_value = m_config.get(dv_path)
if name in s:
new_value = s[name]
else:
name = name.replace('_','-')
if name in s:
new_value = self.definitions[s[name]] if s[name] in self.definitions else s[name]
if old_value != new_value and new_value != '(No default value)':
LOG.debug('Replacing %s default value %s with precompiled value %s',
name, old_value, new_value)
m_config.set(path=dv_path, value=new_value, force=True)
except NoPathError:
pass
m_config.write(path)
self._merged_manifest = _CnfManifest(path)
return self._merged_manifest
def _write_manifest(self, snapshot, tranzit_path):
''' Make snapshot manifest '''
manifest_path = os.path.join(tranzit_path, '%s.%s' % (snapshot.id, self.MANIFEST_NAME))
self._logger.info('Writing snapshot manifest file in %s', manifest_path)
config = Configuration('ini')
config.add('snapshot/description', snapshot.description, force=True)
config.add('snapshot/created_at', time.strftime("%Y-%m-%d %H:%M:%S"))
config.add('snapshot/pack_method', 'pigz') # Not used yet
for chunk, md5 in self._chunks_md5.iteritems():
config.add('chunks/%s' % chunk, md5, force=True)
config.write(manifest_path)
return manifest_path
class ApacheConfigManager(object):
_cnf = None
path = None
def __init__(self, path):
self._cnf = Configuration("apache")
self.path = path
def __enter__(self):
self._cnf.read(self.path)
return self._cnf
def __exit__(self, type, value, traceback):
self._cnf.write(self.path)
class ApacheConfigManager(object):
_cnf = None
path = None
def __init__(self, path):
self._cnf = Configuration("apache")
self.path = path
def __enter__(self):
self._cnf.read(self.path)
return self._cnf
def __exit__(self, type, value, traceback):
self._cnf.write(self.path)
def _write_manifest(self, snapshot, tranzit_path):
''' Make snapshot manifest '''
manifest_path = os.path.join(
tranzit_path, '%s.%s' % (snapshot.id, self.MANIFEST_NAME))
self._logger.info('Writing snapshot manifest file in %s',
manifest_path)
config = Configuration('ini')
config.add('snapshot/description', snapshot.description, force=True)
config.add('snapshot/created_at', time.strftime("%Y-%m-%d %H:%M:%S"))
config.add('snapshot/pack_method', 'pigz') # Not used yet
for chunk, md5 in self._chunks_md5.iteritems():
config.add('chunks/%s' % chunk, md5, force=True)
config.write(manifest_path)
return manifest_path
def save(self, preset, preset_type):
'''
@type preset: CnfPreset
@type preset_type: CnfPresetStore.PresetType
@raise ValueError: When `preset` is not an instance of CnfPreset
@raise OSError: When cannot save preset file
'''
if not isinstance(preset, CnfPreset):
raise ValueError('argument `preset` should be a CnfPreset instance, %s is given', type(preset))
self._logger.debug('Saving preset as %s' % preset_type)
ini = Configuration('ini')
ini.add('general')
ini.add('general/name', preset.name if (hasattr(preset, 'name') and preset.name) else 'Noname')
ini.add('settings')
for k, v in preset.settings.items():
ini.add('settings/%s' % k, v)
ini.write(self._filename(preset_type))
def save(self, preset, preset_type):
'''
@type preset: CnfPreset
@type preset_type: CnfPresetStore.PresetType
@raise ValueError: When `preset` is not an instance of CnfPreset
@raise OSError: When cannot save preset file
'''
if not isinstance(preset, CnfPreset):
raise ValueError(
'argument `preset` should be a CnfPreset instance, %s is given',
type(preset))
self._logger.debug('Saving preset as %s' % preset_type)
ini = Configuration('ini')
ini.add('general')
ini.add(
'general/name', preset.name if
(hasattr(preset, 'name') and preset.name) else 'Noname')
ini.add('settings')
for k, v in preset.settings.items():
ini.add('settings/%s' % k, v)
ini.write(self._filename(preset_type))
def _test_bool_option(self, option):
self.assertRaises(ValueError, setattr, self.cnf, option, 'NotBoolValue')
setattr(self.cnf, option, True)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('true', c.get(option))
self.assertEqual(True, getattr(self.cnf, option))
setattr(self.cnf, option, False)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('false', c.get(option))
self.assertEqual(False, getattr(self.cnf, option))
setattr(self.cnf, option, None)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertRaises(NoPathError, c.get, option)
c.set(option, 'NotBool', force=True)
c.write(self.cnf_path)
self.assertRaises(ValueError, getattr, self.cnf, option)
def _test_bool_option(self, option):
self.assertRaises(ValueError, setattr, self.cnf, option, 'NotBoolValue')
setattr(self.cnf, option, True)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('true', c.get(option))
self.assertEqual(True, getattr(self.cnf, option))
setattr(self.cnf, option, False)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('false', c.get(option))
self.assertEqual(False, getattr(self.cnf, option))
setattr(self.cnf, option, None)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertRaises(NoPathError, c.get, option)
c.set(option, 'NotBool', force=True)
c.write(self.cnf_path)
self.assertRaises(ValueError, getattr, self.cnf, option)
class MysqlProxyHandler(ServiceCtlHandler):
def __init__(self):
self._logger = logging.getLogger(__name__)
self.service = initdv2.lookup(BEHAVIOUR)
self._service_name = BEHAVIOUR
bus.on(init=self.on_init)
def accept(self,
message,
queue,
behaviour=None,
platform=None,
os=None,
dist=None):
return message.behaviour and is_mysql_role(
message.behaviour) and message.name in (
Messages.HOST_UP, Messages.HOST_DOWN, NEW_MASTER_UP,
DbMsrMessages.DBMSR_NEW_MASTER_UP)
def on_init(self):
bus.on(start=self.on_start,
before_host_up=self.on_before_host_up,
reload=self.on_reload)
def on_reload(self):
self._reload_backends()
def on_start(self):
cnf = bus.cnf
if cnf.state == config.ScalarizrState.RUNNING:
self._reload_backends()
def on_before_host_up(self, msg):
self._reload_backends()
def _reload_backends(self):
self._logger.info('Updating mysql-proxy backends list')
self.config = Configuration('mysql')
if os.path.exists(CONFIG_FILE_PATH):
self.config.read(CONFIG_FILE_PATH)
self.config.remove('./mysql-proxy/proxy-backend-addresses')
self.config.remove(
'./mysql-proxy/proxy-read-only-backend-addresses')
try:
self.config.get('./mysql-proxy')
except NoPathError:
self.config.add('./mysql-proxy')
queryenv = bus.queryenv_service
roles = queryenv.list_roles()
master = None
slaves = []
for role in roles:
if not is_mysql_role(role.behaviour):
continue
for host in role.hosts:
ip = host.internal_ip or host.external_ip
if host.replication_master:
master = ip
else:
slaves.append(ip)
if master:
self._logger.debug(
'Adding mysql master %s to mysql-proxy defaults file', master)
self.config.add('./mysql-proxy/proxy-backend-addresses',
'%s:3306' % master)
if slaves:
self._logger.debug(
'Adding mysql slaves to mysql-proxy defaults file: %s',
', '.join(slaves))
for slave in slaves:
self.config.add(
'./mysql-proxy/proxy-read-only-backend-addresses',
'%s:3306' % slave)
self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
self.config.set('./mysql-proxy/daemon', 'true', force=True)
self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
if self.service.version > (0, 8, 0):
self.config.set('./mysql-proxy/plugins', 'proxy', force=True)
self._logger.debug('Saving new mysql-proxy defaults file')
self.config.write(CONFIG_FILE_PATH)
os.chmod(CONFIG_FILE_PATH, 0660)
self.service.restart()
def on_HostUp(self, message):
self._reload_backends()
on_DbMsr_NewMasterUp = on_Mysql_NewMasterUp = on_HostDown = on_HostUp
def _manifest(self):
class HeadRequest(urllib2.Request):
def get_method(self):
return "HEAD"
manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour
path = self._manifest_path
url_handle = urllib2.urlopen(HeadRequest(manifest_url))
headers = url_handle.info()
url_last_modified = headers.getdate("Last-Modified")
file_modified = tuple(time.localtime(
os.path.getmtime(path))) if os.path.exists(path) else None
if not file_modified or url_last_modified > file_modified:
self._logger.debug('Fetching %s', manifest_url)
response = urllib2.urlopen(manifest_url)
data = response.read()
if data:
old_manifest = Configuration('ini')
if os.path.exists(path):
old_manifest.read(path)
new_manifest = Configuration('ini')
o = StringIO()
o.write(data)
o.seek(0)
new_manifest.readfp(o)
new_sections = new_manifest.sections('./')
old_sections = old_manifest.sections('./')
diff_path = os.path.join(os.path.dirname(path),
self.behaviour + '.incdiff')
diff = Configuration('ini')
if old_sections and old_sections != new_sections:
#skipping diff if no previous manifest found or it is equal to the new one
if os.path.exists(diff_path):
diff.read(diff_path)
sys_vars = self.get_system_variables()
for section in new_sections:
if section not in old_sections and sys_vars.has_key(
section):
sys_var = sys_vars[section]
if self.definitions:
if self.definitions.has_key(sys_var):
sys_var = self.definitions[sys_var]
diff.add('./%s/default-value' % section,
sys_var,
force=True)
diff.write(diff_path)
if os.path.exists(diff_path):
diff.read(diff_path)
for variable in diff.sections('./'):
sys_value = diff.get('./%s/default-value' % variable)
if sys_value and variable in new_manifest.sections('./'):
new_manifest.set('./%s/default-value' % variable,
sys_value,
force=True)
new_manifest.write(path)
return _CnfManifest(path)
def _manifest(self):
class HeadRequest(urllib2.Request):
def get_method(self):
return "HEAD"
manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour
path = self._manifest_path
url_handle = urllib2.urlopen(HeadRequest(manifest_url))
headers = url_handle.info()
url_last_modified = headers.getdate("Last-Modified")
file_modified = tuple(time.localtime(os.path.getmtime(path))) if os.path.exists(path) else None
if not file_modified or url_last_modified > file_modified:
self._logger.debug('Fetching %s', manifest_url)
response = urllib2.urlopen(manifest_url)
data = response.read()
if data:
old_manifest = Configuration('ini')
if os.path.exists(path):
old_manifest.read(path)
new_manifest = Configuration('ini')
o = StringIO()
o.write(data)
o.seek(0)
new_manifest.readfp(o)
new_sections = new_manifest.sections('./')
old_sections = old_manifest.sections('./')
diff_path = os.path.join(os.path.dirname(path), self.behaviour + '.incdiff')
diff = Configuration('ini')
if old_sections and old_sections != new_sections:
#skipping diff if no previous manifest found or it is equal to the new one
if os.path.exists(diff_path):
diff.read(diff_path)
sys_vars = self.get_system_variables()
for section in new_sections:
if section not in old_sections and sys_vars.has_key(section):
sys_var = sys_vars[section]
if self.definitions:
if self.definitions.has_key(sys_var):
sys_var = self.definitions[sys_var]
diff.add('./%s/default-value' % section, sys_var, force=True)
diff.write(diff_path)
if os.path.exists(diff_path):
diff.read(diff_path)
for variable in diff.sections('./'):
sys_value = diff.get('./%s/default-value' % variable)
if sys_value and variable in new_manifest.sections('./'):
new_manifest.set('./%s/default-value' % variable, sys_value, force=True)
new_manifest.write(path)
return _CnfManifest(path)
class BaseConfig(object):
'''
Parent class for object representations of postgresql.conf and recovery.conf which fortunately both have similar syntax
'''
autosave = None
path = None
data = None
config_name = None
config_type = None
comment_empty = False
def __init__(self, path, autosave=True):
self._logger = logging.getLogger(__name__)
self.autosave = autosave
self.path = path
@classmethod
def find(cls, config_dir):
return cls(os.path.join(config_dir.path, cls.config_name))
def set(self, option, value):
self.apply_dict({option: value})
def set_path_type_option(self, option, path):
if not os.path.exists(path):
raise ValueError('%s %s does not exist' % (option, path))
self.set(option, path)
def set_numeric_option(self, option, number):
try:
assert number is None or type(number) is int
except AssertionError:
raise ValueError('%s must be a number (got %s instead)' %
(option, number))
is_numeric = type(number) is int
self.set(option, str(number) if is_numeric else None)
def get(self, option):
self._init_configuration()
try:
value = self.data.get(option)
except NoPathError:
try:
value = getattr(self, option + '_default')
except AttributeError:
value = None
self._cleanup()
return value
def get_numeric_option(self, option):
value = self.get(option)
return value if value is None else int(value)
def to_dict(self):
self._init_configuration()
result = {}
for section in self.data.sections('./'):
try:
kv = dict(self.data.items(section))
except NoPathError:
kv = {}
for variable, value in kv.items():
path = '%s/%s' % (section, variable)
result[path] = value
'''
variables in root section
'''
for variable, value in self.data.items('.'):
if value and value.strip():
result[variable] = value
self._cleanup()
return result
def apply_dict(self, kv):
self._init_configuration()
for path, value in kv.items():
if not value and self.comment_empty:
self.data.comment(path)
else:
self.data.set(path, str(value), force=True)
self._cleanup(True)
def delete_options(self, options):
self._init_configuration()
for path in options:
self.data.remove(path)
self._cleanup(True)
def _init_configuration(self):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
def _cleanup(self, save_data=False):
if self.autosave:
if save_data and self.data:
self.data.write(self.path)
self.data = None
if disttool.is_debian_based():
# Comment /etc/nginx/sites-enabled/*
try:
i = config.get_list('http/include').index('/etc/nginx/sites-enabled/*')
config.comment('http/include[%d]' % (i+1))
self._logger.debug('comment site-enabled include')
except (ValueError, IndexError):
self._logger.debug('site-enabled include already commented')
elif disttool.is_redhat_based():
def_host_path = '/etc/nginx/conf.d/default.conf'
if os.path.exists(def_host_path):
default_host = Configuration('nginx')
default_host.read(def_host_path)
default_host.comment('server')
default_host.write(def_host_path)
if dump == self._dump_config(config):
self._logger.debug("Main nginx config wasn`t changed")
else:
# Write new nginx.conf
shutil.copy(nginx_conf_path, nginx_conf_path + '.bak')
config.write(nginx_conf_path)
if reload_service:
self.api._reload_service()
def _insert_iptables_rules(self, *args, **kwargs):
if iptables.enabled():
iptables.FIREWALL.ensure([
{"jump": "ACCEPT", "protocol": "tcp", "match": "tcp", "dport": "80"},
{"jump": "ACCEPT", "protocol": "tcp", "match": "tcp", "dport": "443"},
class MysqlProxyHandler(ServiceCtlHandler):
def __init__(self):
self._logger = logging.getLogger(__name__)
self.service = initdv2.lookup(BEHAVIOUR)
self._service_name = BEHAVIOUR
bus.on(init=self.on_init)
def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None):
return message.behaviour and is_mysql_role(message.behaviour) and message.name in (
Messages.HOST_UP,
Messages.HOST_DOWN,
NEW_MASTER_UP,
DbMsrMessages.DBMSR_NEW_MASTER_UP
)
def on_init(self):
bus.on(
start=self.on_start,
before_host_up=self.on_before_host_up,
reload=self.on_reload
)
def on_reload(self):
self._reload_backends()
def on_start(self):
cnf = bus.cnf
if cnf.state == config.ScalarizrState.RUNNING:
self._reload_backends()
def on_before_host_up(self, msg):
self._reload_backends()
def _reload_backends(self):
self._logger.info('Updating mysql-proxy backends list')
self.config = Configuration('mysql')
if os.path.exists(CONFIG_FILE_PATH):
self.config.read(CONFIG_FILE_PATH)
self.config.remove('./mysql-proxy/proxy-backend-addresses')
self.config.remove('./mysql-proxy/proxy-read-only-backend-addresses')
try:
self.config.get('./mysql-proxy')
except NoPathError:
self.config.add('./mysql-proxy')
queryenv = bus.queryenv_service
roles = queryenv.list_roles()
master = None
slaves = []
for role in roles:
if not is_mysql_role(role.behaviour):
continue
for host in role.hosts:
ip = host.internal_ip or host.external_ip
if host.replication_master:
master = ip
else:
slaves.append(ip)
if master:
self._logger.debug('Adding mysql master %s to mysql-proxy defaults file', master)
self.config.add('./mysql-proxy/proxy-backend-addresses', '%s:3306' % master)
if slaves:
self._logger.debug('Adding mysql slaves to mysql-proxy defaults file: %s', ', '.join(slaves))
for slave in slaves:
self.config.add('./mysql-proxy/proxy-read-only-backend-addresses', '%s:3306' % slave)
self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
self.config.set('./mysql-proxy/daemon', 'true', force=True)
self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
if self.service.version > (0,8,0):
self.config.set('./mysql-proxy/plugins', 'proxy', force=True)
self._logger.debug('Saving new mysql-proxy defaults file')
self.config.write(CONFIG_FILE_PATH)
os.chmod(CONFIG_FILE_PATH, 0660)
self.service.restart()
def on_HostUp(self, message):
self._reload_backends()
on_DbMsr_NewMasterUp = on_Mysql_NewMasterUp = on_HostDown = on_HostUp
class BaseConfig(object):
'''
Parent class for object representations of postgresql.conf and recovery.conf which fortunately both have similar syntax
'''
autosave = None
path = None
data = None
config_name = None
config_type = None
comment_empty = False
def __init__(self, path, autosave=True):
self._logger = logging.getLogger(__name__)
self.autosave = autosave
self.path = path
@classmethod
def find(cls, config_dir):
return cls(os.path.join(config_dir.path, cls.config_name))
def set(self, option, value):
self.apply_dict({option:value})
def set_path_type_option(self, option, path):
if not os.path.exists(path):
raise ValueError('%s %s does not exist' % (option, path))
self.set(option, path)
def set_numeric_option(self, option, number):
try:
assert number is None or type(number) is int
except AssertionError:
raise ValueError('%s must be a number (got %s instead)' % (option, number))
is_numeric = type(number) is int
self.set(option, str(number) if is_numeric else None)
def get(self, option):
self._init_configuration()
try:
value = self.data.get(option)
except NoPathError:
try:
value = getattr(self, option+'_default')
except AttributeError:
value = None
self._cleanup()
return value
def get_numeric_option(self, option):
value = self.get(option)
return value if value is None else int(value)
def to_dict(self):
self._init_configuration()
result = {}
for section in self.data.sections('./'):
try:
kv = dict(self.data.items(section))
except NoPathError:
kv = {}
for variable, value in kv.items():
path = '%s/%s' % (section,variable)
result[path] = value
'''
variables in root section
'''
for variable,value in self.data.items('.'):
if value and value.strip():
result[variable] = value
self._cleanup()
return result
def apply_dict(self, kv):
self._init_configuration()
for path, value in kv.items():
if not value and self.comment_empty:
self.data.comment(path)
else:
self.data.set(path,str(value), force=True)
self._cleanup(True)
def delete_options(self, options):
self._init_configuration()
for path in options:
self.data.remove(path)
self._cleanup(True)
def _init_configuration(self):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
def _cleanup(self, save_data=False):
if self.autosave:
if save_data and self.data:
self.data.write(self.path)
self.data = None
class BaseConfig(object):
'''
Parent class for object representations of postgresql.conf and recovery.conf which fortunately both have similar syntax
'''
autosave = None
path = None
data = None
config_name = None
config_type = None
comment_empty = False
def __init__(self, path, autosave=True):
self._logger = logging.getLogger(__name__)
self.autosave = autosave
self.path = path
@classmethod
def find(cls, config_dir):
return cls(os.path.join(config_dir.path, cls.config_name))
def set(self, option, value):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
if value:
self.data.set(option,str(value), force=True)
elif self.comment_empty:
self.data.comment(option)
if self.autosave:
self.save_data()
self.data = None
def set_path_type_option(self, option, path):
if not os.path.exists(path):
raise ValueError('%s %s does not exist' % (option, path))
self.set(option, path)
def set_numeric_option(self, option, number):
try:
assert number is None or type(number) is int
except AssertionError:
raise ValueError('%s must be a number (got %s instead)' % (option, number))
is_numeric = type(number) is int
self.set(option, str(number) if is_numeric else None)
def get(self, option):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
try:
value = self.data.get(option)
except NoPathError:
try:
value = getattr(self, option+'_default')
except AttributeError:
value = None
if self.autosave:
self.data = None
return value
def get_numeric_option(self, option):
value = self.get(option)
try:
assert value is None or int(value)
except AssertionError:
raise ValueError('%s must be a number (got %s instead)' % (option, type(value)))
return value if value is None else int(value)
def save_data(self):
if self.data:
self.data.write(self.path)