def _test_path_option(self, option):
self.assertRaises(ValueError, setattr, self.cnf, option, '/not/exists')
setattr(self.cnf, option, '/tmp')
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('/tmp', c.get(option))
self.assertEqual('/tmp', getattr(self.cnf, option))
class Cassandra(object):
def __init__(self):
self._logger = logging.getLogger(__name__)
self._initd = initdv2.lookup(SERVICE_NAME)
bus.on(reload=self.on_reload)
self.on_reload()
def on_reload(self):
self.queryenv = bus.queryenv_service
self.platform = bus.platform
self.private_ip = self.platform.get_private_ip()
self.zone = self.platform.get_avail_zone()
cnf = bus.cnf
self.ini = cnf.rawini
self.role_name = self.ini.get(config.SECT_GENERAL, config.OPT_ROLE_NAME)
self.storage_path = self.ini.get(CNF_SECTION, OPT_STORAGE_PATH)
self.storage_conf_path = self.ini.get(CNF_SECTION, OPT_STORAGE_CNF_PATH)
self.data_file_directory = self.storage_path + "/datafile"
self.commit_log_directory = self.storage_path + "/commitlog"
self.cassandra_conf = Configuration('xml')
try:
self.cassandra_conf.read(self.storage_conf_path)
except (OSError, MetaconfError, ParseError), e:
self._logger.error('Cassandra storage-conf.xml is broken. %s' % e)
def _test_path_option(self, option):
self.assertRaises(ValueError, setattr, self.cnf, option, '/not/exists')
setattr(self.cnf, option, '/tmp')
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('/tmp', c.get(option))
self.assertEqual('/tmp', getattr(self.cnf, option))
class Cassandra(object):
def __init__(self):
self._logger = logging.getLogger(__name__)
self._initd = initdv2.lookup(SERVICE_NAME)
bus.on(reload=self.on_reload)
self.on_reload()
def on_reload(self):
self.queryenv = bus.queryenv_service
self.platform = bus.platform
self.private_ip = self.platform.get_private_ip()
self.zone = self.platform.get_avail_zone()
cnf = bus.cnf
self.ini = cnf.rawini
self.role_name = self.ini.get(config.SECT_GENERAL, config.OPT_ROLE_NAME)
self.storage_path = self.ini.get(CNF_SECTION, OPT_STORAGE_PATH)
self.storage_conf_path = self.ini.get(CNF_SECTION, OPT_STORAGE_CNF_PATH)
self.data_file_directory = self.storage_path + "/datafile"
self.commit_log_directory = self.storage_path + "/commitlog"
self.cassandra_conf = Configuration('xml')
try:
self.cassandra_conf.read(self.storage_conf_path)
except (OSError, MetaconfError, ParseError), e:
self._logger.error('Cassandra storage-conf.xml is broken. %s' % e)
def get_system_variables(self):
#TESTING REQUIRED!
conf = Configuration(self._config_format)
conf.read(self._config_path)
vars = {}
for section in conf.sections('./'):
vars[section] = conf.get(section)
return vars
def __init__(self, body):
config = Configuration("apache")
try:
config.reads(str(body))
except ParseError, e:
LOG.error("MetaConf failed to parse Apache VirtualHost body: \n%s" % body)
e._err = body + "\n" + e._err
raise
def _https_config_exists(self):
config_dir = os.path.dirname(self.api.app_inc_path)
conf_path = os.path.join(config_dir, 'https.include')
config = None
try:
config = Configuration('nginx')
config.read(conf_path)
except (Exception, BaseException), e:
raise HandlerError('Cannot read/parse nginx main configuration file: %s' % str(e))
def __init__(self, body):
config = Configuration("apache")
try:
config.reads(str(body))
except ParseError, e:
LOG.error(
"MetaConf failed to parse Apache VirtualHost body: \n%s" %
body)
e._err = body + "\n" + e._err
raise
def _https_config_exists(self):
config_dir = os.path.dirname(self.api.app_inc_path)
conf_path = os.path.join(config_dir, 'https.include')
config = None
try:
config = Configuration('nginx')
config.read(conf_path)
except (Exception, BaseException), e:
raise HandlerError('Cannot read/parse nginx main configuration file: %s' % str(e))
def _update_main_config(self, remove_server_section=True, reload_service=True):
config_dir = os.path.dirname(self.api.app_inc_path)
nginx_conf_path = os.path.join(config_dir, 'nginx.conf')
config = None
try:
config = Configuration('nginx')
config.read(nginx_conf_path)
except (Exception, BaseException), e:
raise HandlerError('Cannot read/parse nginx main configuration file: %s' % str(e))
def _reload_backends(self):
self._logger.info('Updating mysql-proxy backends list')
self.config = Configuration('mysql')
if os.path.exists(CONFIG_FILE_PATH):
self.config.read(CONFIG_FILE_PATH)
self.config.remove('./mysql-proxy/proxy-backend-addresses')
self.config.remove(
'./mysql-proxy/proxy-read-only-backend-addresses')
try:
self.config.get('./mysql-proxy')
except NoPathError:
self.config.add('./mysql-proxy')
queryenv = bus.queryenv_service
roles = queryenv.list_roles()
master = None
slaves = []
for role in roles:
if not is_mysql_role(role.behaviour):
continue
for host in role.hosts:
ip = host.internal_ip or host.external_ip
if host.replication_master:
master = ip
else:
slaves.append(ip)
if master:
self._logger.debug(
'Adding mysql master %s to mysql-proxy defaults file', master)
self.config.add('./mysql-proxy/proxy-backend-addresses',
'%s:3306' % master)
if slaves:
self._logger.debug(
'Adding mysql slaves to mysql-proxy defaults file: %s',
', '.join(slaves))
for slave in slaves:
self.config.add(
'./mysql-proxy/proxy-read-only-backend-addresses',
'%s:3306' % slave)
self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
self.config.set('./mysql-proxy/daemon', 'true', force=True)
self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
if self.service.version > (0, 8, 0):
self.config.set('./mysql-proxy/plugins', 'proxy', force=True)
self._logger.debug('Saving new mysql-proxy defaults file')
self.config.write(CONFIG_FILE_PATH)
os.chmod(CONFIG_FILE_PATH, 0660)
self.service.restart()
def load(self, preset_type):
'''
@rtype: Preset
@raise OSError: When cannot read preset file
@raise MetaconfError: When experience problems with preset file parsing
'''
self._logger.debug('Loading %s %s preset' % (preset_type, self.service_name))
ini = Configuration('ini')
ini.read(self._filename(preset_type))
return CnfPreset(ini.get('general/name'), dict(ini.items('settings/')))
def _main_config_contains_server(self):
config_dir = os.path.dirname(self.api.app_inc_path)
nginx_conf_path = os.path.join(config_dir, 'nginx.conf')
config = None
result = False
try:
config = Configuration('nginx')
config.read(nginx_conf_path)
except (Exception, BaseException), e:
raise HandlerError('Cannot read/parse nginx main configuration file: %s' % str(e))
def __init__(self, manifest_path):
self._options = []
ini = Configuration('ini')
ini.read(manifest_path)
try:
self._defaults = dict(ini.items('__defaults__'))
except NoPathError:
self._defaults = dict()
for name in ini.sections("./"):
if name == '__defaults__':
continue
self._options.append(_OptionSpec.from_ini(ini, name, self._defaults))
def _test_option(self, option):
setattr(self.cnf, option, 'value')
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('value', c.get(option))
setattr(self.cnf, option, None)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertRaises(NoPathError, c.get, option)
def download_and_restore(self, volume, snapshot, tranzit_path):
# Load manifest
clear_queue(self._writer_queue)
clear_queue(self._download_queue)
self._download_finished.clear()
transfer = self._transfer_cls()
mnf_path = transfer.download(snapshot.path, tranzit_path)
mnf = Configuration('ini')
mnf.read(mnf_path)
volume.fs_created = False
volume.mkfs(snapshot.fstype)
remote_path = os.path.dirname(snapshot.path)
# Get links with md5 sums
links = [(os.path.join(remote_path, chunk[0]), chunk[1])
for chunk in mnf.items('chunks')]
links.sort()
# Download 2 first chunks
for link in links[:2]:
transfer.download(link[0], tranzit_path)
chunk_path = os.path.join(tranzit_path, os.path.basename(link[0]))
if self._md5sum(chunk_path) != link[1]:
raise Exception("md5sum of chunk %s is not correct." %
chunk_path)
self._writer_queue.put(chunk_path)
if hasattr(snapshot,
'snap_strategy') and snapshot.snap_strategy == 'data':
restore_strategy = DataRestoreStrategy(self._logger)
else:
restore_strategy = DeviceRestoreStrategy(self._logger)
writer = threading.Thread(target=restore_strategy.restore,
name='writer',
args=(self._writer_queue, volume,
self._download_finished))
writer.start()
# Add remaining files to download queue
for link in links[2:]:
self._download_queue.put(link)
downloader = threading.Thread(name="Downloader",
target=self._downloader,
args=(tranzit_path, ))
downloader.start()
downloader.join()
writer.join()
class ApacheConfigManager(object):
_cnf = None
path = None
def __init__(self, path):
self._cnf = Configuration("apache")
self.path = path
def __enter__(self):
self._cnf.read(self.path)
return self._cnf
def __exit__(self, type, value, traceback):
self._cnf.write(self.path)
class ApacheConfigManager(object):
_cnf = None
path = None
def __init__(self, path):
self._cnf = Configuration("apache")
self.path = path
def __enter__(self):
self._cnf.read(self.path)
return self._cnf
def __exit__(self, type, value, traceback):
self._cnf.write(self.path)
def _manifest(self):
f_manifest = CnfController._manifest
base_manifest = f_manifest.fget(self)
path = self._manifest_path
s = {}
out = None
if not self._merged_manifest:
cmd = '%s --no-defaults --verbose SU_EXEC' % mysql_svc.MYSQLD_PATH
out = system2('%s - mysql -s %s -c "%s"' % (SU_EXEC, BASH, cmd),
shell=True, raise_exc=False, silent=True)[0]
if out:
raw = out.split(49*'-'+' '+24*'-')
if raw:
a = raw[-1].split('\n')
if len(a) > 5:
b = a[1:-5]
for item in b:
c = item.split()
if len(c) > 1:
key = c[0]
val = ' '.join(c[1:])
s[key.strip()] = val.strip()
if s:
m_config = Configuration('ini')
if os.path.exists(path):
m_config.read(path)
for variable in base_manifest:
name = variable.name
dv_path = './%s/default-value' % name
try:
old_value = m_config.get(dv_path)
if name in s:
new_value = s[name]
else:
name = name.replace('_','-')
if name in s:
new_value = self.definitions[s[name]] if s[name] in self.definitions else s[name]
if old_value != new_value and new_value != '(No default value)':
LOG.debug('Replacing %s default value %s with precompiled value %s',
name, old_value, new_value)
m_config.set(path=dv_path, value=new_value, force=True)
except NoPathError:
pass
m_config.write(path)
self._merged_manifest = _CnfManifest(path)
return self._merged_manifest
def _test_numeric_option(self, option):
self.assertRaises(ValueError, setattr, self.cnf, option, 'NotNumericValue')
setattr(self.cnf, option, 113)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('113', c.get(option))
self.assertEqual(113, getattr(self.cnf, option))
setattr(self.cnf, option, None)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertRaises(NoPathError, c.get, option)
def download_and_restore(self, volume, snapshot, tranzit_path):
# Load manifest
clear_queue(self._writer_queue)
clear_queue(self._download_queue)
self._download_finished.clear()
transfer = self._transfer_cls()
mnf_path = transfer.download(snapshot.path, tranzit_path)
mnf = Configuration('ini')
mnf.read(mnf_path)
volume.fs_created = False
volume.mkfs(snapshot.fstype)
remote_path = os.path.dirname(snapshot.path)
# Get links with md5 sums
links = [(os.path.join(remote_path, chunk[0]), chunk[1]) for chunk in mnf.items('chunks')]
links.sort()
# Download 2 first chunks
for link in links[:2]:
transfer.download(link[0], tranzit_path)
chunk_path = os.path.join(tranzit_path, os.path.basename(link[0]))
if self._md5sum(chunk_path) != link[1]:
raise Exception("md5sum of chunk %s is not correct." % chunk_path)
self._writer_queue.put(chunk_path)
if hasattr(snapshot, 'snap_strategy') and snapshot.snap_strategy == 'data':
restore_strategy = DataRestoreStrategy(self._logger)
else:
restore_strategy = DeviceRestoreStrategy(self._logger)
writer = threading.Thread(target=restore_strategy.restore, name='writer',
args=(self._writer_queue, volume, self._download_finished))
writer.start()
# Add remaining files to download queue
for link in links[2:]:
self._download_queue.put(link)
downloader = threading.Thread(name="Downloader", target=self._downloader,
args=(tranzit_path,))
downloader.start()
downloader.join()
writer.join()
def set(self, option, value): if not self.data: self.data = Configuration(self.config_type) if os.path.exists(self.path): self.data.read(self.path) if value: self.data.set(option,str(value), force=True) elif self.comment_empty: self.data.comment(option) if self.autosave: self.save_data() self.data = None
def _test_option(self, option):
setattr(self.cnf, option, 'value')
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('value', c.get(option))
setattr(self.cnf, option, None)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertRaises(NoPathError, c.get, option)
def set(self, option, value):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
if value:
self.data.set(option, value, force=True)
else:
self.data.remove(option)
if self.autosave:
self.save_data()
self.data = None
def get_system_variables(self):
#TESTING REQUIRED!
conf = Configuration(self._config_format)
conf.read(self._config_path)
vars = {}
for section in conf.sections('./'):
vars[section] = conf.get(section)
return vars
def _test_numeric_option(self, option):
self.assertRaises(ValueError, setattr, self.cnf, option, 'NotNumericValue')
setattr(self.cnf, option, 113)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('113', c.get(option))
self.assertEqual(113, getattr(self.cnf, option))
setattr(self.cnf, option, None)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertRaises(NoPathError, c.get, option)
def get_list(self, option): if not self.data: self.data = Configuration(self.config_type) if os.path.exists(self.path): self.data.read(self.path) try: value = self.data.get_list(option) except NoPathError: try: value = getattr(self, option+'_default') except AttributeError: value = () if self.autosave: self.data = None return value
def set(self, option, value, append=False): if not self.data: self.data = Configuration(self.config_type) if os.path.exists(self.path): self.data.read(self.path) if value: if append: self.data.add(option, str(value)) else: self.data.set(option,str(value), force=True) else: self.data.comment(option) if self.autosave: self.save_data() self.data = None
def _reload_backends(self):
self._logger.info('Updating mysql-proxy backends list')
self.config = Configuration('mysql')
if os.path.exists(CONFIG_FILE_PATH):
self.config.read(CONFIG_FILE_PATH)
self.config.remove('./mysql-proxy/proxy-backend-addresses')
self.config.remove('./mysql-proxy/proxy-read-only-backend-addresses')
try:
self.config.get('./mysql-proxy')
except NoPathError:
self.config.add('./mysql-proxy')
queryenv = bus.queryenv_service
roles = queryenv.list_roles()
master = None
slaves = []
for role in roles:
if not is_mysql_role(role.behaviour):
continue
for host in role.hosts:
ip = host.internal_ip or host.external_ip
if host.replication_master:
master = ip
else:
slaves.append(ip)
if master:
self._logger.debug('Adding mysql master %s to mysql-proxy defaults file', master)
self.config.add('./mysql-proxy/proxy-backend-addresses', '%s:3306' % master)
if slaves:
self._logger.debug('Adding mysql slaves to mysql-proxy defaults file: %s', ', '.join(slaves))
for slave in slaves:
self.config.add('./mysql-proxy/proxy-read-only-backend-addresses', '%s:3306' % slave)
self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
self.config.set('./mysql-proxy/daemon', 'true', force=True)
self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
if self.service.version > (0,8,0):
self.config.set('./mysql-proxy/plugins', 'proxy', force=True)
self._logger.debug('Saving new mysql-proxy defaults file')
self.config.write(CONFIG_FILE_PATH)
os.chmod(CONFIG_FILE_PATH, 0660)
self.service.restart()
def load(self, preset_type):
'''
@rtype: Preset
@raise OSError: When cannot read preset file
@raise MetaconfError: When experience problems with preset file parsing
'''
self._logger.debug('Loading %s %s preset' %
(preset_type, self.service_name))
ini = Configuration('ini')
ini.read(self._filename(preset_type))
return CnfPreset(ini.get('general/name'), dict(ini.items('settings/')))
def __init__(self, manifest_path):
self._options = []
ini = Configuration('ini')
ini.read(manifest_path)
try:
self._defaults = dict(ini.items('__defaults__'))
except NoPathError:
self._defaults = dict()
for name in ini.sections("./"):
if name == '__defaults__':
continue
self._options.append(
_OptionSpec.from_ini(ini, name, self._defaults))
self._logger.debug('Do not removing http/server section')
if not config.get_list('http/server'):
config.read(os.path.join(bus.share_path, "nginx/server.tpl"))
if disttool.is_debian_based():
# Comment /etc/nginx/sites-enabled/*
try:
i = config.get_list('http/include').index('/etc/nginx/sites-enabled/*')
config.comment('http/include[%d]' % (i+1))
self._logger.debug('comment site-enabled include')
except (ValueError, IndexError):
self._logger.debug('site-enabled include already commented')
elif disttool.is_redhat_based():
def_host_path = '/etc/nginx/conf.d/default.conf'
if os.path.exists(def_host_path):
default_host = Configuration('nginx')
default_host.read(def_host_path)
default_host.comment('server')
default_host.write(def_host_path)
if dump == self._dump_config(config):
self._logger.debug("Main nginx config wasn`t changed")
else:
# Write new nginx.conf
shutil.copy(nginx_conf_path, nginx_conf_path + '.bak')
config.write(nginx_conf_path)
if reload_service:
self.api._reload_service()
def _insert_iptables_rules(self, *args, **kwargs):
if iptables.enabled():
def save(self, preset, preset_type):
'''
@type preset: CnfPreset
@type preset_type: CnfPresetStore.PresetType
@raise ValueError: When `preset` is not an instance of CnfPreset
@raise OSError: When cannot save preset file
'''
if not isinstance(preset, CnfPreset):
raise ValueError(
'argument `preset` should be a CnfPreset instance, %s is given',
type(preset))
self._logger.debug('Saving preset as %s' % preset_type)
ini = Configuration('ini')
ini.add('general')
ini.add(
'general/name', preset.name if
(hasattr(preset, 'name') and preset.name) else 'Noname')
ini.add('settings')
for k, v in preset.settings.items():
ini.add('settings/%s' % k, v)
ini.write(self._filename(preset_type))
def __init__(self, path):
self._cnf = Configuration("apache")
self.path = path
def _write_manifest(self, snapshot, tranzit_path):
''' Make snapshot manifest '''
manifest_path = os.path.join(tranzit_path, '%s.%s' % (snapshot.id, self.MANIFEST_NAME))
self._logger.info('Writing snapshot manifest file in %s', manifest_path)
config = Configuration('ini')
config.add('snapshot/description', snapshot.description, force=True)
config.add('snapshot/created_at', time.strftime("%Y-%m-%d %H:%M:%S"))
config.add('snapshot/pack_method', 'pigz') # Not used yet
for chunk, md5 in self._chunks_md5.iteritems():
config.add('chunks/%s' % chunk, md5, force=True)
config.write(manifest_path)
return manifest_path
class BaseConfig(object):
'''
Parent class for object representations of postgresql.conf and recovery.conf which fortunately both have similar syntax
'''
autosave = None
path = None
data = None
config_name = None
config_type = None
comment_empty = False
def __init__(self, path, autosave=True):
self._logger = logging.getLogger(__name__)
self.autosave = autosave
self.path = path
@classmethod
def find(cls, config_dir):
return cls(os.path.join(config_dir.path, cls.config_name))
def set(self, option, value):
self.apply_dict({option:value})
def set_path_type_option(self, option, path):
if not os.path.exists(path):
raise ValueError('%s %s does not exist' % (option, path))
self.set(option, path)
def set_numeric_option(self, option, number):
try:
assert number is None or type(number) is int
except AssertionError:
raise ValueError('%s must be a number (got %s instead)' % (option, number))
is_numeric = type(number) is int
self.set(option, str(number) if is_numeric else None)
def get(self, option):
self._init_configuration()
try:
value = self.data.get(option)
except NoPathError:
try:
value = getattr(self, option+'_default')
except AttributeError:
value = None
self._cleanup()
return value
def get_numeric_option(self, option):
value = self.get(option)
return value if value is None else int(value)
def to_dict(self):
self._init_configuration()
result = {}
for section in self.data.sections('./'):
try:
kv = dict(self.data.items(section))
except NoPathError:
kv = {}
for variable, value in kv.items():
path = '%s/%s' % (section,variable)
result[path] = value
'''
variables in root section
'''
for variable,value in self.data.items('.'):
if value and value.strip():
result[variable] = value
self._cleanup()
return result
def apply_dict(self, kv):
self._init_configuration()
for path, value in kv.items():
if not value and self.comment_empty:
self.data.comment(path)
else:
self.data.set(path,str(value), force=True)
self._cleanup(True)
def delete_options(self, options):
self._init_configuration()
for path in options:
self.data.remove(path)
self._cleanup(True)
def _init_configuration(self):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
def _cleanup(self, save_data=False):
if self.autosave:
if save_data and self.data:
self.data.write(self.path)
self.data = None
def apply_preset(self, preset):
conf = Configuration(self._config_format)
conf.read(self._config_path)
self._before_apply_preset()
ver = self._software_version
for opt in self._manifest:
path = opt.name if not opt.section else '%s/%s' % (opt.section,
opt.name)
try:
value = conf.get(path)
except NoPathError:
value = ''
if opt.name in preset.settings:
new_value = preset.settings[opt.name]
# Skip unsupported
if ver and opt.supported_from and opt.supported_from > ver:
self._logger.debug(
"Skipping option '%s' supported from %s; installed %s"
% (opt.name, opt.supported_from, ver))
continue
if not opt.default_value:
self._logger.debug("Option '%s' has no default value" %
opt.name)
pass
elif new_value == opt.default_value:
if value:
self._logger.debug(
"Option '%s' equal to default. Removing." %
opt.name)
conf.remove(path)
self._after_remove_option(opt)
continue
if self.definitions and new_value in self.definitions:
manifest = Configuration('ini')
if os.path.exists(self._manifest_path):
manifest.read(self._manifest_path)
try:
if manifest.get('%s/type' % opt.name) == 'boolean':
new_value = self.definitions[new_value]
except NoPathError, e:
pass
self._logger.debug("Check that '%s' value changed:'%s'='%s'" %
(opt.name, value, new_value))
if new_value == value:
self._logger.debug("Skip option '%s'. Not changed" %
opt.name)
pass
else:
self._logger.debug("Set option '%s' = '%s'" %
(opt.name, new_value))
self._logger.debug('Set path %s = %s', path, new_value)
conf.set(path, new_value, force=True)
self._after_set_option(opt, new_value)
else:
if value:
self._logger.debug(
"Removing option '%s'. Not found in preset" % opt.name)
conf.remove(path)
self._after_remove_option(opt)
def _manifest(self):
class HeadRequest(urllib2.Request):
def get_method(self):
return "HEAD"
manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour
path = self._manifest_path
url_handle = urllib2.urlopen(HeadRequest(manifest_url))
headers = url_handle.info()
url_last_modified = headers.getdate("Last-Modified")
file_modified = tuple(time.localtime(
os.path.getmtime(path))) if os.path.exists(path) else None
if not file_modified or url_last_modified > file_modified:
self._logger.debug('Fetching %s', manifest_url)
response = urllib2.urlopen(manifest_url)
data = response.read()
if data:
old_manifest = Configuration('ini')
if os.path.exists(path):
old_manifest.read(path)
new_manifest = Configuration('ini')
o = StringIO()
o.write(data)
o.seek(0)
new_manifest.readfp(o)
new_sections = new_manifest.sections('./')
old_sections = old_manifest.sections('./')
diff_path = os.path.join(os.path.dirname(path),
self.behaviour + '.incdiff')
diff = Configuration('ini')
if old_sections and old_sections != new_sections:
#skipping diff if no previous manifest found or it is equal to the new one
if os.path.exists(diff_path):
diff.read(diff_path)
sys_vars = self.get_system_variables()
for section in new_sections:
if section not in old_sections and sys_vars.has_key(
section):
sys_var = sys_vars[section]
if self.definitions:
if self.definitions.has_key(sys_var):
sys_var = self.definitions[sys_var]
diff.add('./%s/default-value' % section,
sys_var,
force=True)
diff.write(diff_path)
if os.path.exists(diff_path):
diff.read(diff_path)
for variable in diff.sections('./'):
sys_value = diff.get('./%s/default-value' % variable)
if sys_value and variable in new_manifest.sections('./'):
new_manifest.set('./%s/default-value' % variable,
sys_value,
force=True)
new_manifest.write(path)
return _CnfManifest(path)
def _test_bool_option(self, option):
self.assertRaises(ValueError, setattr, self.cnf, option, 'NotBoolValue')
setattr(self.cnf, option, True)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('true', c.get(option))
self.assertEqual(True, getattr(self.cnf, option))
setattr(self.cnf, option, False)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('false', c.get(option))
self.assertEqual(False, getattr(self.cnf, option))
setattr(self.cnf, option, None)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertRaises(NoPathError, c.get, option)
c.set(option, 'NotBool', force=True)
c.write(self.cnf_path)
self.assertRaises(ValueError, getattr, self.cnf, option)
def __init__(self, path):
self._cnf = Configuration("apache")
self.path = path
def _test_bool_option(self, option):
self.assertRaises(ValueError, setattr, self.cnf, option, 'NotBoolValue')
setattr(self.cnf, option, True)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('true', c.get(option))
self.assertEqual(True, getattr(self.cnf, option))
setattr(self.cnf, option, False)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertEqual('false', c.get(option))
self.assertEqual(False, getattr(self.cnf, option))
setattr(self.cnf, option, None)
c = Configuration('mongodb')
c.read(self.cnf_path)
self.assertRaises(NoPathError, c.get, option)
c.set(option, 'NotBool', force=True)
c.write(self.cnf_path)
self.assertRaises(ValueError, getattr, self.cnf, option)
class MysqlProxyHandler(ServiceCtlHandler):
def __init__(self):
self._logger = logging.getLogger(__name__)
self.service = initdv2.lookup(BEHAVIOUR)
self._service_name = BEHAVIOUR
bus.on(init=self.on_init)
def accept(self,
message,
queue,
behaviour=None,
platform=None,
os=None,
dist=None):
return message.behaviour and is_mysql_role(
message.behaviour) and message.name in (
Messages.HOST_UP, Messages.HOST_DOWN, NEW_MASTER_UP,
DbMsrMessages.DBMSR_NEW_MASTER_UP)
def on_init(self):
bus.on(start=self.on_start,
before_host_up=self.on_before_host_up,
reload=self.on_reload)
def on_reload(self):
self._reload_backends()
def on_start(self):
cnf = bus.cnf
if cnf.state == config.ScalarizrState.RUNNING:
self._reload_backends()
def on_before_host_up(self, msg):
self._reload_backends()
def _reload_backends(self):
self._logger.info('Updating mysql-proxy backends list')
self.config = Configuration('mysql')
if os.path.exists(CONFIG_FILE_PATH):
self.config.read(CONFIG_FILE_PATH)
self.config.remove('./mysql-proxy/proxy-backend-addresses')
self.config.remove(
'./mysql-proxy/proxy-read-only-backend-addresses')
try:
self.config.get('./mysql-proxy')
except NoPathError:
self.config.add('./mysql-proxy')
queryenv = bus.queryenv_service
roles = queryenv.list_roles()
master = None
slaves = []
for role in roles:
if not is_mysql_role(role.behaviour):
continue
for host in role.hosts:
ip = host.internal_ip or host.external_ip
if host.replication_master:
master = ip
else:
slaves.append(ip)
if master:
self._logger.debug(
'Adding mysql master %s to mysql-proxy defaults file', master)
self.config.add('./mysql-proxy/proxy-backend-addresses',
'%s:3306' % master)
if slaves:
self._logger.debug(
'Adding mysql slaves to mysql-proxy defaults file: %s',
', '.join(slaves))
for slave in slaves:
self.config.add(
'./mysql-proxy/proxy-read-only-backend-addresses',
'%s:3306' % slave)
self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
self.config.set('./mysql-proxy/daemon', 'true', force=True)
self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
if self.service.version > (0, 8, 0):
self.config.set('./mysql-proxy/plugins', 'proxy', force=True)
self._logger.debug('Saving new mysql-proxy defaults file')
self.config.write(CONFIG_FILE_PATH)
os.chmod(CONFIG_FILE_PATH, 0660)
self.service.restart()
def on_HostUp(self, message):
self._reload_backends()
on_DbMsr_NewMasterUp = on_Mysql_NewMasterUp = on_HostDown = on_HostUp
def _manifest(self):
class HeadRequest(urllib2.Request):
def get_method(self):
return "HEAD"
manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour
path = self._manifest_path
url_handle = urllib2.urlopen(HeadRequest(manifest_url))
headers = url_handle.info()
url_last_modified = headers.getdate("Last-Modified")
file_modified = tuple(time.localtime(os.path.getmtime(path))) if os.path.exists(path) else None
if not file_modified or url_last_modified > file_modified:
self._logger.debug('Fetching %s', manifest_url)
response = urllib2.urlopen(manifest_url)
data = response.read()
if data:
old_manifest = Configuration('ini')
if os.path.exists(path):
old_manifest.read(path)
new_manifest = Configuration('ini')
o = StringIO()
o.write(data)
o.seek(0)
new_manifest.readfp(o)
new_sections = new_manifest.sections('./')
old_sections = old_manifest.sections('./')
diff_path = os.path.join(os.path.dirname(path), self.behaviour + '.incdiff')
diff = Configuration('ini')
if old_sections and old_sections != new_sections:
#skipping diff if no previous manifest found or it is equal to the new one
if os.path.exists(diff_path):
diff.read(diff_path)
sys_vars = self.get_system_variables()
for section in new_sections:
if section not in old_sections and sys_vars.has_key(section):
sys_var = sys_vars[section]
if self.definitions:
if self.definitions.has_key(sys_var):
sys_var = self.definitions[sys_var]
diff.add('./%s/default-value' % section, sys_var, force=True)
diff.write(diff_path)
if os.path.exists(diff_path):
diff.read(diff_path)
for variable in diff.sections('./'):
sys_value = diff.get('./%s/default-value' % variable)
if sys_value and variable in new_manifest.sections('./'):
new_manifest.set('./%s/default-value' % variable, sys_value, force=True)
new_manifest.write(path)
return _CnfManifest(path)
class BaseRedisConfig(BaseConfig):
config_type = 'redis'
def set(self, option, value, append=False):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
if value:
if append:
self.data.add(option, str(value))
else:
self.data.set(option,str(value), force=True)
else:
self.data.comment(option)
if self.autosave:
self.save_data()
self.data = None
def set_sequential_option(self, option, seq):
is_typle = type(seq) is tuple
try:
assert seq is None or is_typle
except AssertionError:
raise ValueError('%s must be a sequence (got %s instead)' % (option, seq))
self.set(option, ' '.join(map(str,seq)) if is_typle else None)
def get_sequential_option(self, option):
raw = self.get(option)
return raw.split() if raw else ()
def get_list(self, option):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
try:
value = self.data.get_list(option)
except NoPathError:
try:
value = getattr(self, option+'_default')
except AttributeError:
value = ()
if self.autosave:
self.data = None
return value
def get_dict_option(self, option):
raw = self.get_list(option)
d = {}
for raw_value in raw:
k,v = raw_value.split()
if k and v:
d[k] = v
return d
def set_dict_option(self, option, d):
try:
assert d is None or type(d)==dict
#cleaning up
#TODO: make clean process smarter using indexes
for i in self.get_list(option):
self.set(option+'[0]', None)
#adding multiple entries
for k,v in d.items():
val = ' '.join(map(str,'%s %s'%(k,v)))
self.set(option, val, append=True)
except ValueError:
raise ValueError('%s must be a sequence (got %s instead)' % (option, d))
class BaseConfig(object):
'''
Parent class for object representations of postgresql.conf and recovery.conf which fortunately both have similar syntax
'''
autosave = None
path = None
data = None
config_name = None
config_type = None
comment_empty = False
def __init__(self, path, autosave=True):
self._logger = logging.getLogger(__name__)
self.autosave = autosave
self.path = path
@classmethod
def find(cls, config_dir):
return cls(os.path.join(config_dir.path, cls.config_name))
def set(self, option, value):
self.apply_dict({option: value})
def set_path_type_option(self, option, path):
if not os.path.exists(path):
raise ValueError('%s %s does not exist' % (option, path))
self.set(option, path)
def set_numeric_option(self, option, number):
try:
assert number is None or type(number) is int
except AssertionError:
raise ValueError('%s must be a number (got %s instead)' %
(option, number))
is_numeric = type(number) is int
self.set(option, str(number) if is_numeric else None)
def get(self, option):
self._init_configuration()
try:
value = self.data.get(option)
except NoPathError:
try:
value = getattr(self, option + '_default')
except AttributeError:
value = None
self._cleanup()
return value
def get_numeric_option(self, option):
value = self.get(option)
return value if value is None else int(value)
def to_dict(self):
self._init_configuration()
result = {}
for section in self.data.sections('./'):
try:
kv = dict(self.data.items(section))
except NoPathError:
kv = {}
for variable, value in kv.items():
path = '%s/%s' % (section, variable)
result[path] = value
'''
variables in root section
'''
for variable, value in self.data.items('.'):
if value and value.strip():
result[variable] = value
self._cleanup()
return result
def apply_dict(self, kv):
self._init_configuration()
for path, value in kv.items():
if not value and self.comment_empty:
self.data.comment(path)
else:
self.data.set(path, str(value), force=True)
self._cleanup(True)
def delete_options(self, options):
self._init_configuration()
for path in options:
self.data.remove(path)
self._cleanup(True)
def _init_configuration(self):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
def _cleanup(self, save_data=False):
if self.autosave:
if save_data and self.data:
self.data.write(self.path)
self.data = None
def save(self, preset, preset_type):
'''
@type preset: CnfPreset
@type preset_type: CnfPresetStore.PresetType
@raise ValueError: When `preset` is not an instance of CnfPreset
@raise OSError: When cannot save preset file
'''
if not isinstance(preset, CnfPreset):
raise ValueError('argument `preset` should be a CnfPreset instance, %s is given', type(preset))
self._logger.debug('Saving preset as %s' % preset_type)
ini = Configuration('ini')
ini.add('general')
ini.add('general/name', preset.name if (hasattr(preset, 'name') and preset.name) else 'Noname')
ini.add('settings')
for k, v in preset.settings.items():
ini.add('settings/%s' % k, v)
ini.write(self._filename(preset_type))
class MongoDBConfig(BaseConfig):
config_type = "mongodb"
config_name = os.path.basename(CONFIG_PATH_DEFAULT)
@classmethod
def find(cls, config_dir=None):
# conf_path = UBUNTU_CONFIG_PATH if disttool.is_ubuntu() else CENTOS_CONFIG_PATH
return cls(os.path.join(config_dir, cls.config_name) if config_dir else CONFIG_PATH_DEFAULT)
def set(self, option, value):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
if value:
self.data.set(option, value, force=True)
else:
self.data.remove(option)
if self.autosave:
self.save_data()
self.data = None
def set_bool_option(self, option, value):
try:
assert value in (None, True, False)
except AssertionError:
raise ValueError("%s must be a boolean (got %s instead)" % (option, type(value)))
value_to_set = None if value is None else str(value).lower()
self.set(option, value_to_set)
def get_bool_option(self, option):
value = self.get(option)
try:
assert not value or value == "true" or value == "false"
except AssertionError:
raise ValueError("%s must be true or false (got %s instead)" % (option, type(value)))
return True if value == "true" else False
def _get_logpath(self):
return self.get("logpath")
def _set_logpath(self, path):
self.set("logpath", path)
def _get_replSet(self):
return self.get("replSet")
def _set_replSet(self, name):
self.set("replSet", name)
def _get_port(self):
return self.get_numeric_option("port")
def _set_port(self, number):
self.set_numeric_option("port", number)
def _get_logappend(self):
return self.get_bool_option("logappend")
def _set_logappend(self, on=True):
self.set_bool_option("logappend", on)
def _get_dbpath(self):
return self.get("dbpath")
def _set_dbpath(self, path):
self.set_path_type_option("dbpath", path)
def _get_nojournal(self):
return self.get_bool_option("nojournal")
def _set_nojournal(self, on=False):
self.set_bool_option("nojournal", on)
def _get_nohttpinterface(self):
return self.get_bool_option("nohttpinterface")
def _set_nohttpinterface(self, on=False):
self.set_bool_option("nohttpinterface", on)
def _get_rest(self):
return self.get_bool_option("rest")
def _set_rest(self, on=False):
self.set_bool_option("rest", on)
def _set_shardsvr(self, value):
self.set_bool_option("shardsvr", value)
def _get_shardsvr(self):
return self.get_bool_option("shardsvr")
shardsvr = property(_get_shardsvr, _set_shardsvr)
rest = property(_get_rest, _set_rest)
nohttpinterface = property(_get_nohttpinterface, _set_nohttpinterface)
nojournal = property(_get_nojournal, _set_nojournal)
dbpath = property(_get_dbpath, _set_dbpath)
logappend = property(_get_logappend, _set_logappend)
port = property(_get_port, _set_port)
replSet = property(_get_replSet, _set_replSet)
logpath = property(_get_logpath, _set_logpath)
def _write_manifest(self, snapshot, tranzit_path):
''' Make snapshot manifest '''
manifest_path = os.path.join(
tranzit_path, '%s.%s' % (snapshot.id, self.MANIFEST_NAME))
self._logger.info('Writing snapshot manifest file in %s',
manifest_path)
config = Configuration('ini')
config.add('snapshot/description', snapshot.description, force=True)
config.add('snapshot/created_at', time.strftime("%Y-%m-%d %H:%M:%S"))
config.add('snapshot/pack_method', 'pigz') # Not used yet
for chunk, md5 in self._chunks_md5.iteritems():
config.add('chunks/%s' % chunk, md5, force=True)
config.write(manifest_path)
return manifest_path
def _init_configuration(self):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
def _init_configuration(self):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
class MysqlProxyHandler(ServiceCtlHandler):
def __init__(self):
self._logger = logging.getLogger(__name__)
self.service = initdv2.lookup(BEHAVIOUR)
self._service_name = BEHAVIOUR
bus.on(init=self.on_init)
def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None):
return message.behaviour and is_mysql_role(message.behaviour) and message.name in (
Messages.HOST_UP,
Messages.HOST_DOWN,
NEW_MASTER_UP,
DbMsrMessages.DBMSR_NEW_MASTER_UP
)
def on_init(self):
bus.on(
start=self.on_start,
before_host_up=self.on_before_host_up,
reload=self.on_reload
)
def on_reload(self):
self._reload_backends()
def on_start(self):
cnf = bus.cnf
if cnf.state == config.ScalarizrState.RUNNING:
self._reload_backends()
def on_before_host_up(self, msg):
self._reload_backends()
def _reload_backends(self):
self._logger.info('Updating mysql-proxy backends list')
self.config = Configuration('mysql')
if os.path.exists(CONFIG_FILE_PATH):
self.config.read(CONFIG_FILE_PATH)
self.config.remove('./mysql-proxy/proxy-backend-addresses')
self.config.remove('./mysql-proxy/proxy-read-only-backend-addresses')
try:
self.config.get('./mysql-proxy')
except NoPathError:
self.config.add('./mysql-proxy')
queryenv = bus.queryenv_service
roles = queryenv.list_roles()
master = None
slaves = []
for role in roles:
if not is_mysql_role(role.behaviour):
continue
for host in role.hosts:
ip = host.internal_ip or host.external_ip
if host.replication_master:
master = ip
else:
slaves.append(ip)
if master:
self._logger.debug('Adding mysql master %s to mysql-proxy defaults file', master)
self.config.add('./mysql-proxy/proxy-backend-addresses', '%s:3306' % master)
if slaves:
self._logger.debug('Adding mysql slaves to mysql-proxy defaults file: %s', ', '.join(slaves))
for slave in slaves:
self.config.add('./mysql-proxy/proxy-read-only-backend-addresses', '%s:3306' % slave)
self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
self.config.set('./mysql-proxy/daemon', 'true', force=True)
self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
if self.service.version > (0,8,0):
self.config.set('./mysql-proxy/plugins', 'proxy', force=True)
self._logger.debug('Saving new mysql-proxy defaults file')
self.config.write(CONFIG_FILE_PATH)
os.chmod(CONFIG_FILE_PATH, 0660)
self.service.restart()
def on_HostUp(self, message):
self._reload_backends()
on_DbMsr_NewMasterUp = on_Mysql_NewMasterUp = on_HostDown = on_HostUp