def apply_preset(self, preset): conf = Configuration(self._config_format) conf.read(self._config_path) self._before_apply_preset() ver = self._software_version for opt in self._manifest: path = opt.name if not opt.section else '%s/%s' % (opt.section, opt.name) try: value = conf.get(path) except NoPathError: value = '' if opt.name in preset.settings: new_value = preset.settings[opt.name] # Skip unsupported if ver and opt.supported_from and opt.supported_from > ver: self._logger.debug("Skipping option '%s' supported from %s; installed %s" % (opt.name, opt.supported_from, ver)) continue if not opt.default_value: self._logger.debug("Option '%s' has no default value" % opt.name) pass elif new_value == opt.default_value: if value: self._logger.debug("Option '%s' equal to default. Removing." % opt.name) conf.remove(path) self._after_remove_option(opt) continue if self.definitions and new_value in self.definitions: manifest = Configuration('ini') if os.path.exists(self._manifest_path): manifest.read(self._manifest_path) try: if manifest.get('%s/type' % opt.name) == 'boolean': new_value = self.definitions[new_value] except NoPathError, e: pass self._logger.debug("Check that '%s' value changed:'%s'='%s'"%(opt.name, value, new_value)) if new_value == value: self._logger.debug("Skip option '%s'. Not changed" % opt.name) pass else: self._logger.debug("Set option '%s' = '%s'" % (opt.name, new_value)) self._logger.debug('Set path %s = %s', path, new_value) conf.set(path, new_value, force=True) self._after_set_option(opt, new_value) else: if value: self._logger.debug("Removing option '%s'. Not found in preset" % opt.name) conf.remove(path) self._after_remove_option(opt)
def _manifest(self): f_manifest = CnfController._manifest base_manifest = f_manifest.fget(self) path = self._manifest_path s = {} out = None if not self._merged_manifest: cmd = '%s --no-defaults --verbose SU_EXEC' % mysql_svc.MYSQLD_PATH out = system2('%s - mysql -s %s -c "%s"' % (SU_EXEC, BASH, cmd), shell=True, raise_exc=False, silent=True)[0] if out: raw = out.split(49*'-'+' '+24*'-') if raw: a = raw[-1].split('\n') if len(a) > 5: b = a[1:-5] for item in b: c = item.split() if len(c) > 1: key = c[0] val = ' '.join(c[1:]) s[key.strip()] = val.strip() if s: m_config = Configuration('ini') if os.path.exists(path): m_config.read(path) for variable in base_manifest: name = variable.name dv_path = './%s/default-value' % name try: old_value = m_config.get(dv_path) if name in s: new_value = s[name] else: name = name.replace('_','-') if name in s: new_value = self.definitions[s[name]] if s[name] in self.definitions else s[name] if old_value != new_value and new_value != '(No default value)': LOG.debug('Replacing %s default value %s with precompiled value %s', name, old_value, new_value) m_config.set(path=dv_path, value=new_value, force=True) except NoPathError: pass m_config.write(path) self._merged_manifest = _CnfManifest(path) return self._merged_manifest
def _test_bool_option(self, option): self.assertRaises(ValueError, setattr, self.cnf, option, 'NotBoolValue') setattr(self.cnf, option, True) c = Configuration('mongodb') c.read(self.cnf_path) self.assertEqual('true', c.get(option)) self.assertEqual(True, getattr(self.cnf, option)) setattr(self.cnf, option, False) c = Configuration('mongodb') c.read(self.cnf_path) self.assertEqual('false', c.get(option)) self.assertEqual(False, getattr(self.cnf, option)) setattr(self.cnf, option, None) c = Configuration('mongodb') c.read(self.cnf_path) self.assertRaises(NoPathError, c.get, option) c.set(option, 'NotBool', force=True) c.write(self.cnf_path) self.assertRaises(ValueError, getattr, self.cnf, option)
class MysqlProxyHandler(ServiceCtlHandler): def __init__(self): self._logger = logging.getLogger(__name__) self.service = initdv2.lookup(BEHAVIOUR) self._service_name = BEHAVIOUR bus.on(init=self.on_init) def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None): return message.behaviour and is_mysql_role( message.behaviour) and message.name in ( Messages.HOST_UP, Messages.HOST_DOWN, NEW_MASTER_UP, DbMsrMessages.DBMSR_NEW_MASTER_UP) def on_init(self): bus.on(start=self.on_start, before_host_up=self.on_before_host_up, reload=self.on_reload) def on_reload(self): self._reload_backends() def on_start(self): cnf = bus.cnf if cnf.state == config.ScalarizrState.RUNNING: self._reload_backends() def on_before_host_up(self, msg): self._reload_backends() def _reload_backends(self): self._logger.info('Updating mysql-proxy backends list') self.config = Configuration('mysql') if os.path.exists(CONFIG_FILE_PATH): self.config.read(CONFIG_FILE_PATH) self.config.remove('./mysql-proxy/proxy-backend-addresses') self.config.remove( './mysql-proxy/proxy-read-only-backend-addresses') try: self.config.get('./mysql-proxy') except NoPathError: self.config.add('./mysql-proxy') queryenv = bus.queryenv_service roles = queryenv.list_roles() master = None slaves = [] for role in roles: if not is_mysql_role(role.behaviour): continue for host in role.hosts: ip = host.internal_ip or host.external_ip if host.replication_master: master = ip else: slaves.append(ip) if master: self._logger.debug( 'Adding mysql master %s to mysql-proxy defaults file', master) self.config.add('./mysql-proxy/proxy-backend-addresses', '%s:3306' % master) if slaves: self._logger.debug( 'Adding mysql slaves to mysql-proxy defaults file: %s', ', '.join(slaves)) for slave in slaves: self.config.add( './mysql-proxy/proxy-read-only-backend-addresses', '%s:3306' % slave) self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True) self.config.set('./mysql-proxy/daemon', 'true', force=True) self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True) if self.service.version > (0, 8, 0): self.config.set('./mysql-proxy/plugins', 'proxy', force=True) self._logger.debug('Saving new mysql-proxy defaults file') self.config.write(CONFIG_FILE_PATH) os.chmod(CONFIG_FILE_PATH, 0660) self.service.restart() def on_HostUp(self, message): self._reload_backends() on_DbMsr_NewMasterUp = on_Mysql_NewMasterUp = on_HostDown = on_HostUp
def _manifest(self): class HeadRequest(urllib2.Request): def get_method(self): return "HEAD" manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour path = self._manifest_path url_handle = urllib2.urlopen(HeadRequest(manifest_url)) headers = url_handle.info() url_last_modified = headers.getdate("Last-Modified") file_modified = tuple(time.localtime( os.path.getmtime(path))) if os.path.exists(path) else None if not file_modified or url_last_modified > file_modified: self._logger.debug('Fetching %s', manifest_url) response = urllib2.urlopen(manifest_url) data = response.read() if data: old_manifest = Configuration('ini') if os.path.exists(path): old_manifest.read(path) new_manifest = Configuration('ini') o = StringIO() o.write(data) o.seek(0) new_manifest.readfp(o) new_sections = new_manifest.sections('./') old_sections = old_manifest.sections('./') diff_path = os.path.join(os.path.dirname(path), self.behaviour + '.incdiff') diff = Configuration('ini') if old_sections and old_sections != new_sections: #skipping diff if no previous manifest found or it is equal to the new one if os.path.exists(diff_path): diff.read(diff_path) sys_vars = self.get_system_variables() for section in new_sections: if section not in old_sections and sys_vars.has_key( section): sys_var = sys_vars[section] if self.definitions: if self.definitions.has_key(sys_var): sys_var = self.definitions[sys_var] diff.add('./%s/default-value' % section, sys_var, force=True) diff.write(diff_path) if os.path.exists(diff_path): diff.read(diff_path) for variable in diff.sections('./'): sys_value = diff.get('./%s/default-value' % variable) if sys_value and variable in new_manifest.sections('./'): new_manifest.set('./%s/default-value' % variable, sys_value, force=True) new_manifest.write(path) return _CnfManifest(path)
def apply_preset(self, preset): conf = Configuration(self._config_format) conf.read(self._config_path) self._before_apply_preset() ver = self._software_version for opt in self._manifest: path = opt.name if not opt.section else '%s/%s' % (opt.section, opt.name) try: value = conf.get(path) except NoPathError: value = '' if opt.name in preset.settings: new_value = preset.settings[opt.name] # Skip unsupported if ver and opt.supported_from and opt.supported_from > ver: self._logger.debug( "Skipping option '%s' supported from %s; installed %s" % (opt.name, opt.supported_from, ver)) continue if not opt.default_value: self._logger.debug("Option '%s' has no default value" % opt.name) pass elif new_value == opt.default_value: if value: self._logger.debug( "Option '%s' equal to default. Removing." % opt.name) conf.remove(path) self._after_remove_option(opt) continue if self.definitions and new_value in self.definitions: manifest = Configuration('ini') if os.path.exists(self._manifest_path): manifest.read(self._manifest_path) try: if manifest.get('%s/type' % opt.name) == 'boolean': new_value = self.definitions[new_value] except NoPathError, e: pass self._logger.debug("Check that '%s' value changed:'%s'='%s'" % (opt.name, value, new_value)) if new_value == value: self._logger.debug("Skip option '%s'. Not changed" % opt.name) pass else: self._logger.debug("Set option '%s' = '%s'" % (opt.name, new_value)) self._logger.debug('Set path %s = %s', path, new_value) conf.set(path, new_value, force=True) self._after_set_option(opt, new_value) else: if value: self._logger.debug( "Removing option '%s'. Not found in preset" % opt.name) conf.remove(path) self._after_remove_option(opt)
class MysqlProxyHandler(ServiceCtlHandler): def __init__(self): self._logger = logging.getLogger(__name__) self.service = initdv2.lookup(BEHAVIOUR) self._service_name = BEHAVIOUR bus.on(init=self.on_init) def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None): return message.behaviour and is_mysql_role(message.behaviour) and message.name in ( Messages.HOST_UP, Messages.HOST_DOWN, NEW_MASTER_UP, DbMsrMessages.DBMSR_NEW_MASTER_UP ) def on_init(self): bus.on( start=self.on_start, before_host_up=self.on_before_host_up, reload=self.on_reload ) def on_reload(self): self._reload_backends() def on_start(self): cnf = bus.cnf if cnf.state == config.ScalarizrState.RUNNING: self._reload_backends() def on_before_host_up(self, msg): self._reload_backends() def _reload_backends(self): self._logger.info('Updating mysql-proxy backends list') self.config = Configuration('mysql') if os.path.exists(CONFIG_FILE_PATH): self.config.read(CONFIG_FILE_PATH) self.config.remove('./mysql-proxy/proxy-backend-addresses') self.config.remove('./mysql-proxy/proxy-read-only-backend-addresses') try: self.config.get('./mysql-proxy') except NoPathError: self.config.add('./mysql-proxy') queryenv = bus.queryenv_service roles = queryenv.list_roles() master = None slaves = [] for role in roles: if not is_mysql_role(role.behaviour): continue for host in role.hosts: ip = host.internal_ip or host.external_ip if host.replication_master: master = ip else: slaves.append(ip) if master: self._logger.debug('Adding mysql master %s to mysql-proxy defaults file', master) self.config.add('./mysql-proxy/proxy-backend-addresses', '%s:3306' % master) if slaves: self._logger.debug('Adding mysql slaves to mysql-proxy defaults file: %s', ', '.join(slaves)) for slave in slaves: self.config.add('./mysql-proxy/proxy-read-only-backend-addresses', '%s:3306' % slave) self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True) self.config.set('./mysql-proxy/daemon', 'true', force=True) self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True) if self.service.version > (0,8,0): self.config.set('./mysql-proxy/plugins', 'proxy', force=True) self._logger.debug('Saving new mysql-proxy defaults file') self.config.write(CONFIG_FILE_PATH) os.chmod(CONFIG_FILE_PATH, 0660) self.service.restart() def on_HostUp(self, message): self._reload_backends() on_DbMsr_NewMasterUp = on_Mysql_NewMasterUp = on_HostDown = on_HostUp
class MongoDBConfig(BaseConfig): config_type = "mongodb" config_name = os.path.basename(CONFIG_PATH_DEFAULT) @classmethod def find(cls, config_dir=None): # conf_path = UBUNTU_CONFIG_PATH if disttool.is_ubuntu() else CENTOS_CONFIG_PATH return cls(os.path.join(config_dir, cls.config_name) if config_dir else CONFIG_PATH_DEFAULT) def set(self, option, value): if not self.data: self.data = Configuration(self.config_type) if os.path.exists(self.path): self.data.read(self.path) if value: self.data.set(option, value, force=True) else: self.data.remove(option) if self.autosave: self.save_data() self.data = None def set_bool_option(self, option, value): try: assert value in (None, True, False) except AssertionError: raise ValueError("%s must be a boolean (got %s instead)" % (option, type(value))) value_to_set = None if value is None else str(value).lower() self.set(option, value_to_set) def get_bool_option(self, option): value = self.get(option) try: assert not value or value == "true" or value == "false" except AssertionError: raise ValueError("%s must be true or false (got %s instead)" % (option, type(value))) return True if value == "true" else False def _get_logpath(self): return self.get("logpath") def _set_logpath(self, path): self.set("logpath", path) def _get_replSet(self): return self.get("replSet") def _set_replSet(self, name): self.set("replSet", name) def _get_port(self): return self.get_numeric_option("port") def _set_port(self, number): self.set_numeric_option("port", number) def _get_logappend(self): return self.get_bool_option("logappend") def _set_logappend(self, on=True): self.set_bool_option("logappend", on) def _get_dbpath(self): return self.get("dbpath") def _set_dbpath(self, path): self.set_path_type_option("dbpath", path) def _get_nojournal(self): return self.get_bool_option("nojournal") def _set_nojournal(self, on=False): self.set_bool_option("nojournal", on) def _get_nohttpinterface(self): return self.get_bool_option("nohttpinterface") def _set_nohttpinterface(self, on=False): self.set_bool_option("nohttpinterface", on) def _get_rest(self): return self.get_bool_option("rest") def _set_rest(self, on=False): self.set_bool_option("rest", on) def _set_shardsvr(self, value): self.set_bool_option("shardsvr", value) def _get_shardsvr(self): return self.get_bool_option("shardsvr") shardsvr = property(_get_shardsvr, _set_shardsvr) rest = property(_get_rest, _set_rest) nohttpinterface = property(_get_nohttpinterface, _set_nohttpinterface) nojournal = property(_get_nojournal, _set_nojournal) dbpath = property(_get_dbpath, _set_dbpath) logappend = property(_get_logappend, _set_logappend) port = property(_get_port, _set_port) replSet = property(_get_replSet, _set_replSet) logpath = property(_get_logpath, _set_logpath)
class BaseRedisConfig(BaseConfig): config_type = 'redis' def set(self, option, value, append=False): if not self.data: self.data = Configuration(self.config_type) if os.path.exists(self.path): self.data.read(self.path) if value: if append: self.data.add(option, str(value)) else: self.data.set(option,str(value), force=True) else: self.data.comment(option) if self.autosave: self.save_data() self.data = None def set_sequential_option(self, option, seq): is_typle = type(seq) is tuple try: assert seq is None or is_typle except AssertionError: raise ValueError('%s must be a sequence (got %s instead)' % (option, seq)) self.set(option, ' '.join(map(str,seq)) if is_typle else None) def get_sequential_option(self, option): raw = self.get(option) return raw.split() if raw else () def get_list(self, option): if not self.data: self.data = Configuration(self.config_type) if os.path.exists(self.path): self.data.read(self.path) try: value = self.data.get_list(option) except NoPathError: try: value = getattr(self, option+'_default') except AttributeError: value = () if self.autosave: self.data = None return value def get_dict_option(self, option): raw = self.get_list(option) d = {} for raw_value in raw: k,v = raw_value.split() if k and v: d[k] = v return d def set_dict_option(self, option, d): try: assert d is None or type(d)==dict #cleaning up #TODO: make clean process smarter using indexes for i in self.get_list(option): self.set(option+'[0]', None) #adding multiple entries for k,v in d.items(): val = ' '.join(map(str,'%s %s'%(k,v))) self.set(option, val, append=True) except ValueError: raise ValueError('%s must be a sequence (got %s instead)' % (option, d))
def _manifest(self): class HeadRequest(urllib2.Request): def get_method(self): return "HEAD" manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour path = self._manifest_path url_handle = urllib2.urlopen(HeadRequest(manifest_url)) headers = url_handle.info() url_last_modified = headers.getdate("Last-Modified") file_modified = tuple(time.localtime(os.path.getmtime(path))) if os.path.exists(path) else None if not file_modified or url_last_modified > file_modified: self._logger.debug('Fetching %s', manifest_url) response = urllib2.urlopen(manifest_url) data = response.read() if data: old_manifest = Configuration('ini') if os.path.exists(path): old_manifest.read(path) new_manifest = Configuration('ini') o = StringIO() o.write(data) o.seek(0) new_manifest.readfp(o) new_sections = new_manifest.sections('./') old_sections = old_manifest.sections('./') diff_path = os.path.join(os.path.dirname(path), self.behaviour + '.incdiff') diff = Configuration('ini') if old_sections and old_sections != new_sections: #skipping diff if no previous manifest found or it is equal to the new one if os.path.exists(diff_path): diff.read(diff_path) sys_vars = self.get_system_variables() for section in new_sections: if section not in old_sections and sys_vars.has_key(section): sys_var = sys_vars[section] if self.definitions: if self.definitions.has_key(sys_var): sys_var = self.definitions[sys_var] diff.add('./%s/default-value' % section, sys_var, force=True) diff.write(diff_path) if os.path.exists(diff_path): diff.read(diff_path) for variable in diff.sections('./'): sys_value = diff.get('./%s/default-value' % variable) if sys_value and variable in new_manifest.sections('./'): new_manifest.set('./%s/default-value' % variable, sys_value, force=True) new_manifest.write(path) return _CnfManifest(path)
class BaseConfig(object): ''' Parent class for object representations of postgresql.conf and recovery.conf which fortunately both have similar syntax ''' autosave = None path = None data = None config_name = None config_type = None comment_empty = False def __init__(self, path, autosave=True): self._logger = logging.getLogger(__name__) self.autosave = autosave self.path = path @classmethod def find(cls, config_dir): return cls(os.path.join(config_dir.path, cls.config_name)) def set(self, option, value): if not self.data: self.data = Configuration(self.config_type) if os.path.exists(self.path): self.data.read(self.path) if value: self.data.set(option,str(value), force=True) elif self.comment_empty: self.data.comment(option) if self.autosave: self.save_data() self.data = None def set_path_type_option(self, option, path): if not os.path.exists(path): raise ValueError('%s %s does not exist' % (option, path)) self.set(option, path) def set_numeric_option(self, option, number): try: assert number is None or type(number) is int except AssertionError: raise ValueError('%s must be a number (got %s instead)' % (option, number)) is_numeric = type(number) is int self.set(option, str(number) if is_numeric else None) def get(self, option): if not self.data: self.data = Configuration(self.config_type) if os.path.exists(self.path): self.data.read(self.path) try: value = self.data.get(option) except NoPathError: try: value = getattr(self, option+'_default') except AttributeError: value = None if self.autosave: self.data = None return value def get_numeric_option(self, option): value = self.get(option) try: assert value is None or int(value) except AssertionError: raise ValueError('%s must be a number (got %s instead)' % (option, type(value))) return value if value is None else int(value) def save_data(self): if self.data: self.data.write(self.path)