def _write_manifest(self, snapshot, tranzit_path): ''' Make snapshot manifest ''' manifest_path = os.path.join(tranzit_path, '%s.%s' % (snapshot.id, self.MANIFEST_NAME)) self._logger.info('Writing snapshot manifest file in %s', manifest_path) config = Configuration('ini') config.add('snapshot/description', snapshot.description, force=True) config.add('snapshot/created_at', time.strftime("%Y-%m-%d %H:%M:%S")) config.add('snapshot/pack_method', 'pigz') # Not used yet for chunk, md5 in self._chunks_md5.iteritems(): config.add('chunks/%s' % chunk, md5, force=True) config.write(manifest_path) return manifest_path
def _write_manifest(self, snapshot, tranzit_path): ''' Make snapshot manifest ''' manifest_path = os.path.join( tranzit_path, '%s.%s' % (snapshot.id, self.MANIFEST_NAME)) self._logger.info('Writing snapshot manifest file in %s', manifest_path) config = Configuration('ini') config.add('snapshot/description', snapshot.description, force=True) config.add('snapshot/created_at', time.strftime("%Y-%m-%d %H:%M:%S")) config.add('snapshot/pack_method', 'pigz') # Not used yet for chunk, md5 in self._chunks_md5.iteritems(): config.add('chunks/%s' % chunk, md5, force=True) config.write(manifest_path) return manifest_path
def save(self, preset, preset_type): ''' @type preset: CnfPreset @type preset_type: CnfPresetStore.PresetType @raise ValueError: When `preset` is not an instance of CnfPreset @raise OSError: When cannot save preset file ''' if not isinstance(preset, CnfPreset): raise ValueError('argument `preset` should be a CnfPreset instance, %s is given', type(preset)) self._logger.debug('Saving preset as %s' % preset_type) ini = Configuration('ini') ini.add('general') ini.add('general/name', preset.name if (hasattr(preset, 'name') and preset.name) else 'Noname') ini.add('settings') for k, v in preset.settings.items(): ini.add('settings/%s' % k, v) ini.write(self._filename(preset_type))
def save(self, preset, preset_type): ''' @type preset: CnfPreset @type preset_type: CnfPresetStore.PresetType @raise ValueError: When `preset` is not an instance of CnfPreset @raise OSError: When cannot save preset file ''' if not isinstance(preset, CnfPreset): raise ValueError( 'argument `preset` should be a CnfPreset instance, %s is given', type(preset)) self._logger.debug('Saving preset as %s' % preset_type) ini = Configuration('ini') ini.add('general') ini.add( 'general/name', preset.name if (hasattr(preset, 'name') and preset.name) else 'Noname') ini.add('settings') for k, v in preset.settings.items(): ini.add('settings/%s' % k, v) ini.write(self._filename(preset_type))
class MysqlProxyHandler(ServiceCtlHandler): def __init__(self): self._logger = logging.getLogger(__name__) self.service = initdv2.lookup(BEHAVIOUR) self._service_name = BEHAVIOUR bus.on(init=self.on_init) def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None): return message.behaviour and is_mysql_role( message.behaviour) and message.name in ( Messages.HOST_UP, Messages.HOST_DOWN, NEW_MASTER_UP, DbMsrMessages.DBMSR_NEW_MASTER_UP) def on_init(self): bus.on(start=self.on_start, before_host_up=self.on_before_host_up, reload=self.on_reload) def on_reload(self): self._reload_backends() def on_start(self): cnf = bus.cnf if cnf.state == config.ScalarizrState.RUNNING: self._reload_backends() def on_before_host_up(self, msg): self._reload_backends() def _reload_backends(self): self._logger.info('Updating mysql-proxy backends list') self.config = Configuration('mysql') if os.path.exists(CONFIG_FILE_PATH): self.config.read(CONFIG_FILE_PATH) self.config.remove('./mysql-proxy/proxy-backend-addresses') self.config.remove( './mysql-proxy/proxy-read-only-backend-addresses') try: self.config.get('./mysql-proxy') except NoPathError: self.config.add('./mysql-proxy') queryenv = bus.queryenv_service roles = queryenv.list_roles() master = None slaves = [] for role in roles: if not is_mysql_role(role.behaviour): continue for host in role.hosts: ip = host.internal_ip or host.external_ip if host.replication_master: master = ip else: slaves.append(ip) if master: self._logger.debug( 'Adding mysql master %s to mysql-proxy defaults file', master) self.config.add('./mysql-proxy/proxy-backend-addresses', '%s:3306' % master) if slaves: self._logger.debug( 'Adding mysql slaves to mysql-proxy defaults file: %s', ', '.join(slaves)) for slave in slaves: self.config.add( './mysql-proxy/proxy-read-only-backend-addresses', '%s:3306' % slave) self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True) self.config.set('./mysql-proxy/daemon', 'true', force=True) self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True) if self.service.version > (0, 8, 0): self.config.set('./mysql-proxy/plugins', 'proxy', force=True) self._logger.debug('Saving new mysql-proxy defaults file') self.config.write(CONFIG_FILE_PATH) os.chmod(CONFIG_FILE_PATH, 0660) self.service.restart() def on_HostUp(self, message): self._reload_backends() on_DbMsr_NewMasterUp = on_Mysql_NewMasterUp = on_HostDown = on_HostUp
def _manifest(self): class HeadRequest(urllib2.Request): def get_method(self): return "HEAD" manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour path = self._manifest_path url_handle = urllib2.urlopen(HeadRequest(manifest_url)) headers = url_handle.info() url_last_modified = headers.getdate("Last-Modified") file_modified = tuple(time.localtime( os.path.getmtime(path))) if os.path.exists(path) else None if not file_modified or url_last_modified > file_modified: self._logger.debug('Fetching %s', manifest_url) response = urllib2.urlopen(manifest_url) data = response.read() if data: old_manifest = Configuration('ini') if os.path.exists(path): old_manifest.read(path) new_manifest = Configuration('ini') o = StringIO() o.write(data) o.seek(0) new_manifest.readfp(o) new_sections = new_manifest.sections('./') old_sections = old_manifest.sections('./') diff_path = os.path.join(os.path.dirname(path), self.behaviour + '.incdiff') diff = Configuration('ini') if old_sections and old_sections != new_sections: #skipping diff if no previous manifest found or it is equal to the new one if os.path.exists(diff_path): diff.read(diff_path) sys_vars = self.get_system_variables() for section in new_sections: if section not in old_sections and sys_vars.has_key( section): sys_var = sys_vars[section] if self.definitions: if self.definitions.has_key(sys_var): sys_var = self.definitions[sys_var] diff.add('./%s/default-value' % section, sys_var, force=True) diff.write(diff_path) if os.path.exists(diff_path): diff.read(diff_path) for variable in diff.sections('./'): sys_value = diff.get('./%s/default-value' % variable) if sys_value and variable in new_manifest.sections('./'): new_manifest.set('./%s/default-value' % variable, sys_value, force=True) new_manifest.write(path) return _CnfManifest(path)
class MysqlProxyHandler(ServiceCtlHandler): def __init__(self): self._logger = logging.getLogger(__name__) self.service = initdv2.lookup(BEHAVIOUR) self._service_name = BEHAVIOUR bus.on(init=self.on_init) def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None): return message.behaviour and is_mysql_role(message.behaviour) and message.name in ( Messages.HOST_UP, Messages.HOST_DOWN, NEW_MASTER_UP, DbMsrMessages.DBMSR_NEW_MASTER_UP ) def on_init(self): bus.on( start=self.on_start, before_host_up=self.on_before_host_up, reload=self.on_reload ) def on_reload(self): self._reload_backends() def on_start(self): cnf = bus.cnf if cnf.state == config.ScalarizrState.RUNNING: self._reload_backends() def on_before_host_up(self, msg): self._reload_backends() def _reload_backends(self): self._logger.info('Updating mysql-proxy backends list') self.config = Configuration('mysql') if os.path.exists(CONFIG_FILE_PATH): self.config.read(CONFIG_FILE_PATH) self.config.remove('./mysql-proxy/proxy-backend-addresses') self.config.remove('./mysql-proxy/proxy-read-only-backend-addresses') try: self.config.get('./mysql-proxy') except NoPathError: self.config.add('./mysql-proxy') queryenv = bus.queryenv_service roles = queryenv.list_roles() master = None slaves = [] for role in roles: if not is_mysql_role(role.behaviour): continue for host in role.hosts: ip = host.internal_ip or host.external_ip if host.replication_master: master = ip else: slaves.append(ip) if master: self._logger.debug('Adding mysql master %s to mysql-proxy defaults file', master) self.config.add('./mysql-proxy/proxy-backend-addresses', '%s:3306' % master) if slaves: self._logger.debug('Adding mysql slaves to mysql-proxy defaults file: %s', ', '.join(slaves)) for slave in slaves: self.config.add('./mysql-proxy/proxy-read-only-backend-addresses', '%s:3306' % slave) self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True) self.config.set('./mysql-proxy/daemon', 'true', force=True) self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True) if self.service.version > (0,8,0): self.config.set('./mysql-proxy/plugins', 'proxy', force=True) self._logger.debug('Saving new mysql-proxy defaults file') self.config.write(CONFIG_FILE_PATH) os.chmod(CONFIG_FILE_PATH, 0660) self.service.restart() def on_HostUp(self, message): self._reload_backends() on_DbMsr_NewMasterUp = on_Mysql_NewMasterUp = on_HostDown = on_HostUp
class BaseRedisConfig(BaseConfig): config_type = 'redis' def set(self, option, value, append=False): if not self.data: self.data = Configuration(self.config_type) if os.path.exists(self.path): self.data.read(self.path) if value: if append: self.data.add(option, str(value)) else: self.data.set(option,str(value), force=True) else: self.data.comment(option) if self.autosave: self.save_data() self.data = None def set_sequential_option(self, option, seq): is_typle = type(seq) is tuple try: assert seq is None or is_typle except AssertionError: raise ValueError('%s must be a sequence (got %s instead)' % (option, seq)) self.set(option, ' '.join(map(str,seq)) if is_typle else None) def get_sequential_option(self, option): raw = self.get(option) return raw.split() if raw else () def get_list(self, option): if not self.data: self.data = Configuration(self.config_type) if os.path.exists(self.path): self.data.read(self.path) try: value = self.data.get_list(option) except NoPathError: try: value = getattr(self, option+'_default') except AttributeError: value = () if self.autosave: self.data = None return value def get_dict_option(self, option): raw = self.get_list(option) d = {} for raw_value in raw: k,v = raw_value.split() if k and v: d[k] = v return d def set_dict_option(self, option, d): try: assert d is None or type(d)==dict #cleaning up #TODO: make clean process smarter using indexes for i in self.get_list(option): self.set(option+'[0]', None) #adding multiple entries for k,v in d.items(): val = ' '.join(map(str,'%s %s'%(k,v))) self.set(option, val, append=True) except ValueError: raise ValueError('%s must be a sequence (got %s instead)' % (option, d))
def _manifest(self): class HeadRequest(urllib2.Request): def get_method(self): return "HEAD" manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour path = self._manifest_path url_handle = urllib2.urlopen(HeadRequest(manifest_url)) headers = url_handle.info() url_last_modified = headers.getdate("Last-Modified") file_modified = tuple(time.localtime(os.path.getmtime(path))) if os.path.exists(path) else None if not file_modified or url_last_modified > file_modified: self._logger.debug('Fetching %s', manifest_url) response = urllib2.urlopen(manifest_url) data = response.read() if data: old_manifest = Configuration('ini') if os.path.exists(path): old_manifest.read(path) new_manifest = Configuration('ini') o = StringIO() o.write(data) o.seek(0) new_manifest.readfp(o) new_sections = new_manifest.sections('./') old_sections = old_manifest.sections('./') diff_path = os.path.join(os.path.dirname(path), self.behaviour + '.incdiff') diff = Configuration('ini') if old_sections and old_sections != new_sections: #skipping diff if no previous manifest found or it is equal to the new one if os.path.exists(diff_path): diff.read(diff_path) sys_vars = self.get_system_variables() for section in new_sections: if section not in old_sections and sys_vars.has_key(section): sys_var = sys_vars[section] if self.definitions: if self.definitions.has_key(sys_var): sys_var = self.definitions[sys_var] diff.add('./%s/default-value' % section, sys_var, force=True) diff.write(diff_path) if os.path.exists(diff_path): diff.read(diff_path) for variable in diff.sections('./'): sys_value = diff.get('./%s/default-value' % variable) if sys_value and variable in new_manifest.sections('./'): new_manifest.set('./%s/default-value' % variable, sys_value, force=True) new_manifest.write(path) return _CnfManifest(path)