def _write_manifest(self, snapshot, tranzit_path):
''' Make snapshot manifest '''
manifest_path = os.path.join(tranzit_path, '%s.%s' % (snapshot.id, self.MANIFEST_NAME))
self._logger.info('Writing snapshot manifest file in %s', manifest_path)
config = Configuration('ini')
config.add('snapshot/description', snapshot.description, force=True)
config.add('snapshot/created_at', time.strftime("%Y-%m-%d %H:%M:%S"))
config.add('snapshot/pack_method', 'pigz') # Not used yet
for chunk, md5 in self._chunks_md5.iteritems():
config.add('chunks/%s' % chunk, md5, force=True)
config.write(manifest_path)
return manifest_path
def _write_manifest(self, snapshot, tranzit_path):
''' Make snapshot manifest '''
manifest_path = os.path.join(
tranzit_path, '%s.%s' % (snapshot.id, self.MANIFEST_NAME))
self._logger.info('Writing snapshot manifest file in %s',
manifest_path)
config = Configuration('ini')
config.add('snapshot/description', snapshot.description, force=True)
config.add('snapshot/created_at', time.strftime("%Y-%m-%d %H:%M:%S"))
config.add('snapshot/pack_method', 'pigz') # Not used yet
for chunk, md5 in self._chunks_md5.iteritems():
config.add('chunks/%s' % chunk, md5, force=True)
config.write(manifest_path)
return manifest_path
def save(self, preset, preset_type):
'''
@type preset: CnfPreset
@type preset_type: CnfPresetStore.PresetType
@raise ValueError: When `preset` is not an instance of CnfPreset
@raise OSError: When cannot save preset file
'''
if not isinstance(preset, CnfPreset):
raise ValueError('argument `preset` should be a CnfPreset instance, %s is given', type(preset))
self._logger.debug('Saving preset as %s' % preset_type)
ini = Configuration('ini')
ini.add('general')
ini.add('general/name', preset.name if (hasattr(preset, 'name') and preset.name) else 'Noname')
ini.add('settings')
for k, v in preset.settings.items():
ini.add('settings/%s' % k, v)
ini.write(self._filename(preset_type))
def save(self, preset, preset_type):
'''
@type preset: CnfPreset
@type preset_type: CnfPresetStore.PresetType
@raise ValueError: When `preset` is not an instance of CnfPreset
@raise OSError: When cannot save preset file
'''
if not isinstance(preset, CnfPreset):
raise ValueError(
'argument `preset` should be a CnfPreset instance, %s is given',
type(preset))
self._logger.debug('Saving preset as %s' % preset_type)
ini = Configuration('ini')
ini.add('general')
ini.add(
'general/name', preset.name if
(hasattr(preset, 'name') and preset.name) else 'Noname')
ini.add('settings')
for k, v in preset.settings.items():
ini.add('settings/%s' % k, v)
ini.write(self._filename(preset_type))
class MysqlProxyHandler(ServiceCtlHandler):
def __init__(self):
self._logger = logging.getLogger(__name__)
self.service = initdv2.lookup(BEHAVIOUR)
self._service_name = BEHAVIOUR
bus.on(init=self.on_init)
def accept(self,
message,
queue,
behaviour=None,
platform=None,
os=None,
dist=None):
return message.behaviour and is_mysql_role(
message.behaviour) and message.name in (
Messages.HOST_UP, Messages.HOST_DOWN, NEW_MASTER_UP,
DbMsrMessages.DBMSR_NEW_MASTER_UP)
def on_init(self):
bus.on(start=self.on_start,
before_host_up=self.on_before_host_up,
reload=self.on_reload)
def on_reload(self):
self._reload_backends()
def on_start(self):
cnf = bus.cnf
if cnf.state == config.ScalarizrState.RUNNING:
self._reload_backends()
def on_before_host_up(self, msg):
self._reload_backends()
def _reload_backends(self):
self._logger.info('Updating mysql-proxy backends list')
self.config = Configuration('mysql')
if os.path.exists(CONFIG_FILE_PATH):
self.config.read(CONFIG_FILE_PATH)
self.config.remove('./mysql-proxy/proxy-backend-addresses')
self.config.remove(
'./mysql-proxy/proxy-read-only-backend-addresses')
try:
self.config.get('./mysql-proxy')
except NoPathError:
self.config.add('./mysql-proxy')
queryenv = bus.queryenv_service
roles = queryenv.list_roles()
master = None
slaves = []
for role in roles:
if not is_mysql_role(role.behaviour):
continue
for host in role.hosts:
ip = host.internal_ip or host.external_ip
if host.replication_master:
master = ip
else:
slaves.append(ip)
if master:
self._logger.debug(
'Adding mysql master %s to mysql-proxy defaults file', master)
self.config.add('./mysql-proxy/proxy-backend-addresses',
'%s:3306' % master)
if slaves:
self._logger.debug(
'Adding mysql slaves to mysql-proxy defaults file: %s',
', '.join(slaves))
for slave in slaves:
self.config.add(
'./mysql-proxy/proxy-read-only-backend-addresses',
'%s:3306' % slave)
self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
self.config.set('./mysql-proxy/daemon', 'true', force=True)
self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
if self.service.version > (0, 8, 0):
self.config.set('./mysql-proxy/plugins', 'proxy', force=True)
self._logger.debug('Saving new mysql-proxy defaults file')
self.config.write(CONFIG_FILE_PATH)
os.chmod(CONFIG_FILE_PATH, 0660)
self.service.restart()
def on_HostUp(self, message):
self._reload_backends()
on_DbMsr_NewMasterUp = on_Mysql_NewMasterUp = on_HostDown = on_HostUp
def _manifest(self):
class HeadRequest(urllib2.Request):
def get_method(self):
return "HEAD"
manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour
path = self._manifest_path
url_handle = urllib2.urlopen(HeadRequest(manifest_url))
headers = url_handle.info()
url_last_modified = headers.getdate("Last-Modified")
file_modified = tuple(time.localtime(
os.path.getmtime(path))) if os.path.exists(path) else None
if not file_modified or url_last_modified > file_modified:
self._logger.debug('Fetching %s', manifest_url)
response = urllib2.urlopen(manifest_url)
data = response.read()
if data:
old_manifest = Configuration('ini')
if os.path.exists(path):
old_manifest.read(path)
new_manifest = Configuration('ini')
o = StringIO()
o.write(data)
o.seek(0)
new_manifest.readfp(o)
new_sections = new_manifest.sections('./')
old_sections = old_manifest.sections('./')
diff_path = os.path.join(os.path.dirname(path),
self.behaviour + '.incdiff')
diff = Configuration('ini')
if old_sections and old_sections != new_sections:
#skipping diff if no previous manifest found or it is equal to the new one
if os.path.exists(diff_path):
diff.read(diff_path)
sys_vars = self.get_system_variables()
for section in new_sections:
if section not in old_sections and sys_vars.has_key(
section):
sys_var = sys_vars[section]
if self.definitions:
if self.definitions.has_key(sys_var):
sys_var = self.definitions[sys_var]
diff.add('./%s/default-value' % section,
sys_var,
force=True)
diff.write(diff_path)
if os.path.exists(diff_path):
diff.read(diff_path)
for variable in diff.sections('./'):
sys_value = diff.get('./%s/default-value' % variable)
if sys_value and variable in new_manifest.sections('./'):
new_manifest.set('./%s/default-value' % variable,
sys_value,
force=True)
new_manifest.write(path)
return _CnfManifest(path)
class MysqlProxyHandler(ServiceCtlHandler):
def __init__(self):
self._logger = logging.getLogger(__name__)
self.service = initdv2.lookup(BEHAVIOUR)
self._service_name = BEHAVIOUR
bus.on(init=self.on_init)
def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None):
return message.behaviour and is_mysql_role(message.behaviour) and message.name in (
Messages.HOST_UP,
Messages.HOST_DOWN,
NEW_MASTER_UP,
DbMsrMessages.DBMSR_NEW_MASTER_UP
)
def on_init(self):
bus.on(
start=self.on_start,
before_host_up=self.on_before_host_up,
reload=self.on_reload
)
def on_reload(self):
self._reload_backends()
def on_start(self):
cnf = bus.cnf
if cnf.state == config.ScalarizrState.RUNNING:
self._reload_backends()
def on_before_host_up(self, msg):
self._reload_backends()
def _reload_backends(self):
self._logger.info('Updating mysql-proxy backends list')
self.config = Configuration('mysql')
if os.path.exists(CONFIG_FILE_PATH):
self.config.read(CONFIG_FILE_PATH)
self.config.remove('./mysql-proxy/proxy-backend-addresses')
self.config.remove('./mysql-proxy/proxy-read-only-backend-addresses')
try:
self.config.get('./mysql-proxy')
except NoPathError:
self.config.add('./mysql-proxy')
queryenv = bus.queryenv_service
roles = queryenv.list_roles()
master = None
slaves = []
for role in roles:
if not is_mysql_role(role.behaviour):
continue
for host in role.hosts:
ip = host.internal_ip or host.external_ip
if host.replication_master:
master = ip
else:
slaves.append(ip)
if master:
self._logger.debug('Adding mysql master %s to mysql-proxy defaults file', master)
self.config.add('./mysql-proxy/proxy-backend-addresses', '%s:3306' % master)
if slaves:
self._logger.debug('Adding mysql slaves to mysql-proxy defaults file: %s', ', '.join(slaves))
for slave in slaves:
self.config.add('./mysql-proxy/proxy-read-only-backend-addresses', '%s:3306' % slave)
self.config.set('./mysql-proxy/pid-file', PID_FILE, force=True)
self.config.set('./mysql-proxy/daemon', 'true', force=True)
self.config.set('./mysql-proxy/log-file', LOG_FILE, force=True)
if self.service.version > (0,8,0):
self.config.set('./mysql-proxy/plugins', 'proxy', force=True)
self._logger.debug('Saving new mysql-proxy defaults file')
self.config.write(CONFIG_FILE_PATH)
os.chmod(CONFIG_FILE_PATH, 0660)
self.service.restart()
def on_HostUp(self, message):
self._reload_backends()
on_DbMsr_NewMasterUp = on_Mysql_NewMasterUp = on_HostDown = on_HostUp
class BaseRedisConfig(BaseConfig):
config_type = 'redis'
def set(self, option, value, append=False):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
if value:
if append:
self.data.add(option, str(value))
else:
self.data.set(option,str(value), force=True)
else:
self.data.comment(option)
if self.autosave:
self.save_data()
self.data = None
def set_sequential_option(self, option, seq):
is_typle = type(seq) is tuple
try:
assert seq is None or is_typle
except AssertionError:
raise ValueError('%s must be a sequence (got %s instead)' % (option, seq))
self.set(option, ' '.join(map(str,seq)) if is_typle else None)
def get_sequential_option(self, option):
raw = self.get(option)
return raw.split() if raw else ()
def get_list(self, option):
if not self.data:
self.data = Configuration(self.config_type)
if os.path.exists(self.path):
self.data.read(self.path)
try:
value = self.data.get_list(option)
except NoPathError:
try:
value = getattr(self, option+'_default')
except AttributeError:
value = ()
if self.autosave:
self.data = None
return value
def get_dict_option(self, option):
raw = self.get_list(option)
d = {}
for raw_value in raw:
k,v = raw_value.split()
if k and v:
d[k] = v
return d
def set_dict_option(self, option, d):
try:
assert d is None or type(d)==dict
#cleaning up
#TODO: make clean process smarter using indexes
for i in self.get_list(option):
self.set(option+'[0]', None)
#adding multiple entries
for k,v in d.items():
val = ' '.join(map(str,'%s %s'%(k,v)))
self.set(option, val, append=True)
except ValueError:
raise ValueError('%s must be a sequence (got %s instead)' % (option, d))
def _manifest(self):
class HeadRequest(urllib2.Request):
def get_method(self):
return "HEAD"
manifest_url = bus.scalr_url + '/storage/service-configuration-manifests/%s.ini' % self.behaviour
path = self._manifest_path
url_handle = urllib2.urlopen(HeadRequest(manifest_url))
headers = url_handle.info()
url_last_modified = headers.getdate("Last-Modified")
file_modified = tuple(time.localtime(os.path.getmtime(path))) if os.path.exists(path) else None
if not file_modified or url_last_modified > file_modified:
self._logger.debug('Fetching %s', manifest_url)
response = urllib2.urlopen(manifest_url)
data = response.read()
if data:
old_manifest = Configuration('ini')
if os.path.exists(path):
old_manifest.read(path)
new_manifest = Configuration('ini')
o = StringIO()
o.write(data)
o.seek(0)
new_manifest.readfp(o)
new_sections = new_manifest.sections('./')
old_sections = old_manifest.sections('./')
diff_path = os.path.join(os.path.dirname(path), self.behaviour + '.incdiff')
diff = Configuration('ini')
if old_sections and old_sections != new_sections:
#skipping diff if no previous manifest found or it is equal to the new one
if os.path.exists(diff_path):
diff.read(diff_path)
sys_vars = self.get_system_variables()
for section in new_sections:
if section not in old_sections and sys_vars.has_key(section):
sys_var = sys_vars[section]
if self.definitions:
if self.definitions.has_key(sys_var):
sys_var = self.definitions[sys_var]
diff.add('./%s/default-value' % section, sys_var, force=True)
diff.write(diff_path)
if os.path.exists(diff_path):
diff.read(diff_path)
for variable in diff.sections('./'):
sys_value = diff.get('./%s/default-value' % variable)
if sys_value and variable in new_manifest.sections('./'):
new_manifest.set('./%s/default-value' % variable, sys_value, force=True)
new_manifest.write(path)
return _CnfManifest(path)
