def _init_slave(self, message):
"""
Initialize redis slave
@type message: scalarizr.messaging.Message
@param message: HostUp message
"""
LOG.info("Initializing %s slave" % BEHAVIOUR)
with bus.initialization_op as op:
with op.step(self._step_create_storage):
LOG.debug("Initializing slave storage")
self.storage_vol = self._plug_storage(self._storage_path,
dict(snapshot=Storage.restore_config(self._snapshot_config_path)))
Storage.backup_config(self.storage_vol.config(), self._volume_config_path)
with op.step(self._step_init_slave):
# Change replication master
master_host = self._get_master_host()
LOG.debug("Master server obtained (local_ip: %s, public_ip: %s)",
master_host.internal_ip, master_host.external_ip)
host = master_host.internal_ip or master_host.external_ip
instance = self.redis_instances.get_instance(port=redis.DEFAULT_PORT)
instance.init_slave(self._storage_path, host, redis.DEFAULT_PORT)
op.progress(50)
instance.wait_for_sync()
with op.step(self._step_collect_host_up_data):
# Update HostUp message
message.redis = self._compat_storage_data(self.storage_vol)
message.db_type = BEHAVIOUR
def _init_slave(self, message):
"""
Initialize postgresql slave
@type message: scalarizr.messaging.Message
@param message: HostUp message
"""
self._logger.info("Initializing postgresql slave")
with bus.initialization_op as op:
with op.step(self._step_create_storage):
self._logger.debug("Initialize slave storage")
self.storage_vol = self._plug_storage(self._storage_path,
dict(snapshot=Storage.restore_config(self._snapshot_config_path)))
Storage.backup_config(self.storage_vol.config(), self._volume_config_path)
with op.step(self._step_init_slave):
# Change replication master
master_host = self._get_master_host()
self._logger.debug("Master server obtained (local_ip: %s, public_ip: %s)",
master_host.internal_ip, master_host.external_ip)
host = master_host.internal_ip or master_host.external_ip
self.postgresql.init_slave(self._storage_path, host, POSTGRESQL_DEFAULT_PORT, self.root_password)
with op.step(self._step_collect_host_up_data):
# Update HostUp message
message.postgresql = self._compat_storage_data(self.storage_vol)
message.db_type = BEHAVIOUR
def _init_master(self, message):
"""
Initialize postgresql master
@type message: scalarizr.messaging.Message
@param message: HostUp message
"""
self._logger.info("Initializing PostgreSQL master")
with bus.initialization_op as op:
with op.step(self._step_create_storage):
# Plug storage
volume_cnf = Storage.restore_config(self._volume_config_path)
try:
snap_cnf = Storage.restore_config(self._snapshot_config_path)
volume_cnf['snapshot'] = snap_cnf
except IOError:
pass
self.storage_vol = self._plug_storage(mpoint=self._storage_path, vol=volume_cnf)
Storage.backup_config(self.storage_vol.config(), self._volume_config_path)
with op.step(self._step_init_master):
self.postgresql.init_master(mpoint=self._storage_path, password=self.root_password)
msg_data = dict()
msg_data.update({OPT_REPLICATION_MASTER : str(int(self.is_replication_master)),
OPT_ROOT_USER : self.postgresql.root_user.name,
OPT_ROOT_PASSWORD : self.root_password,
OPT_CURRENT_XLOG_LOCATION : None})
with op.step(self._step_create_data_bundle):
# Create snapshot
snap = self._create_snapshot()
Storage.backup_config(snap.config(), self._snapshot_config_path)
with op.step(self._step_collect_host_up_data):
# Update HostUp message
msg_data.update(self._compat_storage_data(self.storage_vol, snap))
if msg_data:
message.db_type = BEHAVIOUR
message.postgresql = msg_data.copy()
message.postgresql.update({
OPT_ROOT_SSH_PRIVATE_KEY : self.postgresql.root_user.private_key,
OPT_ROOT_SSH_PUBLIC_KEY : self.postgresql.root_user.public_key
})
try:
del msg_data[OPT_SNAPSHOT_CNF], msg_data[OPT_VOLUME_CNF]
except KeyError:
pass
self._update_config(msg_data)
def _init_master(self, message):
"""
Initialize redis master
@type message: scalarizr.messaging.Message
@param message: HostUp message
"""
with bus.initialization_op as op:
with op.step(self._step_create_storage):
LOG.info("Initializing %s master" % BEHAVIOUR)
# Plug storage
volume_cnf = Storage.restore_config(self._volume_config_path)
try:
snap_cnf = Storage.restore_config(self._snapshot_config_path)
volume_cnf['snapshot'] = snap_cnf
except IOError:
pass
self.storage_vol = self._plug_storage(mpoint=self._storage_path, vol=volume_cnf)
Storage.backup_config(self.storage_vol.config(), self._volume_config_path)
with op.step(self._step_init_master):
password = self.get_main_password()
ri = self.redis_instances.get_instance(port=redis.DEFAULT_PORT)
ri.init_master(mpoint=self._storage_path)
msg_data = dict()
msg_data.update({OPT_REPLICATION_MASTER : '1',
OPT_MASTER_PASSWORD : password})
with op.step(self._step_create_data_bundle):
# Create snapshot
snap = self._create_snapshot()
Storage.backup_config(snap.config(), self._snapshot_config_path)
with op.step(self._step_collect_host_up_data):
# Update HostUp message
msg_data.update(self._compat_storage_data(self.storage_vol, snap))
if msg_data:
message.db_type = BEHAVIOUR
message.redis = msg_data.copy()
try:
del msg_data[OPT_SNAPSHOT_CNF], msg_data[OPT_VOLUME_CNF]
except KeyError:
pass
self._update_config(msg_data)
def on_host_init_response(self, message):
"""
Check postgresql data in host init response
@type message: scalarizr.messaging.Message
@param message: HostInitResponse
"""
with bus.initialization_op as op:
with op.phase(self._phase_postgresql):
with op.step(self._step_accept_scalr_conf):
if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR:
raise HandlerError("HostInitResponse message for PostgreSQL behaviour must have 'postgresql' property and db_type 'postgresql'")
'''
if message.postgresql[OPT_REPLICATION_MASTER] != '1' and \
(not message.body.has_key(OPT_ROOT_SSH_PUBLIC_KEY) or not
message.body.has_key(OPT_ROOT_SSH_PRIVATE_KEY)):
raise HandlerError("HostInitResponse message for PostgreSQL slave must contain both public and private ssh keys")
'''
dir = os.path.dirname(self._volume_config_path)
if not os.path.exists(dir):
os.makedirs(dir)
postgresql_data = message.postgresql.copy()
root = PgUser(ROOT_USER, self.pg_keys_dir)
root.store_keys(postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY], postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY])
del postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY]
del postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY]
for key, file in ((OPT_VOLUME_CNF, self._volume_config_path),
(OPT_SNAPSHOT_CNF, self._snapshot_config_path)):
if os.path.exists(file):
os.remove(file)
if key in postgresql_data:
if postgresql_data[key]:
Storage.backup_config(postgresql_data[key], file)
del postgresql_data[key]
root_user= postgresql_data[OPT_ROOT_USER] or ROOT_USER
postgresql_data['%s_password' % root_user] = postgresql_data.get(OPT_ROOT_PASSWORD) or cryptotool.pwgen(10)
del postgresql_data[OPT_ROOT_PASSWORD]
self._logger.debug("Update postgresql config with %s", postgresql_data)
self._update_config(postgresql_data)
def on_host_init_response(self, message):
"""
Check redis data in host init response
@type message: scalarizr.messaging.Message
@param message: HostInitResponse
"""
with bus.initialization_op as op:
with op.phase(self._phase_redis):
with op.step(self._step_accept_scalr_conf):
if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR:
raise HandlerError("HostInitResponse message for %s behaviour must have '%s' property and db_type '%s'"
% (BEHAVIOUR, BEHAVIOUR, BEHAVIOUR))
config_dir = os.path.dirname(self._volume_config_path)
if not os.path.exists(config_dir):
os.makedirs(config_dir)
redis_data = message.redis.copy()
LOG.info('Got Redis part of HostInitResponse: %s' % redis_data)
'''
XXX: following line enables support for old scalr installations
use_password shoud be set by postinstall script for old servers
'''
redis_data[OPT_USE_PASSWORD] = redis_data.get(OPT_USE_PASSWORD, '1')
for key, config_file in ((OPT_VOLUME_CNF, self._volume_config_path),
(OPT_SNAPSHOT_CNF, self._snapshot_config_path)):
if os.path.exists(config_file):
os.remove(config_file)
if key in redis_data:
if redis_data[key]:
Storage.backup_config(redis_data[key], config_file)
del redis_data[key]
LOG.debug("Update redis config with %s", redis_data)
self._update_config(redis_data)
if self.default_service.running:
self.default_service.stop('Treminating default redis instance')
self.redis_instances = redis.RedisInstances(self.is_replication_master, self.persistence_type)
self.redis_instances.init_processes(ports=[redis.DEFAULT_PORT,], passwords=[self.get_main_password(),])
def on_DbMsr_NewMasterUp(self, message):
"""
Switch replication to a new master server
@type message: scalarizr.messaging.Message
@param message: DbMsr_NewMasterUp
"""
if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR:
raise HandlerError("DbMsr_NewMasterUp message for PostgreSQL behaviour must have 'postgresql' property and db_type 'postgresql'")
postgresql_data = message.postgresql.copy()
if self.is_replication_master:
self._logger.debug('Skipping NewMasterUp. My replication role is master')
return
host = message.local_ip or message.remote_ip
self._logger.info("Switching replication to a new postgresql master %s", host)
bus.fire('before_postgresql_change_master', host=host)
if OPT_SNAPSHOT_CNF in postgresql_data and postgresql_data[OPT_SNAPSHOT_CNF]['type'] != 'eph':
snap_data = postgresql_data[OPT_SNAPSHOT_CNF]
self._logger.info('Reinitializing Slave from the new snapshot %s',
snap_data['id'])
self.postgresql.service.stop()
self._logger.debug('Destroying old storage')
self.storage_vol.destroy()
self._logger.debug('Storage destroyed')
self._logger.debug('Plugging new storage')
vol = Storage.create(snapshot=snap_data.copy(), tags=self.postgres_tags)
self._plug_storage(self._storage_path, vol)
self._logger.debug('Storage plugged')
Storage.backup_config(vol.config(), self._volume_config_path)
Storage.backup_config(snap_data, self._snapshot_config_path)
self.storage_vol = vol
self.postgresql.init_slave(self._storage_path, host, POSTGRESQL_DEFAULT_PORT, self.root_password)
self._logger.debug("Replication switched")
bus.fire('postgresql_change_master', host=host)
def on_DbMsr_PromoteToMaster(self, message):
"""
Promote slave to master
@type message: scalarizr.messaging.Message
@param message: redis_PromoteToMaster
"""
if message.db_type != BEHAVIOUR:
LOG.error('Wrong db_type in DbMsr_PromoteToMaster message: %s' % message.db_type)
return
if self.is_replication_master:
LOG.warning('Cannot promote to master. Already master')
return
bus.fire('before_slave_promote_to_master')
master_storage_conf = message.body.get('volume_config')
tx_complete = False
old_conf = None
new_storage_vol = None
try:
msg_data = dict(
db_type=BEHAVIOUR,
status="ok",
)
if master_storage_conf and master_storage_conf['type'] != 'eph':
self.redis_instances.stop('Unplugging slave storage and then plugging master one')
old_conf = self.storage_vol.detach(force=True) # ??????
new_storage_vol = self._plug_storage(self._storage_path, master_storage_conf)
'''
#This code was removed because redis master storage can be empty yet valid
for r in self.redis_instances:
# Continue if master storage is a valid redis storage
if not r.working_directory.is_initialized(self._storage_path):
raise HandlerError("%s is not a valid %s storage" % (self._storage_path, BEHAVIOUR))
Storage.backup_config(new_storage_vol.config(), self._volume_config_path)
'''
Storage.backup_config(new_storage_vol.config(), self._volume_config_path)
msg_data[BEHAVIOUR] = self._compat_storage_data(vol=new_storage_vol)
self.redis_instances.init_as_masters(self._storage_path)
self._update_config({OPT_REPLICATION_MASTER : "1"})
if not master_storage_conf or master_storage_conf['type'] == 'eph':
snap = self._create_snapshot()
Storage.backup_config(snap.config(), self._snapshot_config_path)
msg_data[BEHAVIOUR] = self._compat_storage_data(self.storage_vol, snap)
self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT, msg_data)
tx_complete = True
bus.fire('slave_promote_to_master')
except (Exception, BaseException), e:
LOG.exception(e)
if new_storage_vol and not new_storage_vol.detached:
new_storage_vol.detach()
# Get back slave storage
if old_conf:
self._plug_storage(self._storage_path, old_conf)
self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT, dict(
db_type=BEHAVIOUR,
status="error",
last_error=str(e)
))
# Start redis
self.redis_instances.start()
self._plug_storage(self._storage_path, old_conf)
self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT, dict(
db_type=BEHAVIOUR,
status="error",
last_error=str(e)
))
# Start redis
self.redis_instances.start()
if tx_complete and master_storage_conf and master_storage_conf['type'] != 'eph':
# Delete slave EBS
self.storage_vol.destroy(remove_disks=True)
self.storage_vol = new_storage_vol
Storage.backup_config(self.storage_vol.config(), self._volume_config_path)
def on_DbMsr_NewMasterUp(self, message):
"""
Switch replication to a new master server
@type message: scalarizr.messaging.Message
@param message: DbMsr__NewMasterUp
"""
if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR:
raise HandlerError("DbMsr_NewMasterUp message for %s behaviour must have '%s' property and db_type '%s'" %
BEHAVIOUR, BEHAVIOUR, BEHAVIOUR)
if self.is_replication_master:
LOG.debug('Skipping NewMasterUp. My replication role is master')
def on_DbMsr_PromoteToMaster(self, message):
"""
Promote slave to master
@type message: scalarizr.messaging.Message
@param message: postgresql_PromoteToMaster
"""
if message.db_type != BEHAVIOUR:
self._logger.error('Wrong db_type in DbMsr_PromoteToMaster message: %s' % message.db_type)
return
if self.is_replication_master:
self._logger.warning('Cannot promote to master. Already master')
return
bus.fire('before_slave_promote_to_master')
master_storage_conf = message.body.get('volume_config')
tx_complete = False
old_conf = None
new_storage_vol = None
try:
msg_data = dict(
db_type=BEHAVIOUR,
status="ok",
)
self.postgresql.stop_replication()
if master_storage_conf and master_storage_conf['type'] != 'eph':
self.postgresql.service.stop('Unplugging slave storage and then plugging master one')
old_conf = self.storage_vol.detach(force=True) # ??????
new_storage_vol = self._plug_storage(self._storage_path, master_storage_conf)
# Continue if master storage is a valid postgresql storage
if not self.postgresql.cluster_dir.is_initialized(self._storage_path):
raise HandlerError("%s is not a valid postgresql storage" % self._storage_path)
Storage.backup_config(new_storage_vol.config(), self._volume_config_path)
msg_data[BEHAVIOUR] = self._compat_storage_data(vol=new_storage_vol)
slaves = [host.internal_ip for host in self._get_slave_hosts()]
self.postgresql.init_master(self._storage_path, self.root_password, slaves)
self._update_config({OPT_REPLICATION_MASTER : "1"})
if not master_storage_conf or master_storage_conf['type'] == 'eph':
snap = self._create_snapshot()
Storage.backup_config(snap.config(), self._snapshot_config_path)
msg_data[BEHAVIOUR] = self._compat_storage_data(self.storage_vol, snap)
msg_data[BEHAVIOUR].update({OPT_CURRENT_XLOG_LOCATION: None})
self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT, msg_data)
tx_complete = True
bus.fire('slave_promote_to_master')
except (Exception, BaseException), e:
self._logger.exception(e)
if new_storage_vol:
new_storage_vol.detach()
# Get back slave storage
if old_conf:
self._plug_storage(self._storage_path, old_conf)
self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT, dict(
db_type=BEHAVIOUR,
status="error",
last_error=str(e)
))
# Start postgresql
self.postgresql.service.start()
