def _init_slave(self, message): """ Initialize redis slave @type message: scalarizr.messaging.Message @param message: HostUp message """ LOG.info("Initializing %s slave" % BEHAVIOUR) with bus.initialization_op as op: with op.step(self._step_create_storage): LOG.debug("Initializing slave storage") self.storage_vol = self._plug_storage(self._storage_path, dict(snapshot=Storage.restore_config(self._snapshot_config_path))) Storage.backup_config(self.storage_vol.config(), self._volume_config_path) with op.step(self._step_init_slave): # Change replication master master_host = self._get_master_host() LOG.debug("Master server obtained (local_ip: %s, public_ip: %s)", master_host.internal_ip, master_host.external_ip) host = master_host.internal_ip or master_host.external_ip instance = self.redis_instances.get_instance(port=redis.DEFAULT_PORT) instance.init_slave(self._storage_path, host, redis.DEFAULT_PORT) op.progress(50) instance.wait_for_sync() with op.step(self._step_collect_host_up_data): # Update HostUp message message.redis = self._compat_storage_data(self.storage_vol) message.db_type = BEHAVIOUR
def _init_slave(self, message): """ Initialize postgresql slave @type message: scalarizr.messaging.Message @param message: HostUp message """ self._logger.info("Initializing postgresql slave") with bus.initialization_op as op: with op.step(self._step_create_storage): self._logger.debug("Initialize slave storage") self.storage_vol = self._plug_storage(self._storage_path, dict(snapshot=Storage.restore_config(self._snapshot_config_path))) Storage.backup_config(self.storage_vol.config(), self._volume_config_path) with op.step(self._step_init_slave): # Change replication master master_host = self._get_master_host() self._logger.debug("Master server obtained (local_ip: %s, public_ip: %s)", master_host.internal_ip, master_host.external_ip) host = master_host.internal_ip or master_host.external_ip self.postgresql.init_slave(self._storage_path, host, POSTGRESQL_DEFAULT_PORT, self.root_password) with op.step(self._step_collect_host_up_data): # Update HostUp message message.postgresql = self._compat_storage_data(self.storage_vol) message.db_type = BEHAVIOUR
def _init_master(self, message): """ Initialize postgresql master @type message: scalarizr.messaging.Message @param message: HostUp message """ self._logger.info("Initializing PostgreSQL master") with bus.initialization_op as op: with op.step(self._step_create_storage): # Plug storage volume_cnf = Storage.restore_config(self._volume_config_path) try: snap_cnf = Storage.restore_config(self._snapshot_config_path) volume_cnf['snapshot'] = snap_cnf except IOError: pass self.storage_vol = self._plug_storage(mpoint=self._storage_path, vol=volume_cnf) Storage.backup_config(self.storage_vol.config(), self._volume_config_path) with op.step(self._step_init_master): self.postgresql.init_master(mpoint=self._storage_path, password=self.root_password) msg_data = dict() msg_data.update({OPT_REPLICATION_MASTER : str(int(self.is_replication_master)), OPT_ROOT_USER : self.postgresql.root_user.name, OPT_ROOT_PASSWORD : self.root_password, OPT_CURRENT_XLOG_LOCATION : None}) with op.step(self._step_create_data_bundle): # Create snapshot snap = self._create_snapshot() Storage.backup_config(snap.config(), self._snapshot_config_path) with op.step(self._step_collect_host_up_data): # Update HostUp message msg_data.update(self._compat_storage_data(self.storage_vol, snap)) if msg_data: message.db_type = BEHAVIOUR message.postgresql = msg_data.copy() message.postgresql.update({ OPT_ROOT_SSH_PRIVATE_KEY : self.postgresql.root_user.private_key, OPT_ROOT_SSH_PUBLIC_KEY : self.postgresql.root_user.public_key }) try: del msg_data[OPT_SNAPSHOT_CNF], msg_data[OPT_VOLUME_CNF] except KeyError: pass self._update_config(msg_data)
def _init_master(self, message): """ Initialize redis master @type message: scalarizr.messaging.Message @param message: HostUp message """ with bus.initialization_op as op: with op.step(self._step_create_storage): LOG.info("Initializing %s master" % BEHAVIOUR) # Plug storage volume_cnf = Storage.restore_config(self._volume_config_path) try: snap_cnf = Storage.restore_config(self._snapshot_config_path) volume_cnf['snapshot'] = snap_cnf except IOError: pass self.storage_vol = self._plug_storage(mpoint=self._storage_path, vol=volume_cnf) Storage.backup_config(self.storage_vol.config(), self._volume_config_path) with op.step(self._step_init_master): password = self.get_main_password() ri = self.redis_instances.get_instance(port=redis.DEFAULT_PORT) ri.init_master(mpoint=self._storage_path) msg_data = dict() msg_data.update({OPT_REPLICATION_MASTER : '1', OPT_MASTER_PASSWORD : password}) with op.step(self._step_create_data_bundle): # Create snapshot snap = self._create_snapshot() Storage.backup_config(snap.config(), self._snapshot_config_path) with op.step(self._step_collect_host_up_data): # Update HostUp message msg_data.update(self._compat_storage_data(self.storage_vol, snap)) if msg_data: message.db_type = BEHAVIOUR message.redis = msg_data.copy() try: del msg_data[OPT_SNAPSHOT_CNF], msg_data[OPT_VOLUME_CNF] except KeyError: pass self._update_config(msg_data)
def on_host_init_response(self, message): """ Check postgresql data in host init response @type message: scalarizr.messaging.Message @param message: HostInitResponse """ with bus.initialization_op as op: with op.phase(self._phase_postgresql): with op.step(self._step_accept_scalr_conf): if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR: raise HandlerError("HostInitResponse message for PostgreSQL behaviour must have 'postgresql' property and db_type 'postgresql'") ''' if message.postgresql[OPT_REPLICATION_MASTER] != '1' and \ (not message.body.has_key(OPT_ROOT_SSH_PUBLIC_KEY) or not message.body.has_key(OPT_ROOT_SSH_PRIVATE_KEY)): raise HandlerError("HostInitResponse message for PostgreSQL slave must contain both public and private ssh keys") ''' dir = os.path.dirname(self._volume_config_path) if not os.path.exists(dir): os.makedirs(dir) postgresql_data = message.postgresql.copy() root = PgUser(ROOT_USER, self.pg_keys_dir) root.store_keys(postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY], postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY]) del postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY] del postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY] for key, file in ((OPT_VOLUME_CNF, self._volume_config_path), (OPT_SNAPSHOT_CNF, self._snapshot_config_path)): if os.path.exists(file): os.remove(file) if key in postgresql_data: if postgresql_data[key]: Storage.backup_config(postgresql_data[key], file) del postgresql_data[key] root_user= postgresql_data[OPT_ROOT_USER] or ROOT_USER postgresql_data['%s_password' % root_user] = postgresql_data.get(OPT_ROOT_PASSWORD) or cryptotool.pwgen(10) del postgresql_data[OPT_ROOT_PASSWORD] self._logger.debug("Update postgresql config with %s", postgresql_data) self._update_config(postgresql_data)
def on_host_init_response(self, message): """ Check redis data in host init response @type message: scalarizr.messaging.Message @param message: HostInitResponse """ with bus.initialization_op as op: with op.phase(self._phase_redis): with op.step(self._step_accept_scalr_conf): if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR: raise HandlerError("HostInitResponse message for %s behaviour must have '%s' property and db_type '%s'" % (BEHAVIOUR, BEHAVIOUR, BEHAVIOUR)) config_dir = os.path.dirname(self._volume_config_path) if not os.path.exists(config_dir): os.makedirs(config_dir) redis_data = message.redis.copy() LOG.info('Got Redis part of HostInitResponse: %s' % redis_data) ''' XXX: following line enables support for old scalr installations use_password shoud be set by postinstall script for old servers ''' redis_data[OPT_USE_PASSWORD] = redis_data.get(OPT_USE_PASSWORD, '1') for key, config_file in ((OPT_VOLUME_CNF, self._volume_config_path), (OPT_SNAPSHOT_CNF, self._snapshot_config_path)): if os.path.exists(config_file): os.remove(config_file) if key in redis_data: if redis_data[key]: Storage.backup_config(redis_data[key], config_file) del redis_data[key] LOG.debug("Update redis config with %s", redis_data) self._update_config(redis_data) if self.default_service.running: self.default_service.stop('Treminating default redis instance') self.redis_instances = redis.RedisInstances(self.is_replication_master, self.persistence_type) self.redis_instances.init_processes(ports=[redis.DEFAULT_PORT,], passwords=[self.get_main_password(),])
def on_DbMsr_NewMasterUp(self, message): """ Switch replication to a new master server @type message: scalarizr.messaging.Message @param message: DbMsr_NewMasterUp """ if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR: raise HandlerError("DbMsr_NewMasterUp message for PostgreSQL behaviour must have 'postgresql' property and db_type 'postgresql'") postgresql_data = message.postgresql.copy() if self.is_replication_master: self._logger.debug('Skipping NewMasterUp. My replication role is master') return host = message.local_ip or message.remote_ip self._logger.info("Switching replication to a new postgresql master %s", host) bus.fire('before_postgresql_change_master', host=host) if OPT_SNAPSHOT_CNF in postgresql_data and postgresql_data[OPT_SNAPSHOT_CNF]['type'] != 'eph': snap_data = postgresql_data[OPT_SNAPSHOT_CNF] self._logger.info('Reinitializing Slave from the new snapshot %s', snap_data['id']) self.postgresql.service.stop() self._logger.debug('Destroying old storage') self.storage_vol.destroy() self._logger.debug('Storage destroyed') self._logger.debug('Plugging new storage') vol = Storage.create(snapshot=snap_data.copy(), tags=self.postgres_tags) self._plug_storage(self._storage_path, vol) self._logger.debug('Storage plugged') Storage.backup_config(vol.config(), self._volume_config_path) Storage.backup_config(snap_data, self._snapshot_config_path) self.storage_vol = vol self.postgresql.init_slave(self._storage_path, host, POSTGRESQL_DEFAULT_PORT, self.root_password) self._logger.debug("Replication switched") bus.fire('postgresql_change_master', host=host)
def on_DbMsr_PromoteToMaster(self, message): """ Promote slave to master @type message: scalarizr.messaging.Message @param message: redis_PromoteToMaster """ if message.db_type != BEHAVIOUR: LOG.error('Wrong db_type in DbMsr_PromoteToMaster message: %s' % message.db_type) return if self.is_replication_master: LOG.warning('Cannot promote to master. Already master') return bus.fire('before_slave_promote_to_master') master_storage_conf = message.body.get('volume_config') tx_complete = False old_conf = None new_storage_vol = None try: msg_data = dict( db_type=BEHAVIOUR, status="ok", ) if master_storage_conf and master_storage_conf['type'] != 'eph': self.redis_instances.stop('Unplugging slave storage and then plugging master one') old_conf = self.storage_vol.detach(force=True) # ?????? new_storage_vol = self._plug_storage(self._storage_path, master_storage_conf) ''' #This code was removed because redis master storage can be empty yet valid for r in self.redis_instances: # Continue if master storage is a valid redis storage if not r.working_directory.is_initialized(self._storage_path): raise HandlerError("%s is not a valid %s storage" % (self._storage_path, BEHAVIOUR)) Storage.backup_config(new_storage_vol.config(), self._volume_config_path) ''' Storage.backup_config(new_storage_vol.config(), self._volume_config_path) msg_data[BEHAVIOUR] = self._compat_storage_data(vol=new_storage_vol) self.redis_instances.init_as_masters(self._storage_path) self._update_config({OPT_REPLICATION_MASTER : "1"}) if not master_storage_conf or master_storage_conf['type'] == 'eph': snap = self._create_snapshot() Storage.backup_config(snap.config(), self._snapshot_config_path) msg_data[BEHAVIOUR] = self._compat_storage_data(self.storage_vol, snap) self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT, msg_data) tx_complete = True bus.fire('slave_promote_to_master') except (Exception, BaseException), e: LOG.exception(e) if new_storage_vol and not new_storage_vol.detached: new_storage_vol.detach() # Get back slave storage if old_conf: self._plug_storage(self._storage_path, old_conf) self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT, dict( db_type=BEHAVIOUR, status="error", last_error=str(e) )) # Start redis self.redis_instances.start()
self._plug_storage(self._storage_path, old_conf) self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT, dict( db_type=BEHAVIOUR, status="error", last_error=str(e) )) # Start redis self.redis_instances.start() if tx_complete and master_storage_conf and master_storage_conf['type'] != 'eph': # Delete slave EBS self.storage_vol.destroy(remove_disks=True) self.storage_vol = new_storage_vol Storage.backup_config(self.storage_vol.config(), self._volume_config_path) def on_DbMsr_NewMasterUp(self, message): """ Switch replication to a new master server @type message: scalarizr.messaging.Message @param message: DbMsr__NewMasterUp """ if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR: raise HandlerError("DbMsr_NewMasterUp message for %s behaviour must have '%s' property and db_type '%s'" % BEHAVIOUR, BEHAVIOUR, BEHAVIOUR) if self.is_replication_master: LOG.debug('Skipping NewMasterUp. My replication role is master')
def on_DbMsr_PromoteToMaster(self, message): """ Promote slave to master @type message: scalarizr.messaging.Message @param message: postgresql_PromoteToMaster """ if message.db_type != BEHAVIOUR: self._logger.error('Wrong db_type in DbMsr_PromoteToMaster message: %s' % message.db_type) return if self.is_replication_master: self._logger.warning('Cannot promote to master. Already master') return bus.fire('before_slave_promote_to_master') master_storage_conf = message.body.get('volume_config') tx_complete = False old_conf = None new_storage_vol = None try: msg_data = dict( db_type=BEHAVIOUR, status="ok", ) self.postgresql.stop_replication() if master_storage_conf and master_storage_conf['type'] != 'eph': self.postgresql.service.stop('Unplugging slave storage and then plugging master one') old_conf = self.storage_vol.detach(force=True) # ?????? new_storage_vol = self._plug_storage(self._storage_path, master_storage_conf) # Continue if master storage is a valid postgresql storage if not self.postgresql.cluster_dir.is_initialized(self._storage_path): raise HandlerError("%s is not a valid postgresql storage" % self._storage_path) Storage.backup_config(new_storage_vol.config(), self._volume_config_path) msg_data[BEHAVIOUR] = self._compat_storage_data(vol=new_storage_vol) slaves = [host.internal_ip for host in self._get_slave_hosts()] self.postgresql.init_master(self._storage_path, self.root_password, slaves) self._update_config({OPT_REPLICATION_MASTER : "1"}) if not master_storage_conf or master_storage_conf['type'] == 'eph': snap = self._create_snapshot() Storage.backup_config(snap.config(), self._snapshot_config_path) msg_data[BEHAVIOUR] = self._compat_storage_data(self.storage_vol, snap) msg_data[BEHAVIOUR].update({OPT_CURRENT_XLOG_LOCATION: None}) self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT, msg_data) tx_complete = True bus.fire('slave_promote_to_master') except (Exception, BaseException), e: self._logger.exception(e) if new_storage_vol: new_storage_vol.detach() # Get back slave storage if old_conf: self._plug_storage(self._storage_path, old_conf) self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT, dict( db_type=BEHAVIOUR, status="error", last_error=str(e) )) # Start postgresql self.postgresql.service.start()