Exemple #1
0
class PostgreSqlHander(ServiceCtlHandler):
    _logger = None

    _queryenv = None
    """ @type _queryenv: scalarizr.queryenv.QueryEnvService """

    _platform = None
    """ @type _platform: scalarizr.platform.Ec2Platform """

    _cnf = None
    ''' @type _cnf: scalarizr.config.ScalarizrCnf '''

    preset_provider = None

    def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None):
        return BEHAVIOUR in behaviour and (
            message.name == DbMsrMessages.DBMSR_NEW_MASTER_UP
            or message.name == DbMsrMessages.DBMSR_PROMOTE_TO_MASTER
            or message.name == DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE
            or message.name == DbMsrMessages.DBMSR_CREATE_BACKUP
            or message.name == Messages.UPDATE_SERVICE_CONFIGURATION
            or message.name == Messages.HOST_INIT
            or message.name == Messages.BEFORE_HOST_TERMINATE
            or message.name == Messages.HOST_UP
            or message.name == Messages.HOST_DOWN)

    
    def get_initialization_phases(self, hir_message):
        if BEHAVIOUR in hir_message.body:
            steps = [self._step_accept_scalr_conf, self._step_create_storage]
            if hir_message.body[BEHAVIOUR]['replication_master'] == '1':
                steps += [self._step_init_master, self._step_create_data_bundle]
            else:
                steps += [self._step_init_slave]
            steps += [self._step_collect_host_up_data]
            
            return {'before_host_up': [{
                'name': self._phase_postgresql, 
                'steps': steps
            }]}
    
    
    def __init__(self):
        self._service_name = SERVICE_NAME
        ServiceCtlHandler.__init__(self, SERVICE_NAME, initdv2.lookup(SERVICE_NAME))
        bus.on("init", self.on_init)
        bus.define_events(
            'before_postgresql_data_bundle',
            
            'postgresql_data_bundle',
            
            # @param host: New master hostname 
            'before_postgresql_change_master',
            
            # @param host: New master hostname 
            'postgresql_change_master',
            
            'before_slave_promote_to_master',
            
            'slave_promote_to_master'
        )   
        
        self._phase_postgresql = 'Configure PostgreSQL'
        self._phase_data_bundle = self._op_data_bundle = 'PostgreSQL data bundle'
        self._phase_backup = self._op_backup = 'PostgreSQL backup'
        self._step_upload_to_cloud_storage = 'Upload data to cloud storage'
        self._step_accept_scalr_conf = 'Accept Scalr configuration'
        self._step_patch_conf = 'Patch configuration files'
        self._step_create_storage = 'Create storage'
        self._step_init_master = 'Initialize Master'
        self._step_init_slave = 'Initialize Slave'
        self._step_create_data_bundle = 'Create data bundle'
        self._step_change_replication_master = 'Change replication Master'
        self._step_collect_host_up_data = 'Collect HostUp data'
        
        self.on_reload()        


    def on_init(self):      
        #temporary fix for starting-after-rebundle issue
        if not os.path.exists(PG_SOCKET_DIR):
            os.makedirs(PG_SOCKET_DIR)
            chown_r(PG_SOCKET_DIR, 'postgres')
            
        bus.on("host_init_response", self.on_host_init_response)
        bus.on("before_host_up", self.on_before_host_up)
        bus.on("before_reboot_start", self.on_before_reboot_start)

        self._insert_iptables_rules()       

        if __node__['state'] == ScalarizrState.BOOTSTRAPPING:
            
            if disttool.is_redhat_based():      
                    
                checkmodule_path = software.which('checkmodule')
                semodule_package_path = software.which('semodule_package')
                semodule_path = software.which('semodule')
            
                if all((checkmodule_path, semodule_package_path, semodule_path)):
                    
                    with open('/tmp/sshkeygen.te', 'w') as fp:
                        fp.write(SSH_KEYGEN_SELINUX_MODULE)
                    
                    self._logger.debug('Compiling SELinux policy for ssh-keygen')
                    system2((checkmodule_path, '-M', '-m', '-o',
                             '/tmp/sshkeygen.mod', '/tmp/sshkeygen.te'), logger=self._logger)
                    
                    self._logger.debug('Building SELinux package for ssh-keygen')
                    system2((semodule_package_path, '-o', '/tmp/sshkeygen.pp',
                             '-m', '/tmp/sshkeygen.mod'), logger=self._logger)
                    
                    self._logger.debug('Loading ssh-keygen SELinux package')                    
                    system2((semodule_path, '-i', '/tmp/sshkeygen.pp'), logger=self._logger)


        if __node__['state'] == 'running':

            vol = storage2.volume(__postgresql__['volume'])
            vol.ensure(mount=True)
            
            self.postgresql.service.start()
            self.accept_all_clients()
            
            self._logger.debug("Checking presence of Scalr's PostgreSQL root user.")
            root_password = self.root_password
            
            if not self.postgresql.root_user.exists():
                self._logger.debug("Scalr's PostgreSQL root user does not exist. Recreating")
                self.postgresql.root_user = self.postgresql.create_linux_user(ROOT_USER, root_password)
            else:
                try:
                    self.postgresql.root_user.check_system_password(root_password)
                    self._logger.debug("Scalr's root PgSQL user is present. Password is correct.")              
                except ValueError:
                    self._logger.warning("Scalr's root PgSQL user was changed. Recreating.")
                    self.postgresql.root_user.change_system_password(root_password)
                    
            if self.is_replication_master:  
                #ALTER ROLE cannot be executed in a read-only transaction
                self._logger.debug("Checking password for pg_role scalr.")      
                if not self.postgresql.root_user.check_role_password(root_password):
                    LOG.warning("Scalr's root PgSQL role was changed. Recreating.")
                    self.postgresql.root_user.change_role_password(root_password)
            

    def on_reload(self):
        self._queryenv = bus.queryenv_service
        self._platform = bus.platform
        self.postgresql = PostgreSql()
        self.preset_provider = PgSQLPresetProvider(self.postgresql.postgresql_conf)
        preset_service.services[BEHAVIOUR] = self.preset_provider

    
    def on_HostInit(self, message):
        if message.local_ip != self._platform.get_private_ip() and message.local_ip in self.pg_hosts:
            LOG.debug('Got new slave IP: %s. Registering in pg_hba.conf' % message.local_ip)
            self.postgresql.register_slave(message.local_ip)
            

    def on_HostUp(self, message):
        if message.local_ip == self._platform.get_private_ip():
            self.accept_all_clients()
        elif message.local_ip in self.farm_hosts:
            self.postgresql.register_client(message.local_ip)
        
    
    
    def on_HostDown(self, message):
        if  message.local_ip != self._platform.get_private_ip():
            self.postgresql.unregister_client(message.local_ip)
            if self.is_replication_master and self.farmrole_id == message.farm_role_id:
                self.postgresql.unregister_slave(message.local_ip)
    
    @property           
    def farm_hosts(self):
        list_roles = self._queryenv.list_roles()
        servers = []
        for serv in list_roles:
            for host in serv.hosts :
                servers.append(host.internal_ip or host.external_ip)
        LOG.debug("QueryEnv returned list of servers within farm: %s" % servers)
        return servers              
        
        
    @property
    def pg_hosts(self):
        '''
        All pg instances including those in Initializing state
        '''
        list_roles = self._queryenv.list_roles(behaviour=BEHAVIOUR, with_init=True)
        servers = []
        for pg_serv in list_roles:
            for pg_host in pg_serv.hosts:
                servers.append(pg_host.internal_ip or pg_host.external_ip)
        LOG.debug("QueryEnv returned list of %s servers: %s" % (BEHAVIOUR, servers))
        return servers
    
    
    def accept_all_clients(self):
        farm_hosts = self.farm_hosts
        for ip in farm_hosts:
                self.postgresql.register_client(ip, force=False)
        if farm_hosts:
            self.postgresql.service.reload('Granting access to all servers within farm.', force=True)
                

    @property
    def root_password(self):
        return __postgresql__['%s_password' % ROOT_USER]


    @property   
    def farmrole_id(self):
        return __node__[config.OPT_FARMROLE_ID]
    
            
    def store_password(self, name, password):
        __postgresql__['%s_password' % name] = password


    @property
    def _tmp_path(self):
        return os.path.join(__postgresql__['storage_dir'], 'tmp')


    @property
    def is_replication_master(self):
        return True if int(__postgresql__[OPT_REPLICATION_MASTER]) else False


    def resource_tags(self):
        purpose = '%s-'%BEHAVIOUR + ('master' if self.is_replication_master else 'slave')
        return build_tags(purpose, 'active')


    def on_host_init_response(self, message):
        """
        Check postgresql data in host init response
        @type message: scalarizr.messaging.Message
        @param message: HostInitResponse
        """
        
        with bus.initialization_op as op:
            with op.phase(self._phase_postgresql):
                with op.step(self._step_accept_scalr_conf):
        
                    if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR:
                        raise HandlerError("HostInitResponse message for PostgreSQL behaviour must have 'postgresql' property and db_type 'postgresql'")

                    postgresql_data = message.postgresql.copy()

                    #Extracting service configuration preset from message
                    if 'preset' in postgresql_data:
                        self.initial_preset = postgresql_data['preset']
                        LOG.debug('Scalr sent current preset: %s' % self.initial_preset)
                        del postgresql_data['preset']

                    #Extracting or generating postgresql root password
                    postgresql_data['%s_password' % ROOT_USER] = postgresql_data.get(OPT_ROOT_PASSWORD) or cryptotool.pwgen(10)
                    del postgresql_data[OPT_ROOT_PASSWORD]

                    #Extracting replication ssh keys from message
                    root = PgUser(ROOT_USER, self.postgresql.pg_keys_dir)
                    root.store_keys(postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY], postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY])
                    del postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY]
                    del postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY]


                    if postgresql_data.get('volume'):
                        # New format
                        postgresql_data['compat_prior_backup_restore'] = False
                        postgresql_data['volume'] = storage2.volume(postgresql_data['volume'])
                        LOG.debug("message.pg['volume']:", postgresql_data['volume'])
                        if 'backup' in postgresql_data:
                            postgresql_data['backup'] = backup.backup(postgresql_data['backup'])
                            LOG.debug("message.pg['backup']:", postgresql_data['backup'])
                        if 'restore' in postgresql_data:
                            postgresql_data['restore'] = backup.restore(postgresql_data['restore'])
                            LOG.debug("message.pg['restore']:", postgresql_data['restore'])
                    else:

                        # Compatibility transformation
                        # - volume_config -> volume
                        # - master n'th start, type=ebs - del snapshot_config
                        # - snapshot_config -> restore
                        # - create backup object on master 1'st start

                        postgresql_data['compat_prior_backup_restore'] = True
                        if postgresql_data.get(OPT_VOLUME_CNF):
                            postgresql_data['volume'] = storage2.volume(
                                postgresql_data.pop(OPT_VOLUME_CNF))

                        elif postgresql_data.get(OPT_SNAPSHOT_CNF):
                            postgresql_data['volume'] = storage2.volume(
                                type=postgresql_data[OPT_SNAPSHOT_CNF]['type'])

                        else:
                            raise HandlerError('No volume config or snapshot config provided')

                        if postgresql_data['volume'].device and \
                                        postgresql_data['volume'].type in ('ebs', 'csvol', 'cinder', 'raid'):
                            LOG.debug("Master n'th start detected. Removing snapshot config from message")
                            postgresql_data.pop(OPT_SNAPSHOT_CNF, None)

                        if postgresql_data.get(OPT_SNAPSHOT_CNF):
                            postgresql_data['restore'] = backup.restore(
                                type='snap_postgresql',
                                snapshot=postgresql_data.pop(OPT_SNAPSHOT_CNF),
                                volume=postgresql_data['volume'])

                        if int(postgresql_data['replication_master']):
                            postgresql_data['backup'] = backup.backup(
                                type='snap_postgresql',
                                volume=postgresql_data['volume'])

                    LOG.debug("Update postgresql config with %s", postgresql_data)
                    __postgresql__.update(postgresql_data)
                    __postgresql__['volume'].mpoint = __postgresql__['storage_dir']
                    __postgresql__['volume'].tags = self.resource_tags()
                    if 'backup' in __postgresql__:
                        __postgresql__['backup'].tags = self.resource_tags()


    def on_before_host_up(self, message):
        """
        Configure PostgreSQL behaviour
        @type message: scalarizr.messaging.Message      
        @param message: HostUp message
        """

        repl = 'master' if self.is_replication_master else 'slave'
        #bus.fire('before_postgresql_configure', replication=repl)
        
        if self.is_replication_master:
            self._init_master(message)                                    
        else:
            self._init_slave(message)
        # Force to resave volume settings
        __postgresql__['volume'] = storage2.volume(__postgresql__['volume'])
        bus.fire('service_configured', service_name=SERVICE_NAME, replication=repl, preset=self.initial_preset)
                    
                
    def on_before_reboot_start(self, *args, **kwargs):
        """
        Stop PostgreSQL and unplug storage
        """
        self.postgresql.service.stop('rebooting')


    def on_BeforeHostTerminate(self, message):
        LOG.info('Handling BeforeHostTerminate message from %s' % message.local_ip)
        if message.local_ip == self._platform.get_private_ip():
            LOG.info('Stopping %s service' % BEHAVIOUR)
            self.postgresql.service.stop('Server will be terminated')
            if not self.is_replication_master:
                LOG.info('Destroying volume %s' % __postgresql__['volume'].id)
                __postgresql__['volume'].destroy(remove_disks=True)
                LOG.info('Volume %s has been destroyed.' % __postgresql__['volume'].id)
            else:
                __postgresql__['volume'].umount()


    def on_DbMsr_CreateDataBundle(self, message):
        
        try:
            op = operation(name=self._op_data_bundle, phases=[{
                'name': self._phase_data_bundle, 
                'steps': [self._step_create_data_bundle]
            }])
            op.define()
            
            with op.phase(self._phase_data_bundle):
                with op.step(self._step_create_data_bundle):
                    
                    bus.fire('before_postgresql_data_bundle')
                    snap = self._create_snapshot()
                    used_size = int(system2(('df', '-P', '--block-size=M', STORAGE_PATH))[0].split('\n')[1].split()[2][:-1])
                    bus.fire('postgresql_data_bundle', snapshot_id=snap.id)

                    # Notify scalr
                    msg_data = {
                    'db_type': BEHAVIOUR,
                    'status': 'ok',
                    'used_size' : '%.3f' % (float(used_size) / 1000,),
                    BEHAVIOUR: {OPT_SNAPSHOT_CNF: dict(snap)}
                    }
                    self.send_message(DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE_RESULT, msg_data)

            op.ok()

        except (Exception, BaseException), e:
            LOG.exception(e)
            
            # Notify Scalr about error
            self.send_message(DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE_RESULT, dict(
                db_type     = BEHAVIOUR,
                status      ='error',
                last_error  = str(e)
            ))
class PostgreSqlHander(ServiceCtlHandler):
    _logger = None

    _queryenv = None
    """ @type _queryenv: scalarizr.queryenv.QueryEnvService """

    _platform = None
    """ @type _platform: scalarizr.platform.Ec2Platform """

    _cnf = None
    ''' @type _cnf: scalarizr.config.ScalarizrCnf '''

    preset_provider = None

    def accept(self,
               message,
               queue,
               behaviour=None,
               platform=None,
               os=None,
               dist=None):
        return BEHAVIOUR in behaviour and (
            message.name == DbMsrMessages.DBMSR_NEW_MASTER_UP
            or message.name == DbMsrMessages.DBMSR_PROMOTE_TO_MASTER
            or message.name == DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE
            or message.name == DbMsrMessages.DBMSR_CREATE_BACKUP
            or message.name == Messages.UPDATE_SERVICE_CONFIGURATION
            or message.name == Messages.HOST_INIT
            or message.name == Messages.BEFORE_HOST_TERMINATE or message.name
            == Messages.HOST_UP or message.name == Messages.HOST_DOWN)

    def __init__(self):
        self._service_name = SERVICE_NAME
        ServiceCtlHandler.__init__(self, SERVICE_NAME,
                                   initdv2.lookup(SERVICE_NAME))
        bus.on("init", self.on_init)
        bus.define_events(
            'before_postgresql_data_bundle',
            'postgresql_data_bundle',

            # @param host: New master hostname
            'before_postgresql_change_master',

            # @param host: New master hostname
            'postgresql_change_master',
            'before_slave_promote_to_master',
            'slave_promote_to_master')

        self._hir_volume_growth = None
        self._postgresql_api = postgresql_api.PostgreSQLAPI()

        self.on_reload()

    def on_init(self):
        #temporary fix for starting-after-rebundle issue
        if not os.path.exists(PG_SOCKET_DIR):
            os.makedirs(PG_SOCKET_DIR)
            chown_r(PG_SOCKET_DIR, 'postgres')

        bus.on("host_init_response", self.on_host_init_response)
        bus.on("before_host_up", self.on_before_host_up)
        bus.on("before_reboot_start", self.on_before_reboot_start)

        self._insert_iptables_rules()

        if __node__['state'] == ScalarizrState.BOOTSTRAPPING:

            if linux.os.redhat_family:

                def selinux_enabled():
                    selinuxenabled = software.which('selinuxenabled')
                    if selinuxenabled:
                        _, _, ret_code = system2((selinuxenabled, ),
                                                 raise_exc=False)
                        return 0 == ret_code
                    # Consider it enabled by default
                    return True

                checkmodule_path = software.which('checkmodule')
                semodule_package_path = software.which('semodule_package')
                semodule_path = software.which('semodule')

                if all(
                    (checkmodule_path, semodule_package_path, semodule_path)):
                    if selinux_enabled():
                        with open('/tmp/sshkeygen.te', 'w') as fp:
                            fp.write(SSH_KEYGEN_SELINUX_MODULE)

                        self._logger.debug(
                            'Compiling SELinux policy for ssh-keygen')
                        system2((checkmodule_path, '-M', '-m', '-o',
                                 '/tmp/sshkeygen.mod', '/tmp/sshkeygen.te'),
                                logger=self._logger)

                        self._logger.debug(
                            'Building SELinux package for ssh-keygen')
                        system2(
                            (semodule_package_path, '-o', '/tmp/sshkeygen.pp',
                             '-m', '/tmp/sshkeygen.mod'),
                            logger=self._logger)

                        self._logger.debug(
                            'Loading ssh-keygen SELinux package')
                        system2((semodule_path, '-i', '/tmp/sshkeygen.pp'),
                                logger=self._logger)

        if __node__['state'] == 'running':

            vol = storage2.volume(__postgresql__['volume'])
            vol.ensure(mount=True)

            self.postgresql.service.start()
            self.accept_all_clients()

            self._logger.debug(
                "Checking presence of Scalr's PostgreSQL root user.")
            root_password = self.root_password

            if not self.postgresql.root_user.exists():
                self._logger.debug(
                    "Scalr's PostgreSQL root user does not exist. Recreating")
                self.postgresql.root_user = self.postgresql.create_linux_user(
                    ROOT_USER, root_password)
            else:
                try:
                    self.postgresql.root_user.check_system_password(
                        root_password)
                    self._logger.debug(
                        "Scalr's root PgSQL user is present. Password is correct."
                    )
                except ValueError:
                    self._logger.warning(
                        "Scalr's root PgSQL user was changed. Recreating.")
                    self.postgresql.root_user.change_system_password(
                        root_password)

            if self.is_replication_master:
                #ALTER ROLE cannot be executed in a read-only transaction
                self._logger.debug("Checking password for pg_role scalr.")
                if not self.postgresql.root_user.check_role_password(
                        root_password):
                    LOG.warning(
                        "Scalr's root PgSQL role was changed. Recreating.")
                    self.postgresql.root_user.change_role_password(
                        root_password)

    def on_reload(self):
        self._queryenv = bus.queryenv_service
        self._platform = bus.platform
        self.postgresql = PostgreSql()
        self.preset_provider = PgSQLPresetProvider()

    def on_HostInit(self, message):
        if message.local_ip != self._platform.get_private_ip(
        ) and message.local_ip in self.pg_hosts:
            LOG.debug('Got new slave IP: %s. Registering in pg_hba.conf' %
                      message.local_ip)
            self.postgresql.register_slave(message.local_ip)

    def on_HostUp(self, message):
        if message.local_ip == self._platform.get_private_ip():
            self.accept_all_clients()
        elif message.local_ip in self.farm_hosts:
            self.postgresql.register_client(message.local_ip)

    def on_HostDown(self, message):
        if message.local_ip != self._platform.get_private_ip():
            self.postgresql.unregister_client(message.local_ip)
            if self.is_replication_master and self.farmrole_id == message.farm_role_id:
                self.postgresql.unregister_slave(message.local_ip)

    @property
    def farm_hosts(self):
        list_roles = self._queryenv.list_roles()
        servers = []
        for serv in list_roles:
            for host in serv.hosts:
                servers.append(host.internal_ip or host.external_ip)
        LOG.debug("QueryEnv returned list of servers within farm: %s" %
                  servers)
        return servers

    @property
    def pg_hosts(self):
        '''
        All pg instances including those in Initializing state
        '''
        list_roles = self._queryenv.list_roles(behaviour=BEHAVIOUR,
                                               with_init=True)
        servers = []
        for pg_serv in list_roles:
            for pg_host in pg_serv.hosts:
                servers.append(pg_host.internal_ip or pg_host.external_ip)
        LOG.debug("QueryEnv returned list of %s servers: %s" %
                  (BEHAVIOUR, servers))
        return servers

    def accept_all_clients(self):
        farm_hosts = self.farm_hosts
        for ip in farm_hosts:
            self.postgresql.register_client(ip, force=False)
        if farm_hosts:
            self.postgresql.service.reload(
                'Granting access to all servers within farm.', force=True)

    @property
    def root_password(self):
        return __postgresql__['%s_password' % ROOT_USER]

    @property
    def farmrole_id(self):
        return __node__[config.OPT_FARMROLE_ID]

    def store_password(self, name, password):
        __postgresql__['%s_password' % name] = password

    @property
    def _tmp_path(self):
        return os.path.join(__postgresql__['storage_dir'], 'tmp')

    @property
    def is_replication_master(self):
        return True if int(__postgresql__[OPT_REPLICATION_MASTER]) else False

    def on_host_init_response(self, message):
        """
        Check postgresql data in host init response
        @type message: scalarizr.messaging.Message
        @param message: HostInitResponse
        """
        log = bus.init_op.logger
        log.info('Accept Scalr configuration')

        if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR:
            raise HandlerError(
                "HostInitResponse message for PostgreSQL behaviour must have 'postgresql' property and db_type 'postgresql'"
            )

        postgresql_data = message.postgresql.copy()

        #Extracting service configuration preset from message
        if 'preset' in postgresql_data:
            self.initial_preset = postgresql_data['preset']
            LOG.debug('Scalr sent current preset: %s' % self.initial_preset)
            del postgresql_data['preset']

        #Extracting or generating postgresql root password
        postgresql_data['%s_password' % ROOT_USER] = postgresql_data.get(
            OPT_ROOT_PASSWORD) or cryptotool.pwgen(10)
        del postgresql_data[OPT_ROOT_PASSWORD]

        #Extracting replication ssh keys from message
        root = PgUser(ROOT_USER, self.postgresql.pg_keys_dir)
        root.store_keys(postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY],
                        postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY])
        del postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY]
        del postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY]

        if postgresql_data.get('volume'):
            # New format
            postgresql_data['compat_prior_backup_restore'] = False
            postgresql_data['volume'] = storage2.volume(
                postgresql_data['volume'])

            LOG.debug("message.pg['volume']: %s", postgresql_data['volume'])
            if 'backup' in postgresql_data:
                postgresql_data['backup'] = backup.backup(
                    postgresql_data['backup'])
                LOG.debug("message.pg['backup']: %s",
                          postgresql_data['backup'])
            if 'restore' in postgresql_data:
                postgresql_data['restore'] = backup.restore(
                    postgresql_data['restore'])
                LOG.debug("message.pg['restore']: %s",
                          postgresql_data['restore'])
        else:

            # Compatibility transformation
            # - volume_config -> volume
            # - master n'th start, type=ebs - del snapshot_config
            # - snapshot_config -> restore
            # - create backup object on master 1'st start

            postgresql_data['compat_prior_backup_restore'] = True
            if postgresql_data.get(OPT_VOLUME_CNF):
                postgresql_data['volume'] = storage2.volume(
                    postgresql_data.pop(OPT_VOLUME_CNF))

            elif postgresql_data.get(OPT_SNAPSHOT_CNF):
                postgresql_data['volume'] = storage2.volume(
                    type=postgresql_data[OPT_SNAPSHOT_CNF]['type'])

            else:
                raise HandlerError(
                    'No volume config or snapshot config provided')

            if postgresql_data['volume'].device and \
                            postgresql_data['volume'].type in ('ebs', 'csvol', 'cinder', 'raid', 'gce_persistent'):
                LOG.debug(
                    "Master n'th start detected. Removing snapshot config from message"
                )
                postgresql_data.pop(OPT_SNAPSHOT_CNF, None)

            if postgresql_data.get(OPT_SNAPSHOT_CNF):
                postgresql_data['restore'] = backup.restore(
                    type='snap_postgresql',
                    snapshot=postgresql_data.pop(OPT_SNAPSHOT_CNF),
                    volume=postgresql_data['volume'])

            if int(postgresql_data['replication_master']):
                postgresql_data['backup'] = backup.backup(
                    type='snap_postgresql', volume=postgresql_data['volume'])

        self._hir_volume_growth = postgresql_data.pop('volume_growth', None)

        LOG.debug("Update postgresql config with %s", postgresql_data)
        __postgresql__.update(postgresql_data)
        __postgresql__['volume'].mpoint = __postgresql__['storage_dir']

    def on_before_host_up(self, message):
        """
        Configure PostgreSQL behaviour
        @type message: scalarizr.messaging.Message
        @param message: HostUp message
        """

        repl = 'master' if self.is_replication_master else 'slave'
        #bus.fire('before_postgresql_configure', replication=repl)

        if self.is_replication_master:
            self._init_master(message)
        else:
            self._init_slave(message)
        # Force to resave volume settings
        __postgresql__['volume'] = storage2.volume(__postgresql__['volume'])
        bus.fire('service_configured',
                 service_name=SERVICE_NAME,
                 replication=repl,
                 preset=self.initial_preset)

    def on_before_reboot_start(self, *args, **kwargs):
        """
        Stop PostgreSQL and unplug storage
        """
        self.postgresql.service.stop('rebooting')

    def on_BeforeHostTerminate(self, message):
        LOG.info('Handling BeforeHostTerminate message from %s' %
                 message.local_ip)
        if message.local_ip == self._platform.get_private_ip():
            LOG.info('Stopping %s service' % BEHAVIOUR)
            self.postgresql.service.stop('Server will be terminated')
            if not self.is_replication_master:
                LOG.info('Destroying volume %s' % __postgresql__['volume'].id)
                __postgresql__['volume'].destroy(remove_disks=True)
                LOG.info('Volume %s has been destroyed.' %
                         __postgresql__['volume'].id)
            else:
                __postgresql__['volume'].umount()

    def on_DbMsr_CreateDataBundle(self, message):
        LOG.debug("on_DbMsr_CreateDataBundle")
        self._postgresql_api.create_databundle(async=True)

    def on_DbMsr_PromoteToMaster(self, message):
        """
        Promote slave to master
        @type message: scalarizr.messaging.Message
        @param message: postgresql_PromoteToMaster
        """
        LOG.debug("on_DbMsr_PromoteToMaster")
        postgresql = message.body[BEHAVIOUR]

        if int(__postgresql__['replication_master']):
            LOG.warning('Cannot promote to master. Already master')
            return

        LOG.info('Starting Slave -> Master promotion')
        bus.fire('before_slave_promote_to_master')

        msg_data = {'db_type': BEHAVIOUR, 'status': 'ok', BEHAVIOUR: {}}

        tx_complete = False

        new_vol = None
        if postgresql.get('volume_config'):
            new_vol = storage2.volume(postgresql.get('volume_config'))

        try:
            self.postgresql.stop_replication()

            if new_vol and new_vol.type not in ('eph', 'lvm'):
                self.postgresql.service.stop(
                    'Unplugging slave storage and then plugging master one')

                old_vol = storage2.volume(__postgresql__['volume'])
                old_vol.detach(force=True)

                new_vol.mpoint = __postgresql__['storage_dir']
                new_vol.ensure(mount=True)

                if not self.postgresql.cluster_dir.is_initialized(
                        STORAGE_PATH):
                    raise HandlerError("%s is not a valid postgresql storage" %
                                       STORAGE_PATH)

                __postgresql__['volume'] = new_vol
                msg_data[BEHAVIOUR] = {'volume_config': dict(new_vol)}

            slaves = [host.internal_ip for host in self._get_slave_hosts()]
            self.postgresql.init_master(STORAGE_PATH, self.root_password,
                                        slaves)
            self.postgresql.start_replication()
            __postgresql__[OPT_REPLICATION_MASTER] = 1

            if not new_vol or new_vol.type in ('eph', 'lvm'):
                snap = self._create_snapshot()
                __postgresql__['snapshot'] = snap
                msg_data[BEHAVIOUR].update({OPT_SNAPSHOT_CNF: dict(snap)})

            msg_data[
                OPT_CURRENT_XLOG_LOCATION] = None  # useless but required by Scalr

            self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT,
                              msg_data)

            tx_complete = True
            bus.fire('slave_promote_to_master')

        except (Exception, BaseException), e:
            LOG.exception(e)

            self.send_message(
                DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT,
                dict(db_type=BEHAVIOUR, status="error", last_error=str(e)))

            self.postgresql.service.stop(
                'Unplugging broken storage and then plugging the old one')
            if new_vol:
                new_vol.detach()
            # Get back slave storage
            if old_vol:
                old_vol.ensure(mount=True)
            self.postgresql.service.start()

        if tx_complete and new_vol and new_vol.type not in ('eph', 'lvm'):
            # Delete slave EBS
            old_vol.destroy(remove_disks=True)
Exemple #3
0
class PostgreSqlHander(ServiceCtlHandler):	
	_logger = None
		
	_queryenv = None
	""" @type _queryenv: scalarizr.queryenv.QueryEnvService	"""
	
	_platform = None
	""" @type _platform: scalarizr.platform.Ec2Platform """
	
	_cnf = None
	''' @type _cnf: scalarizr.config.ScalarizrCnf '''
	
	storage_vol = None	
		
		
	def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None):
		return BEHAVIOUR in behaviour and (
					message.name == DbMsrMessages.DBMSR_NEW_MASTER_UP
				or 	message.name == DbMsrMessages.DBMSR_PROMOTE_TO_MASTER
				or 	message.name == DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE
				or 	message.name == DbMsrMessages.DBMSR_CREATE_BACKUP
				or  message.name == Messages.UPDATE_SERVICE_CONFIGURATION
				or  message.name == Messages.HOST_INIT
				or  message.name == Messages.BEFORE_HOST_TERMINATE
				or  message.name == Messages.HOST_UP
				or  message.name == Messages.HOST_DOWN)	

	
	def get_initialization_phases(self, hir_message):
		if BEHAVIOUR in hir_message.body:
			steps = [self._step_accept_scalr_conf, self._step_create_storage]
			if hir_message.body[BEHAVIOUR]['replication_master'] == '1':
				steps += [self._step_init_master, self._step_create_data_bundle]
			else:
				steps += [self._step_init_slave]
			steps += [self._step_collect_host_up_data]
			
			return {'before_host_up': [{
				'name': self._phase_postgresql, 
				'steps': steps
			}]}
	
	
	def __init__(self):
		self._logger = logging.getLogger(__name__)
		self._service_name = SERVICE_NAME
		Handler.__init__(self)
		bus.on("init", self.on_init)
		bus.define_events(
			'before_postgresql_data_bundle',
			
			'postgresql_data_bundle',
			
			# @param host: New master hostname 
			'before_postgresql_change_master',
			
			# @param host: New master hostname 
			'postgresql_change_master',
			
			'before_slave_promote_to_master',
			
			'slave_promote_to_master'
		)	
		
		self._phase_postgresql = 'Configure PostgreSQL'
		self._phase_data_bundle = self._op_data_bundle = 'PostgreSQL data bundle'
		self._phase_backup = self._op_backup = 'PostgreSQL backup'
		self._step_upload_to_cloud_storage = 'Upload data to cloud storage'
		self._step_accept_scalr_conf = 'Accept Scalr configuration'
		self._step_patch_conf = 'Patch configuration files'
		self._step_create_storage = 'Create storage'
		self._step_init_master = 'Initialize Master'
		self._step_init_slave = 'Initialize Slave'
		self._step_create_data_bundle = 'Create data bundle'
		self._step_change_replication_master = 'Change replication Master'
		self._step_collect_host_up_data = 'Collect HostUp data'
		
		self.on_reload()		


	def on_init(self):		
		#temporary fix for starting-after-rebundle issue
		if not os.path.exists(PG_SOCKET_DIR):
			os.makedirs(PG_SOCKET_DIR)
			rchown(user='******', path=PG_SOCKET_DIR)
			
		bus.on("host_init_response", self.on_host_init_response)
		bus.on("before_host_up", self.on_before_host_up)
		bus.on("before_reboot_start", self.on_before_reboot_start)
		bus.on("before_reboot_finish", self.on_before_reboot_finish)
		
		if self._cnf.state == ScalarizrState.BOOTSTRAPPING:
			self._insert_iptables_rules()
			
			if disttool.is_redhat_based():		
					
				checkmodule_paths = software.whereis('checkmodule')
				semodule_package_paths = software.whereis('semodule_package')
				semodule_paths = software.whereis('semodule')
			
				if all((checkmodule_paths, semodule_package_paths, semodule_paths)):
					
					filetool.write_file('/tmp/sshkeygen.te',
								SSH_KEYGEN_SELINUX_MODULE, logger=self._logger)
					
					self._logger.debug('Compiling SELinux policy for ssh-keygen')
					system2((checkmodule_paths[0], '-M', '-m', '-o',
							 '/tmp/sshkeygen.mod', '/tmp/sshkeygen.te'), logger=self._logger)
					
					self._logger.debug('Building SELinux package for ssh-keygen')
					system2((semodule_package_paths[0], '-o', '/tmp/sshkeygen.pp',
							 '-m', '/tmp/sshkeygen.mod'), logger=self._logger)
					
					self._logger.debug('Loading ssh-keygen SELinux package')					
					system2((semodule_paths[0], '-i', '/tmp/sshkeygen.pp'), logger=self._logger)
				
		
		if self._cnf.state == ScalarizrState.RUNNING:

			storage_conf = Storage.restore_config(self._volume_config_path)
			storage_conf['tags'] = self.postgres_tags
			self.storage_vol = Storage.create(storage_conf)
			if not self.storage_vol.mounted():
				self.storage_vol.mount()
			
			self.postgresql.service.start()
			self.accept_all_clients()
			
			self._logger.debug("Checking presence of Scalr's PostgreSQL root user.")
			root_password = self.root_password
			
			if not self.postgresql.root_user.exists():
				self._logger.debug("Scalr's PostgreSQL root user does not exist. Recreating")
				self.postgresql.root_user = self.postgresql.create_user(ROOT_USER, root_password)
			else:
				try:
					self.postgresql.root_user.check_system_password(root_password)
					self._logger.debug("Scalr's root PgSQL user is present. Password is correct.")				
				except ValueError:
					self._logger.warning("Scalr's root PgSQL user was changed. Recreating.")
					self.postgresql.root_user.change_system_password(root_password)
					
			if self.is_replication_master:	
				#ALTER ROLE cannot be executed in a read-only transaction
				self._logger.debug("Checking password for pg_role scalr.")		
				if not self.postgresql.root_user.check_role_password(root_password):
					self._logger.warning("Scalr's root PgSQL role was changed. Recreating.")
					self.postgresql.root_user.change_role_password(root_password)
			

	def on_reload(self):
		self._queryenv = bus.queryenv_service
		self._platform = bus.platform
		self._cnf = bus.cnf
		ini = self._cnf.rawini
		self._role_name = ini.get(config.SECT_GENERAL, config.OPT_ROLE_NAME)
		self._storage_path = STORAGE_PATH
		self._tmp_path = os.path.join(self._storage_path, 'tmp')
		
		self._volume_config_path  = self._cnf.private_path(os.path.join('storage', STORAGE_VOLUME_CNF))
		self._snapshot_config_path = self._cnf.private_path(os.path.join('storage', STORAGE_SNAPSHOT_CNF))
		
		self.pg_keys_dir = self._cnf.private_path('keys')
		self.postgresql = PostgreSql(self.version, self.pg_keys_dir)
		
		
	@property
	def version(self):
		ver = None
		if self._cnf.rawini.has_option(CNF_SECTION, OPT_PG_VERSION):
			ver = self._cnf.rawini.get(CNF_SECTION, OPT_PG_VERSION)
			
		if not ver:
			try:
				path_list = glob.glob('/var/lib/p*sql/9.*')
				path_list.sort()
				path = path_list[-1]
				ver = os.path.basename(path)
			except IndexError:
				self._logger.warning('Postgresql default directory not found. Assuming that PostgreSQL 9.0 is installed.')
				ver = '9.0'
			finally:
				self._update_config({OPT_PG_VERSION : ver})
		return ver
		
	
	def on_HostInit(self, message):
		if message.local_ip != self._platform.get_private_ip() and message.local_ip in self.pg_hosts:
			self._logger.debug('Got new slave IP: %s. Registering in pg_hba.conf' % message.local_ip)
			self.postgresql.register_slave(message.local_ip)
			

	def on_HostUp(self, message):
		if message.local_ip == self._platform.get_private_ip():
			self.accept_all_clients()
		elif message.local_ip in self.farm_hosts:
			self.postgresql.register_client(message.local_ip)
		
	
	
	def on_HostDown(self, message):
		if  message.local_ip != self._platform.get_private_ip():
			self.postgresql.unregister_client(message.local_ip)
			if self.is_replication_master and self.farmrole_id == message.farm_role_id:
				self.postgresql.unregister_slave(message.local_ip)
	
	@property			
	def farm_hosts(self):
		list_roles = self._queryenv.list_roles()
		servers = []
		for serv in list_roles:
			for host in serv.hosts :
				servers.append(host.internal_ip or host.external_ip)
		self._logger.debug("QueryEnv returned list of servers within farm: %s" % servers)
		return servers				
		
		
	@property
	def pg_hosts(self):
		'''
		All pg instances including those in Initializing state
		'''
		list_roles = self._queryenv.list_roles(behaviour=BEHAVIOUR, with_init=True)
		servers = []
		for pg_serv in list_roles:
			for pg_host in pg_serv.hosts:
				servers.append(pg_host.internal_ip or pg_host.external_ip)
		self._logger.debug("QueryEnv returned list of %s servers: %s" % (BEHAVIOUR, servers))
		return servers
	
	
	def accept_all_clients(self):
		farm_hosts = self.farm_hosts
		for ip in farm_hosts:
				self.postgresql.register_client(ip, force=False)
		if farm_hosts:
			self.postgresql.service.reload('Granting access to all servers within farm.', force=True)
				
	
	@property
	def root_password(self):
		password = None 
		
		opt_pwd = '%s_password' % ROOT_USER
		if self._cnf.rawini.has_option(CNF_SECTION, opt_pwd):
			password = self._cnf.rawini.get(CNF_SECTION, opt_pwd)
		return password


	@property	
	def farmrole_id(self):
		id = None
		if self._cnf.rawini.has_option(config.SECT_GENERAL, config.OPT_FARMROLE_ID):
			id = self._cnf.rawini.get(config.SECT_GENERAL, config.OPT_FARMROLE_ID)
		return id
	
			
	def store_password(self, name, password):
		opt_user_password = '******' % name
		self._cnf.update_ini(BEHAVIOUR, {CNF_SECTION: {opt_user_password:password}})
			
			
	@property
	def is_replication_master(self):
		value = self._cnf.rawini.get(CNF_SECTION, OPT_REPLICATION_MASTER)
		self._logger.debug('Got %s : %s' % (OPT_REPLICATION_MASTER, value))
		return True if int(value) else False
				
				
	@property
	def postgres_tags(self):
		return prepare_tags(BEHAVIOUR, db_replication_role=self.is_replication_master)
		
				
	def on_host_init_response(self, message):
		"""
		Check postgresql data in host init response
		@type message: scalarizr.messaging.Message
		@param message: HostInitResponse
		"""
		
		with bus.initialization_op as op:
			with op.phase(self._phase_postgresql):
				with op.step(self._step_accept_scalr_conf):
		
					if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR:
						raise HandlerError("HostInitResponse message for PostgreSQL behaviour must have 'postgresql' property and db_type 'postgresql'")
					
					'''
					if message.postgresql[OPT_REPLICATION_MASTER] != '1'  and \
							(not message.body.has_key(OPT_ROOT_SSH_PUBLIC_KEY) or not 
							message.body.has_key(OPT_ROOT_SSH_PRIVATE_KEY)):
						raise HandlerError("HostInitResponse message for PostgreSQL slave must contain both public and private ssh keys")
					'''
					
					dir = os.path.dirname(self._volume_config_path)
					if not os.path.exists(dir):
						os.makedirs(dir)
					
					postgresql_data = message.postgresql.copy()
			
					root = PgUser(ROOT_USER, self.pg_keys_dir)
					root.store_keys(postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY], postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY])
					del postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY]
					del postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY]		
					
					for key, file in ((OPT_VOLUME_CNF, self._volume_config_path), 
									(OPT_SNAPSHOT_CNF, self._snapshot_config_path)):
						if os.path.exists(file):
							os.remove(file)
						
						if key in postgresql_data:
							if postgresql_data[key]:
								Storage.backup_config(postgresql_data[key], file)
							del postgresql_data[key]
					
					root_user= postgresql_data[OPT_ROOT_USER] or ROOT_USER
					postgresql_data['%s_password' % root_user] = postgresql_data.get(OPT_ROOT_PASSWORD) or cryptotool.pwgen(10)
					del postgresql_data[OPT_ROOT_PASSWORD]
					
					self._logger.debug("Update postgresql config with %s", postgresql_data)
					self._update_config(postgresql_data)


	def on_before_host_up(self, message):
		"""
		Configure PostgreSQL behaviour
		@type message: scalarizr.messaging.Message		
		@param message: HostUp message
		"""

		repl = 'master' if self.is_replication_master else 'slave'
		#bus.fire('before_postgresql_configure', replication=repl)
		
		if self.is_replication_master:
			self._init_master(message)									  
		else:
			self._init_slave(message)	
		bus.fire('service_configured', service_name=SERVICE_NAME, replication=repl)
					
				
	def on_before_reboot_start(self, *args, **kwargs):
		"""
		Stop MySQL and unplug storage
		"""
		self.postgresql.service.stop('rebooting')


	def on_before_reboot_finish(self, *args, **kwargs):
		self._insert_iptables_rules()


	def on_BeforeHostTerminate(self, message):
		self._logger.info('Handling BeforeHostTerminate message from %s' % message.local_ip)
		if message.local_ip == self._platform.get_private_ip():
			self._logger.info('Stopping %s service' % BEHAVIOUR)
			self.postgresql.service.stop('Server will be terminated')
			if not self.is_replication_master:
				self._logger.info('Destroying volume %s' % self.storage_vol.id)
				self.storage_vol.destroy(remove_disks=True)
				self._logger.info('Volume %s has been destroyed.' % self.storage_vol.id)


	def on_DbMsr_CreateDataBundle(self, message):
		
		try:
			op = operation(name=self._op_data_bundle, phases=[{
				'name': self._phase_data_bundle, 
				'steps': [self._step_create_data_bundle]
			}])
			op.define()
			
			with op.phase(self._phase_data_bundle):
				with op.step(self._step_create_data_bundle):
					
					bus.fire('before_postgresql_data_bundle')
					# Retrieve password for scalr postgresql root user
					# Creating snapshot		
					snap = self._create_snapshot()
					used_size = int(system2(('df', '-P', '--block-size=M', self._storage_path))[0].split('\n')[1].split()[2][:-1])
					bus.fire('postgresql_data_bundle', snapshot_id=snap.id)			
					
					# Notify scalr
					msg_data = dict(
						db_type 	= BEHAVIOUR,
						used_size	= '%.3f' % (float(used_size) / 1000,),
						status		= 'ok'
					)
					msg_data[BEHAVIOUR] = self._compat_storage_data(snap=snap)
					self.send_message(DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE_RESULT, msg_data)

			op.ok()

		except (Exception, BaseException), e:
			self._logger.exception(e)
			
			# Notify Scalr about error
			self.send_message(DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE_RESULT, dict(
				db_type 	= BEHAVIOUR,
				status		='error',
				last_error	= str(e)
			))
Exemple #4
0
class PostgreSqlHander(ServiceCtlHandler):
    _logger = None

    _queryenv = None
    """ @type _queryenv: scalarizr.queryenv.QueryEnvService """

    _platform = None
    """ @type _platform: scalarizr.platform.Ec2Platform """

    _cnf = None
    ''' @type _cnf: scalarizr.config.ScalarizrCnf '''

    preset_provider = None

    def accept(self,
               message,
               queue,
               behaviour=None,
               platform=None,
               os=None,
               dist=None):
        return BEHAVIOUR in behaviour and (
            message.name == DbMsrMessages.DBMSR_NEW_MASTER_UP
            or message.name == DbMsrMessages.DBMSR_PROMOTE_TO_MASTER
            or message.name == DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE
            or message.name == DbMsrMessages.DBMSR_CREATE_BACKUP
            or message.name == Messages.UPDATE_SERVICE_CONFIGURATION
            or message.name == Messages.HOST_INIT
            or message.name == Messages.BEFORE_HOST_TERMINATE or message.name
            == Messages.HOST_UP or message.name == Messages.HOST_DOWN)

    def get_initialization_phases(self, hir_message):
        if BEHAVIOUR in hir_message.body:
            steps = [self._step_accept_scalr_conf, self._step_create_storage]
            if hir_message.body[BEHAVIOUR]['replication_master'] == '1':
                steps += [
                    self._step_init_master, self._step_create_data_bundle
                ]
            else:
                steps += [self._step_init_slave]
            steps += [self._step_collect_host_up_data]

            return {
                'before_host_up': [{
                    'name': self._phase_postgresql,
                    'steps': steps
                }]
            }

    def __init__(self):
        self._service_name = SERVICE_NAME
        ServiceCtlHandler.__init__(self, SERVICE_NAME,
                                   initdv2.lookup(SERVICE_NAME))
        bus.on("init", self.on_init)
        bus.define_events(
            'before_postgresql_data_bundle',
            'postgresql_data_bundle',

            # @param host: New master hostname
            'before_postgresql_change_master',

            # @param host: New master hostname
            'postgresql_change_master',
            'before_slave_promote_to_master',
            'slave_promote_to_master')

        self._phase_postgresql = 'Configure PostgreSQL'
        self._phase_data_bundle = self._op_data_bundle = 'PostgreSQL data bundle'
        self._phase_backup = self._op_backup = 'PostgreSQL backup'
        self._step_upload_to_cloud_storage = 'Upload data to cloud storage'
        self._step_accept_scalr_conf = 'Accept Scalr configuration'
        self._step_patch_conf = 'Patch configuration files'
        self._step_create_storage = 'Create storage'
        self._step_init_master = 'Initialize Master'
        self._step_init_slave = 'Initialize Slave'
        self._step_create_data_bundle = 'Create data bundle'
        self._step_change_replication_master = 'Change replication Master'
        self._step_collect_host_up_data = 'Collect HostUp data'

        self.on_reload()

    def on_init(self):
        #temporary fix for starting-after-rebundle issue
        if not os.path.exists(PG_SOCKET_DIR):
            os.makedirs(PG_SOCKET_DIR)
            chown_r(PG_SOCKET_DIR, 'postgres')

        bus.on("host_init_response", self.on_host_init_response)
        bus.on("before_host_up", self.on_before_host_up)
        bus.on("before_reboot_start", self.on_before_reboot_start)

        self._insert_iptables_rules()

        if __node__['state'] == ScalarizrState.BOOTSTRAPPING:

            if disttool.is_redhat_based():

                checkmodule_paths = software.whereis('checkmodule')
                semodule_package_paths = software.whereis('semodule_package')
                semodule_paths = software.whereis('semodule')

                if all((checkmodule_paths, semodule_package_paths,
                        semodule_paths)):

                    with open('/tmp/sshkeygen.te', 'w') as fp:
                        fp.write(SSH_KEYGEN_SELINUX_MODULE)

                    self._logger.debug(
                        'Compiling SELinux policy for ssh-keygen')
                    system2((checkmodule_paths[0], '-M', '-m', '-o',
                             '/tmp/sshkeygen.mod', '/tmp/sshkeygen.te'),
                            logger=self._logger)

                    self._logger.debug(
                        'Building SELinux package for ssh-keygen')
                    system2((semodule_package_paths[0], '-o',
                             '/tmp/sshkeygen.pp', '-m', '/tmp/sshkeygen.mod'),
                            logger=self._logger)

                    self._logger.debug('Loading ssh-keygen SELinux package')
                    system2((semodule_paths[0], '-i', '/tmp/sshkeygen.pp'),
                            logger=self._logger)

        if __node__['state'] == 'running':

            vol = storage2.volume(__postgresql__['volume'])
            if not vol.tags:
                vol.tags = self.resource_tags()
            vol.ensure(mount=True)

            self.postgresql.service.start()
            self.accept_all_clients()

            self._logger.debug(
                "Checking presence of Scalr's PostgreSQL root user.")
            root_password = self.root_password

            if not self.postgresql.root_user.exists():
                self._logger.debug(
                    "Scalr's PostgreSQL root user does not exist. Recreating")
                self.postgresql.root_user = self.postgresql.create_linux_user(
                    ROOT_USER, root_password)
            else:
                try:
                    self.postgresql.root_user.check_system_password(
                        root_password)
                    self._logger.debug(
                        "Scalr's root PgSQL user is present. Password is correct."
                    )
                except ValueError:
                    self._logger.warning(
                        "Scalr's root PgSQL user was changed. Recreating.")
                    self.postgresql.root_user.change_system_password(
                        root_password)

            if self.is_replication_master:
                #ALTER ROLE cannot be executed in a read-only transaction
                self._logger.debug("Checking password for pg_role scalr.")
                if not self.postgresql.root_user.check_role_password(
                        root_password):
                    LOG.warning(
                        "Scalr's root PgSQL role was changed. Recreating.")
                    self.postgresql.root_user.change_role_password(
                        root_password)

    def on_reload(self):
        self._queryenv = bus.queryenv_service
        self._platform = bus.platform
        self.postgresql = PostgreSql()
        self.preset_provider = PgSQLPresetProvider(
            self.postgresql.postgresql_conf)
        preset_service.services[BEHAVIOUR] = self.preset_provider

    def on_HostInit(self, message):
        if message.local_ip != self._platform.get_private_ip(
        ) and message.local_ip in self.pg_hosts:
            LOG.debug('Got new slave IP: %s. Registering in pg_hba.conf' %
                      message.local_ip)
            self.postgresql.register_slave(message.local_ip)

    def on_HostUp(self, message):
        if message.local_ip == self._platform.get_private_ip():
            self.accept_all_clients()
        elif message.local_ip in self.farm_hosts:
            self.postgresql.register_client(message.local_ip)

    def on_HostDown(self, message):
        if message.local_ip != self._platform.get_private_ip():
            self.postgresql.unregister_client(message.local_ip)
            if self.is_replication_master and self.farmrole_id == message.farm_role_id:
                self.postgresql.unregister_slave(message.local_ip)

    @property
    def farm_hosts(self):
        list_roles = self._queryenv.list_roles()
        servers = []
        for serv in list_roles:
            for host in serv.hosts:
                servers.append(host.internal_ip or host.external_ip)
        LOG.debug("QueryEnv returned list of servers within farm: %s" %
                  servers)
        return servers

    @property
    def pg_hosts(self):
        '''
        All pg instances including those in Initializing state
        '''
        list_roles = self._queryenv.list_roles(behaviour=BEHAVIOUR,
                                               with_init=True)
        servers = []
        for pg_serv in list_roles:
            for pg_host in pg_serv.hosts:
                servers.append(pg_host.internal_ip or pg_host.external_ip)
        LOG.debug("QueryEnv returned list of %s servers: %s" %
                  (BEHAVIOUR, servers))
        return servers

    def accept_all_clients(self):
        farm_hosts = self.farm_hosts
        for ip in farm_hosts:
            self.postgresql.register_client(ip, force=False)
        if farm_hosts:
            self.postgresql.service.reload(
                'Granting access to all servers within farm.', force=True)

    @property
    def root_password(self):
        return __postgresql__['%s_password' % ROOT_USER]

    @property
    def farmrole_id(self):
        return __node__[config.OPT_FARMROLE_ID]

    def store_password(self, name, password):
        __postgresql__['%s_password' % name] = password

    @property
    def _tmp_path(self):
        return os.path.join(__postgresql__['storage_dir'], 'tmp')

    @property
    def is_replication_master(self):
        return True if int(__postgresql__[OPT_REPLICATION_MASTER]) else False

    def resource_tags(self):
        return prepare_tags(BEHAVIOUR,
                            db_replication_role=self.is_replication_master)

    def on_host_init_response(self, message):
        """
        Check postgresql data in host init response
        @type message: scalarizr.messaging.Message
        @param message: HostInitResponse
        """

        with bus.initialization_op as op:
            with op.phase(self._phase_postgresql):
                with op.step(self._step_accept_scalr_conf):

                    if not message.body.has_key(
                            BEHAVIOUR) or message.db_type != BEHAVIOUR:
                        raise HandlerError(
                            "HostInitResponse message for PostgreSQL behaviour must have 'postgresql' property and db_type 'postgresql'"
                        )

                    postgresql_data = message.postgresql.copy()

                    #Extracting service configuration preset from message
                    if 'preset' in postgresql_data:
                        self.initial_preset = postgresql_data['preset']
                        LOG.debug('Scalr sent current preset: %s' %
                                  self.initial_preset)
                        del postgresql_data['preset']

                    #Extracting or generating postgresql root password
                    postgresql_data[
                        '%s_password' % ROOT_USER] = postgresql_data.get(
                            OPT_ROOT_PASSWORD) or cryptotool.pwgen(10)
                    del postgresql_data[OPT_ROOT_PASSWORD]

                    #Extracting replication ssh keys from message
                    root = PgUser(ROOT_USER, self.postgresql.pg_keys_dir)
                    root.store_keys(postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY],
                                    postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY])
                    del postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY]
                    del postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY]

                    if postgresql_data.get('volume'):
                        # New format
                        postgresql_data['compat_prior_backup_restore'] = False
                        postgresql_data['volume'] = storage2.volume(
                            postgresql_data['volume'])
                        LOG.debug("message.pg['volume']:",
                                  postgresql_data['volume'])
                        if 'backup' in postgresql_data:
                            postgresql_data['backup'] = backup.backup(
                                postgresql_data['backup'])
                            LOG.debug("message.pg['backup']:",
                                      postgresql_data['backup'])
                        if 'restore' in postgresql_data:
                            postgresql_data['restore'] = backup.restore(
                                postgresql_data['restore'])
                            LOG.debug("message.pg['restore']:",
                                      postgresql_data['restore'])
                    else:

                        # Compatibility transformation
                        # - volume_config -> volume
                        # - master n'th start, type=ebs - del snapshot_config
                        # - snapshot_config -> restore
                        # - create backup object on master 1'st start

                        postgresql_data['compat_prior_backup_restore'] = True
                        if postgresql_data.get(OPT_VOLUME_CNF):
                            postgresql_data['volume'] = storage2.volume(
                                postgresql_data.pop(OPT_VOLUME_CNF))

                        elif postgresql_data.get(OPT_SNAPSHOT_CNF):
                            postgresql_data['volume'] = storage2.volume(
                                type=postgresql_data[OPT_SNAPSHOT_CNF]['type'])

                        else:
                            raise HandlerError(
                                'No volume config or snapshot config provided')

                        if postgresql_data['volume'].device and \
                                        postgresql_data['volume'].type in ('ebs', 'csvol', 'cinder', 'raid'):
                            LOG.debug(
                                "Master n'th start detected. Removing snapshot config from message"
                            )
                            postgresql_data.pop(OPT_SNAPSHOT_CNF, None)

                        if postgresql_data.get(OPT_SNAPSHOT_CNF):
                            postgresql_data['restore'] = backup.restore(
                                type='snap_postgresql',
                                snapshot=postgresql_data.pop(OPT_SNAPSHOT_CNF),
                                volume=postgresql_data['volume'])

                        if int(postgresql_data['replication_master']):
                            postgresql_data['backup'] = backup.backup(
                                type='snap_postgresql',
                                volume=postgresql_data['volume'])

                    LOG.debug("Update postgresql config with %s",
                              postgresql_data)
                    __postgresql__.update(postgresql_data)
                    __postgresql__['volume'].mpoint = __postgresql__[
                        'storage_dir']
                    __postgresql__['volume'].tags = self.resource_tags()
                    if 'backup' in __postgresql__:
                        __postgresql__['backup'].tags = self.resource_tags()

    def on_before_host_up(self, message):
        """
        Configure PostgreSQL behaviour
        @type message: scalarizr.messaging.Message      
        @param message: HostUp message
        """

        repl = 'master' if self.is_replication_master else 'slave'
        #bus.fire('before_postgresql_configure', replication=repl)

        if self.is_replication_master:
            self._init_master(message)
        else:
            self._init_slave(message)
        # Force to resave volume settings
        __postgresql__['volume'] = storage2.volume(__postgresql__['volume'])
        bus.fire('service_configured',
                 service_name=SERVICE_NAME,
                 replication=repl,
                 preset=self.initial_preset)

    def on_before_reboot_start(self, *args, **kwargs):
        """
        Stop PostgreSQL and unplug storage
        """
        self.postgresql.service.stop('rebooting')

    def on_BeforeHostTerminate(self, message):
        LOG.info('Handling BeforeHostTerminate message from %s' %
                 message.local_ip)
        if message.local_ip == self._platform.get_private_ip():
            LOG.info('Stopping %s service' % BEHAVIOUR)
            self.postgresql.service.stop('Server will be terminated')
            if not self.is_replication_master:
                LOG.info('Destroying volume %s' % __postgresql__['volume'].id)
                __postgresql__['volume'].destroy(remove_disks=True)
                LOG.info('Volume %s has been destroyed.' %
                         __postgresql__['volume'].id)
            else:
                __postgresql__['volume'].umount()

    def on_DbMsr_CreateDataBundle(self, message):

        try:
            op = operation(name=self._op_data_bundle,
                           phases=[{
                               'name': self._phase_data_bundle,
                               'steps': [self._step_create_data_bundle]
                           }])
            op.define()

            with op.phase(self._phase_data_bundle):
                with op.step(self._step_create_data_bundle):

                    bus.fire('before_postgresql_data_bundle')
                    snap = self._create_snapshot()
                    used_size = int(
                        system2(
                            ('df', '-P', '--block-size=M',
                             STORAGE_PATH))[0].split('\n')[1].split()[2][:-1])
                    bus.fire('postgresql_data_bundle', snapshot_id=snap.id)

                    # Notify scalr
                    msg_data = {
                        'db_type': BEHAVIOUR,
                        'status': 'ok',
                        'used_size': '%.3f' % (float(used_size) / 1000, ),
                        BEHAVIOUR: {
                            OPT_SNAPSHOT_CNF: dict(snap)
                        }
                    }
                    self.send_message(
                        DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE_RESULT,
                        msg_data)

            op.ok()

        except (Exception, BaseException), e:
            LOG.exception(e)

            # Notify Scalr about error
            self.send_message(
                DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE_RESULT,
                dict(db_type=BEHAVIOUR, status='error', last_error=str(e)))
Exemple #5
0
class PostgreSqlHander(ServiceCtlHandler):
    _logger = None

    _queryenv = None
    """ @type _queryenv: scalarizr.queryenv.QueryEnvService """

    _platform = None
    """ @type _platform: scalarizr.platform.Ec2Platform """

    _cnf = None
    ''' @type _cnf: scalarizr.config.ScalarizrCnf '''

    preset_provider = None

    def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None):
        return BEHAVIOUR in behaviour and (
            message.name == DbMsrMessages.DBMSR_NEW_MASTER_UP
            or message.name == DbMsrMessages.DBMSR_PROMOTE_TO_MASTER
            or message.name == DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE
            or message.name == DbMsrMessages.DBMSR_CREATE_BACKUP
            or message.name == Messages.UPDATE_SERVICE_CONFIGURATION
            or message.name == Messages.HOST_INIT
            or message.name == Messages.BEFORE_HOST_TERMINATE
            or message.name == Messages.HOST_UP
            or message.name == Messages.HOST_DOWN)
    
    
    def __init__(self):
        self._service_name = SERVICE_NAME
        ServiceCtlHandler.__init__(self, SERVICE_NAME, initdv2.lookup(SERVICE_NAME))
        bus.on("init", self.on_init)
        bus.define_events(
            'before_postgresql_data_bundle',
            
            'postgresql_data_bundle',
            
            # @param host: New master hostname 
            'before_postgresql_change_master',
            
            # @param host: New master hostname 
            'postgresql_change_master',
            
            'before_slave_promote_to_master',
            
            'slave_promote_to_master'
        )

        self._hir_volume_growth = None
        self._postgresql_api = postgresql_api.PostgreSQLAPI()

        self.on_reload()        


    def on_init(self):      
        #temporary fix for starting-after-rebundle issue
        if not os.path.exists(PG_SOCKET_DIR):
            os.makedirs(PG_SOCKET_DIR)
            chown_r(PG_SOCKET_DIR, 'postgres')
            
        bus.on("host_init_response", self.on_host_init_response)
        bus.on("before_host_up", self.on_before_host_up)
        bus.on("before_reboot_start", self.on_before_reboot_start)

        self._insert_iptables_rules()       

        if __node__['state'] == ScalarizrState.BOOTSTRAPPING:
            
            if linux.os.redhat_family:      
                    
                checkmodule_path = software.which('checkmodule')
                semodule_package_path = software.which('semodule_package')
                semodule_path = software.which('semodule')
            
                if all((checkmodule_path, semodule_package_path, semodule_path)):
                    
                    with open('/tmp/sshkeygen.te', 'w') as fp:
                        fp.write(SSH_KEYGEN_SELINUX_MODULE)
                    
                    self._logger.debug('Compiling SELinux policy for ssh-keygen')
                    system2((checkmodule_path, '-M', '-m', '-o',
                             '/tmp/sshkeygen.mod', '/tmp/sshkeygen.te'), logger=self._logger)
                    
                    self._logger.debug('Building SELinux package for ssh-keygen')
                    system2((semodule_package_path, '-o', '/tmp/sshkeygen.pp',
                             '-m', '/tmp/sshkeygen.mod'), logger=self._logger)
                    
                    self._logger.debug('Loading ssh-keygen SELinux package')                    
                    system2((semodule_path, '-i', '/tmp/sshkeygen.pp'), logger=self._logger)


        if __node__['state'] == 'running':

            vol = storage2.volume(__postgresql__['volume'])
            vol.ensure(mount=True)
            
            self.postgresql.service.start()
            self.accept_all_clients()
            
            self._logger.debug("Checking presence of Scalr's PostgreSQL root user.")
            root_password = self.root_password
            
            if not self.postgresql.root_user.exists():
                self._logger.debug("Scalr's PostgreSQL root user does not exist. Recreating")
                self.postgresql.root_user = self.postgresql.create_linux_user(ROOT_USER, root_password)
            else:
                try:
                    self.postgresql.root_user.check_system_password(root_password)
                    self._logger.debug("Scalr's root PgSQL user is present. Password is correct.")              
                except ValueError:
                    self._logger.warning("Scalr's root PgSQL user was changed. Recreating.")
                    self.postgresql.root_user.change_system_password(root_password)
                    
            if self.is_replication_master:  
                #ALTER ROLE cannot be executed in a read-only transaction
                self._logger.debug("Checking password for pg_role scalr.")      
                if not self.postgresql.root_user.check_role_password(root_password):
                    LOG.warning("Scalr's root PgSQL role was changed. Recreating.")
                    self.postgresql.root_user.change_role_password(root_password)
            

    def on_reload(self):
        self._queryenv = bus.queryenv_service
        self._platform = bus.platform
        self.postgresql = PostgreSql()
        self.preset_provider = PgSQLPresetProvider()

    
    def on_HostInit(self, message):
        if message.local_ip != self._platform.get_private_ip() and message.local_ip in self.pg_hosts:
            LOG.debug('Got new slave IP: %s. Registering in pg_hba.conf' % message.local_ip)
            self.postgresql.register_slave(message.local_ip)
            

    def on_HostUp(self, message):
        if message.local_ip == self._platform.get_private_ip():
            self.accept_all_clients()
        elif message.local_ip in self.farm_hosts:
            self.postgresql.register_client(message.local_ip)
        
    
    
    def on_HostDown(self, message):
        if  message.local_ip != self._platform.get_private_ip():
            self.postgresql.unregister_client(message.local_ip)
            if self.is_replication_master and self.farmrole_id == message.farm_role_id:
                self.postgresql.unregister_slave(message.local_ip)
    
    @property           
    def farm_hosts(self):
        list_roles = self._queryenv.list_roles()
        servers = []
        for serv in list_roles:
            for host in serv.hosts :
                servers.append(host.internal_ip or host.external_ip)
        LOG.debug("QueryEnv returned list of servers within farm: %s" % servers)
        return servers              
        
        
    @property
    def pg_hosts(self):
        '''
        All pg instances including those in Initializing state
        '''
        list_roles = self._queryenv.list_roles(behaviour=BEHAVIOUR, with_init=True)
        servers = []
        for pg_serv in list_roles:
            for pg_host in pg_serv.hosts:
                servers.append(pg_host.internal_ip or pg_host.external_ip)
        LOG.debug("QueryEnv returned list of %s servers: %s" % (BEHAVIOUR, servers))
        return servers
    
    
    def accept_all_clients(self):
        farm_hosts = self.farm_hosts
        for ip in farm_hosts:
                self.postgresql.register_client(ip, force=False)
        if farm_hosts:
            self.postgresql.service.reload('Granting access to all servers within farm.', force=True)
                

    @property
    def root_password(self):
        return __postgresql__['%s_password' % ROOT_USER]


    @property   
    def farmrole_id(self):
        return __node__[config.OPT_FARMROLE_ID]
    
            
    def store_password(self, name, password):
        __postgresql__['%s_password' % name] = password


    @property
    def _tmp_path(self):
        return os.path.join(__postgresql__['storage_dir'], 'tmp')


    @property
    def is_replication_master(self):
        return True if int(__postgresql__[OPT_REPLICATION_MASTER]) else False


    def resource_tags(self):
        purpose = '%s-' % BEHAVIOUR + ('master' if self.is_replication_master else 'slave')
        return build_tags(purpose, 'active')


    def on_host_init_response(self, message):
        """
        Check postgresql data in host init response
        @type message: scalarizr.messaging.Message
        @param message: HostInitResponse
        """
        log = bus.init_op.logger
        log.info('Accept Scalr configuration')

        if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR:
            raise HandlerError("HostInitResponse message for PostgreSQL behaviour must have 'postgresql' property and db_type 'postgresql'")

        postgresql_data = message.postgresql.copy()

        #Extracting service configuration preset from message
        if 'preset' in postgresql_data:
            self.initial_preset = postgresql_data['preset']
            LOG.debug('Scalr sent current preset: %s' % self.initial_preset)
            del postgresql_data['preset']

        #Extracting or generating postgresql root password
        postgresql_data['%s_password' % ROOT_USER] = postgresql_data.get(OPT_ROOT_PASSWORD) or cryptotool.pwgen(10)
        del postgresql_data[OPT_ROOT_PASSWORD]

        #Extracting replication ssh keys from message
        root = PgUser(ROOT_USER, self.postgresql.pg_keys_dir)
        root.store_keys(postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY], postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY])
        del postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY]
        del postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY]


        if postgresql_data.get('volume'):
            # New format
            postgresql_data['compat_prior_backup_restore'] = False
            postgresql_data['volume'] = storage2.volume(postgresql_data['volume'])

            LOG.debug("message.pg['volume']: %s", postgresql_data['volume'])
            if 'backup' in postgresql_data:
                postgresql_data['backup'] = backup.backup(postgresql_data['backup'])
                LOG.debug("message.pg['backup']: %s", postgresql_data['backup'])
            if 'restore' in postgresql_data:
                postgresql_data['restore'] = backup.restore(postgresql_data['restore'])
                LOG.debug("message.pg['restore']: %s", postgresql_data['restore'])
        else:

            # Compatibility transformation
            # - volume_config -> volume
            # - master n'th start, type=ebs - del snapshot_config
            # - snapshot_config -> restore
            # - create backup object on master 1'st start

            postgresql_data['compat_prior_backup_restore'] = True
            if postgresql_data.get(OPT_VOLUME_CNF):
                postgresql_data['volume'] = storage2.volume(
                    postgresql_data.pop(OPT_VOLUME_CNF))

            elif postgresql_data.get(OPT_SNAPSHOT_CNF):
                postgresql_data['volume'] = storage2.volume(
                    type=postgresql_data[OPT_SNAPSHOT_CNF]['type'])

            else:
                raise HandlerError('No volume config or snapshot config provided')

            if postgresql_data['volume'].device and \
                            postgresql_data['volume'].type in ('ebs', 'csvol', 'cinder', 'raid', 'gce_persistent'):
                LOG.debug("Master n'th start detected. Removing snapshot config from message")
                postgresql_data.pop(OPT_SNAPSHOT_CNF, None)

            if postgresql_data.get(OPT_SNAPSHOT_CNF):
                postgresql_data['restore'] = backup.restore(
                    type='snap_postgresql',
                    snapshot=postgresql_data.pop(OPT_SNAPSHOT_CNF),
                    volume=postgresql_data['volume'])

            if int(postgresql_data['replication_master']):
                postgresql_data['backup'] = backup.backup(
                    type='snap_postgresql',
                    volume=postgresql_data['volume'])

        self._hir_volume_growth = postgresql_data.pop('volume_growth', None)

        LOG.debug("Update postgresql config with %s", postgresql_data)
        __postgresql__.update(postgresql_data)
        __postgresql__['volume'].mpoint = __postgresql__['storage_dir']
        __postgresql__['volume'].tags = self.resource_tags()
        if 'backup' in __postgresql__:
            __postgresql__['backup'].tags = self.resource_tags()


    def on_before_host_up(self, message):
        """
        Configure PostgreSQL behaviour
        @type message: scalarizr.messaging.Message      
        @param message: HostUp message
        """

        repl = 'master' if self.is_replication_master else 'slave'
        #bus.fire('before_postgresql_configure', replication=repl)
        
        if self.is_replication_master:
            self._init_master(message)                                    
        else:
            self._init_slave(message)
        # Force to resave volume settings
        __postgresql__['volume'] = storage2.volume(__postgresql__['volume'])
        bus.fire('service_configured', service_name=SERVICE_NAME, replication=repl, preset=self.initial_preset)
                    
                
    def on_before_reboot_start(self, *args, **kwargs):
        """
        Stop PostgreSQL and unplug storage
        """
        self.postgresql.service.stop('rebooting')


    def on_BeforeHostTerminate(self, message):
        LOG.info('Handling BeforeHostTerminate message from %s' % message.local_ip)
        if message.local_ip == self._platform.get_private_ip():
            LOG.info('Stopping %s service' % BEHAVIOUR)
            self.postgresql.service.stop('Server will be terminated')
            if not self.is_replication_master:
                LOG.info('Destroying volume %s' % __postgresql__['volume'].id)
                __postgresql__['volume'].destroy(remove_disks=True)
                LOG.info('Volume %s has been destroyed.' % __postgresql__['volume'].id)
            else:
                __postgresql__['volume'].umount()


    def on_DbMsr_CreateDataBundle(self, message):
        LOG.debug("on_DbMsr_CreateDataBundle")
        self._postgresql_api.create_databundle(async=True)


    def on_DbMsr_PromoteToMaster(self, message):
        """
        Promote slave to master
        @type message: scalarizr.messaging.Message
        @param message: postgresql_PromoteToMaster
        """
        LOG.debug("on_DbMsr_PromoteToMaster")
        postgresql = message.body[BEHAVIOUR]

        if int(__postgresql__['replication_master']):
            LOG.warning('Cannot promote to master. Already master')
            return

        LOG.info('Starting Slave -> Master promotion')
        bus.fire('before_slave_promote_to_master')

        msg_data = {
            'db_type' : BEHAVIOUR,
            'status' : 'ok',
            BEHAVIOUR : {}
        }

        tx_complete = False

        new_vol = None
        if postgresql.get('volume_config'):
            new_vol = storage2.volume(postgresql.get('volume_config'))

        try:
            self.postgresql.stop_replication()

            if new_vol and new_vol.type not in ('eph', 'lvm'):
                self.postgresql.service.stop('Unplugging slave storage and then plugging master one')

                old_vol = storage2.volume(__postgresql__['volume'])
                old_vol.detach(force=True)

                new_vol.mpoint = __postgresql__['storage_dir']
                new_vol.ensure(mount=True)

                if not self.postgresql.cluster_dir.is_initialized(STORAGE_PATH):
                    raise HandlerError("%s is not a valid postgresql storage" % STORAGE_PATH)

                __postgresql__['volume'] = new_vol
                msg_data[BEHAVIOUR] = {'volume_config': dict(new_vol)}

            slaves = [host.internal_ip for host in self._get_slave_hosts()]
            self.postgresql.init_master(STORAGE_PATH, self.root_password, slaves)
            self.postgresql.start_replication()
            __postgresql__[OPT_REPLICATION_MASTER] = 1

            if not new_vol or new_vol.type in ('eph', 'lvm'):
                snap = self._create_snapshot()
                __postgresql__['snapshot'] = snap
                msg_data[BEHAVIOUR].update({OPT_SNAPSHOT_CNF : dict(snap)})

            msg_data[OPT_CURRENT_XLOG_LOCATION] = None # useless but required by Scalr

            self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT, msg_data)

            tx_complete = True
            bus.fire('slave_promote_to_master')

        except (Exception, BaseException), e:
            LOG.exception(e)

            self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT, dict(
                db_type=BEHAVIOUR,
                status="error",
                last_error=str(e)
            ))

            self.postgresql.service.stop('Unplugging broken storage and then plugging the old one')
            if new_vol:
                new_vol.detach()
            # Get back slave storage
            if old_vol:
                old_vol.ensure(mount=True)
            self.postgresql.service.start()
        
        if tx_complete and new_vol and new_vol.type not in ('eph', 'lvm'):
            # Delete slave EBS
            old_vol.destroy(remove_disks=True)