def on_reload(self):
self._queryenv = bus.queryenv_service
self._platform = bus.platform
self.postgresql = PostgreSql()
self.preset_provider = PgSQLPresetProvider(
self.postgresql.postgresql_conf)
preset_service.services[BEHAVIOUR] = self.preset_provider
def on_reload(self):
self._queryenv = bus.queryenv_service
self._platform = bus.platform
self._cnf = bus.cnf
ini = self._cnf.rawini
self._role_name = ini.get(config.SECT_GENERAL, config.OPT_ROLE_NAME)
self._storage_path = STORAGE_PATH
self._tmp_path = os.path.join(self._storage_path, 'tmp')
self._volume_config_path = self._cnf.private_path(os.path.join('storage', STORAGE_VOLUME_CNF))
self._snapshot_config_path = self._cnf.private_path(os.path.join('storage', STORAGE_SNAPSHOT_CNF))
self.pg_keys_dir = self._cnf.private_path('keys')
self.postgresql = PostgreSql(self.version, self.pg_keys_dir)
class PostgreSqlHander(ServiceCtlHandler):
_logger = None
_queryenv = None
""" @type _queryenv: scalarizr.queryenv.QueryEnvService """
_platform = None
""" @type _platform: scalarizr.platform.Ec2Platform """
_cnf = None
''' @type _cnf: scalarizr.config.ScalarizrCnf '''
preset_provider = None
def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None):
return BEHAVIOUR in behaviour and (
message.name == DbMsrMessages.DBMSR_NEW_MASTER_UP
or message.name == DbMsrMessages.DBMSR_PROMOTE_TO_MASTER
or message.name == DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE
or message.name == DbMsrMessages.DBMSR_CREATE_BACKUP
or message.name == Messages.UPDATE_SERVICE_CONFIGURATION
or message.name == Messages.HOST_INIT
or message.name == Messages.BEFORE_HOST_TERMINATE
or message.name == Messages.HOST_UP
or message.name == Messages.HOST_DOWN)
def get_initialization_phases(self, hir_message):
if BEHAVIOUR in hir_message.body:
steps = [self._step_accept_scalr_conf, self._step_create_storage]
if hir_message.body[BEHAVIOUR]['replication_master'] == '1':
steps += [self._step_init_master, self._step_create_data_bundle]
else:
steps += [self._step_init_slave]
steps += [self._step_collect_host_up_data]
return {'before_host_up': [{
'name': self._phase_postgresql,
'steps': steps
}]}
def __init__(self):
self._service_name = SERVICE_NAME
ServiceCtlHandler.__init__(self, SERVICE_NAME, initdv2.lookup(SERVICE_NAME))
bus.on("init", self.on_init)
bus.define_events(
'before_postgresql_data_bundle',
'postgresql_data_bundle',
# @param host: New master hostname
'before_postgresql_change_master',
# @param host: New master hostname
'postgresql_change_master',
'before_slave_promote_to_master',
'slave_promote_to_master'
)
self._phase_postgresql = 'Configure PostgreSQL'
self._phase_data_bundle = self._op_data_bundle = 'PostgreSQL data bundle'
self._phase_backup = self._op_backup = 'PostgreSQL backup'
self._step_upload_to_cloud_storage = 'Upload data to cloud storage'
self._step_accept_scalr_conf = 'Accept Scalr configuration'
self._step_patch_conf = 'Patch configuration files'
self._step_create_storage = 'Create storage'
self._step_init_master = 'Initialize Master'
self._step_init_slave = 'Initialize Slave'
self._step_create_data_bundle = 'Create data bundle'
self._step_change_replication_master = 'Change replication Master'
self._step_collect_host_up_data = 'Collect HostUp data'
self.on_reload()
def on_init(self):
#temporary fix for starting-after-rebundle issue
if not os.path.exists(PG_SOCKET_DIR):
os.makedirs(PG_SOCKET_DIR)
chown_r(PG_SOCKET_DIR, 'postgres')
bus.on("host_init_response", self.on_host_init_response)
bus.on("before_host_up", self.on_before_host_up)
bus.on("before_reboot_start", self.on_before_reboot_start)
self._insert_iptables_rules()
if __node__['state'] == ScalarizrState.BOOTSTRAPPING:
if disttool.is_redhat_based():
checkmodule_path = software.which('checkmodule')
semodule_package_path = software.which('semodule_package')
semodule_path = software.which('semodule')
if all((checkmodule_path, semodule_package_path, semodule_path)):
with open('/tmp/sshkeygen.te', 'w') as fp:
fp.write(SSH_KEYGEN_SELINUX_MODULE)
self._logger.debug('Compiling SELinux policy for ssh-keygen')
system2((checkmodule_path, '-M', '-m', '-o',
'/tmp/sshkeygen.mod', '/tmp/sshkeygen.te'), logger=self._logger)
self._logger.debug('Building SELinux package for ssh-keygen')
system2((semodule_package_path, '-o', '/tmp/sshkeygen.pp',
'-m', '/tmp/sshkeygen.mod'), logger=self._logger)
self._logger.debug('Loading ssh-keygen SELinux package')
system2((semodule_path, '-i', '/tmp/sshkeygen.pp'), logger=self._logger)
if __node__['state'] == 'running':
vol = storage2.volume(__postgresql__['volume'])
vol.ensure(mount=True)
self.postgresql.service.start()
self.accept_all_clients()
self._logger.debug("Checking presence of Scalr's PostgreSQL root user.")
root_password = self.root_password
if not self.postgresql.root_user.exists():
self._logger.debug("Scalr's PostgreSQL root user does not exist. Recreating")
self.postgresql.root_user = self.postgresql.create_linux_user(ROOT_USER, root_password)
else:
try:
self.postgresql.root_user.check_system_password(root_password)
self._logger.debug("Scalr's root PgSQL user is present. Password is correct.")
except ValueError:
self._logger.warning("Scalr's root PgSQL user was changed. Recreating.")
self.postgresql.root_user.change_system_password(root_password)
if self.is_replication_master:
#ALTER ROLE cannot be executed in a read-only transaction
self._logger.debug("Checking password for pg_role scalr.")
if not self.postgresql.root_user.check_role_password(root_password):
LOG.warning("Scalr's root PgSQL role was changed. Recreating.")
self.postgresql.root_user.change_role_password(root_password)
def on_reload(self):
self._queryenv = bus.queryenv_service
self._platform = bus.platform
self.postgresql = PostgreSql()
self.preset_provider = PgSQLPresetProvider(self.postgresql.postgresql_conf)
preset_service.services[BEHAVIOUR] = self.preset_provider
def on_HostInit(self, message):
if message.local_ip != self._platform.get_private_ip() and message.local_ip in self.pg_hosts:
LOG.debug('Got new slave IP: %s. Registering in pg_hba.conf' % message.local_ip)
self.postgresql.register_slave(message.local_ip)
def on_HostUp(self, message):
if message.local_ip == self._platform.get_private_ip():
self.accept_all_clients()
elif message.local_ip in self.farm_hosts:
self.postgresql.register_client(message.local_ip)
def on_HostDown(self, message):
if message.local_ip != self._platform.get_private_ip():
self.postgresql.unregister_client(message.local_ip)
if self.is_replication_master and self.farmrole_id == message.farm_role_id:
self.postgresql.unregister_slave(message.local_ip)
@property
def farm_hosts(self):
list_roles = self._queryenv.list_roles()
servers = []
for serv in list_roles:
for host in serv.hosts :
servers.append(host.internal_ip or host.external_ip)
LOG.debug("QueryEnv returned list of servers within farm: %s" % servers)
return servers
@property
def pg_hosts(self):
'''
All pg instances including those in Initializing state
'''
list_roles = self._queryenv.list_roles(behaviour=BEHAVIOUR, with_init=True)
servers = []
for pg_serv in list_roles:
for pg_host in pg_serv.hosts:
servers.append(pg_host.internal_ip or pg_host.external_ip)
LOG.debug("QueryEnv returned list of %s servers: %s" % (BEHAVIOUR, servers))
return servers
def accept_all_clients(self):
farm_hosts = self.farm_hosts
for ip in farm_hosts:
self.postgresql.register_client(ip, force=False)
if farm_hosts:
self.postgresql.service.reload('Granting access to all servers within farm.', force=True)
@property
def root_password(self):
return __postgresql__['%s_password' % ROOT_USER]
@property
def farmrole_id(self):
return __node__[config.OPT_FARMROLE_ID]
def store_password(self, name, password):
__postgresql__['%s_password' % name] = password
@property
def _tmp_path(self):
return os.path.join(__postgresql__['storage_dir'], 'tmp')
@property
def is_replication_master(self):
return True if int(__postgresql__[OPT_REPLICATION_MASTER]) else False
def resource_tags(self):
purpose = '%s-'%BEHAVIOUR + ('master' if self.is_replication_master else 'slave')
return build_tags(purpose, 'active')
def on_host_init_response(self, message):
"""
Check postgresql data in host init response
@type message: scalarizr.messaging.Message
@param message: HostInitResponse
"""
with bus.initialization_op as op:
with op.phase(self._phase_postgresql):
with op.step(self._step_accept_scalr_conf):
if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR:
raise HandlerError("HostInitResponse message for PostgreSQL behaviour must have 'postgresql' property and db_type 'postgresql'")
postgresql_data = message.postgresql.copy()
#Extracting service configuration preset from message
if 'preset' in postgresql_data:
self.initial_preset = postgresql_data['preset']
LOG.debug('Scalr sent current preset: %s' % self.initial_preset)
del postgresql_data['preset']
#Extracting or generating postgresql root password
postgresql_data['%s_password' % ROOT_USER] = postgresql_data.get(OPT_ROOT_PASSWORD) or cryptotool.pwgen(10)
del postgresql_data[OPT_ROOT_PASSWORD]
#Extracting replication ssh keys from message
root = PgUser(ROOT_USER, self.postgresql.pg_keys_dir)
root.store_keys(postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY], postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY])
del postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY]
del postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY]
if postgresql_data.get('volume'):
# New format
postgresql_data['compat_prior_backup_restore'] = False
postgresql_data['volume'] = storage2.volume(postgresql_data['volume'])
LOG.debug("message.pg['volume']:", postgresql_data['volume'])
if 'backup' in postgresql_data:
postgresql_data['backup'] = backup.backup(postgresql_data['backup'])
LOG.debug("message.pg['backup']:", postgresql_data['backup'])
if 'restore' in postgresql_data:
postgresql_data['restore'] = backup.restore(postgresql_data['restore'])
LOG.debug("message.pg['restore']:", postgresql_data['restore'])
else:
# Compatibility transformation
# - volume_config -> volume
# - master n'th start, type=ebs - del snapshot_config
# - snapshot_config -> restore
# - create backup object on master 1'st start
postgresql_data['compat_prior_backup_restore'] = True
if postgresql_data.get(OPT_VOLUME_CNF):
postgresql_data['volume'] = storage2.volume(
postgresql_data.pop(OPT_VOLUME_CNF))
elif postgresql_data.get(OPT_SNAPSHOT_CNF):
postgresql_data['volume'] = storage2.volume(
type=postgresql_data[OPT_SNAPSHOT_CNF]['type'])
else:
raise HandlerError('No volume config or snapshot config provided')
if postgresql_data['volume'].device and \
postgresql_data['volume'].type in ('ebs', 'csvol', 'cinder', 'raid'):
LOG.debug("Master n'th start detected. Removing snapshot config from message")
postgresql_data.pop(OPT_SNAPSHOT_CNF, None)
if postgresql_data.get(OPT_SNAPSHOT_CNF):
postgresql_data['restore'] = backup.restore(
type='snap_postgresql',
snapshot=postgresql_data.pop(OPT_SNAPSHOT_CNF),
volume=postgresql_data['volume'])
if int(postgresql_data['replication_master']):
postgresql_data['backup'] = backup.backup(
type='snap_postgresql',
volume=postgresql_data['volume'])
LOG.debug("Update postgresql config with %s", postgresql_data)
__postgresql__.update(postgresql_data)
__postgresql__['volume'].mpoint = __postgresql__['storage_dir']
__postgresql__['volume'].tags = self.resource_tags()
if 'backup' in __postgresql__:
__postgresql__['backup'].tags = self.resource_tags()
def on_before_host_up(self, message):
"""
Configure PostgreSQL behaviour
@type message: scalarizr.messaging.Message
@param message: HostUp message
"""
repl = 'master' if self.is_replication_master else 'slave'
#bus.fire('before_postgresql_configure', replication=repl)
if self.is_replication_master:
self._init_master(message)
else:
self._init_slave(message)
# Force to resave volume settings
__postgresql__['volume'] = storage2.volume(__postgresql__['volume'])
bus.fire('service_configured', service_name=SERVICE_NAME, replication=repl, preset=self.initial_preset)
def on_before_reboot_start(self, *args, **kwargs):
"""
Stop PostgreSQL and unplug storage
"""
self.postgresql.service.stop('rebooting')
def on_BeforeHostTerminate(self, message):
LOG.info('Handling BeforeHostTerminate message from %s' % message.local_ip)
if message.local_ip == self._platform.get_private_ip():
LOG.info('Stopping %s service' % BEHAVIOUR)
self.postgresql.service.stop('Server will be terminated')
if not self.is_replication_master:
LOG.info('Destroying volume %s' % __postgresql__['volume'].id)
__postgresql__['volume'].destroy(remove_disks=True)
LOG.info('Volume %s has been destroyed.' % __postgresql__['volume'].id)
else:
__postgresql__['volume'].umount()
def on_DbMsr_CreateDataBundle(self, message):
try:
op = operation(name=self._op_data_bundle, phases=[{
'name': self._phase_data_bundle,
'steps': [self._step_create_data_bundle]
}])
op.define()
with op.phase(self._phase_data_bundle):
with op.step(self._step_create_data_bundle):
bus.fire('before_postgresql_data_bundle')
snap = self._create_snapshot()
used_size = int(system2(('df', '-P', '--block-size=M', STORAGE_PATH))[0].split('\n')[1].split()[2][:-1])
bus.fire('postgresql_data_bundle', snapshot_id=snap.id)
# Notify scalr
msg_data = {
'db_type': BEHAVIOUR,
'status': 'ok',
'used_size' : '%.3f' % (float(used_size) / 1000,),
BEHAVIOUR: {OPT_SNAPSHOT_CNF: dict(snap)}
}
self.send_message(DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE_RESULT, msg_data)
op.ok()
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
self.send_message(DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE_RESULT, dict(
db_type = BEHAVIOUR,
status ='error',
last_error = str(e)
))
def on_reload(self):
self._queryenv = bus.queryenv_service
self._platform = bus.platform
self.postgresql = PostgreSql()
self.preset_provider = PgSQLPresetProvider(self.postgresql.postgresql_conf)
preset_service.services[BEHAVIOUR] = self.preset_provider
class PostgreSqlHander(ServiceCtlHandler):
_logger = None
_queryenv = None
""" @type _queryenv: scalarizr.queryenv.QueryEnvService """
_platform = None
""" @type _platform: scalarizr.platform.Ec2Platform """
_cnf = None
''' @type _cnf: scalarizr.config.ScalarizrCnf '''
preset_provider = None
def accept(self,
message,
queue,
behaviour=None,
platform=None,
os=None,
dist=None):
return BEHAVIOUR in behaviour and (
message.name == DbMsrMessages.DBMSR_NEW_MASTER_UP
or message.name == DbMsrMessages.DBMSR_PROMOTE_TO_MASTER
or message.name == DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE
or message.name == DbMsrMessages.DBMSR_CREATE_BACKUP
or message.name == Messages.UPDATE_SERVICE_CONFIGURATION
or message.name == Messages.HOST_INIT
or message.name == Messages.BEFORE_HOST_TERMINATE or message.name
== Messages.HOST_UP or message.name == Messages.HOST_DOWN)
def __init__(self):
self._service_name = SERVICE_NAME
ServiceCtlHandler.__init__(self, SERVICE_NAME,
initdv2.lookup(SERVICE_NAME))
bus.on("init", self.on_init)
bus.define_events(
'before_postgresql_data_bundle',
'postgresql_data_bundle',
# @param host: New master hostname
'before_postgresql_change_master',
# @param host: New master hostname
'postgresql_change_master',
'before_slave_promote_to_master',
'slave_promote_to_master')
self._hir_volume_growth = None
self._postgresql_api = postgresql_api.PostgreSQLAPI()
self.on_reload()
def on_init(self):
#temporary fix for starting-after-rebundle issue
if not os.path.exists(PG_SOCKET_DIR):
os.makedirs(PG_SOCKET_DIR)
chown_r(PG_SOCKET_DIR, 'postgres')
bus.on("host_init_response", self.on_host_init_response)
bus.on("before_host_up", self.on_before_host_up)
bus.on("before_reboot_start", self.on_before_reboot_start)
self._insert_iptables_rules()
if __node__['state'] == ScalarizrState.BOOTSTRAPPING:
if linux.os.redhat_family:
def selinux_enabled():
selinuxenabled = software.which('selinuxenabled')
if selinuxenabled:
_, _, ret_code = system2((selinuxenabled, ),
raise_exc=False)
return 0 == ret_code
# Consider it enabled by default
return True
checkmodule_path = software.which('checkmodule')
semodule_package_path = software.which('semodule_package')
semodule_path = software.which('semodule')
if all(
(checkmodule_path, semodule_package_path, semodule_path)):
if selinux_enabled():
with open('/tmp/sshkeygen.te', 'w') as fp:
fp.write(SSH_KEYGEN_SELINUX_MODULE)
self._logger.debug(
'Compiling SELinux policy for ssh-keygen')
system2((checkmodule_path, '-M', '-m', '-o',
'/tmp/sshkeygen.mod', '/tmp/sshkeygen.te'),
logger=self._logger)
self._logger.debug(
'Building SELinux package for ssh-keygen')
system2(
(semodule_package_path, '-o', '/tmp/sshkeygen.pp',
'-m', '/tmp/sshkeygen.mod'),
logger=self._logger)
self._logger.debug(
'Loading ssh-keygen SELinux package')
system2((semodule_path, '-i', '/tmp/sshkeygen.pp'),
logger=self._logger)
if __node__['state'] == 'running':
vol = storage2.volume(__postgresql__['volume'])
vol.ensure(mount=True)
self.postgresql.service.start()
self.accept_all_clients()
self._logger.debug(
"Checking presence of Scalr's PostgreSQL root user.")
root_password = self.root_password
if not self.postgresql.root_user.exists():
self._logger.debug(
"Scalr's PostgreSQL root user does not exist. Recreating")
self.postgresql.root_user = self.postgresql.create_linux_user(
ROOT_USER, root_password)
else:
try:
self.postgresql.root_user.check_system_password(
root_password)
self._logger.debug(
"Scalr's root PgSQL user is present. Password is correct."
)
except ValueError:
self._logger.warning(
"Scalr's root PgSQL user was changed. Recreating.")
self.postgresql.root_user.change_system_password(
root_password)
if self.is_replication_master:
#ALTER ROLE cannot be executed in a read-only transaction
self._logger.debug("Checking password for pg_role scalr.")
if not self.postgresql.root_user.check_role_password(
root_password):
LOG.warning(
"Scalr's root PgSQL role was changed. Recreating.")
self.postgresql.root_user.change_role_password(
root_password)
def on_reload(self):
self._queryenv = bus.queryenv_service
self._platform = bus.platform
self.postgresql = PostgreSql()
self.preset_provider = PgSQLPresetProvider()
def on_HostInit(self, message):
if message.local_ip != self._platform.get_private_ip(
) and message.local_ip in self.pg_hosts:
LOG.debug('Got new slave IP: %s. Registering in pg_hba.conf' %
message.local_ip)
self.postgresql.register_slave(message.local_ip)
def on_HostUp(self, message):
if message.local_ip == self._platform.get_private_ip():
self.accept_all_clients()
elif message.local_ip in self.farm_hosts:
self.postgresql.register_client(message.local_ip)
def on_HostDown(self, message):
if message.local_ip != self._platform.get_private_ip():
self.postgresql.unregister_client(message.local_ip)
if self.is_replication_master and self.farmrole_id == message.farm_role_id:
self.postgresql.unregister_slave(message.local_ip)
@property
def farm_hosts(self):
list_roles = self._queryenv.list_roles()
servers = []
for serv in list_roles:
for host in serv.hosts:
servers.append(host.internal_ip or host.external_ip)
LOG.debug("QueryEnv returned list of servers within farm: %s" %
servers)
return servers
@property
def pg_hosts(self):
'''
All pg instances including those in Initializing state
'''
list_roles = self._queryenv.list_roles(behaviour=BEHAVIOUR,
with_init=True)
servers = []
for pg_serv in list_roles:
for pg_host in pg_serv.hosts:
servers.append(pg_host.internal_ip or pg_host.external_ip)
LOG.debug("QueryEnv returned list of %s servers: %s" %
(BEHAVIOUR, servers))
return servers
def accept_all_clients(self):
farm_hosts = self.farm_hosts
for ip in farm_hosts:
self.postgresql.register_client(ip, force=False)
if farm_hosts:
self.postgresql.service.reload(
'Granting access to all servers within farm.', force=True)
@property
def root_password(self):
return __postgresql__['%s_password' % ROOT_USER]
@property
def farmrole_id(self):
return __node__[config.OPT_FARMROLE_ID]
def store_password(self, name, password):
__postgresql__['%s_password' % name] = password
@property
def _tmp_path(self):
return os.path.join(__postgresql__['storage_dir'], 'tmp')
@property
def is_replication_master(self):
return True if int(__postgresql__[OPT_REPLICATION_MASTER]) else False
def on_host_init_response(self, message):
"""
Check postgresql data in host init response
@type message: scalarizr.messaging.Message
@param message: HostInitResponse
"""
log = bus.init_op.logger
log.info('Accept Scalr configuration')
if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR:
raise HandlerError(
"HostInitResponse message for PostgreSQL behaviour must have 'postgresql' property and db_type 'postgresql'"
)
postgresql_data = message.postgresql.copy()
#Extracting service configuration preset from message
if 'preset' in postgresql_data:
self.initial_preset = postgresql_data['preset']
LOG.debug('Scalr sent current preset: %s' % self.initial_preset)
del postgresql_data['preset']
#Extracting or generating postgresql root password
postgresql_data['%s_password' % ROOT_USER] = postgresql_data.get(
OPT_ROOT_PASSWORD) or cryptotool.pwgen(10)
del postgresql_data[OPT_ROOT_PASSWORD]
#Extracting replication ssh keys from message
root = PgUser(ROOT_USER, self.postgresql.pg_keys_dir)
root.store_keys(postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY],
postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY])
del postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY]
del postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY]
if postgresql_data.get('volume'):
# New format
postgresql_data['compat_prior_backup_restore'] = False
postgresql_data['volume'] = storage2.volume(
postgresql_data['volume'])
LOG.debug("message.pg['volume']: %s", postgresql_data['volume'])
if 'backup' in postgresql_data:
postgresql_data['backup'] = backup.backup(
postgresql_data['backup'])
LOG.debug("message.pg['backup']: %s",
postgresql_data['backup'])
if 'restore' in postgresql_data:
postgresql_data['restore'] = backup.restore(
postgresql_data['restore'])
LOG.debug("message.pg['restore']: %s",
postgresql_data['restore'])
else:
# Compatibility transformation
# - volume_config -> volume
# - master n'th start, type=ebs - del snapshot_config
# - snapshot_config -> restore
# - create backup object on master 1'st start
postgresql_data['compat_prior_backup_restore'] = True
if postgresql_data.get(OPT_VOLUME_CNF):
postgresql_data['volume'] = storage2.volume(
postgresql_data.pop(OPT_VOLUME_CNF))
elif postgresql_data.get(OPT_SNAPSHOT_CNF):
postgresql_data['volume'] = storage2.volume(
type=postgresql_data[OPT_SNAPSHOT_CNF]['type'])
else:
raise HandlerError(
'No volume config or snapshot config provided')
if postgresql_data['volume'].device and \
postgresql_data['volume'].type in ('ebs', 'csvol', 'cinder', 'raid', 'gce_persistent'):
LOG.debug(
"Master n'th start detected. Removing snapshot config from message"
)
postgresql_data.pop(OPT_SNAPSHOT_CNF, None)
if postgresql_data.get(OPT_SNAPSHOT_CNF):
postgresql_data['restore'] = backup.restore(
type='snap_postgresql',
snapshot=postgresql_data.pop(OPT_SNAPSHOT_CNF),
volume=postgresql_data['volume'])
if int(postgresql_data['replication_master']):
postgresql_data['backup'] = backup.backup(
type='snap_postgresql', volume=postgresql_data['volume'])
self._hir_volume_growth = postgresql_data.pop('volume_growth', None)
LOG.debug("Update postgresql config with %s", postgresql_data)
__postgresql__.update(postgresql_data)
__postgresql__['volume'].mpoint = __postgresql__['storage_dir']
def on_before_host_up(self, message):
"""
Configure PostgreSQL behaviour
@type message: scalarizr.messaging.Message
@param message: HostUp message
"""
repl = 'master' if self.is_replication_master else 'slave'
#bus.fire('before_postgresql_configure', replication=repl)
if self.is_replication_master:
self._init_master(message)
else:
self._init_slave(message)
# Force to resave volume settings
__postgresql__['volume'] = storage2.volume(__postgresql__['volume'])
bus.fire('service_configured',
service_name=SERVICE_NAME,
replication=repl,
preset=self.initial_preset)
def on_before_reboot_start(self, *args, **kwargs):
"""
Stop PostgreSQL and unplug storage
"""
self.postgresql.service.stop('rebooting')
def on_BeforeHostTerminate(self, message):
LOG.info('Handling BeforeHostTerminate message from %s' %
message.local_ip)
if message.local_ip == self._platform.get_private_ip():
LOG.info('Stopping %s service' % BEHAVIOUR)
self.postgresql.service.stop('Server will be terminated')
if not self.is_replication_master:
LOG.info('Destroying volume %s' % __postgresql__['volume'].id)
__postgresql__['volume'].destroy(remove_disks=True)
LOG.info('Volume %s has been destroyed.' %
__postgresql__['volume'].id)
else:
__postgresql__['volume'].umount()
def on_DbMsr_CreateDataBundle(self, message):
LOG.debug("on_DbMsr_CreateDataBundle")
self._postgresql_api.create_databundle(async=True)
def on_DbMsr_PromoteToMaster(self, message):
"""
Promote slave to master
@type message: scalarizr.messaging.Message
@param message: postgresql_PromoteToMaster
"""
LOG.debug("on_DbMsr_PromoteToMaster")
postgresql = message.body[BEHAVIOUR]
if int(__postgresql__['replication_master']):
LOG.warning('Cannot promote to master. Already master')
return
LOG.info('Starting Slave -> Master promotion')
bus.fire('before_slave_promote_to_master')
msg_data = {'db_type': BEHAVIOUR, 'status': 'ok', BEHAVIOUR: {}}
tx_complete = False
new_vol = None
if postgresql.get('volume_config'):
new_vol = storage2.volume(postgresql.get('volume_config'))
try:
self.postgresql.stop_replication()
if new_vol and new_vol.type not in ('eph', 'lvm'):
self.postgresql.service.stop(
'Unplugging slave storage and then plugging master one')
old_vol = storage2.volume(__postgresql__['volume'])
old_vol.detach(force=True)
new_vol.mpoint = __postgresql__['storage_dir']
new_vol.ensure(mount=True)
if not self.postgresql.cluster_dir.is_initialized(
STORAGE_PATH):
raise HandlerError("%s is not a valid postgresql storage" %
STORAGE_PATH)
__postgresql__['volume'] = new_vol
msg_data[BEHAVIOUR] = {'volume_config': dict(new_vol)}
slaves = [host.internal_ip for host in self._get_slave_hosts()]
self.postgresql.init_master(STORAGE_PATH, self.root_password,
slaves)
self.postgresql.start_replication()
__postgresql__[OPT_REPLICATION_MASTER] = 1
if not new_vol or new_vol.type in ('eph', 'lvm'):
snap = self._create_snapshot()
__postgresql__['snapshot'] = snap
msg_data[BEHAVIOUR].update({OPT_SNAPSHOT_CNF: dict(snap)})
msg_data[
OPT_CURRENT_XLOG_LOCATION] = None # useless but required by Scalr
self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT,
msg_data)
tx_complete = True
bus.fire('slave_promote_to_master')
except (Exception, BaseException), e:
LOG.exception(e)
self.send_message(
DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT,
dict(db_type=BEHAVIOUR, status="error", last_error=str(e)))
self.postgresql.service.stop(
'Unplugging broken storage and then plugging the old one')
if new_vol:
new_vol.detach()
# Get back slave storage
if old_vol:
old_vol.ensure(mount=True)
self.postgresql.service.start()
if tx_complete and new_vol and new_vol.type not in ('eph', 'lvm'):
# Delete slave EBS
old_vol.destroy(remove_disks=True)
def on_reload(self):
self._queryenv = bus.queryenv_service
self._platform = bus.platform
self.postgresql = PostgreSql()
self.preset_provider = PgSQLPresetProvider()
class PostgreSqlHander(ServiceCtlHandler):
_logger = None
_queryenv = None
""" @type _queryenv: scalarizr.queryenv.QueryEnvService """
_platform = None
""" @type _platform: scalarizr.platform.Ec2Platform """
_cnf = None
''' @type _cnf: scalarizr.config.ScalarizrCnf '''
preset_provider = None
def accept(self,
message,
queue,
behaviour=None,
platform=None,
os=None,
dist=None):
return BEHAVIOUR in behaviour and (
message.name == DbMsrMessages.DBMSR_NEW_MASTER_UP
or message.name == DbMsrMessages.DBMSR_PROMOTE_TO_MASTER
or message.name == DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE
or message.name == DbMsrMessages.DBMSR_CREATE_BACKUP
or message.name == Messages.UPDATE_SERVICE_CONFIGURATION
or message.name == Messages.HOST_INIT
or message.name == Messages.BEFORE_HOST_TERMINATE or message.name
== Messages.HOST_UP or message.name == Messages.HOST_DOWN)
def get_initialization_phases(self, hir_message):
if BEHAVIOUR in hir_message.body:
steps = [self._step_accept_scalr_conf, self._step_create_storage]
if hir_message.body[BEHAVIOUR]['replication_master'] == '1':
steps += [
self._step_init_master, self._step_create_data_bundle
]
else:
steps += [self._step_init_slave]
steps += [self._step_collect_host_up_data]
return {
'before_host_up': [{
'name': self._phase_postgresql,
'steps': steps
}]
}
def __init__(self):
self._service_name = SERVICE_NAME
ServiceCtlHandler.__init__(self, SERVICE_NAME,
initdv2.lookup(SERVICE_NAME))
bus.on("init", self.on_init)
bus.define_events(
'before_postgresql_data_bundle',
'postgresql_data_bundle',
# @param host: New master hostname
'before_postgresql_change_master',
# @param host: New master hostname
'postgresql_change_master',
'before_slave_promote_to_master',
'slave_promote_to_master')
self._phase_postgresql = 'Configure PostgreSQL'
self._phase_data_bundle = self._op_data_bundle = 'PostgreSQL data bundle'
self._phase_backup = self._op_backup = 'PostgreSQL backup'
self._step_upload_to_cloud_storage = 'Upload data to cloud storage'
self._step_accept_scalr_conf = 'Accept Scalr configuration'
self._step_patch_conf = 'Patch configuration files'
self._step_create_storage = 'Create storage'
self._step_init_master = 'Initialize Master'
self._step_init_slave = 'Initialize Slave'
self._step_create_data_bundle = 'Create data bundle'
self._step_change_replication_master = 'Change replication Master'
self._step_collect_host_up_data = 'Collect HostUp data'
self.on_reload()
def on_init(self):
#temporary fix for starting-after-rebundle issue
if not os.path.exists(PG_SOCKET_DIR):
os.makedirs(PG_SOCKET_DIR)
chown_r(PG_SOCKET_DIR, 'postgres')
bus.on("host_init_response", self.on_host_init_response)
bus.on("before_host_up", self.on_before_host_up)
bus.on("before_reboot_start", self.on_before_reboot_start)
self._insert_iptables_rules()
if __node__['state'] == ScalarizrState.BOOTSTRAPPING:
if disttool.is_redhat_based():
checkmodule_paths = software.whereis('checkmodule')
semodule_package_paths = software.whereis('semodule_package')
semodule_paths = software.whereis('semodule')
if all((checkmodule_paths, semodule_package_paths,
semodule_paths)):
with open('/tmp/sshkeygen.te', 'w') as fp:
fp.write(SSH_KEYGEN_SELINUX_MODULE)
self._logger.debug(
'Compiling SELinux policy for ssh-keygen')
system2((checkmodule_paths[0], '-M', '-m', '-o',
'/tmp/sshkeygen.mod', '/tmp/sshkeygen.te'),
logger=self._logger)
self._logger.debug(
'Building SELinux package for ssh-keygen')
system2((semodule_package_paths[0], '-o',
'/tmp/sshkeygen.pp', '-m', '/tmp/sshkeygen.mod'),
logger=self._logger)
self._logger.debug('Loading ssh-keygen SELinux package')
system2((semodule_paths[0], '-i', '/tmp/sshkeygen.pp'),
logger=self._logger)
if __node__['state'] == 'running':
vol = storage2.volume(__postgresql__['volume'])
if not vol.tags:
vol.tags = self.resource_tags()
vol.ensure(mount=True)
self.postgresql.service.start()
self.accept_all_clients()
self._logger.debug(
"Checking presence of Scalr's PostgreSQL root user.")
root_password = self.root_password
if not self.postgresql.root_user.exists():
self._logger.debug(
"Scalr's PostgreSQL root user does not exist. Recreating")
self.postgresql.root_user = self.postgresql.create_linux_user(
ROOT_USER, root_password)
else:
try:
self.postgresql.root_user.check_system_password(
root_password)
self._logger.debug(
"Scalr's root PgSQL user is present. Password is correct."
)
except ValueError:
self._logger.warning(
"Scalr's root PgSQL user was changed. Recreating.")
self.postgresql.root_user.change_system_password(
root_password)
if self.is_replication_master:
#ALTER ROLE cannot be executed in a read-only transaction
self._logger.debug("Checking password for pg_role scalr.")
if not self.postgresql.root_user.check_role_password(
root_password):
LOG.warning(
"Scalr's root PgSQL role was changed. Recreating.")
self.postgresql.root_user.change_role_password(
root_password)
def on_reload(self):
self._queryenv = bus.queryenv_service
self._platform = bus.platform
self.postgresql = PostgreSql()
self.preset_provider = PgSQLPresetProvider(
self.postgresql.postgresql_conf)
preset_service.services[BEHAVIOUR] = self.preset_provider
def on_HostInit(self, message):
if message.local_ip != self._platform.get_private_ip(
) and message.local_ip in self.pg_hosts:
LOG.debug('Got new slave IP: %s. Registering in pg_hba.conf' %
message.local_ip)
self.postgresql.register_slave(message.local_ip)
def on_HostUp(self, message):
if message.local_ip == self._platform.get_private_ip():
self.accept_all_clients()
elif message.local_ip in self.farm_hosts:
self.postgresql.register_client(message.local_ip)
def on_HostDown(self, message):
if message.local_ip != self._platform.get_private_ip():
self.postgresql.unregister_client(message.local_ip)
if self.is_replication_master and self.farmrole_id == message.farm_role_id:
self.postgresql.unregister_slave(message.local_ip)
@property
def farm_hosts(self):
list_roles = self._queryenv.list_roles()
servers = []
for serv in list_roles:
for host in serv.hosts:
servers.append(host.internal_ip or host.external_ip)
LOG.debug("QueryEnv returned list of servers within farm: %s" %
servers)
return servers
@property
def pg_hosts(self):
'''
All pg instances including those in Initializing state
'''
list_roles = self._queryenv.list_roles(behaviour=BEHAVIOUR,
with_init=True)
servers = []
for pg_serv in list_roles:
for pg_host in pg_serv.hosts:
servers.append(pg_host.internal_ip or pg_host.external_ip)
LOG.debug("QueryEnv returned list of %s servers: %s" %
(BEHAVIOUR, servers))
return servers
def accept_all_clients(self):
farm_hosts = self.farm_hosts
for ip in farm_hosts:
self.postgresql.register_client(ip, force=False)
if farm_hosts:
self.postgresql.service.reload(
'Granting access to all servers within farm.', force=True)
@property
def root_password(self):
return __postgresql__['%s_password' % ROOT_USER]
@property
def farmrole_id(self):
return __node__[config.OPT_FARMROLE_ID]
def store_password(self, name, password):
__postgresql__['%s_password' % name] = password
@property
def _tmp_path(self):
return os.path.join(__postgresql__['storage_dir'], 'tmp')
@property
def is_replication_master(self):
return True if int(__postgresql__[OPT_REPLICATION_MASTER]) else False
def resource_tags(self):
return prepare_tags(BEHAVIOUR,
db_replication_role=self.is_replication_master)
def on_host_init_response(self, message):
"""
Check postgresql data in host init response
@type message: scalarizr.messaging.Message
@param message: HostInitResponse
"""
with bus.initialization_op as op:
with op.phase(self._phase_postgresql):
with op.step(self._step_accept_scalr_conf):
if not message.body.has_key(
BEHAVIOUR) or message.db_type != BEHAVIOUR:
raise HandlerError(
"HostInitResponse message for PostgreSQL behaviour must have 'postgresql' property and db_type 'postgresql'"
)
postgresql_data = message.postgresql.copy()
#Extracting service configuration preset from message
if 'preset' in postgresql_data:
self.initial_preset = postgresql_data['preset']
LOG.debug('Scalr sent current preset: %s' %
self.initial_preset)
del postgresql_data['preset']
#Extracting or generating postgresql root password
postgresql_data[
'%s_password' % ROOT_USER] = postgresql_data.get(
OPT_ROOT_PASSWORD) or cryptotool.pwgen(10)
del postgresql_data[OPT_ROOT_PASSWORD]
#Extracting replication ssh keys from message
root = PgUser(ROOT_USER, self.postgresql.pg_keys_dir)
root.store_keys(postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY],
postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY])
del postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY]
del postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY]
if postgresql_data.get('volume'):
# New format
postgresql_data['compat_prior_backup_restore'] = False
postgresql_data['volume'] = storage2.volume(
postgresql_data['volume'])
LOG.debug("message.pg['volume']:",
postgresql_data['volume'])
if 'backup' in postgresql_data:
postgresql_data['backup'] = backup.backup(
postgresql_data['backup'])
LOG.debug("message.pg['backup']:",
postgresql_data['backup'])
if 'restore' in postgresql_data:
postgresql_data['restore'] = backup.restore(
postgresql_data['restore'])
LOG.debug("message.pg['restore']:",
postgresql_data['restore'])
else:
# Compatibility transformation
# - volume_config -> volume
# - master n'th start, type=ebs - del snapshot_config
# - snapshot_config -> restore
# - create backup object on master 1'st start
postgresql_data['compat_prior_backup_restore'] = True
if postgresql_data.get(OPT_VOLUME_CNF):
postgresql_data['volume'] = storage2.volume(
postgresql_data.pop(OPT_VOLUME_CNF))
elif postgresql_data.get(OPT_SNAPSHOT_CNF):
postgresql_data['volume'] = storage2.volume(
type=postgresql_data[OPT_SNAPSHOT_CNF]['type'])
else:
raise HandlerError(
'No volume config or snapshot config provided')
if postgresql_data['volume'].device and \
postgresql_data['volume'].type in ('ebs', 'csvol', 'cinder', 'raid'):
LOG.debug(
"Master n'th start detected. Removing snapshot config from message"
)
postgresql_data.pop(OPT_SNAPSHOT_CNF, None)
if postgresql_data.get(OPT_SNAPSHOT_CNF):
postgresql_data['restore'] = backup.restore(
type='snap_postgresql',
snapshot=postgresql_data.pop(OPT_SNAPSHOT_CNF),
volume=postgresql_data['volume'])
if int(postgresql_data['replication_master']):
postgresql_data['backup'] = backup.backup(
type='snap_postgresql',
volume=postgresql_data['volume'])
LOG.debug("Update postgresql config with %s",
postgresql_data)
__postgresql__.update(postgresql_data)
__postgresql__['volume'].mpoint = __postgresql__[
'storage_dir']
__postgresql__['volume'].tags = self.resource_tags()
if 'backup' in __postgresql__:
__postgresql__['backup'].tags = self.resource_tags()
def on_before_host_up(self, message):
"""
Configure PostgreSQL behaviour
@type message: scalarizr.messaging.Message
@param message: HostUp message
"""
repl = 'master' if self.is_replication_master else 'slave'
#bus.fire('before_postgresql_configure', replication=repl)
if self.is_replication_master:
self._init_master(message)
else:
self._init_slave(message)
# Force to resave volume settings
__postgresql__['volume'] = storage2.volume(__postgresql__['volume'])
bus.fire('service_configured',
service_name=SERVICE_NAME,
replication=repl,
preset=self.initial_preset)
def on_before_reboot_start(self, *args, **kwargs):
"""
Stop PostgreSQL and unplug storage
"""
self.postgresql.service.stop('rebooting')
def on_BeforeHostTerminate(self, message):
LOG.info('Handling BeforeHostTerminate message from %s' %
message.local_ip)
if message.local_ip == self._platform.get_private_ip():
LOG.info('Stopping %s service' % BEHAVIOUR)
self.postgresql.service.stop('Server will be terminated')
if not self.is_replication_master:
LOG.info('Destroying volume %s' % __postgresql__['volume'].id)
__postgresql__['volume'].destroy(remove_disks=True)
LOG.info('Volume %s has been destroyed.' %
__postgresql__['volume'].id)
else:
__postgresql__['volume'].umount()
def on_DbMsr_CreateDataBundle(self, message):
try:
op = operation(name=self._op_data_bundle,
phases=[{
'name': self._phase_data_bundle,
'steps': [self._step_create_data_bundle]
}])
op.define()
with op.phase(self._phase_data_bundle):
with op.step(self._step_create_data_bundle):
bus.fire('before_postgresql_data_bundle')
snap = self._create_snapshot()
used_size = int(
system2(
('df', '-P', '--block-size=M',
STORAGE_PATH))[0].split('\n')[1].split()[2][:-1])
bus.fire('postgresql_data_bundle', snapshot_id=snap.id)
# Notify scalr
msg_data = {
'db_type': BEHAVIOUR,
'status': 'ok',
'used_size': '%.3f' % (float(used_size) / 1000, ),
BEHAVIOUR: {
OPT_SNAPSHOT_CNF: dict(snap)
}
}
self.send_message(
DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE_RESULT,
msg_data)
op.ok()
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
self.send_message(
DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE_RESULT,
dict(db_type=BEHAVIOUR, status='error', last_error=str(e)))
class PostgreSqlHander(ServiceCtlHandler):
_logger = None
_queryenv = None
""" @type _queryenv: scalarizr.queryenv.QueryEnvService """
_platform = None
""" @type _platform: scalarizr.platform.Ec2Platform """
_cnf = None
''' @type _cnf: scalarizr.config.ScalarizrCnf '''
storage_vol = None
def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None):
return BEHAVIOUR in behaviour and (
message.name == DbMsrMessages.DBMSR_NEW_MASTER_UP
or message.name == DbMsrMessages.DBMSR_PROMOTE_TO_MASTER
or message.name == DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE
or message.name == DbMsrMessages.DBMSR_CREATE_BACKUP
or message.name == Messages.UPDATE_SERVICE_CONFIGURATION
or message.name == Messages.HOST_INIT
or message.name == Messages.BEFORE_HOST_TERMINATE
or message.name == Messages.HOST_UP
or message.name == Messages.HOST_DOWN)
def get_initialization_phases(self, hir_message):
if BEHAVIOUR in hir_message.body:
steps = [self._step_accept_scalr_conf, self._step_create_storage]
if hir_message.body[BEHAVIOUR]['replication_master'] == '1':
steps += [self._step_init_master, self._step_create_data_bundle]
else:
steps += [self._step_init_slave]
steps += [self._step_collect_host_up_data]
return {'before_host_up': [{
'name': self._phase_postgresql,
'steps': steps
}]}
def __init__(self):
self._logger = logging.getLogger(__name__)
self._service_name = SERVICE_NAME
Handler.__init__(self)
bus.on("init", self.on_init)
bus.define_events(
'before_postgresql_data_bundle',
'postgresql_data_bundle',
# @param host: New master hostname
'before_postgresql_change_master',
# @param host: New master hostname
'postgresql_change_master',
'before_slave_promote_to_master',
'slave_promote_to_master'
)
self._phase_postgresql = 'Configure PostgreSQL'
self._phase_data_bundle = self._op_data_bundle = 'PostgreSQL data bundle'
self._phase_backup = self._op_backup = 'PostgreSQL backup'
self._step_upload_to_cloud_storage = 'Upload data to cloud storage'
self._step_accept_scalr_conf = 'Accept Scalr configuration'
self._step_patch_conf = 'Patch configuration files'
self._step_create_storage = 'Create storage'
self._step_init_master = 'Initialize Master'
self._step_init_slave = 'Initialize Slave'
self._step_create_data_bundle = 'Create data bundle'
self._step_change_replication_master = 'Change replication Master'
self._step_collect_host_up_data = 'Collect HostUp data'
self.on_reload()
def on_init(self):
#temporary fix for starting-after-rebundle issue
if not os.path.exists(PG_SOCKET_DIR):
os.makedirs(PG_SOCKET_DIR)
rchown(user='******', path=PG_SOCKET_DIR)
bus.on("host_init_response", self.on_host_init_response)
bus.on("before_host_up", self.on_before_host_up)
bus.on("before_reboot_start", self.on_before_reboot_start)
bus.on("before_reboot_finish", self.on_before_reboot_finish)
if self._cnf.state == ScalarizrState.BOOTSTRAPPING:
self._insert_iptables_rules()
if disttool.is_redhat_based():
checkmodule_paths = software.whereis('checkmodule')
semodule_package_paths = software.whereis('semodule_package')
semodule_paths = software.whereis('semodule')
if all((checkmodule_paths, semodule_package_paths, semodule_paths)):
filetool.write_file('/tmp/sshkeygen.te',
SSH_KEYGEN_SELINUX_MODULE, logger=self._logger)
self._logger.debug('Compiling SELinux policy for ssh-keygen')
system2((checkmodule_paths[0], '-M', '-m', '-o',
'/tmp/sshkeygen.mod', '/tmp/sshkeygen.te'), logger=self._logger)
self._logger.debug('Building SELinux package for ssh-keygen')
system2((semodule_package_paths[0], '-o', '/tmp/sshkeygen.pp',
'-m', '/tmp/sshkeygen.mod'), logger=self._logger)
self._logger.debug('Loading ssh-keygen SELinux package')
system2((semodule_paths[0], '-i', '/tmp/sshkeygen.pp'), logger=self._logger)
if self._cnf.state == ScalarizrState.RUNNING:
storage_conf = Storage.restore_config(self._volume_config_path)
storage_conf['tags'] = self.postgres_tags
self.storage_vol = Storage.create(storage_conf)
if not self.storage_vol.mounted():
self.storage_vol.mount()
self.postgresql.service.start()
self.accept_all_clients()
self._logger.debug("Checking presence of Scalr's PostgreSQL root user.")
root_password = self.root_password
if not self.postgresql.root_user.exists():
self._logger.debug("Scalr's PostgreSQL root user does not exist. Recreating")
self.postgresql.root_user = self.postgresql.create_user(ROOT_USER, root_password)
else:
try:
self.postgresql.root_user.check_system_password(root_password)
self._logger.debug("Scalr's root PgSQL user is present. Password is correct.")
except ValueError:
self._logger.warning("Scalr's root PgSQL user was changed. Recreating.")
self.postgresql.root_user.change_system_password(root_password)
if self.is_replication_master:
#ALTER ROLE cannot be executed in a read-only transaction
self._logger.debug("Checking password for pg_role scalr.")
if not self.postgresql.root_user.check_role_password(root_password):
self._logger.warning("Scalr's root PgSQL role was changed. Recreating.")
self.postgresql.root_user.change_role_password(root_password)
def on_reload(self):
self._queryenv = bus.queryenv_service
self._platform = bus.platform
self._cnf = bus.cnf
ini = self._cnf.rawini
self._role_name = ini.get(config.SECT_GENERAL, config.OPT_ROLE_NAME)
self._storage_path = STORAGE_PATH
self._tmp_path = os.path.join(self._storage_path, 'tmp')
self._volume_config_path = self._cnf.private_path(os.path.join('storage', STORAGE_VOLUME_CNF))
self._snapshot_config_path = self._cnf.private_path(os.path.join('storage', STORAGE_SNAPSHOT_CNF))
self.pg_keys_dir = self._cnf.private_path('keys')
self.postgresql = PostgreSql(self.version, self.pg_keys_dir)
@property
def version(self):
ver = None
if self._cnf.rawini.has_option(CNF_SECTION, OPT_PG_VERSION):
ver = self._cnf.rawini.get(CNF_SECTION, OPT_PG_VERSION)
if not ver:
try:
path_list = glob.glob('/var/lib/p*sql/9.*')
path_list.sort()
path = path_list[-1]
ver = os.path.basename(path)
except IndexError:
self._logger.warning('Postgresql default directory not found. Assuming that PostgreSQL 9.0 is installed.')
ver = '9.0'
finally:
self._update_config({OPT_PG_VERSION : ver})
return ver
def on_HostInit(self, message):
if message.local_ip != self._platform.get_private_ip() and message.local_ip in self.pg_hosts:
self._logger.debug('Got new slave IP: %s. Registering in pg_hba.conf' % message.local_ip)
self.postgresql.register_slave(message.local_ip)
def on_HostUp(self, message):
if message.local_ip == self._platform.get_private_ip():
self.accept_all_clients()
elif message.local_ip in self.farm_hosts:
self.postgresql.register_client(message.local_ip)
def on_HostDown(self, message):
if message.local_ip != self._platform.get_private_ip():
self.postgresql.unregister_client(message.local_ip)
if self.is_replication_master and self.farmrole_id == message.farm_role_id:
self.postgresql.unregister_slave(message.local_ip)
@property
def farm_hosts(self):
list_roles = self._queryenv.list_roles()
servers = []
for serv in list_roles:
for host in serv.hosts :
servers.append(host.internal_ip or host.external_ip)
self._logger.debug("QueryEnv returned list of servers within farm: %s" % servers)
return servers
@property
def pg_hosts(self):
'''
All pg instances including those in Initializing state
'''
list_roles = self._queryenv.list_roles(behaviour=BEHAVIOUR, with_init=True)
servers = []
for pg_serv in list_roles:
for pg_host in pg_serv.hosts:
servers.append(pg_host.internal_ip or pg_host.external_ip)
self._logger.debug("QueryEnv returned list of %s servers: %s" % (BEHAVIOUR, servers))
return servers
def accept_all_clients(self):
farm_hosts = self.farm_hosts
for ip in farm_hosts:
self.postgresql.register_client(ip, force=False)
if farm_hosts:
self.postgresql.service.reload('Granting access to all servers within farm.', force=True)
@property
def root_password(self):
password = None
opt_pwd = '%s_password' % ROOT_USER
if self._cnf.rawini.has_option(CNF_SECTION, opt_pwd):
password = self._cnf.rawini.get(CNF_SECTION, opt_pwd)
return password
@property
def farmrole_id(self):
id = None
if self._cnf.rawini.has_option(config.SECT_GENERAL, config.OPT_FARMROLE_ID):
id = self._cnf.rawini.get(config.SECT_GENERAL, config.OPT_FARMROLE_ID)
return id
def store_password(self, name, password):
opt_user_password = '******' % name
self._cnf.update_ini(BEHAVIOUR, {CNF_SECTION: {opt_user_password:password}})
@property
def is_replication_master(self):
value = self._cnf.rawini.get(CNF_SECTION, OPT_REPLICATION_MASTER)
self._logger.debug('Got %s : %s' % (OPT_REPLICATION_MASTER, value))
return True if int(value) else False
@property
def postgres_tags(self):
return prepare_tags(BEHAVIOUR, db_replication_role=self.is_replication_master)
def on_host_init_response(self, message):
"""
Check postgresql data in host init response
@type message: scalarizr.messaging.Message
@param message: HostInitResponse
"""
with bus.initialization_op as op:
with op.phase(self._phase_postgresql):
with op.step(self._step_accept_scalr_conf):
if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR:
raise HandlerError("HostInitResponse message for PostgreSQL behaviour must have 'postgresql' property and db_type 'postgresql'")
'''
if message.postgresql[OPT_REPLICATION_MASTER] != '1' and \
(not message.body.has_key(OPT_ROOT_SSH_PUBLIC_KEY) or not
message.body.has_key(OPT_ROOT_SSH_PRIVATE_KEY)):
raise HandlerError("HostInitResponse message for PostgreSQL slave must contain both public and private ssh keys")
'''
dir = os.path.dirname(self._volume_config_path)
if not os.path.exists(dir):
os.makedirs(dir)
postgresql_data = message.postgresql.copy()
root = PgUser(ROOT_USER, self.pg_keys_dir)
root.store_keys(postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY], postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY])
del postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY]
del postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY]
for key, file in ((OPT_VOLUME_CNF, self._volume_config_path),
(OPT_SNAPSHOT_CNF, self._snapshot_config_path)):
if os.path.exists(file):
os.remove(file)
if key in postgresql_data:
if postgresql_data[key]:
Storage.backup_config(postgresql_data[key], file)
del postgresql_data[key]
root_user= postgresql_data[OPT_ROOT_USER] or ROOT_USER
postgresql_data['%s_password' % root_user] = postgresql_data.get(OPT_ROOT_PASSWORD) or cryptotool.pwgen(10)
del postgresql_data[OPT_ROOT_PASSWORD]
self._logger.debug("Update postgresql config with %s", postgresql_data)
self._update_config(postgresql_data)
def on_before_host_up(self, message):
"""
Configure PostgreSQL behaviour
@type message: scalarizr.messaging.Message
@param message: HostUp message
"""
repl = 'master' if self.is_replication_master else 'slave'
#bus.fire('before_postgresql_configure', replication=repl)
if self.is_replication_master:
self._init_master(message)
else:
self._init_slave(message)
bus.fire('service_configured', service_name=SERVICE_NAME, replication=repl)
def on_before_reboot_start(self, *args, **kwargs):
"""
Stop MySQL and unplug storage
"""
self.postgresql.service.stop('rebooting')
def on_before_reboot_finish(self, *args, **kwargs):
self._insert_iptables_rules()
def on_BeforeHostTerminate(self, message):
self._logger.info('Handling BeforeHostTerminate message from %s' % message.local_ip)
if message.local_ip == self._platform.get_private_ip():
self._logger.info('Stopping %s service' % BEHAVIOUR)
self.postgresql.service.stop('Server will be terminated')
if not self.is_replication_master:
self._logger.info('Destroying volume %s' % self.storage_vol.id)
self.storage_vol.destroy(remove_disks=True)
self._logger.info('Volume %s has been destroyed.' % self.storage_vol.id)
def on_DbMsr_CreateDataBundle(self, message):
try:
op = operation(name=self._op_data_bundle, phases=[{
'name': self._phase_data_bundle,
'steps': [self._step_create_data_bundle]
}])
op.define()
with op.phase(self._phase_data_bundle):
with op.step(self._step_create_data_bundle):
bus.fire('before_postgresql_data_bundle')
# Retrieve password for scalr postgresql root user
# Creating snapshot
snap = self._create_snapshot()
used_size = int(system2(('df', '-P', '--block-size=M', self._storage_path))[0].split('\n')[1].split()[2][:-1])
bus.fire('postgresql_data_bundle', snapshot_id=snap.id)
# Notify scalr
msg_data = dict(
db_type = BEHAVIOUR,
used_size = '%.3f' % (float(used_size) / 1000,),
status = 'ok'
)
msg_data[BEHAVIOUR] = self._compat_storage_data(snap=snap)
self.send_message(DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE_RESULT, msg_data)
op.ok()
except (Exception, BaseException), e:
self._logger.exception(e)
# Notify Scalr about error
self.send_message(DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE_RESULT, dict(
db_type = BEHAVIOUR,
status ='error',
last_error = str(e)
))
class PostgreSqlHander(ServiceCtlHandler):
_logger = None
_queryenv = None
""" @type _queryenv: scalarizr.queryenv.QueryEnvService """
_platform = None
""" @type _platform: scalarizr.platform.Ec2Platform """
_cnf = None
''' @type _cnf: scalarizr.config.ScalarizrCnf '''
preset_provider = None
def accept(self, message, queue, behaviour=None, platform=None, os=None, dist=None):
return BEHAVIOUR in behaviour and (
message.name == DbMsrMessages.DBMSR_NEW_MASTER_UP
or message.name == DbMsrMessages.DBMSR_PROMOTE_TO_MASTER
or message.name == DbMsrMessages.DBMSR_CREATE_DATA_BUNDLE
or message.name == DbMsrMessages.DBMSR_CREATE_BACKUP
or message.name == Messages.UPDATE_SERVICE_CONFIGURATION
or message.name == Messages.HOST_INIT
or message.name == Messages.BEFORE_HOST_TERMINATE
or message.name == Messages.HOST_UP
or message.name == Messages.HOST_DOWN)
def __init__(self):
self._service_name = SERVICE_NAME
ServiceCtlHandler.__init__(self, SERVICE_NAME, initdv2.lookup(SERVICE_NAME))
bus.on("init", self.on_init)
bus.define_events(
'before_postgresql_data_bundle',
'postgresql_data_bundle',
# @param host: New master hostname
'before_postgresql_change_master',
# @param host: New master hostname
'postgresql_change_master',
'before_slave_promote_to_master',
'slave_promote_to_master'
)
self._hir_volume_growth = None
self._postgresql_api = postgresql_api.PostgreSQLAPI()
self.on_reload()
def on_init(self):
#temporary fix for starting-after-rebundle issue
if not os.path.exists(PG_SOCKET_DIR):
os.makedirs(PG_SOCKET_DIR)
chown_r(PG_SOCKET_DIR, 'postgres')
bus.on("host_init_response", self.on_host_init_response)
bus.on("before_host_up", self.on_before_host_up)
bus.on("before_reboot_start", self.on_before_reboot_start)
self._insert_iptables_rules()
if __node__['state'] == ScalarizrState.BOOTSTRAPPING:
if linux.os.redhat_family:
checkmodule_path = software.which('checkmodule')
semodule_package_path = software.which('semodule_package')
semodule_path = software.which('semodule')
if all((checkmodule_path, semodule_package_path, semodule_path)):
with open('/tmp/sshkeygen.te', 'w') as fp:
fp.write(SSH_KEYGEN_SELINUX_MODULE)
self._logger.debug('Compiling SELinux policy for ssh-keygen')
system2((checkmodule_path, '-M', '-m', '-o',
'/tmp/sshkeygen.mod', '/tmp/sshkeygen.te'), logger=self._logger)
self._logger.debug('Building SELinux package for ssh-keygen')
system2((semodule_package_path, '-o', '/tmp/sshkeygen.pp',
'-m', '/tmp/sshkeygen.mod'), logger=self._logger)
self._logger.debug('Loading ssh-keygen SELinux package')
system2((semodule_path, '-i', '/tmp/sshkeygen.pp'), logger=self._logger)
if __node__['state'] == 'running':
vol = storage2.volume(__postgresql__['volume'])
vol.ensure(mount=True)
self.postgresql.service.start()
self.accept_all_clients()
self._logger.debug("Checking presence of Scalr's PostgreSQL root user.")
root_password = self.root_password
if not self.postgresql.root_user.exists():
self._logger.debug("Scalr's PostgreSQL root user does not exist. Recreating")
self.postgresql.root_user = self.postgresql.create_linux_user(ROOT_USER, root_password)
else:
try:
self.postgresql.root_user.check_system_password(root_password)
self._logger.debug("Scalr's root PgSQL user is present. Password is correct.")
except ValueError:
self._logger.warning("Scalr's root PgSQL user was changed. Recreating.")
self.postgresql.root_user.change_system_password(root_password)
if self.is_replication_master:
#ALTER ROLE cannot be executed in a read-only transaction
self._logger.debug("Checking password for pg_role scalr.")
if not self.postgresql.root_user.check_role_password(root_password):
LOG.warning("Scalr's root PgSQL role was changed. Recreating.")
self.postgresql.root_user.change_role_password(root_password)
def on_reload(self):
self._queryenv = bus.queryenv_service
self._platform = bus.platform
self.postgresql = PostgreSql()
self.preset_provider = PgSQLPresetProvider()
def on_HostInit(self, message):
if message.local_ip != self._platform.get_private_ip() and message.local_ip in self.pg_hosts:
LOG.debug('Got new slave IP: %s. Registering in pg_hba.conf' % message.local_ip)
self.postgresql.register_slave(message.local_ip)
def on_HostUp(self, message):
if message.local_ip == self._platform.get_private_ip():
self.accept_all_clients()
elif message.local_ip in self.farm_hosts:
self.postgresql.register_client(message.local_ip)
def on_HostDown(self, message):
if message.local_ip != self._platform.get_private_ip():
self.postgresql.unregister_client(message.local_ip)
if self.is_replication_master and self.farmrole_id == message.farm_role_id:
self.postgresql.unregister_slave(message.local_ip)
@property
def farm_hosts(self):
list_roles = self._queryenv.list_roles()
servers = []
for serv in list_roles:
for host in serv.hosts :
servers.append(host.internal_ip or host.external_ip)
LOG.debug("QueryEnv returned list of servers within farm: %s" % servers)
return servers
@property
def pg_hosts(self):
'''
All pg instances including those in Initializing state
'''
list_roles = self._queryenv.list_roles(behaviour=BEHAVIOUR, with_init=True)
servers = []
for pg_serv in list_roles:
for pg_host in pg_serv.hosts:
servers.append(pg_host.internal_ip or pg_host.external_ip)
LOG.debug("QueryEnv returned list of %s servers: %s" % (BEHAVIOUR, servers))
return servers
def accept_all_clients(self):
farm_hosts = self.farm_hosts
for ip in farm_hosts:
self.postgresql.register_client(ip, force=False)
if farm_hosts:
self.postgresql.service.reload('Granting access to all servers within farm.', force=True)
@property
def root_password(self):
return __postgresql__['%s_password' % ROOT_USER]
@property
def farmrole_id(self):
return __node__[config.OPT_FARMROLE_ID]
def store_password(self, name, password):
__postgresql__['%s_password' % name] = password
@property
def _tmp_path(self):
return os.path.join(__postgresql__['storage_dir'], 'tmp')
@property
def is_replication_master(self):
return True if int(__postgresql__[OPT_REPLICATION_MASTER]) else False
def resource_tags(self):
purpose = '%s-' % BEHAVIOUR + ('master' if self.is_replication_master else 'slave')
return build_tags(purpose, 'active')
def on_host_init_response(self, message):
"""
Check postgresql data in host init response
@type message: scalarizr.messaging.Message
@param message: HostInitResponse
"""
log = bus.init_op.logger
log.info('Accept Scalr configuration')
if not message.body.has_key(BEHAVIOUR) or message.db_type != BEHAVIOUR:
raise HandlerError("HostInitResponse message for PostgreSQL behaviour must have 'postgresql' property and db_type 'postgresql'")
postgresql_data = message.postgresql.copy()
#Extracting service configuration preset from message
if 'preset' in postgresql_data:
self.initial_preset = postgresql_data['preset']
LOG.debug('Scalr sent current preset: %s' % self.initial_preset)
del postgresql_data['preset']
#Extracting or generating postgresql root password
postgresql_data['%s_password' % ROOT_USER] = postgresql_data.get(OPT_ROOT_PASSWORD) or cryptotool.pwgen(10)
del postgresql_data[OPT_ROOT_PASSWORD]
#Extracting replication ssh keys from message
root = PgUser(ROOT_USER, self.postgresql.pg_keys_dir)
root.store_keys(postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY], postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY])
del postgresql_data[OPT_ROOT_SSH_PUBLIC_KEY]
del postgresql_data[OPT_ROOT_SSH_PRIVATE_KEY]
if postgresql_data.get('volume'):
# New format
postgresql_data['compat_prior_backup_restore'] = False
postgresql_data['volume'] = storage2.volume(postgresql_data['volume'])
LOG.debug("message.pg['volume']: %s", postgresql_data['volume'])
if 'backup' in postgresql_data:
postgresql_data['backup'] = backup.backup(postgresql_data['backup'])
LOG.debug("message.pg['backup']: %s", postgresql_data['backup'])
if 'restore' in postgresql_data:
postgresql_data['restore'] = backup.restore(postgresql_data['restore'])
LOG.debug("message.pg['restore']: %s", postgresql_data['restore'])
else:
# Compatibility transformation
# - volume_config -> volume
# - master n'th start, type=ebs - del snapshot_config
# - snapshot_config -> restore
# - create backup object on master 1'st start
postgresql_data['compat_prior_backup_restore'] = True
if postgresql_data.get(OPT_VOLUME_CNF):
postgresql_data['volume'] = storage2.volume(
postgresql_data.pop(OPT_VOLUME_CNF))
elif postgresql_data.get(OPT_SNAPSHOT_CNF):
postgresql_data['volume'] = storage2.volume(
type=postgresql_data[OPT_SNAPSHOT_CNF]['type'])
else:
raise HandlerError('No volume config or snapshot config provided')
if postgresql_data['volume'].device and \
postgresql_data['volume'].type in ('ebs', 'csvol', 'cinder', 'raid', 'gce_persistent'):
LOG.debug("Master n'th start detected. Removing snapshot config from message")
postgresql_data.pop(OPT_SNAPSHOT_CNF, None)
if postgresql_data.get(OPT_SNAPSHOT_CNF):
postgresql_data['restore'] = backup.restore(
type='snap_postgresql',
snapshot=postgresql_data.pop(OPT_SNAPSHOT_CNF),
volume=postgresql_data['volume'])
if int(postgresql_data['replication_master']):
postgresql_data['backup'] = backup.backup(
type='snap_postgresql',
volume=postgresql_data['volume'])
self._hir_volume_growth = postgresql_data.pop('volume_growth', None)
LOG.debug("Update postgresql config with %s", postgresql_data)
__postgresql__.update(postgresql_data)
__postgresql__['volume'].mpoint = __postgresql__['storage_dir']
__postgresql__['volume'].tags = self.resource_tags()
if 'backup' in __postgresql__:
__postgresql__['backup'].tags = self.resource_tags()
def on_before_host_up(self, message):
"""
Configure PostgreSQL behaviour
@type message: scalarizr.messaging.Message
@param message: HostUp message
"""
repl = 'master' if self.is_replication_master else 'slave'
#bus.fire('before_postgresql_configure', replication=repl)
if self.is_replication_master:
self._init_master(message)
else:
self._init_slave(message)
# Force to resave volume settings
__postgresql__['volume'] = storage2.volume(__postgresql__['volume'])
bus.fire('service_configured', service_name=SERVICE_NAME, replication=repl, preset=self.initial_preset)
def on_before_reboot_start(self, *args, **kwargs):
"""
Stop PostgreSQL and unplug storage
"""
self.postgresql.service.stop('rebooting')
def on_BeforeHostTerminate(self, message):
LOG.info('Handling BeforeHostTerminate message from %s' % message.local_ip)
if message.local_ip == self._platform.get_private_ip():
LOG.info('Stopping %s service' % BEHAVIOUR)
self.postgresql.service.stop('Server will be terminated')
if not self.is_replication_master:
LOG.info('Destroying volume %s' % __postgresql__['volume'].id)
__postgresql__['volume'].destroy(remove_disks=True)
LOG.info('Volume %s has been destroyed.' % __postgresql__['volume'].id)
else:
__postgresql__['volume'].umount()
def on_DbMsr_CreateDataBundle(self, message):
LOG.debug("on_DbMsr_CreateDataBundle")
self._postgresql_api.create_databundle(async=True)
def on_DbMsr_PromoteToMaster(self, message):
"""
Promote slave to master
@type message: scalarizr.messaging.Message
@param message: postgresql_PromoteToMaster
"""
LOG.debug("on_DbMsr_PromoteToMaster")
postgresql = message.body[BEHAVIOUR]
if int(__postgresql__['replication_master']):
LOG.warning('Cannot promote to master. Already master')
return
LOG.info('Starting Slave -> Master promotion')
bus.fire('before_slave_promote_to_master')
msg_data = {
'db_type' : BEHAVIOUR,
'status' : 'ok',
BEHAVIOUR : {}
}
tx_complete = False
new_vol = None
if postgresql.get('volume_config'):
new_vol = storage2.volume(postgresql.get('volume_config'))
try:
self.postgresql.stop_replication()
if new_vol and new_vol.type not in ('eph', 'lvm'):
self.postgresql.service.stop('Unplugging slave storage and then plugging master one')
old_vol = storage2.volume(__postgresql__['volume'])
old_vol.detach(force=True)
new_vol.mpoint = __postgresql__['storage_dir']
new_vol.ensure(mount=True)
if not self.postgresql.cluster_dir.is_initialized(STORAGE_PATH):
raise HandlerError("%s is not a valid postgresql storage" % STORAGE_PATH)
__postgresql__['volume'] = new_vol
msg_data[BEHAVIOUR] = {'volume_config': dict(new_vol)}
slaves = [host.internal_ip for host in self._get_slave_hosts()]
self.postgresql.init_master(STORAGE_PATH, self.root_password, slaves)
self.postgresql.start_replication()
__postgresql__[OPT_REPLICATION_MASTER] = 1
if not new_vol or new_vol.type in ('eph', 'lvm'):
snap = self._create_snapshot()
__postgresql__['snapshot'] = snap
msg_data[BEHAVIOUR].update({OPT_SNAPSHOT_CNF : dict(snap)})
msg_data[OPT_CURRENT_XLOG_LOCATION] = None # useless but required by Scalr
self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT, msg_data)
tx_complete = True
bus.fire('slave_promote_to_master')
except (Exception, BaseException), e:
LOG.exception(e)
self.send_message(DbMsrMessages.DBMSR_PROMOTE_TO_MASTER_RESULT, dict(
db_type=BEHAVIOUR,
status="error",
last_error=str(e)
))
self.postgresql.service.stop('Unplugging broken storage and then plugging the old one')
if new_vol:
new_vol.detach()
# Get back slave storage
if old_vol:
old_vol.ensure(mount=True)
self.postgresql.service.start()
if tx_complete and new_vol and new_vol.type not in ('eph', 'lvm'):
# Delete slave EBS
old_vol.destroy(remove_disks=True)
def on_reload(self):
self._queryenv = bus.queryenv_service
self._platform = bus.platform
self.postgresql = PostgreSql()
self.preset_provider = PgSQLPresetProvider()
