def move_to(self, dst):
if not os.path.exists(dst):
LOG.debug("creating %s" % dst)
os.makedirs(dst)
for config in ['postgresql.conf', 'pg_ident.conf', 'pg_hba.conf']:
old_config = os.path.join(self.path, config)
new_config = os.path.join(dst, config)
if os.path.exists(old_config):
LOG.debug('Moving %s' % config)
shutil.move(old_config, new_config)
elif os.path.exists(new_config):
LOG.debug('%s is already in place. Skipping.' % config)
else:
raise BaseException('Postgresql config file not found: %s' %
old_config)
chown_r(new_config, DEFAULT_USER)
#the following block needs revision
#self._make_symlinks(dst)
self._patch_sysconfig(dst)
self.path = dst
LOG.debug("configuring pid")
conf = PostgresqlConf.find(self)
conf.pid_file = os.path.join(dst, 'postmaster.pid')
def start(cls):
if not cls.is_running():
cls._logger.info("Starting %s process" % MONGOS)
args = [
"sudo",
"-u",
DEFAULT_USER,
MONGOS,
"--fork",
"--logpath",
ROUTER_LOG_PATH,
"--configdb",
"mongo-0-0:%s" % CONFIG_SERVER_DEFAULT_PORT,
]
if cls.keyfile and os.path.exists(cls.keyfile):
chown_r(cls.keyfile, DEFAULT_USER)
args.append("--keyFile=%s" % cls.keyfile)
if cls.verbose and isinstance(cls.verbose, int) and 0 < cls.verbose < 6:
args.append("-" + "v" * cls.verbose)
if os.path.exists(ROUTER_LOG_PATH):
chown_r(ROUTER_LOG_PATH, DEFAULT_USER)
system2(args, close_fds=True, preexec_fn=mongo_preexec_fn)
wait_until(lambda: cls.is_running, timeout=MAX_START_TIMEOUT)
wait_until(lambda: cls.get_cli().has_connection, timeout=MAX_START_TIMEOUT)
cls._logger.debug("%s process has been started." % MONGOS)
def get_pidfile(port=__redis__['defaults']['port']):
pid_file = os.path.join(__redis__['pid_dir'], 'redis-server.%s.pid' % port)
if not os.path.exists(pid_file): # fix for ubuntu1004
open(pid_file, 'w').close()
chown_r(pid_file, 'redis')
return pid_file
def move_to(self, dst, move_files=True):
new_cluster_dir = os.path.join(dst, STORAGE_DATA_DIR)
if not os.path.exists(dst):
LOG.debug('Creating directory structure for postgresql cluster: %s' % dst)
os.makedirs(dst)
if move_files:
source = self.path
if not os.path.exists(self.path):
source = self.default_path
LOG.debug('data_directory in postgresql.conf points to non-existing location, using %s instead' % source)
if source != new_cluster_dir:
LOG.debug("copying cluster files from %s into %s" % (source, new_cluster_dir))
shutil.copytree(source, new_cluster_dir)
LOG.debug("changing directory owner to %s" % self.user)
chown_r(dst, self.user)
LOG.debug("Changing postgres user`s home directory")
if disttool.is_redhat_based():
#looks like ubuntu doesn`t need this
system2([USERMOD, '-d', new_cluster_dir, self.user])
self.path = new_cluster_dir
return new_cluster_dir
def move_to(self, dst, move_files=True):
new_cluster_dir = os.path.join(dst, STORAGE_DATA_DIR)
if not os.path.exists(dst):
LOG.debug(
'Creating directory structure for postgresql cluster: %s' %
dst)
os.makedirs(dst)
if move_files:
source = self.path
if not os.path.exists(self.path):
source = self.default_path
LOG.debug(
'data_directory in postgresql.conf points to non-existing location, using %s instead'
% source)
if source != new_cluster_dir:
LOG.debug("copying cluster files from %s into %s" %
(source, new_cluster_dir))
shutil.copytree(source, new_cluster_dir)
LOG.debug("changing directory owner to %s" % self.user)
chown_r(dst, self.user)
LOG.debug("Changing postgres user`s home directory")
if linux.os.redhat_family:
#looks like ubuntu doesn`t need this
system2([USERMOD, '-d', new_cluster_dir, self.user])
self.path = new_cluster_dir
return new_cluster_dir
def move_mysqldir_to(self, storage_path):
LOG.info('Moving mysql dir to %s' % storage_path)
for directive, dirname in (
('mysqld/log_bin', os.path.join(storage_path,STORAGE_BINLOG)),
('mysqld/datadir', os.path.join(storage_path,STORAGE_DATA_DIR) + '/')
):
dest = os.path.dirname(dirname)
if os.path.isdir(dest):
LOG.info('No need to move %s to %s: already in place.' % (directive, dest))
else:
os.makedirs(dest)
raw_value = self.my_cnf.get(directive)
LOG.debug('directive %s:%s' % (directive, raw_value))
if raw_value:
src_dir = os.path.dirname(raw_value + "/") + "/"
LOG.debug('source path: %s' % src_dir)
if os.path.isdir(src_dir) and src_dir != dest:
selinuxenabled = software.which('selinuxenabled')
if selinuxenabled:
if not system2((selinuxenabled, ), raise_exc=False)[2]:
if not system2((software.which('getsebool'), 'mysqld_disable_trans'), raise_exc=False)[2]:
LOG.debug('Make SELinux rule for rsync')
system2((software.which('setsebool'), '-P', 'mysqld_disable_trans', '1'))
LOG.info('Copying mysql directory \'%s\' to \'%s\'', src_dir, dest)
rsync(src_dir, dest, archive=True, exclude=['ib_logfile*', '*.sock'])
self.my_cnf.set(directive, dirname)
chown_r(dest, "mysql", "mysql")
# Adding rules to apparmor config
if disttool.is_debian_based():
_add_apparmor_rules(dest)
def move_to(self, dst):
if not os.path.exists(dst):
LOG.debug("creating %s" % dst)
os.makedirs(dst)
for config in ['postgresql.conf', 'pg_ident.conf', 'pg_hba.conf']:
old_config = os.path.join(self.path, config)
new_config = os.path.join(dst, config)
if os.path.exists(old_config):
LOG.debug('Moving %s' % config)
shutil.move(old_config, new_config)
elif os.path.exists(new_config):
LOG.debug('%s is already in place. Skipping.' % config)
else:
raise BaseException('Postgresql config file not found: %s' % old_config)
chown_r(new_config, DEFAULT_USER)
#the following block needs revision
#self._make_symlinks(dst)
self._patch_sysconfig(dst)
self.path = dst
LOG.debug("configuring pid")
conf = PostgresqlConf.find(self)
conf.pid_file = os.path.join(dst, 'postmaster.pid')
def init_service(self, mpoint):
if not os.path.exists(mpoint):
os.makedirs(mpoint)
LOG.debug('Created directory structure for redis db files: %s' %
mpoint)
chown_r(mpoint, __redis__['defaults']['user'])
self.redis_conf.requirepass = self.password
self.redis_conf.daemonize = True
self.redis_conf.dir = mpoint
self.redis_conf.bind = None
self.redis_conf.port = self.port
self.redis_conf.pidfile = get_pidfile(self.port)
persistence_type = __redis__["persistence_type"]
if persistence_type == 'snapshotting':
self.redis_conf.appendonly = False
self.redis_conf.dbfilename = get_snap_db_filename(self.port)
self.redis_conf.appendfilename = None
elif persistence_type == 'aof':
aof_path = get_aof_db_filename(self.port)
self.redis_conf.appendonly = True
self.redis_conf.appendfilename = aof_path
self.redis_conf.dbfilename = None
self.redis_conf.save = {}
elif persistence_type == 'nopersistence':
self.redis_conf.dbfilename = get_snap_db_filename(self.port)
self.redis_conf.appendonly = False
self.redis_conf.appendfsync = 'no'
self.redis_conf.save = {}
assert not self.redis_conf.save
LOG.debug('Persistence type is set to %s' % persistence_type)
def init_service(self, mpoint):
if not os.path.exists(mpoint):
os.makedirs(mpoint)
LOG.debug('Created directory structure for redis db files: %s' % mpoint)
chown_r(mpoint, __redis__['defaults']['user'])
self.redis_conf.requirepass = self.password
self.redis_conf.daemonize = True
self.redis_conf.dir = mpoint
self.redis_conf.bind = None
self.redis_conf.port = self.port
self.redis_conf.pidfile = get_pidfile(self.port)
persistence_type = __redis__["persistence_type"]
if persistence_type == 'snapshotting':
self.redis_conf.appendonly = False
self.redis_conf.dbfilename = get_snap_db_filename(self.port)
self.redis_conf.appendfilename = None
elif persistence_type == 'aof':
aof_path = get_aof_db_filename(self.port)
self.redis_conf.appendonly = True
self.redis_conf.appendfilename = aof_path
self.redis_conf.dbfilename = None
self.redis_conf.save = {}
elif persistence_type == 'nopersistence':
self.redis_conf.dbfilename = get_snap_db_filename(self.port)
self.redis_conf.appendonly = False
self.redis_conf.appendfsync = 'no'
self.redis_conf.save = {}
assert not self.redis_conf.save
LOG.debug('Persistence type is set to %s' % persistence_type)
def do_backup(op):
tmpdir = None
dumps = []
tmp_path = os.path.join(__postgresql__['storage_dir'], 'tmp')
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (PG_DUMP, db_name, dump_path)
su_args = [SU_EXEC, '-', self.postgresql.root_user.name, '-c', pg_args]
err = system2(su_args)[1]
if err:
raise HandlerError('Error while dumping database %s: %s' % (db_name, err)) #?
dumps.append(dump_path)
for db_name in databases:
_single_backup(db_name)
cloud_storage_path = __node__.platform.scalrfs.backups(BEHAVIOUR)
suffix = 'master' if int(__postgresql__[OPT_REPLICATION_MASTER]) else 'slave'
backup_tags = {'scalr-purpose': 'postgresql-%s' % suffix}
LOG.info("Uploading backup to %s with tags %s" % (cloud_storage_path, backup_tags))
trn = LargeTransfer(dumps, cloud_storage_path, tags=backup_tags)
manifest = trn.run()
LOG.info("Postgresql backup uploaded to cloud storage under %s", cloud_storage_path)
# Notify Scalr
result = transfer_result_to_backup_result(manifest)
__node__.messaging.send(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR,
status='ok',
backup_parts=result))
return result
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
__node__.messaging.send(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR,
status='error',
last_error=str(e)))
def rollback_hook(self):
try:
pwd.getpwnam("apache")
uname = "apache"
except:
uname = "www-data"
for obj in self.config_data:
coreutils.chown_r(obj.path, uname)
def do_backup(self):
tmpdir = None
dumps = []
tmp_path = os.path.join(__postgresql__['storage_dir'], 'tmp')
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (PG_DUMP, db_name, dump_path)
su_args = [SU_EXEC, '-', self.postgresql.root_user.name, '-c', pg_args]
err = system2(su_args)[1]
if err:
raise HandlerError('Error while dumping database %s: %s' % (db_name, err)) # ?
dumps.append(dump_path)
for db_name in databases:
_single_backup(db_name)
cloud_storage_path = __node__.platform.scalrfs.backups(BEHAVIOUR)
suffix = 'master' if int(__postgresql__[OPT_REPLICATION_MASTER]) else 'slave'
backup_tags = {'scalr-purpose': 'postgresql-%s' % suffix}
LOG.info("Uploading backup to %s with tags %s" % (cloud_storage_path, backup_tags))
def progress_cb(progress):
LOG.debug('Uploading %s bytes' % progress)
uploader = largetransfer.Upload(dumps, cloud_storage_path, progress_cb=progress_cb)
try:
uploader.apply_async()
uploader.join()
manifest = uploader.manifest
LOG.info("Postgresql backup uploaded to cloud storage under %s", cloud_storage_path)
LOG.debug(manifest.data)
return transfer_result_to_backup_result(manifest)
except:
uploader.terminate()
raise
finally:
if tmpdir:
shutil.rmtree(tmpdir, ignore_errors=True)
def create_redis_conf_copy(port=__redis__['defaults']['port']):
if not os.path.exists(__redis__['defaults']['redis.conf']):
raise ServiceError('Default redis config %s does not exist' % __redis__['defaults']['redis.conf'])
dst = get_redis_conf_path(port)
if not os.path.exists(dst):
LOG.debug('Copying %s to %s.' % (__redis__['defaults']['redis.conf'],dst))
shutil.copy(__redis__['defaults']['redis.conf'], dst)
chown_r(dst, 'redis')
else:
LOG.debug('%s already exists.' % dst)
def create_redis_conf_copy(port=__redis__['defaults']['port']):
if not os.path.exists(__redis__['defaults']['redis.conf']):
raise ServiceError('Default redis config %s does not exist' % __redis__['defaults']['redis.conf'])
dst = get_redis_conf_path(port)
if not os.path.exists(dst):
LOG.debug('Copying %s to %s.' % (__redis__['defaults']['redis.conf'],dst))
shutil.copy(__redis__['defaults']['redis.conf'], dst)
chown_r(dst, 'redis')
else:
LOG.debug('%s already exists.' % dst)
def get_pidfile(port=__redis__['defaults']['port']):
pid_file = os.path.join(__redis__['pid_dir'], 'redis-server.%s.pid' % port)
'''
fix for ubuntu1004
'''
if not os.path.exists(pid_file):
open(pid_file, 'w').close()
chown_r(pid_file, 'redis')
return pid_file
def get_pidfile(port=DEFAULT_PORT):
pid_file = os.path.join(DEFAULT_PID_DIR,'redis-server.%s.pid' % port)
'''
fix for ubuntu1004
'''
if not os.path.exists(pid_file):
open(pid_file, 'w').close()
chown_r(pid_file, 'redis')
return pid_file
def apply_private_ssh_key(self,source_path=None):
source_path = source_path or self.private_key_path
if not os.path.exists(source_path):
LOG.error('Cannot apply private ssh key: source %s not found' % source_path)
else:
if not os.path.exists(self.ssh_dir):
os.makedirs(self.ssh_dir)
chown_r(self.ssh_dir, self.name)
dst = os.path.join(self.ssh_dir, 'id_rsa')
shutil.copyfile(source_path, dst)
os.chmod(dst, 0400)
chown_r(dst, self.name)
def _prepare_arbiter(self, rs_name):
if os.path.isdir(ARBITER_DATA_DIR):
shutil.rmtree(ARBITER_DATA_DIR)
self._logger.debug("Creating datadir for arbiter: %s" % ARBITER_DATA_DIR)
os.makedirs(ARBITER_DATA_DIR)
chown_r(ARBITER_DATA_DIR, DEFAULT_USER)
self._logger.debug("Preparing arbiter's config file")
self.arbiter_conf.dbpath = ARBITER_DATA_DIR
self.arbiter_conf.replSet = rs_name
self.arbiter_conf.shardsvr = True
self.arbiter_conf.port = ARBITER_DEFAULT_PORT
self.arbiter_conf.logpath = ARBITER_LOG_PATH
self.arbiter_conf.nojournal = True
def _prepare_arbiter(self, rs_name):
if os.path.isdir(ARBITER_DATA_DIR):
shutil.rmtree(ARBITER_DATA_DIR)
self._logger.debug('Creating datadir for arbiter: %s' %
ARBITER_DATA_DIR)
os.makedirs(ARBITER_DATA_DIR)
chown_r(ARBITER_DATA_DIR, DEFAULT_USER)
self._logger.debug("Preparing arbiter's config file")
self.arbiter_conf.dbpath = ARBITER_DATA_DIR
self.arbiter_conf.replSet = rs_name
self.arbiter_conf.shardsvr = True
self.arbiter_conf.port = ARBITER_DEFAULT_PORT
self.arbiter_conf.logpath = ARBITER_LOG_PATH
self.arbiter_conf.nojournal = True
def move_mysqldir_to(self, storage_path):
LOG.info('Moving mysql dir to %s' % storage_path)
for directive, dirname in (
('mysqld/log_bin', os.path.join(storage_path, STORAGE_BINLOG)),
('mysqld/datadir',
os.path.join(storage_path, STORAGE_DATA_DIR) + '/')):
dest = os.path.dirname(dirname)
if os.path.isdir(dest):
LOG.info('No need to move %s to %s: already in place.' %
(directive, dest))
else:
os.makedirs(dest)
raw_value = self.my_cnf.get(directive)
LOG.debug('directive %s:%s' % (directive, raw_value))
if raw_value and node.__node__['platform'] != 'openstack':
src_dir = os.path.dirname(raw_value + "/") + "/"
LOG.debug('source path: %s' % src_dir)
if os.path.isdir(src_dir) and src_dir != dest:
selinuxenabled = software.which('selinuxenabled')
if selinuxenabled:
if not system2(
(selinuxenabled, ), raise_exc=False)[2]:
if not system2((software.which('getsebool'),
'mysqld_disable_trans'),
raise_exc=False)[2]:
LOG.debug('Make SELinux rule for rsync')
system2((software.which('setsebool'), '-P',
'mysqld_disable_trans', '1'))
else:
semanage = get_semanage()
system2((semanage, 'fcontext', '-a', '-t',
'bin_t', '/usr/bin/rsync'))
system2((software.which('restorecon'),
'-v', '/usr/bin/rsync'))
LOG.info('Copying mysql directory \'%s\' to \'%s\'',
src_dir, dest)
rsync(src_dir,
dest,
archive=True,
exclude=['ib_logfile*', '*.sock'])
self.my_cnf.set(directive, dirname)
chown_r(dest, "mysql", "mysql")
# Adding rules to apparmor config
if disttool.is_debian_based():
_add_apparmor_rules(dest)
def args(self):
s = ["--fork"]
if self.configpath:
s.append("--config=%s" % self.configpath)
if self.dbpath:
s.append("--dbpath=%s" % self.dbpath)
if self.port:
s.append("--port=%s" % self.port)
if self.keyfile and os.path.exists(self.keyfile):
chown_r(self.keyfile, DEFAULT_USER)
s.append("--keyFile=%s" % self.keyfile)
if self.verbose and isinstance(self.verbose, int) and 0 < self.verbose < 6:
s.append("-" + "v" * self.verbose)
return s
def move_to(self, dst, move_files=True):
new_db_path = os.path.join(dst, os.path.basename(self.db_path))
if not os.path.exists(dst):
LOG.debug('Creating directory structure for redis db files: %s' % dst)
os.makedirs(dst)
if move_files and os.path.exists(os.path.dirname(self.db_path)) and os.path.isfile(self.db_path):
LOG.debug("copying db file %s into %s" % (os.path.dirname(self.db_path), dst))
shutil.copyfile(self.db_path, new_db_path)
LOG.debug("changing directory owner to %s" % self.user)
chown_r(dst, self.user)
self.db_path = new_db_path
return new_db_path
def make_symlinks(source_dir, dst_dir, username='******'):
#Vital hack for getting CentOS init script to work
for obj in ['base', 'PG_VERSION', 'postmaster.pid']:
src = os.path.join(source_dir, obj)
dst = os.path.join(dst_dir, obj)
if os.path.islink(dst):
os.unlink(dst)
elif os.path.exists(dst):
shutil.rmtree(dst)
os.symlink(src, dst)
if os.path.exists(src):
chown_r(dst, username)
def make_symlinks(source_dir, dst_dir, username='******'):
#Vital hack to get init script to work on CentOS 5x/6x
for obj in ['base', 'PG_VERSION', 'postmaster.pid']:
src = os.path.join(source_dir, obj)
dst = os.path.join(dst_dir, obj)
if os.path.islink(dst):
os.unlink(dst)
elif os.path.exists(dst):
shutil.rmtree(dst)
os.symlink(src, dst)
if os.path.exists(src):
chown_r(dst, username)
def args(self):
s = ['--fork']
if self.configpath:
s.append('--config=%s' % self.configpath)
if self.dbpath:
s.append('--dbpath=%s' % self.dbpath)
if self.port:
s.append('--port=%s' % self.port)
if self.keyfile and os.path.exists(self.keyfile):
chown_r(self.keyfile, DEFAULT_USER)
s.append('--keyFile=%s' % self.keyfile)
if self.verbose and isinstance(self.verbose,
int) and 0 < self.verbose < 6:
s.append('-' + 'v' * self.verbose)
return s
def _prepare_config_server(self):
self._logger.debug('Preparing config server')
if not os.path.exists(CONFIG_SERVER_DATA_DIR):
os.makedirs(CONFIG_SERVER_DATA_DIR)
chown_r(CONFIG_SERVER_DATA_DIR, DEFAULT_USER)
'''
configsvr changes the default port and turns on the diaglog,
a log that keeps every action the config database performs
in a replayable format, just in case.
For mongo 1.8+ use --port 27019 and --journal (instead of --diaglog).
Journaling gives mostly the same effect as the diaglog with better performance.
P.S. Assume that mongodb roles Scalr will be build on x64 platform only
Wchich means journal option by default will be on.
'''
self.config_server_conf.configsvr = True
self.config_server_conf.port = CONFIG_SERVER_DEFAULT_PORT
self.config_server_conf.logpath = CONFIG_SERVER_LOG_PATH
def _prepare_config_server(self):
self._logger.debug("Preparing config server")
if not os.path.exists(CONFIG_SERVER_DATA_DIR):
os.makedirs(CONFIG_SERVER_DATA_DIR)
chown_r(CONFIG_SERVER_DATA_DIR, DEFAULT_USER)
"""
configsvr changes the default port and turns on the diaglog,
a log that keeps every action the config database performs
in a replayable format, just in case.
For mongo 1.8+ use --port 27019 and --journal (instead of --diaglog).
Journaling gives mostly the same effect as the diaglog with better performance.
P.S. Assume that mongodb roles Scalr will be build on x64 platform only
Wchich means journal option by default will be on.
"""
self.config_server_conf.configsvr = True
self.config_server_conf.port = CONFIG_SERVER_DEFAULT_PORT
self.config_server_conf.logpath = CONFIG_SERVER_LOG_PATH
def move_to(self, dst):
datadir = os.path.join(__postgresql__['storage_dir'], STORAGE_DATA_DIR)
centos7 = "centos" in linux.os['name'].lower(
) and linux.os["release"].version[0] == 7
if not os.path.exists(dst):
LOG.debug("creating %s" % dst)
os.makedirs(dst)
for config in ['postgresql.conf', 'pg_ident.conf', 'pg_hba.conf']:
old_config = os.path.join(self.path, config)
new_config = os.path.join(dst, config)
if os.path.exists(
old_config) and not os.path.islink(old_config): #???
LOG.debug('Moving %s' % config)
shutil.move(old_config, new_config)
elif os.path.exists(new_config):
LOG.debug('%s is already in place. Skipping.' % config)
else:
raise BaseException('Postgresql config file not found: %s' %
old_config)
chown_r(new_config, DEFAULT_USER)
if centos7:
new_link = os.path.join(datadir, config)
if os.path.exists(new_link) and os.path.isfile(new_link):
os.remove(new_link)
LOG.debug("Duplicate config %s removed." % new_link)
os.symlink(new_config, new_link)
chown_r(new_link, DEFAULT_USER)
#the following block needs revision
if centos7:
self._systemd_change_pgdata(datadir)
system2([software.which("systemctl"),
"daemon-reload"]) # [SCALARIZR-1627]
else:
self._patch_sysconfig(dst)
self.path = dst
LOG.debug("configuring pid")
conf = PostgresqlConf.find(self)
if not centos7:
conf.pid_file = os.path.join(dst,
'postmaster.pid') # [SCALARIZR-1685]
def apply_public_ssh_key(self, source_path=None):
source_path = source_path or self.public_key_path
if not os.path.exists(self.ssh_dir):
os.makedirs(self.ssh_dir)
chown_r(self.ssh_dir, self.name)
pub_key = ''
with open(source_path, 'r') as fp:
pub_key = fp.read()
path = os.path.join(self.ssh_dir, 'authorized_keys')
keys = ''
if os.path.exists(path):
with open(path, 'r') as fp:
keys = fp.read()
if not keys or not pub_key in keys:
with open(path, 'a') as fp:
fp.write('\n%s %s\n' % (pub_key, self.name))
chown_r(path, self.name)
def apply_public_ssh_key(self, source_path=None):
source_path = source_path or self.public_key_path
if not os.path.exists(self.ssh_dir):
os.makedirs(self.ssh_dir)
chown_r(self.ssh_dir, self.name)
pub_key = ''
with open(source_path, 'r') as fp:
pub_key = fp.read()
path = os.path.join(self.ssh_dir, 'authorized_keys')
keys = ''
if os.path.exists(path):
with open(path, 'r') as fp:
keys = fp.read()
if not keys or not pub_key in keys:
with open(path, 'a') as fp:
fp.write('\n%s %s\n' % (pub_key, self.name))
chown_r(path, self.name)
def start(cls):
if not cls.is_running():
cls._logger.info('Starting %s process' % MONGOS)
args = ['sudo', '-u', DEFAULT_USER, MONGOS, '--fork',
'--logpath', ROUTER_LOG_PATH, '--configdb',
'mongo-0-0:%s' % CONFIG_SERVER_DEFAULT_PORT]
if cls.keyfile and os.path.exists(cls.keyfile):
chown_r(cls.keyfile, DEFAULT_USER)
args.append('--keyFile=%s' % cls.keyfile)
if cls.verbose and isinstance(cls.verbose, int) and 0<cls.verbose<6:
args.append('-'+'v'*cls.verbose)
if os.path.exists(ROUTER_LOG_PATH):
chown_r(ROUTER_LOG_PATH, DEFAULT_USER)
system2(args, close_fds=True, preexec_fn=mongo_preexec_fn)
wait_until(lambda: cls.is_running, timeout=MAX_START_TIMEOUT)
wait_until(lambda: cls.get_cli().has_connection, timeout=MAX_START_TIMEOUT)
cls._logger.debug('%s process has been started.' % MONGOS)
def start(cls):
if not cls.is_running():
cls._logger.info('Starting %s process' % MONGOS)
args = [
'sudo', '-u', DEFAULT_USER, MONGOS, '--fork', '--logpath',
ROUTER_LOG_PATH, '--configdb',
'mongo-0-0:%s' % CONFIG_SERVER_DEFAULT_PORT
]
if cls.keyfile and os.path.exists(cls.keyfile):
chown_r(cls.keyfile, DEFAULT_USER)
args.append('--keyFile=%s' % cls.keyfile)
if cls.verbose and isinstance(cls.verbose,
int) and 0 < cls.verbose < 6:
args.append('-' + 'v' * cls.verbose)
if os.path.exists(ROUTER_LOG_PATH):
chown_r(ROUTER_LOG_PATH, DEFAULT_USER)
system2(args, close_fds=True, preexec_fn=mongo_preexec_fn)
wait_until(lambda: cls.is_running, timeout=MAX_START_TIMEOUT)
wait_until(lambda: cls.get_cli().has_connection,
timeout=MAX_START_TIMEOUT)
cls._logger.debug('%s process has been started.' % MONGOS)
def _run(self):
# Apply resource's meta
mnf = cloudfs.Manifest(cloudfs_path=self.cloudfs_source)
bak = backup.restore(**mnf.meta)
incrementals = []
if bak.backup_type == 'incremental':
incrementals = [bak]
while bak.prev_cloudfs_source:
tmpmnf = cloudfs.Manifest(cloudfs_path=bak.prev_cloudfs_source)
bak = backup.restore(**tmpmnf.meta)
if bak.backup_type == 'incremental':
incrementals.insert(0, bak)
self.incrementals = incrementals
if self.incrementals:
self.log_file = self.incrementals[-1].log_file
self.log_pos = self.incrementals[-1].log_pos
else:
self.log_file = bak.log_file
self.log_pos = bak.log_pos
coreutils.clean_dir(__mysql__['data_dir'])
LOG.info('Downloading the base backup (LSN: 0..%s)', bak.to_lsn)
trn = largetransfer.Download(bak.cloudfs_source)
trn.apply_async()
streamer = xbstream.args(extract=True, directory=__mysql__['data_dir'])
streamer.popen(stdin=trn.output)
trn.join()
LOG.info('Preparing the base backup')
innobackupex(__mysql__['data_dir'],
apply_log=True,
redo_only=True,
ibbackup='xtrabackup',
user=__mysql__['root_user'],
password=__mysql__['root_password'])
if incrementals:
inc_dir = os.path.join(__mysql__['tmp_dir'],
'xtrabackup-restore-inc')
i = 0
for inc in incrementals:
try:
os.makedirs(inc_dir)
inc = backup.restore(inc)
LOG.info(
'Downloading incremental backup #%d (LSN: %s..%s)', i,
inc.from_lsn, inc.to_lsn)
trn = largetransfer.Download(inc.cloudfs_source)
trn.apply_async()
streamer = xbstream.args(extract=True, directory=inc_dir)
streamer.popen(stdin=trn.output)
trn.join()
LOG.info('Preparing incremental backup #%d', i)
innobackupex(__mysql__['data_dir'],
apply_log=True,
redo_only=True,
incremental_dir=inc_dir,
ibbackup='xtrabackup',
user=__mysql__['root_user'],
password=__mysql__['root_password'])
i += 1
finally:
coreutils.remove(inc_dir)
LOG.info('Preparing the full backup')
innobackupex(__mysql__['data_dir'],
apply_log=True,
user=__mysql__['root_user'],
password=__mysql__['root_password'])
coreutils.chown_r(__mysql__['data_dir'], 'mysql', 'mysql')
self._mysql_init.start()
if int(__mysql__['replication_master']):
LOG.info("Master will reset it's binary logs, "
"so updating binary log position in backup manifest")
log_file, log_pos = self._client().master_status()
meta = mnf.meta
meta.update({'log_file': log_file, 'log_pos': log_pos})
mnf.meta = meta
mnf.save()
def create_vhost(self,
hostname,
port,
template,
ssl,
ssl_certificate_id=None,
reload=True,
allow_port=False):
"""
Creates Name-Based Apache VirtualHost
@param hostname: Server Name
@param port: port to listen to
@param template: VirtualHost body with no certificate paths
@param ssl: True if VirtualHost uses SSL certificate
@param ssl_certificate_id: ID of SSL certificate
@param reload: True if immediate apache reload is required.
@return: path to VirtualHost file
"""
#TODO: add Listen and NameVirtualHost directives to httpd.conf or ports.conf if needed
name = "%s:%s" % (hostname, port)
LOG.info("Creating Apache VirtualHost %s" % name)
v_host = VirtualHost(template)
if ssl:
ssl_certificate = SSLCertificate(ssl_certificate_id)
if not ssl_certificate.exists():
ssl_certificate.ensure()
v_host.use_certificate(
ssl_certificate.cert_path, ssl_certificate.key_path,
ssl_certificate.chain_path
if os.path.exists(ssl_certificate.chain_path) else None)
LOG.info("Certificate %s is set to VirtualHost %s" %
(ssl_certificate_id, name))
#Compatibility with old apache handler
if not self.mod_ssl.has_valid_certificate(
) or self.mod_ssl.is_system_certificate_used():
self.mod_ssl.set_default_certificate(ssl_certificate)
for directory in v_host.document_root_paths:
docroot_parent_path = os.path.dirname(directory)
if not os.path.exists(docroot_parent_path):
os.makedirs(docroot_parent_path, 0755)
LOG.info(
"Created parent directory of document root %s for %s" %
(directory, name))
if not os.path.exists(directory):
shutil.copytree(os.path.join(bus.share_path, "apache/html"),
directory)
files = ", ".join(os.listdir(directory))
LOG.debug("Copied document root files: %s" % files)
try:
pwd.getpwnam("apache")
uname = "apache"
except KeyError:
uname = "www-data"
coreutils.chown_r(directory, uname)
LOG.debug("Changed owner to %s: %s" %
(uname, ", ".join(os.listdir(directory))))
else:
LOG.debug("Document root %s already exists." % directory)
try:
clog_path = os.path.dirname(v_host.custom_log_path)
if not os.path.exists(clog_path):
os.makedirs(clog_path, 0755)
LOG.info(
"Created CustomLog directory for VirtualHost %s:%s: %s" % (
hostname,
port,
clog_path,
))
except NoPathError:
LOG.debug("Directive 'CustomLog' not found in %s" % name)
try:
errlog_path = os.path.dirname(v_host.error_log_path)
if not os.path.exists(errlog_path):
os.makedirs(errlog_path, 0755)
LOG.info(
"Created ErrorLog directory for VirtualHost %s:%s: %s" % (
hostname,
port,
errlog_path,
))
except NoPathError:
LOG.debug("Directive 'ErrorLog' not found in %s" % name)
v_host_changed = True
v_host_path = get_virtual_host_path(hostname, port)
if os.path.exists(v_host_path):
with open(v_host_path, "r") as old_v_host:
if old_v_host.read() == v_host.body:
v_host_changed = False
if v_host_changed:
with open(v_host_path, "w") as fp:
fp.write(v_host.body)
LOG.info("VirtualHost %s configuration saved to %s" %
(name, v_host_path))
else:
LOG.info("VirtualHost %s configuration (%s) has no changes." %
(name, v_host_path))
if allow_port:
self._open_ports([port])
if reload:
try:
self.configtest()
except initdv2.InitdError, e:
LOG.error("ConfigTest failed with error: '%s'." % str(e))
raise
else:
self.reload_service("Applying Apache VirtualHost %s" % name)
def on_DbMsr_CreateBackup(self, message):
#TODO: Think how to move the most part of it into Postgresql class
# Retrieve password for scalr pg user
tmpdir = backup_path = None
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
op = operation(name=self._op_backup, phases=[{
'name': self._phase_backup
}])
op.define()
with op.phase(self._phase_backup):
if not os.path.exists(self._tmp_path):
os.makedirs(self._tmp_path)
# Defining archive name and path
backup_filename = time.strftime('%Y-%m-%d-%H:%M:%S')+'.tar.gz'
backup_path = os.path.join(self._tmp_path, backup_filename)
# Creating archive
backup = tarfile.open(backup_path, 'w:gz')
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=self._tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (PG_DUMP, db_name, dump_path)
su_args = [SU_EXEC, '-', self.postgresql.root_user.name, '-c', pg_args]
err = system2(su_args)[1]
if err:
raise HandlerError('Error while dumping database %s: %s' % (db_name, err))
backup.add(dump_path, os.path.basename(dump_path))
make_backup_steps(databases, op, _single_backup)
backup.close()
with op.step(self._step_upload_to_cloud_storage):
# Creating list of full paths to archive chunks
#if os.path.getsize(backup_path) > __postgresql__['pgdump_chunk_size']:
# parts = [os.path.join(tmpdir, file) for file in split(backup_path, backup_filename, __postgresql__['pgdump_chunk_size'], tmpdir)]
#else:
# parts = [backup_path]
#sizes = [os.path.getsize(file) for file in parts]
cloud_storage_path = self._platform.scalrfs.backups(BEHAVIOUR)
LOG.info("Uploading backup to cloud storage (%s)", cloud_storage_path)
trn = LargeTransfer(backup_path, cloud_storage_path)
manifest = trn.run()
LOG.info("Postgresql backup uploaded to cloud storage under %s/%s",
cloud_storage_path, backup_filename)
result = list(dict(path=os.path.join(cloud_storage_path, c[0]), size=c[2]) for c in
manifest['files'][0]['chunks'])
op.ok(data=result)
# Notify Scalr
self.send_message(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(
db_type = BEHAVIOUR,
status = 'ok',
backup_parts = result
))
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
self.send_message(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(
db_type = BEHAVIOUR,
status = 'error',
last_error = str(e)
))
def on_init(self):
#temporary fix for starting-after-rebundle issue
if not os.path.exists(PG_SOCKET_DIR):
os.makedirs(PG_SOCKET_DIR)
chown_r(PG_SOCKET_DIR, 'postgres')
bus.on("host_init_response", self.on_host_init_response)
bus.on("before_host_up", self.on_before_host_up)
bus.on("before_reboot_start", self.on_before_reboot_start)
self._insert_iptables_rules()
if __node__['state'] == ScalarizrState.BOOTSTRAPPING:
if disttool.is_redhat_based():
checkmodule_path = software.which('checkmodule')
semodule_package_path = software.which('semodule_package')
semodule_path = software.which('semodule')
if all((checkmodule_path, semodule_package_path, semodule_path)):
with open('/tmp/sshkeygen.te', 'w') as fp:
fp.write(SSH_KEYGEN_SELINUX_MODULE)
self._logger.debug('Compiling SELinux policy for ssh-keygen')
system2((checkmodule_path, '-M', '-m', '-o',
'/tmp/sshkeygen.mod', '/tmp/sshkeygen.te'), logger=self._logger)
self._logger.debug('Building SELinux package for ssh-keygen')
system2((semodule_package_path, '-o', '/tmp/sshkeygen.pp',
'-m', '/tmp/sshkeygen.mod'), logger=self._logger)
self._logger.debug('Loading ssh-keygen SELinux package')
system2((semodule_path, '-i', '/tmp/sshkeygen.pp'), logger=self._logger)
if __node__['state'] == 'running':
vol = storage2.volume(__postgresql__['volume'])
vol.ensure(mount=True)
self.postgresql.service.start()
self.accept_all_clients()
self._logger.debug("Checking presence of Scalr's PostgreSQL root user.")
root_password = self.root_password
if not self.postgresql.root_user.exists():
self._logger.debug("Scalr's PostgreSQL root user does not exist. Recreating")
self.postgresql.root_user = self.postgresql.create_linux_user(ROOT_USER, root_password)
else:
try:
self.postgresql.root_user.check_system_password(root_password)
self._logger.debug("Scalr's root PgSQL user is present. Password is correct.")
except ValueError:
self._logger.warning("Scalr's root PgSQL user was changed. Recreating.")
self.postgresql.root_user.change_system_password(root_password)
if self.is_replication_master:
#ALTER ROLE cannot be executed in a read-only transaction
self._logger.debug("Checking password for pg_role scalr.")
if not self.postgresql.root_user.check_role_password(root_password):
LOG.warning("Scalr's root PgSQL role was changed. Recreating.")
self.postgresql.root_user.change_role_password(root_password)
def _run(self):
if self.backup_type:
self._check_backup_type()
rst_volume = None
exc_info = None
'''
# Create custom my.cnf
# XXX: it's not a good think to do, but we should create this hacks,
# cause when handler calls restore.run() my.cnf is not patched yet
shutil.copy(__mysql__['my.cnf'], '/tmp/my.cnf')
mycnf = metaconf.Configuration('mysql')
mycnf.read('/tmp/my.cnf')
try:
mycnf.options('mysqld')
except metaconf.NoPathError:
mycnf.add('mysqld')
mycnf.set('mysqld/datadir', __mysql__['data_dir'])
mycnf.set('mysqld/log-bin', __mysql__['binlog_dir'])
mycnf.write('/tmp/my.cnf')
'''
my_defaults = my_print_defaults('mysqld')
rst_volume = None
self._data_dir = os.path.normpath(my_defaults['datadir'])
LOG.info('_run: datadir is "%s"' % self._data_dir)
self._log_bin = os.path.normpath(my_defaults['log_bin'])
if self._log_bin.startswith('/'):
self._binlog_dir = os.path.dirname(self._log_bin)
try:
if self.snapshot:
LOG.info('Creating restore volume from snapshot')
if self.volume:
# Clone volume object
self.volume = storage2.volume(self.volume)
rst_volume = self.volume.clone()
rst_volume.snap = self.snapshot
else:
self.snapshot = storage2.snapshot(self.snapshot)
rst_volume = storage2.volume(type=self.snapshot.type,
snap=self.snapshot)
rst_volume.tags.update({'tmp': 1})
rst_volume.mpoint = self.backup_dir
rst_volume.ensure(mount=True)
if not os.listdir(self.backup_dir):
msg = 'Failed to find any backups in %s'
raise Error(msg, self.backup_dir)
backups = sorted(os.listdir(self.backup_dir))
LOG.info('Preparing the base backup')
base = backups.pop(0)
target_dir = os.path.join(self.backup_dir, base)
innobackupex(target_dir,
apply_log=True,
redo_only=True,
user=__mysql__['root_user'],
password=__mysql__['root_password'])
for inc in backups:
LOG.info('Preparing incremental backup %s', inc)
innobackupex(target_dir,
apply_log=True,
redo_only=True,
incremental_dir=os.path.join(self.backup_dir, inc),
user=__mysql__['root_user'],
password=__mysql__['root_password'])
LOG.info('Preparing the full backup')
innobackupex(target_dir,
apply_log=True,
user=__mysql__['root_user'],
password=__mysql__['root_password'])
LOG.info('Copying backup to datadir')
self._mysql_init.stop()
self._start_copyback()
try:
innobackupex(target_dir, copy_back=True)
coreutils.chown_r(self._data_dir,
'mysql', 'mysql')
self._mysql_init.start()
self._commit_copyback()
except:
self._rollback_copyback()
raise
except:
exc_info = sys.exc_info()
finally:
if rst_volume:
LOG.info('Destroying restore volume')
try:
rst_volume.destroy(force=True)
except:
msg = 'Failed to destroy volume %s: %s'
LOG.warn(msg, rst_volume.id, sys.exc_info()[1])
if exc_info:
raise exc_info[0], exc_info[1], exc_info[2]
def set_permissions(self):
self._logger.debug("Changing working directory owner to %s" %
self.user)
chown_r(self.path, self.user)
def on_DbMsr_CreateBackup(self, message):
#TODO: Think how to move the most part of it into Postgresql class
# Retrieve password for scalr pg user
tmpdir = backup_path = None
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
op = operation(name=self._op_backup,
phases=[{
'name': self._phase_backup
}])
op.define()
with op.phase(self._phase_backup):
if not os.path.exists(self._tmp_path):
os.makedirs(self._tmp_path)
# Defining archive name and path
backup_filename = time.strftime(
'%Y-%m-%d-%H:%M:%S') + '.tar.gz'
backup_path = os.path.join(self._tmp_path, backup_filename)
# Creating archive
backup = tarfile.open(backup_path, 'w:gz')
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=self._tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (
PG_DUMP, db_name, dump_path)
su_args = [
SU_EXEC, '-', self.postgresql.root_user.name, '-c',
pg_args
]
err = system2(su_args)[1]
if err:
raise HandlerError(
'Error while dumping database %s: %s' %
(db_name, err))
backup.add(dump_path, os.path.basename(dump_path))
make_backup_steps(databases, op, _single_backup)
backup.close()
with op.step(self._step_upload_to_cloud_storage):
# Creating list of full paths to archive chunks
if os.path.getsize(
backup_path) > __postgresql__['pgdump_chunk_size']:
parts = [
os.path.join(tmpdir, file) for file in split(
backup_path, backup_filename,
__postgresql__['pgdump_chunk_size'], tmpdir)
]
else:
parts = [backup_path]
sizes = [os.path.getsize(file) for file in parts]
cloud_storage_path = self._platform.scalrfs.backups(
BEHAVIOUR)
LOG.info("Uploading backup to cloud storage (%s)",
cloud_storage_path)
trn = transfer.Transfer()
cloud_files = trn.upload(parts, cloud_storage_path)
LOG.info(
"Postgresql backup uploaded to cloud storage under %s/%s",
cloud_storage_path, backup_filename)
result = list(
dict(path=path, size=size)
for path, size in zip(cloud_files, sizes))
op.ok(data=result)
# Notify Scalr
self.send_message(
DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR, status='ok', backup_parts=result))
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
self.send_message(
DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR, status='error', last_error=str(e)))
def create_vhost(self, hostname, port, template, ssl, ssl_certificate_id=None, reload=True, allow_port=False):
"""
Creates Name-Based Apache VirtualHost
@param hostname: Server Name
@param port: port to listen to
@param template: VirtualHost body with no certificate paths
@param ssl: True if VirtualHost uses SSL certificate
@param ssl_certificate_id: ID of SSL certificate
@param reload: True if immediate apache reload is required.
@return: path to VirtualHost file
"""
#TODO: add Listen and NameVirtualHost directives to httpd.conf or ports.conf if needed
name = "%s:%s" % (hostname, port)
LOG.info("Creating Apache VirtualHost %s" % name)
v_host = VirtualHost(template)
if ssl:
ssl_certificate = SSLCertificate(ssl_certificate_id)
if not ssl_certificate.exists():
ssl_certificate.ensure()
v_host.use_certificate(
ssl_certificate.cert_path,
ssl_certificate.key_path,
ssl_certificate.chain_path if os.path.exists(ssl_certificate.chain_path) else None
)
LOG.info("Certificate %s is set to VirtualHost %s" % (ssl_certificate_id, name))
#Compatibility with old apache handler
if not self.mod_ssl.has_valid_certificate() or self.mod_ssl.is_system_certificate_used():
self.mod_ssl.set_default_certificate(ssl_certificate)
for directory in v_host.document_root_paths:
docroot_parent_path = os.path.dirname(directory)
if not os.path.exists(docroot_parent_path):
os.makedirs(docroot_parent_path, 0755)
LOG.info("Created parent directory of document root %s for %s" % (directory, name))
if not os.path.exists(directory):
shutil.copytree(os.path.join(bus.share_path, "apache/html"), directory)
files = ", ".join(os.listdir(directory))
LOG.debug("Copied document root files: %s" % files)
try:
pwd.getpwnam("apache")
uname = "apache"
except KeyError:
uname = "www-data"
coreutils.chown_r(directory, uname)
LOG.debug("Changed owner to %s: %s" % (
uname, ", ".join(os.listdir(directory))))
else:
LOG.debug("Document root %s already exists." % directory)
try:
clog_path = os.path.dirname(v_host.custom_log_path)
if not os.path.exists(clog_path):
os.makedirs(clog_path, 0755)
LOG.info("Created CustomLog directory for VirtualHost %s:%s: %s" % (
hostname,
port,
clog_path,
))
except NoPathError:
LOG.debug("Directive 'CustomLog' not found in %s" % name)
try:
errlog_path = os.path.dirname(v_host.error_log_path)
if not os.path.exists(errlog_path):
os.makedirs(errlog_path, 0755)
LOG.info("Created ErrorLog directory for VirtualHost %s:%s: %s" % (
hostname,
port,
errlog_path,
))
except NoPathError:
LOG.debug("Directive 'ErrorLog' not found in %s" % name)
v_host_changed = True
v_host_path = get_virtual_host_path(hostname, port)
if os.path.exists(v_host_path):
with open(v_host_path, "r") as old_v_host:
if old_v_host.read() == v_host.body:
v_host_changed = False
if v_host_changed:
with open(v_host_path, "w") as fp:
fp.write(v_host.body)
LOG.info("VirtualHost %s configuration saved to %s" % (name, v_host_path))
else:
LOG.info("VirtualHost %s configuration (%s) has no changes." % (name, v_host_path))
if allow_port:
self._open_ports([port])
if reload:
try:
self.configtest()
except initdv2.InitdError, e:
LOG.error("ConfigTest failed with error: '%s'." % str(e))
raise
else:
self.reload_service("Applying Apache VirtualHost %s" % name)
MONGO_DUMP = software.which('mongodump')
MONGOS = software.which('mongos')
ROUTER_DEFAULT_PORT = 27017
ARBITER_DEFAULT_PORT = 27020
REPLICA_DEFAULT_PORT = 27018
CONFIG_SERVER_DEFAULT_PORT = 27019
SERVICE_NAME = BuiltinBehaviours.MONGODB
STORAGE_PATH = "/mnt/mongodb-storage"
DEFAULT_USER = '******' if disttool.is_debian_based() else 'mongod'
LOG_DIR = '/var/log/mongodb'
if not os.path.isdir(LOG_DIR):
os.makedirs(LOG_DIR)
chown_r(LOG_DIR, DEFAULT_USER)
LOG_PATH_DEFAULT = os.path.join(LOG_DIR, 'mongodb.shardsrv.log')
DEFAULT_UBUNTU_DB_PATH = '/var/lib/mongodb'
DEFAULT_CENTOS_DB_PATH = '/var/lib/mongo'
LOCK_FILE = 'mongod.lock'
SCALR_USER = '******'
STORAGE_DATA_DIR = os.path.join(STORAGE_PATH, 'data')
UBUNTU_CONFIG_PATH = '/etc/mongodb.conf'
CENTOS_CONFIG_PATH = '/etc/mongod.conf'
CONFIG_PATH_DEFAULT = '/etc/mongodb.shardsrv.conf'
ARBITER_DATA_DIR = '/tmp/arbiter'
ARBITER_LOG_PATH = os.path.join(LOG_DIR, 'mongodb.arbiter.log')
ARBITER_CONF_PATH = '/etc/mongodb.arbiter.conf'
def _run(self):
# Apply resource's meta
mnf = cloudfs.Manifest(cloudfs_path=self.cloudfs_source)
bak = backup.restore(**mnf.meta)
incrementals = []
if bak.backup_type == "incremental":
incrementals = [bak]
while bak.prev_cloudfs_source:
tmpmnf = cloudfs.Manifest(cloudfs_path=bak.prev_cloudfs_source)
bak = backup.restore(**tmpmnf.meta)
if bak.backup_type == "incremental":
incrementals.insert(0, bak)
self.incrementals = incrementals
if self.incrementals:
self.log_file = self.incrementals[-1].log_file
self.log_pos = self.incrementals[-1].log_pos
else:
self.log_file = bak.log_file
self.log_pos = bak.log_pos
coreutils.clean_dir(__mysql__["data_dir"])
LOG.info("Downloading the base backup (LSN: 0..%s)", bak.to_lsn)
trn = cloudfs.LargeTransfer(
bak.cloudfs_source,
__mysql__["data_dir"],
streamer=xbstream.args(extract=True, directory=__mysql__["data_dir"]),
)
trn.run()
LOG.info("Preparing the base backup")
innobackupex(
__mysql__["data_dir"],
apply_log=True,
redo_only=True,
ibbackup="xtrabackup",
user=__mysql__["root_user"],
password=__mysql__["root_password"],
)
if incrementals:
inc_dir = os.path.join(__mysql__["tmp_dir"], "xtrabackup-restore-inc")
i = 0
for inc in incrementals:
try:
os.makedirs(inc_dir)
inc = backup.restore(inc)
LOG.info("Downloading incremental backup #%d (LSN: %s..%s)", i, inc.from_lsn, inc.to_lsn)
trn = cloudfs.LargeTransfer(
inc.cloudfs_source, inc_dir, streamer=xbstream.args(extract=True, directory=inc_dir)
)
trn.run() # todo: Largetransfer should support custom decompressor proc
LOG.info("Preparing incremental backup #%d", i)
innobackupex(
__mysql__["data_dir"],
apply_log=True,
redo_only=True,
incremental_dir=inc_dir,
ibbackup="xtrabackup",
user=__mysql__["root_user"],
password=__mysql__["root_password"],
)
i += 1
finally:
coreutils.remove(inc_dir)
LOG.info("Preparing the full backup")
innobackupex(
__mysql__["data_dir"], apply_log=True, user=__mysql__["root_user"], password=__mysql__["root_password"]
)
coreutils.chown_r(__mysql__["data_dir"], "mysql", "mysql")
self._mysql_init.start()
if int(__mysql__["replication_master"]):
LOG.info("Master will reset it's binary logs, " "so updating binary log position in backup manifest")
log_file, log_pos = self._client().master_status()
meta = mnf.meta
meta.update({"log_file": log_file, "log_pos": log_pos})
mnf.meta = meta
mnf.save()
def do_backup(op):
tmpdir = None
dumps = []
tmp_path = os.path.join(__postgresql__['storage_dir'], 'tmp')
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (
PG_DUMP, db_name, dump_path)
su_args = [
SU_EXEC, '-', self.postgresql.root_user.name, '-c',
pg_args
]
err = system2(su_args)[1]
if err:
raise HandlerError(
'Error while dumping database %s: %s' %
(db_name, err)) #?
dumps.append(dump_path)
for db_name in databases:
_single_backup(db_name)
cloud_storage_path = __node__.platform.scalrfs.backups(
BEHAVIOUR)
suffix = 'master' if int(
__postgresql__[OPT_REPLICATION_MASTER]) else 'slave'
backup_tags = {'scalr-purpose': 'postgresql-%s' % suffix}
LOG.info("Uploading backup to %s with tags %s" %
(cloud_storage_path, backup_tags))
def progress_cb(progress):
LOG.debug('Uploading %s bytes' % progress)
uploader = largetransfer.Upload(dumps,
cloud_storage_path,
progress_cb=progress_cb)
uploader.apply_async()
uploader.join()
manifest = uploader.manifest
LOG.info(
"Postgresql backup uploaded to cloud storage under %s",
cloud_storage_path)
LOG.debug(manifest.data)
# Notify Scalr
result = transfer_result_to_backup_result(manifest)
__node__.messaging.send(
DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR, status='ok', backup_parts=result))
return result
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
__node__.messaging.send(
DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR, status='error', last_error=str(e)))
def do_backup(op):
tmpdir = backup_path = None
tmp_path = os.path.join(__postgresql__['storage_dir'], 'tmp')
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
# Defining archive name and path
backup_filename = time.strftime('%Y-%m-%d-%H:%M:%S')+'.tar.gz'
backup_path = os.path.join(tmp_path, backup_filename)
# Creating archive
backup_obj = tarfile.open(backup_path, 'w:gz')
# Dump all databases
LOG.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=tmp_path)
chown_r(tmpdir, self.postgresql.root_user.name)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (PG_DUMP, db_name, dump_path)
su_args = [SU_EXEC, '-', self.postgresql.root_user.name, '-c', pg_args]
err = system2(su_args)[1]
if err:
raise HandlerError('Error while dumping database %s: %s' % (db_name, err)) #?
backup_obj.add(dump_path, os.path.basename(dump_path))
for db_name in databases:
_single_backup(db_name)
backup_obj.close()
# Creating list of full paths to archive chunks
#if os.path.getsize(backup_path) > __postgresql__['pgdump_chunk_size']:
# parts = [os.path.join(tmpdir, file) for file in split(backup_path, backup_filename, __postgresql__['pgdump_chunk_size'], tmpdir)]
#else:
# parts = [backup_path]
#sizes = [os.path.getsize(file) for file in parts]
cloud_storage_path = __node__.platform.scalrfs.backups(BEHAVIOUR)
suffix = 'master' if int(__postgresql__[OPT_REPLICATION_MASTER]) else 'slave'
backup_tags = {'scalr-purpose': 'postgresql-%s' % suffix}
LOG.info("Uploading backup to %s with tags %s" % (cloud_storage_path, backup_tags))
trn = LargeTransfer(backup_path, cloud_storage_path, tags=backup_tags)
manifest = trn.run()
LOG.info("Postgresql backup uploaded to cloud storage under %s/%s",
cloud_storage_path, backup_filename)
result = list(dict(path=os.path.join(os.path.dirname(manifest.cloudfs_path), c[0]), size=c[2]) for c in
manifest['files'][0]['chunks'])
# Notify Scalr
__node__.messaging.send(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR,
status='ok',
backup_parts=result))
return result #?
except (Exception, BaseException), e:
LOG.exception(e)
# Notify Scalr about error
__node__.messaging.send(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT,
dict(db_type=BEHAVIOUR,
status='error',
last_error=str(e)))
MONGO_DUMP = software.which('mongodump')
MONGOS = software.which('mongos')
ROUTER_DEFAULT_PORT = 27017
ARBITER_DEFAULT_PORT = 27020
REPLICA_DEFAULT_PORT = 27018
CONFIG_SERVER_DEFAULT_PORT = 27019
SERVICE_NAME = BuiltinBehaviours.MONGODB
STORAGE_PATH = "/mnt/mongodb-storage"
DEFAULT_USER = '******' if disttool.is_debian_based() else 'mongod'
LOG_DIR = '/var/log/mongodb'
if not os.path.isdir(LOG_DIR):
os.makedirs(LOG_DIR)
chown_r(LOG_DIR, DEFAULT_USER)
LOG_PATH_DEFAULT = os.path.join(LOG_DIR, 'mongodb.shardsrv.log')
DEFAULT_UBUNTU_DB_PATH = '/var/lib/mongodb'
DEFAULT_CENTOS_DB_PATH = '/var/lib/mongo'
LOCK_FILE = 'mongod.lock'
SCALR_USER = '******'
STORAGE_DATA_DIR = os.path.join(STORAGE_PATH, 'data')
UBUNTU_CONFIG_PATH = '/etc/mongodb.conf'
CENTOS_CONFIG_PATH = '/etc/mongod.conf'
CONFIG_PATH_DEFAULT = '/etc/mongodb.shardsrv.conf'
ARBITER_DATA_DIR = '/tmp/arbiter'
ARBITER_LOG_PATH = os.path.join(LOG_DIR, 'mongodb.arbiter.log')
ARBITER_CONF_PATH = '/etc/mongodb.arbiter.conf'
def on_init(self):
#temporary fix for starting-after-rebundle issue
if not os.path.exists(PG_SOCKET_DIR):
os.makedirs(PG_SOCKET_DIR)
chown_r(PG_SOCKET_DIR, 'postgres')
bus.on("host_init_response", self.on_host_init_response)
bus.on("before_host_up", self.on_before_host_up)
bus.on("before_reboot_start", self.on_before_reboot_start)
self._insert_iptables_rules()
if __node__['state'] == ScalarizrState.BOOTSTRAPPING:
if linux.os.redhat_family:
def selinux_enabled():
selinuxenabled = software.which('selinuxenabled')
if selinuxenabled:
_, _, ret_code = system2((selinuxenabled, ),
raise_exc=False)
return 0 == ret_code
# Consider it enabled by default
return True
checkmodule_path = software.which('checkmodule')
semodule_package_path = software.which('semodule_package')
semodule_path = software.which('semodule')
if all(
(checkmodule_path, semodule_package_path, semodule_path)):
if selinux_enabled():
with open('/tmp/sshkeygen.te', 'w') as fp:
fp.write(SSH_KEYGEN_SELINUX_MODULE)
self._logger.debug(
'Compiling SELinux policy for ssh-keygen')
system2((checkmodule_path, '-M', '-m', '-o',
'/tmp/sshkeygen.mod', '/tmp/sshkeygen.te'),
logger=self._logger)
self._logger.debug(
'Building SELinux package for ssh-keygen')
system2(
(semodule_package_path, '-o', '/tmp/sshkeygen.pp',
'-m', '/tmp/sshkeygen.mod'),
logger=self._logger)
self._logger.debug(
'Loading ssh-keygen SELinux package')
system2((semodule_path, '-i', '/tmp/sshkeygen.pp'),
logger=self._logger)
if __node__['state'] == 'running':
vol = storage2.volume(__postgresql__['volume'])
vol.ensure(mount=True)
self.postgresql.service.start()
self.accept_all_clients()
self._logger.debug(
"Checking presence of Scalr's PostgreSQL root user.")
root_password = self.root_password
if not self.postgresql.root_user.exists():
self._logger.debug(
"Scalr's PostgreSQL root user does not exist. Recreating")
self.postgresql.root_user = self.postgresql.create_linux_user(
ROOT_USER, root_password)
else:
try:
self.postgresql.root_user.check_system_password(
root_password)
self._logger.debug(
"Scalr's root PgSQL user is present. Password is correct."
)
except ValueError:
self._logger.warning(
"Scalr's root PgSQL user was changed. Recreating.")
self.postgresql.root_user.change_system_password(
root_password)
if self.is_replication_master:
#ALTER ROLE cannot be executed in a read-only transaction
self._logger.debug("Checking password for pg_role scalr.")
if not self.postgresql.root_user.check_role_password(
root_password):
LOG.warning(
"Scalr's root PgSQL role was changed. Recreating.")
self.postgresql.root_user.change_role_password(
root_password)
def _init_master(self, message):
"""
Initialize MySQL master
@type message: scalarizr.messaging.Message
@param message: HostUp message
"""
LOG.info("Initializing MySQL master")
log = bus.init_op.logger
log.info('Create storage')
if 'restore' in __mysql__ and \
__mysql__['restore'].type == 'snap_mysql':
__mysql__['restore'].run()
else:
if __node__['platform'].name == 'idcf':
if __mysql__['volume'].id:
LOG.info('Cloning volume to workaround reattachment limitations of IDCF')
__mysql__['volume'].snap = __mysql__['volume'].snapshot()
__mysql__['volume'].ensure(mount=True, mkfs=True)
LOG.debug('MySQL volume config after ensure: %s', dict(__mysql__['volume']))
coreutils.clean_dir(__mysql__['defaults']['datadir'])
self.mysql.flush_logs(__mysql__['data_dir'])
self.mysql.move_mysqldir_to(__mysql__['storage_dir'])
self._change_selinux_ctx()
storage_valid = self._storage_valid()
user_creds = self.get_user_creds()
self._fix_percona_debian_cnf()
#datadir = mysql2_svc.my_print_defaults('mysqld').get('datadir', __mysql__['defaults']['datadir'])
#if not storage_valid and datadir.find(__mysql__['data_dir']) == 0:
# # When role was created from another mysql role it contains modified my.cnf settings
# #self.mysql.my_cnf.datadir = '/var/lib/mysql'
# self.mysql.my_cnf.delete_options(['mysqld/log_bin'])
if not storage_valid:
'''
if linux.os['family'] == 'RedHat':
try:
# Check if selinux enabled
selinuxenabled_bin = software.which('selinuxenabled')
if selinuxenabled_bin:
se_enabled = not system2((selinuxenabled_bin, ), raise_exc=False)[2]
if se_enabled:
# Set selinux context for new mysql datadir
semanage = mysql_svc.get_semanage()
linux.system('%s fcontext -a -t mysqld_db_t "%s(/.*)?"'
% (semanage, __mysql__['storage_dir']), shell=True)
# Restore selinux context
restorecon = software.which('restorecon')
linux.system('%s -R -v %s' % (restorecon, __mysql__['storage_dir']), shell=True)
except:
LOG.debug('Selinux context setup failed', exc_info=sys.exc_info())
'''
self.mysql.my_cnf.delete_options(['mysqld/log_bin'])
linux.system(['mysql_install_db', '--user=mysql', '--datadir=%s' % __mysql__['data_dir']])
# Patch configuration
options = {
'bind-address': '0.0.0.0',
'datadir': __mysql__['data_dir'],
'log_bin': os.path.join(__mysql__['binlog_dir'], 'binlog'),
'log-bin-index': os.path.join(__mysql__['binlog_dir'], 'binlog.index'), # MariaDB
'sync_binlog': '1',
'innodb_flush_log_at_trx_commit': '1',
'expire_logs_days': '10'
}
for key, value in options.items():
self.mysql.my_cnf.set('mysqld/' + key, value)
if not storage_valid:
if linux.os.debian_family and os.path.exists(__mysql__['debian.cnf']):
self.mysql.service.start()
debian_cnf = metaconf.Configuration('mysql')
debian_cnf.read(__mysql__['debian.cnf'])
sql = ("GRANT ALL PRIVILEGES ON *.* "
"TO 'debian-sys-maint'@'localhost' "
"IDENTIFIED BY '{0}'").format(debian_cnf.get('client/password'))
linux.system(['mysql', '-u', 'root', '-e', sql])
self.mysql.service.stop()
coreutils.chown_r(__mysql__['data_dir'], 'mysql', 'mysql')
if 'restore' in __mysql__ and \
__mysql__['restore'].type == 'xtrabackup':
# XXX: when restoring data bundle on ephemeral storage, data dir should by empty
# but move_mysqldir_to call required to set several options in my.cnf
coreutils.clean_dir(__mysql__['data_dir'])
#self._change_selinux_ctx()
log.info('Patch my.cnf configuration file')
# Init replication
self.mysql._init_replication(master=True)
if 'restore' in __mysql__ and \
__mysql__['restore'].type == 'xtrabackup':
__mysql__['restore'].run()
# If It's 1st init of mysql master storage
if not storage_valid:
if os.path.exists(__mysql__['debian.cnf']):
log.info("Copying debian.cnf file to mysql storage")
shutil.copy(__mysql__['debian.cnf'], __mysql__['storage_dir'])
# If volume has mysql storage directory structure (N-th init)
else:
log.info('InnoDB recovery')
self._copy_debian_cnf_back()
if 'restore' in __mysql__ and __mysql__['restore'].type != 'xtrabackup':
self._innodb_recovery()
self.mysql.service.start()
log.info('Create Scalr users')
# Check and create mysql system users
self.create_users(**user_creds)
log.info('Create data bundle')
if 'backup' in __mysql__:
__mysql__['restore'] = __mysql__['backup'].run()
# Update HostUp message
log.info('Collect HostUp data')
md = dict(
replication_master=__mysql__['replication_master'],
root_password=__mysql__['root_password'],
repl_password=__mysql__['repl_password'],
stat_password=__mysql__['stat_password'],
master_password=__mysql__['master_password']
)
if __mysql__['compat_prior_backup_restore']:
if 'restore' in __mysql__:
md.update(dict(
log_file=__mysql__['restore'].log_file,
log_pos=__mysql__['restore'].log_pos,
snapshot_config=dict(__mysql__['restore'].snapshot)))
elif 'log_file' in __mysql__:
md.update(dict(
log_file=__mysql__['log_file'],
log_pos=__mysql__['log_pos']))
md.update(dict(
volume_config=dict(__mysql__['volume'])))
else:
md.update(dict(
volume=dict(__mysql__['volume'])
))
for key in ('backup', 'restore'):
if key in __mysql__:
md[key] = dict(__mysql__[key])
message.db_type = __mysql__['behavior']
setattr(message, __mysql__['behavior'], md)
def set_permissions(self):
self._logger.debug("Changing working directory owner to %s" % self.user)
chown_r(self.path, self.user)
