def move_to(self, dst):
if not os.path.exists(dst):
self._logger.debug("creating %s" % dst)
os.makedirs(dst)
for config in ['postgresql.conf', 'pg_ident.conf', 'pg_hba.conf']:
old_config = os.path.join(self.path, config)
new_config = os.path.join(dst, config)
if os.path.exists(old_config):
self._logger.debug('Moving %s' % config)
shutil.move(old_config, new_config)
elif os.path.exists(new_config):
self._logger.debug('%s is already in place. Skipping.' % config)
else:
raise BaseException('Postgresql config file not found: %s' % old_config)
rchown(DEFAULT_USER, new_config)
#the following block needs revision
#self._make_symlinks(dst)
self._patch_sysconfig(dst)
self.path = dst
self._logger.debug("configuring pid")
conf = PostgresqlConf.find(self)
conf.pid_file = os.path.join(dst, 'postmaster.pid')
def move_to(self, dst, move_files=True):
new_cluster_dir = os.path.join(dst, STORAGE_DATA_DIR)
if not os.path.exists(dst):
self._logger.debug('Creating directory structure for postgresql cluster: %s' % dst)
os.makedirs(dst)
if move_files:
source = self.path
if not os.path.exists(self.path):
source = self.default_path
self._logger.debug('data_directory in postgresql.conf points to non-existing location, using %s instead' % source)
if source != new_cluster_dir:
self._logger.debug("copying cluster files from %s into %s" % (source, new_cluster_dir))
shutil.copytree(source, new_cluster_dir)
self._logger.debug("changing directory owner to %s" % self.user)
rchown(self.user, dst)
self._logger.debug("Changing postgres user`s home directory")
if disttool.is_redhat_based():
#looks like ubuntu doesn`t need this
system2([USERMOD, '-d', new_cluster_dir, self.user])
self.path = new_cluster_dir
return new_cluster_dir
def create(self, dst):
if not os.path.exists(dst):
self._logger.debug("Creating directory structure for mongodb files: %s" % dst)
os.makedirs(dst)
self._logger.debug("changing directory owner to %s" % self.user)
rchown(self.user, dst)
self.path = dst
return dst
def get_pidfile(port=DEFAULT_PORT):
pid_file = os.path.join(DEFAULT_PID_DIR,'redis-server.%s.pid' % port)
'''
fix for ubuntu1004
'''
if not os.path.exists(pid_file):
open(pid_file, 'w').close()
rchown('redis', pid_file)
return pid_file
def args(self):
s = ["--fork"]
if self.configpath:
s.append("--config=%s" % self.configpath)
if self.dbpath:
s.append("--dbpath=%s" % self.dbpath)
if self.port:
s.append("--port=%s" % self.port)
if self.keyfile and os.path.exists(self.keyfile):
rchown(DEFAULT_USER, self.keyfile)
s.append("--keyFile=%s" % self.keyfile)
return s
def _prepare_arbiter(self, rs_name):
if os.path.isdir(ARBITER_DATA_DIR):
shutil.rmtree(ARBITER_DATA_DIR)
self._logger.debug("Creating datadir for arbiter: %s" % ARBITER_DATA_DIR)
os.makedirs(ARBITER_DATA_DIR)
rchown(DEFAULT_USER, ARBITER_DATA_DIR)
self._logger.debug("Preparing arbiter's config file")
self.arbiter_conf.dbpath = ARBITER_DATA_DIR
self.arbiter_conf.replSet = rs_name
self.arbiter_conf.shardsvr = True
self.arbiter_conf.port = ARBITER_DEFAULT_PORT
self.arbiter_conf.logpath = ARBITER_LOG_PATH
def apply_private_ssh_key(self,source_path=None):
source_path = source_path or self.private_key_path
if not os.path.exists(source_path):
self._logger.error('Cannot apply private ssh key: source %s not found' % source_path)
else:
if not os.path.exists(self.ssh_dir):
os.makedirs(self.ssh_dir)
rchown(self.name, self.ssh_dir)
dst = os.path.join(self.ssh_dir, 'id_rsa')
shutil.copyfile(source_path, dst)
os.chmod(dst, 0400)
rchown(self.name, dst)
def apply_public_ssh_key(self, source_path=None): source_path = source_path or self.public_key_path if not os.path.exists(self.ssh_dir): os.makedirs(self.ssh_dir) rchown(self.name, self.ssh_dir) pub_key = read_file(source_path,logger=self._logger) path = os.path.join(self.ssh_dir, 'authorized_keys') keys = read_file(path,logger=self._logger) if os.path.exists(path) else '' if not keys or not pub_key in keys: write_file(path, data='\n%s %s\n' % (pub_key, self.name), mode='a', logger=self._logger) rchown(self.name, path)
def move_to(self, dst, move_files=True):
new_db_path = os.path.join(dst, os.path.basename(self.db_path))
if not os.path.exists(dst):
LOG.debug('Creating directory structure for redis db files: %s' % dst)
os.makedirs(dst)
if move_files and os.path.exists(os.path.dirname(self.db_path)) and os.path.isfile(self.db_path):
LOG.debug("copying db file %s into %s" % (os.path.dirname(self.db_path), dst))
shutil.copyfile(self.db_path, new_db_path)
LOG.debug("changing directory owner to %s" % self.user)
rchown(self.user, dst)
self.db_path = new_db_path
return new_db_path
def make_symlinks(source_dir, dst_dir, username='******'): #Vital hack for getting CentOS init script to work for obj in ['base', 'PG_VERSION', 'postmaster.pid']: src = os.path.join(source_dir, obj) dst = os.path.join(dst_dir, obj) if os.path.islink(dst): os.unlink(dst) elif os.path.exists(dst): shutil.rmtree(dst) os.symlink(src, dst) if os.path.exists(src): rchown(username, dst)
def _prepare_config_server(self):
self._logger.debug("Preparing config server")
if not os.path.exists(CONFIG_SERVER_DATA_DIR):
os.makedirs(CONFIG_SERVER_DATA_DIR)
rchown(DEFAULT_USER, CONFIG_SERVER_DATA_DIR)
"""
configsvr changes the default port and turns on the diaglog,
a log that keeps every action the config database performs
in a replayable format, just in case.
For mongo 1.8+ use --port 27019 and --journal (instead of --diaglog).
Journaling gives mostly the same effect as the diaglog with better performance.
P.S. Assume that mongodb roles Scalr will be build on x64 platform only
Wchich means journal option by default will be on.
"""
self.config_server_conf.configsvr = True
self.config_server_conf.port = CONFIG_SERVER_DEFAULT_PORT
self.config_server_conf.logpath = CONFIG_SERVER_LOG_PATH
def start(cls):
if not cls.is_running():
cls._logger.info("Starting %s process" % MONGOS)
args = [
MONGOS,
"--fork",
"--logpath",
ROUTER_LOG_PATH,
"--configdb",
"mongo-0-0:%s" % CONFIG_SERVER_DEFAULT_PORT,
]
if cls.keyfile and os.path.exists(cls.keyfile):
rchown(DEFAULT_USER, cls.keyfile)
args.append("--keyFile=%s" % cls.keyfile)
system2(args, close_fds=True, preexec_fn=os.setsid)
wait_until(lambda: cls.is_running, timeout=MAX_START_TIMEOUT)
wait_until(lambda: cls.get_cli().has_connection, timeout=MAX_START_TIMEOUT)
cls._logger.debug("%s process has been started." % MONGOS)
def move_mysqldir_to(self, storage_path):
LOG.info("Moving mysql dir to %s" % storage_path)
for directive, dirname in (
("mysqld/log_bin", os.path.join(storage_path, STORAGE_BINLOG)),
("mysqld/datadir", os.path.join(storage_path, STORAGE_DATA_DIR) + "/"),
):
dest = os.path.dirname(dirname)
if os.path.isdir(dest):
LOG.info("No need to move %s to %s: already in place." % (directive, dest))
else:
os.makedirs(dest)
raw_value = self.my_cnf.get(directive)
LOG.debug("directive %s:%s" % (directive, raw_value))
if raw_value:
src_dir = os.path.dirname(raw_value + "/") + "/"
LOG.debug("source path: %s" % src_dir)
if os.path.isdir(src_dir) and src_dir != dest:
try:
if not system2((software.which("selinuxenabled"),), raise_exc=False)[2]:
if not system2((software.which("getsebool"), "mysqld_disable_trans"), raise_exc=False)[
2
]:
LOG.debug("Make SELinux rule for rsync")
system2((software.which("setsebool"), "-P", "mysqld_disable_trans", "1"))
except LookupError:
pass
LOG.info("Copying mysql directory '%s' to '%s'", src_dir, dest)
rsync = filetool.Rsync().archive()
rsync.source(src_dir).dest(dest).exclude(["ib_logfile*"])
system2(str(rsync), shell=True)
self.my_cnf.set(directive, dirname)
rchown("mysql", dest)
# Adding rules to apparmor config
if disttool.is_debian_based():
_add_apparmor_rules(dest)
def on_DbMsr_CreateBackup(self, message):
#TODO: Think how to move the most part of it into Postgresql class
# Retrieve password for scalr mysql user
tmpdir = backup_path = None
try:
# Get databases list
psql = PSQL(user=self.postgresql.root_user.name)
databases = psql.list_pg_databases()
if 'template0' in databases:
databases.remove('template0')
op = operation(name=self._op_backup, phases=[{
'name': self._phase_backup
}])
op.define()
with op.phase(self._phase_backup):
if not os.path.exists(self._tmp_path):
os.makedirs(self._tmp_path)
# Defining archive name and path
backup_filename = time.strftime('%Y-%m-%d-%H:%M:%S')+'.tar.gz'
backup_path = os.path.join(self._tmp_path, backup_filename)
# Creating archive
backup = tarfile.open(backup_path, 'w:gz')
# Dump all databases
self._logger.info("Dumping all databases")
tmpdir = tempfile.mkdtemp(dir=self._tmp_path)
rchown(self.postgresql.root_user.name, tmpdir)
def _single_backup(db_name):
dump_path = tmpdir + os.sep + db_name + '.sql'
pg_args = '%s %s --no-privileges -f %s' % (PG_DUMP, db_name, dump_path)
su_args = [SU_EXEC, '-', self.postgresql.root_user.name, '-c', pg_args]
err = system2(su_args)[1]
if err:
raise HandlerError('Error while dumping database %s: %s' % (db_name, err))
backup.add(dump_path, os.path.basename(dump_path))
make_backup_steps(databases, op, _single_backup)
backup.close()
with op.step(self._step_upload_to_cloud_storage):
# Creating list of full paths to archive chunks
if os.path.getsize(backup_path) > BACKUP_CHUNK_SIZE:
parts = [os.path.join(tmpdir, file) for file in split(backup_path, backup_filename, BACKUP_CHUNK_SIZE , tmpdir)]
else:
parts = [backup_path]
sizes = [os.path.getsize(file) for file in parts]
cloud_storage_path = self._platform.scalrfs.backups(BEHAVIOUR)
self._logger.info("Uploading backup to cloud storage (%s)", cloud_storage_path)
trn = transfer.Transfer()
cloud_files = trn.upload(parts, cloud_storage_path)
self._logger.info("Postgresql backup uploaded to cloud storage under %s/%s",
cloud_storage_path, backup_filename)
result = list(dict(path=path, size=size) for path, size in zip(cloud_files, sizes))
op.ok(data=result)
# Notify Scalr
self.send_message(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(
db_type = BEHAVIOUR,
status = 'ok',
backup_parts = result
))
except (Exception, BaseException), e:
self._logger.exception(e)
# Notify Scalr about error
self.send_message(DbMsrMessages.DBMSR_CREATE_BACKUP_RESULT, dict(
db_type = BEHAVIOUR,
status = 'error',
last_error = str(e)
))
def on_init(self):
#temporary fix for starting-after-rebundle issue
if not os.path.exists(PG_SOCKET_DIR):
os.makedirs(PG_SOCKET_DIR)
rchown(user='******', path=PG_SOCKET_DIR)
bus.on("host_init_response", self.on_host_init_response)
bus.on("before_host_up", self.on_before_host_up)
bus.on("before_reboot_start", self.on_before_reboot_start)
bus.on("before_reboot_finish", self.on_before_reboot_finish)
if self._cnf.state == ScalarizrState.BOOTSTRAPPING:
self._insert_iptables_rules()
if disttool.is_redhat_based():
checkmodule_paths = software.whereis('checkmodule')
semodule_package_paths = software.whereis('semodule_package')
semodule_paths = software.whereis('semodule')
if all((checkmodule_paths, semodule_package_paths, semodule_paths)):
filetool.write_file('/tmp/sshkeygen.te',
SSH_KEYGEN_SELINUX_MODULE, logger=self._logger)
self._logger.debug('Compiling SELinux policy for ssh-keygen')
system2((checkmodule_paths[0], '-M', '-m', '-o',
'/tmp/sshkeygen.mod', '/tmp/sshkeygen.te'), logger=self._logger)
self._logger.debug('Building SELinux package for ssh-keygen')
system2((semodule_package_paths[0], '-o', '/tmp/sshkeygen.pp',
'-m', '/tmp/sshkeygen.mod'), logger=self._logger)
self._logger.debug('Loading ssh-keygen SELinux package')
system2((semodule_paths[0], '-i', '/tmp/sshkeygen.pp'), logger=self._logger)
if self._cnf.state == ScalarizrState.RUNNING:
storage_conf = Storage.restore_config(self._volume_config_path)
storage_conf['tags'] = self.postgres_tags
self.storage_vol = Storage.create(storage_conf)
if not self.storage_vol.mounted():
self.storage_vol.mount()
self.postgresql.service.start()
self.accept_all_clients()
self._logger.debug("Checking presence of Scalr's PostgreSQL root user.")
root_password = self.root_password
if not self.postgresql.root_user.exists():
self._logger.debug("Scalr's PostgreSQL root user does not exist. Recreating")
self.postgresql.root_user = self.postgresql.create_user(ROOT_USER, root_password)
else:
try:
self.postgresql.root_user.check_system_password(root_password)
self._logger.debug("Scalr's root PgSQL user is present. Password is correct.")
except ValueError:
self._logger.warning("Scalr's root PgSQL user was changed. Recreating.")
self.postgresql.root_user.change_system_password(root_password)
if self.is_replication_master:
#ALTER ROLE cannot be executed in a read-only transaction
self._logger.debug("Checking password for pg_role scalr.")
if not self.postgresql.root_user.check_role_password(root_password):
self._logger.warning("Scalr's root PgSQL role was changed. Recreating.")
self.postgresql.root_user.change_role_password(root_password)