def lcrp_changelog(log, fsname, lcrp_dir, changelog_user):
"""
Run lcrp_changelog
"""
hostname = socket.gethostname()
host = ssh_host.SSHHost(hostname, local=True)
command = common.c_command_path(LCRP_CMD_CHANGELOG)
command += " -d " + lcrp_dir
command += " -m %s-MDT0000" % fsname
command += " -u " + changelog_user
args = {}
args[watched_io.WATCHEDIO_LOG] = log
args[watched_io.WATCHEDIO_HOSTNAME] = host.sh_hostname
stdout_fd = watched_io.watched_io_open("/dev/null",
watched_io.log_watcher_info_simplified,
args)
stderr_fd = watched_io.watched_io_open("/dev/null",
watched_io.log_watcher_error_simplified,
args)
log.cl_debug("start to run command [%s] on host [%s]",
command, host.sh_hostname)
retval = host.sh_run(log, command, stdout_tee=stdout_fd,
stderr_tee=stderr_fd, return_stdout=False,
return_stderr=False, timeout=None, flush_tee=True)
stdout_fd.close()
stderr_fd.close()
return retval.cr_exit_status
def main():
"""
Install Clownfish
"""
# pylint: disable=unused-variable
log = clog.get_log()
missing_dependencies = []
try:
import yaml
except ImportError:
missing_dependencies.append("PyYAML")
try:
import filelock
except ImportError:
missing_dependencies.append("python2-filelock")
try:
import dateutil
except ImportError:
missing_dependencies.append("python-dateutil")
local_host = ssh_host.SSHHost("localhost", local=True)
for dependent_rpm in install_common.CLOWNFISH_INSTALL_DEPENDENT_RPMS:
ret = local_host.sh_rpm_query(log, dependent_rpm)
if ret != 0:
missing_dependencies.append(dependent_rpm)
if len(missing_dependencies):
log.cl_info("installing dependency RPMs of %s", missing_dependencies)
ret = install_common.dependency_install(log, local_host,
constants.CLOWNFISH_CONFIG,
missing_dependencies,
"Clownfish", "clownfish-*.iso")
if ret:
log.cl_error(
"not able to install Clownfish because some depdendency "
"RPMs are missing and not able to be installed: %s",
missing_dependencies)
sys.exit(-1)
from pyclownfish import clownfish_install_nodeps
clownfish_install_nodeps.main()
def _start_install(log, workspace, install_server, mnt_path,
install_config, arg):
"""
Run the install test
"""
# pylint: disable=too-many-locals,too-many-arguments
# Make sure install server is local host, since this will overwrite the
# local config files
uuid_install = install_server.sh_uuid(log)
if uuid_install is None:
log.cl_error("failed to get the UUID on host [%s]",
install_server.sh_hostname)
return -1
local_host = ssh_host.SSHHost("localhost", local=True)
uuid_local = local_host.sh_uuid(log)
if uuid_local is None:
log.cl_error("failed to get the UUID on localhost")
return -1
if uuid_local == uuid_install:
log.cl_error("please do NOT use host [%s] as the install server, "
"since it is the localhost, and installation test "
"would overwrite the local configuration files",
install_server.sh_hostname)
return -1
ret = install_server.sh_rpm_find_and_uninstall(log, "grep pylcommon")
if ret:
log.cl_error("failed to uninstall pylcommon rpms on host [%s]",
install_server.sh_hostname)
return -1
ret = install_server.sh_rpm_find_and_uninstall(log, "grep clownfish")
if ret:
log.cl_error("failed to uninstall Clownfish rpms on host [%s]",
install_server.sh_hostname)
return -1
package_dir = mnt_path + "/" + cstr.CSTR_PACKAGES
command = ("rpm -ivh %s/clownfish-pylcommon-*.x86_64.rpm "
"%s/clownfish-1.*.x86_64.rpm --nodeps" %
(package_dir, package_dir))
retval = install_server.sh_run(log, command)
if retval.cr_exit_status:
log.cl_error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
install_server.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
cmd_name, config_fname = arg
install_config_fpath = (workspace + "/" + config_fname)
config_string = ("""#
# Configuration file for installing %s from DDN
#
""" % (cmd_name))
config_string += yaml.dump(install_config, Dumper=lyaml.YamlDumper,
default_flow_style=False)
try:
with open(install_config_fpath, 'w') as yaml_file:
yaml_file.write(config_string)
except:
log.cl_error("failed to save the config file to [%s]")
return -1
ret = install_server.sh_send_file(log, install_config_fpath, "/etc")
if ret:
log.cl_error("failed to send file [%s] on local host to "
"/etc on host [%s]",
install_config_fpath,
install_server.sh_hostname)
return -1
args = {}
args["log"] = log
args["hostname"] = install_server.sh_hostname
stdout_file = (workspace + "/" + cmd_name + "_install.stdout")
stderr_file = (workspace + "/" + cmd_name + "_install.stderr")
stdout_fd = watched_io.watched_io_open(stdout_file,
watched_io.log_watcher_info, args)
stderr_fd = watched_io.watched_io_open(stderr_file,
watched_io.log_watcher_error, args)
command = ("%s_install" % (cmd_name))
retval = install_server.sh_run(log, command, stdout_tee=stdout_fd,
stderr_tee=stderr_fd, return_stdout=False,
return_stderr=False, timeout=None,
flush_tee=True)
stdout_fd.close()
stderr_fd.close()
if retval.cr_exit_status:
log.cl_error("failed to run command [%s] on host [%s], "
"ret = [%d]",
command,
install_server.sh_hostname,
retval.cr_exit_status)
return -1
return 0
def start_install(log, workspace, install_server,
install_config, config_fpath, arg):
"""
Start installation
"""
# pylint: disable=too-many-locals,too-many-arguments
command = "mkdir -p %s" % workspace
retval = install_server.sh_run(log, command)
if retval.cr_exit_status:
log.cl_error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
install_server.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
local_host = ssh_host.SSHHost("localhost", local=True)
command = "ls clownfish-*.iso"
retval = local_host.sh_run(log, command)
if retval.cr_exit_status:
log.cl_error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
local_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
current_dir = os.getcwd()
iso_names = retval.cr_stdout.split()
if len(iso_names) != 1:
log.cl_error("found unexpected ISOs [%s] under currect directory "
"[%s]", iso_names, current_dir)
return -1
iso_name = iso_names[0]
iso_path = current_dir + "/" + iso_name
ret = install_server.sh_send_file(log, config_fpath, workspace)
if ret:
log.cl_error("failed to send Clownfish config [%s] on local host to "
"directory [%s] on host [%s]",
config_fpath, workspace,
install_server.sh_hostname)
return -1
config_fname = os.path.basename(config_fpath)
ret = install_server.sh_send_file(log, iso_path, workspace)
if ret:
log.cl_error("failed to send Clownfish ISO [%s] on local host to "
"directory [%s] on host [%s]",
iso_path, workspace,
install_server.sh_hostname)
return -1
host_iso_path = workspace + "/" + iso_name
host_config_fpath = workspace + "/" + config_fname
install_config[cstr.CSTR_ISO_PATH] = host_iso_path
install_config[cstr.CSTR_CONFIG_FPATH] = host_config_fpath
ret = mount_and_run(log, workspace, install_server, host_iso_path,
install_config, _start_install, arg)
if ret:
log.cl_error("failed to test installation on host [%s]",
install_server.sh_hostname)
return -1
return 0
def do_build(log, source_dir, config, config_fpath):
"""
Build the ISO
"""
# pylint: disable=too-many-return-statements,too-many-locals
# pylint: disable=unused-argument
log.cl_info("building using config [%s]", config_fpath)
local_host = ssh_host.SSHHost("localhost", local=True)
distro = local_host.sh_distro(log)
if distro != ssh_host.DISTRO_RHEL7:
log.cl_error("build can only be launched on RHEL7/CentOS7 host")
return -1
iso_cached_dir = source_dir + "/../iso_cached_dir"
command = ("mkdir -p %s" % iso_cached_dir)
retval = local_host.sh_run(log, command)
if retval.cr_exit_status:
log.cl_error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status, retval.cr_stdout,
retval.cr_stderr)
return -1
package_dir = iso_cached_dir + "/" + cstr.CSTR_PACKAGES
ret = prepare_yum_repository(log, local_host, distro)
if ret:
log.cl_error("failed to prepare yum repository")
return -1
ret = download_dependent_rpms(log, local_host, package_dir)
if ret:
log.cl_error("failed to download dependent rpms")
return -1
contents = [cstr.CSTR_PACKAGES]
ret = check_dir_content(log,
local_host,
iso_cached_dir,
contents,
cleanup=True)
if ret:
log.cl_error("directory [%s] doesn't have expected content",
iso_cached_dir)
return -1
log.cl_info("building Clownfish ISO")
command = ("cd %s && rm clownfish-*.tar.bz2 clownfish-*.tar.gz -f && "
"sh autogen.sh && "
"./configure --with-cached-iso=%s && "
"make && "
"make iso" % (source_dir, iso_cached_dir))
retval = local_host.sh_run(log, command)
if retval.cr_exit_status:
log.cl_error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status, retval.cr_stdout,
retval.cr_stderr)
return -1
return 0
def cd_worker_thread(self, worker_index):
"""
Worker routine
"""
# pylint: disable=too-many-nested-blocks,too-many-locals
# pylint: disable=too-many-branches,too-many-statements
# Socket to talk to dispatcher
name = "thread_worker_%s" % worker_index
thread_workspace = self.cd_workspace + "/" + name
if not os.path.exists(thread_workspace):
ret = utils.mkdir(thread_workspace)
if ret:
self.cd_log.cl_error(
"failed to create directory [%s] on local host",
thread_workspace)
return -1
elif not os.path.isdir(thread_workspace):
self.cd_log.cl_error("[%s] is not a directory", thread_workspace)
return -1
log = self.cd_log.cl_get_child(name, resultsdir=thread_workspace)
log.cl_info("starting worker thread [%s]", worker_index)
dispatcher_socket = self.cd_context.socket(zmq.REP)
dispatcher_socket.connect(self.cd_url_worker)
hostname = socket.gethostname()
host = ssh_host.SSHHost(hostname, local=True)
while self.cd_running:
try:
request_message = dispatcher_socket.recv()
except zmq.ContextTerminated:
log.cl_info(
"worker thread [%s] exiting because context has "
"been terminated", worker_index)
break
cmessage = copytoold_pb2.CopytooldMessage
request = cmessage()
request.ParseFromString(request_message)
log.cl_debug("received request with type [%s]", request.cm_type)
reply = cmessage()
reply.cm_protocol_version = cmessage.CPV_ZERO
reply.cm_errno = cmessage.CE_NO_ERROR
if request.cm_type == cmessage.CMT_START_REQUEST:
source = request.cm_start_request.csr_source
dest = request.cm_start_request.csr_dest
log.cl_info(
"received a start request of copytool from [%s] to [%s]",
source, dest)
ret = start_copytool(log, host, source, dest)
if ret:
reply.cm_errno = cmessage.CE_OPERATION_FAILED
reply.cm_type = cmessage.CMT_START_REPLY
else:
reply.cm_type = cmessage.CMT_GENERAL
reply.cm_errno = cmessage.CE_NO_TYPE
log.cl_error(
"received a request with type [%s] that "
"is not supported", request.cm_type)
reply_message = reply.SerializeToString()
dispatcher_socket.send(reply_message)
log.cl_info(
"send reply to a start request of copytool from [%s] to [%s]",
source, dest)
dispatcher_socket.close()
log.cl_info("worker thread [%s] exited", worker_index)
def _iso_mount_and_install(log, workspace, config, config_fpath,
install_funct):
"""
Mount the ISO and install the system
"""
# pylint: disable=bare-except
local_host = ssh_host.SSHHost("localhost", local=True)
fname = utils.config_value(config, cstr.CSTR_ISO_PATH)
if fname is None:
fname = "clownfish-*.iso"
iso_path = install_common.find_iso_path_in_cwd(log, local_host, fname)
if iso_path is None:
log.cl_error(
"failed to find Clownfish ISO [%s] under currect "
"directory", fname)
return -1
log.cl_info(
"no [%s] is configured, use [%s] under current "
"directory", cstr.CSTR_ISO_PATH, iso_path)
else:
# ISO could have wild card, find that
iso_path = install_common.find_iso_path_in_cwd(log, local_host, fname)
if iso_path is None:
log.cl_error(
"failed to find Clownfish ISO [%s] under currect "
"directory", fname)
return -1
mnt_path = "/mnt/" + utils.random_word(8)
command = ("mkdir -p %s && mount -o loop %s %s" %
(mnt_path, iso_path, mnt_path))
retval = local_host.sh_run(log, command)
if retval.cr_exit_status:
log.cl_error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status, retval.cr_stdout,
retval.cr_stderr)
return -1
try:
ret = install_funct(log, workspace, config, config_fpath, mnt_path,
iso_path, local_host)
except:
ret = -1
log.cl_error("exception: %s", traceback.format_exc())
command = ("umount %s" % (mnt_path))
retval = local_host.sh_run(log, command)
if retval.cr_exit_status:
log.cl_error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status, retval.cr_stdout,
retval.cr_stderr)
ret = -1
command = ("rmdir %s" % (mnt_path))
retval = local_host.sh_run(log, command)
if retval.cr_exit_status:
log.cl_error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status, retval.cr_stdout,
retval.cr_stderr)
return -1
return ret
def clownfish_parse_server_hosts(log, config, config_fpath):
"""
Return the server hosts of clownfish
"""
ssh_host_configs = utils.config_value(config, cstr.CSTR_SSH_HOSTS)
if ssh_host_configs is None:
log.cl_error("can NOT find [%s] in the config file, "
"please correct file [%s]",
cstr.CSTR_SSH_HOSTS, config_fpath)
return None
server_config = utils.config_value(config, cstr.CSTR_CLOWNFISH_SERVER)
if server_config is None:
log.cl_error("no [%s] is configured, please correct file [%s]",
cstr.CSTR_CLOWNFISH_SERVER, config_fpath)
return None
hosts = {}
for host_config in ssh_host_configs:
host_id = host_config[cstr.CSTR_HOST_ID]
if host_id is None:
log.cl_error("can NOT find [%s] in the config of a "
"SSH host, please correct file [%s]",
cstr.CSTR_HOST_ID, config_fpath)
return None
hostname = utils.config_value(host_config, cstr.CSTR_HOSTNAME)
if hostname is None:
log.cl_error("can NOT find [%s] in the config of SSH host "
"with ID [%s], please correct file [%s]",
cstr.CSTR_HOSTNAME, host_id, config_fpath)
return None
ssh_identity_file = utils.config_value(host_config, cstr.CSTR_SSH_IDENTITY_FILE)
if host_id in hosts:
log.cl_error("multiple SSH hosts with the same ID [%s], please "
"correct file [%s]", host_id, config_fpath)
return None
host = ssh_host.SSHHost(hostname,
identity_file=ssh_identity_file,
host_id=host_id)
hosts[host_id] = host
server_hosts = utils.config_value(server_config, cstr.CSTR_SERVER_HOSTS)
if server_hosts is None:
log.cl_error("can NOT find [%s] in the config file, "
"please correct file [%s]",
cstr.CSTR_SERVER_HOSTS, config_fpath)
return None
clownfish_hosts = []
for host_config in server_hosts:
host_id = host_config[cstr.CSTR_HOST_ID]
if host_id is None:
log.cl_error("can NOT find [%s/%s] in the config of a "
"SSH host, please correct file [%s]",
cstr.CSTR_HOST_ID, cstr.CSTR_SERVER_HOSTS,
config_fpath)
return None
if host_id not in hosts:
log.cl_error("host with host id is not configured,"
"please correct file [%s]", config_fpath)
return None
clownfish_hosts.append(hosts[host_id])
return clownfish_hosts