def configure_default_port(options):
from log import logger
from installer_io import InstallerIO
from configParser import ConfigParser
from text import text
io = InstallerIO(options.silent)
config_file = os.path.join(options.chorus_path, "shared/chorus.properties")
chorus_config = ConfigParser(config_file)
alpine_config_file = os.path.join(options.chorus_path, "shared/ALPINE_DATA_REPOSITORY/configuration/deploy.properties")
alpine_config = ConfigParser(alpine_config_file)
ports = ["server_port", "solr_port"]
menu = "\n".join(str(i+1) + ". %s: [default: %s]" % (ports[i], chorus_config[ports[i]]) for i in xrange(0, len(ports)))
menu += "\n"
alpine_ports = ["alpine_port"]
menu += "\n".join(str(len(ports)+i+1) + ". %s: [default: %s]" % (alpine_ports[i], alpine_config[alpine_ports[i].replace("_", ".")]) \
for i in xrange(0, len(alpine_ports)))
menu += "\n%d. exit" % (len(ports) + len(alpine_ports) + 1)
num = io.require_menu(text.get("interview_question", "port_menu") % menu,
range(1, len(ports)+len(alpine_ports)+2), default=len(ports)+len(alpine_ports)+1)
if num in range(1, len(ports)+1):
new_port = io.prompt_int(text.get("interview_question", "change_port") % ports[num-1], default=int(chorus_config[ports[num-1]]))
chorus_config[ports[num-1]] = new_port
chorus_config.write(config_file)
logger.info("%s has successfully changed to %d" % (ports[num-1], new_port))
elif num in range(len(ports)+1, len(ports)+1+len(alpine_ports)):
new_port = io.prompt_int(text.get("interview_question", "change_port") % alpine_ports[num-len(ports)-1], default=int(alpine_config[alpine_ports[num-len(ports)-1].replace("_", ".")]))
alpine_config[alpine_ports[num-len(ports)-1].replace("_", ".")] = new_port
alpine_config.write(alpine_config_file)
chorus_config["workflow.url"] = "http://%s:%d" % (alpine_config["alpine.host"], new_port)
chorus_config.write(config_file)
logger.info("%s has successfully changed to %d" % (alpine_ports[num-len(ports)-1], new_port))
def configure_default_port(options):
from log import logger
from installer_io import InstallerIO
from configParser import ConfigParser
from text import text
io = InstallerIO(options.silent)
config_file = os.path.join(options.chorus_path, "shared/chorus.properties")
chorus_config = ConfigParser(config_file)
alpine_config_file = os.path.join(
options.chorus_path,
"shared/ALPINE_DATA_REPOSITORY/configuration/deploy.properties")
alpine_config = ConfigParser(alpine_config_file)
ports = ["server_port", "solr_port"]
menu = "\n".join(
str(i + 1) + ". %s: [default: %s]" %
(ports[i], chorus_config[ports[i]]) for i in xrange(0, len(ports)))
menu += "\n"
alpine_ports = ["alpine_port"]
menu += "\n".join(str(len(ports)+i+1) + ". %s: [default: %s]" % (alpine_ports[i], alpine_config[alpine_ports[i].replace("_", ".")]) \
for i in xrange(0, len(alpine_ports)))
menu += "\n%d. exit" % (len(ports) + len(alpine_ports) + 1)
num = io.require_menu(text.get("interview_question", "port_menu") % menu,
range(1,
len(ports) + len(alpine_ports) + 2),
default=len(ports) + len(alpine_ports) + 1)
if num in range(1, len(ports) + 1):
new_port = io.prompt_int(
text.get("interview_question", "change_port") % ports[num - 1],
default=int(chorus_config[ports[num - 1]]))
chorus_config[ports[num - 1]] = new_port
chorus_config.write(config_file)
logger.info("%s has successfully changed to %d" %
(ports[num - 1], new_port))
elif num in range(len(ports) + 1, len(ports) + 1 + len(alpine_ports)):
new_port = io.prompt_int(
text.get("interview_question", "change_port") %
alpine_ports[num - len(ports) - 1],
default=int(alpine_config[alpine_ports[num - len(ports) -
1].replace("_", ".")]))
alpine_config[alpine_ports[num - len(ports) - 1].replace(
"_", ".")] = new_port
alpine_config.write(alpine_config_file)
chorus_config["workflow.url"] = "http://%s:%d" % (
alpine_config["alpine.host"], new_port)
chorus_config.write(config_file)
logger.info("%s has successfully changed to %d" %
(alpine_ports[num - len(ports) - 1], new_port))
def configure_alpine(self):
@processify(msg=text.get("step_msg", "setting_up_alpine") %
self.alpine_version,
interval=1.5)
def configure():
logger.debug("Extracting %s to %s" %
(self.alpine_installer, self.alpine_release_path))
ret, stdout, stderr = self.executor.run(
"sh %s --target %s --noexec" %
(self.alpine_installer, self.alpine_release_path))
if ret != 0:
raise Exception(stderr)
logger.debug("Preparing Alpine Data Repository")
alpine_data_repo = os.path.join(self.options.chorus_path,
"shared/ALPINE_DATA_REPOSITORY")
if os.path.exists(alpine_data_repo):
logger.debug("Alpine Data Repository existed, skipped")
#if not os.path.exists(os.path.join(alpine_data_repo, "configuration/hadoop_version.properties")):
self._cp_f(os.path.join(self.alpine_release_path, "ALPINE_DATA_REPOSITORY/configuration/hadoop_version.properties"),\
os.path.join(alpine_data_repo, "configuration/hadoop_version.properties"))
migrate_alpine_conf(os.path.join(alpine_data_repo, "configuration/alpine.conf"), \
os.path.join(self.alpine_release_path, "ALPINE_DATA_REPOSITORY/configuration/alpine.conf"))
else:
shutil.copytree(
os.path.join(self.alpine_release_path,
"ALPINE_DATA_REPOSITORY"), alpine_data_repo)
configure()
def c_check_java_version(install_mode=False, user=None):
executor = ChorusExecutor()
@processify(msg=text.get("step_msg", "check_java"))
def check():
if install_mode and user is not None:
ret, stdout, stderr = executor.run("su - %s -c 'echo $JAVA_HOME'" % user)
if ret == 0 and stdout.strip("\n") != "":
command = "su - %s -c '$JAVA_HOME/bin/java -version 2>&1'" % user
else:
command = "su - %s -c 'java -version 2>&1'" % user
else:
if os.getenv("JAVA_HOME") == None:
command = "java -version 2>&1"
else:
command = "$JAVA_HOME/bin/java -version 2>&1"
ret, stdout, stderr = executor.run(command)
if "command not found" in stdout:
raise Exception("no java installed, please install oracle jdk")
elif "openjdk" in stdout.lower():
raise Exception("openjdk not supported, please install oracle jdk!")
else:
java_version = None
for line in stdout.split("\n"):
if line.startswith("java version"):
java_version = line
break
if java_version is None:
raise Exception("java version not detected, please install oracle jdk")
java_version = java_version.split(" ")[2].strip("\"").split(".")[0:2]
java_version = float(".".join(java_version))
if java_version < 1.6:
raise Exception("%s\n only support java version > 1.6, please upgrade" % stdout)
check()
def b_check_runing_user(install_mode=False):
@processify(msg=text.get("step_msg", "check_user"))
def check():
if os.getuid() == 0 and install_mode:
raise Exception("Please don't run this program as root")
check()
def configure_alpine(self):
@processify(msg=text.get("step_msg", "setting_up_alpine") % self.alpine_version, interval=1.5)
def configure():
logger.debug("Extracting %s to %s" % (self.alpine_installer, self.alpine_release_path))
ret, stdout, stderr = self.executor.run(
"sh %s --target %s --noexec" % (self.alpine_installer, self.alpine_release_path)
)
if ret != 0:
raise Exception(stderr)
logger.debug("Preparing Alpine Data Repository")
alpine_data_repo = os.path.join(self.options.chorus_path, "shared/ALPINE_DATA_REPOSITORY")
if os.path.exists(alpine_data_repo):
logger.debug("Alpine Data Repository existed, skipped")
# if not os.path.exists(os.path.join(alpine_data_repo, "configuration/hadoop_version.properties")):
self._cp_f(
os.path.join(
self.alpine_release_path, "ALPINE_DATA_REPOSITORY/configuration/hadoop_version.properties"
),
os.path.join(alpine_data_repo, "configuration/hadoop_version.properties"),
)
migrate_alpine_conf(
os.path.join(alpine_data_repo, "configuration/alpine.conf"),
os.path.join(self.alpine_release_path, "ALPINE_DATA_REPOSITORY/configuration/alpine.conf"),
)
else:
shutil.copytree(os.path.join(self.alpine_release_path, "ALPINE_DATA_REPOSITORY"), alpine_data_repo)
configure()
def enable_https(options):
from installer_io import InstallerIO
from chorus_executor import ChorusExecutor
from log import logger
from configParser import ConfigParser
from text import text
io = InstallerIO(options.silent)
config_file = os.path.join(options.chorus_path, "shared/chorus.properties")
chorus_config = ConfigParser(config_file)
if chorus_config.has_key("ssl.enabled") and chorus_config["ssl.enabled"].lower() == "true":
if not io.require_confirmation(text.get("interview_question", "https_question"), default="no"):
return
executor = ChorusExecutor(options.chorus_path)
server_key = os.path.join(os.path.join(options.chorus_path, "shared/server.key"))
server_csr = os.path.join(os.path.join(options.chorus_path, "shared/server.csr"))
executor.run("openssl genrsa -des3 -out %s 1024" % server_key)
ret = executor.call("openssl req -new -key %s -out %s" % (server_key, server_csr))
if ret != 0:
logger.error("failed to enable https, try again.")
return
server_key_org = os.path.join(os.path.join(options.chorus_path, "shared/server.key.org"))
executor.run("cp %s %s" % (server_key, server_key_org))
executor.run("openssl rsa -in %s -out %s" % (server_key_org, server_key))
server_crt = os.path.join(os.path.join(options.chorus_path, "shared/server.crt"))
executor.run("openssl x509 -req -days 365 -in %s -signkey %s -out %s" % \
(server_csr, server_key, server_crt))
port = io.prompt_int(text.get("interview_question", "https_port"), default=8443)
chorus_config["ssl.enabled"] = "true"
chorus_config["ssl_server_port"] = port
chorus_config["ssl_certificate"] = server_crt
chorus_config["ssl_certificate_key"] = server_key
chorus_config.write(config_file)
alpine_conf = os.path.join(options.chorus_path, "shared/ALPINE_DATA_REPOSITORY/configuration/alpine.conf")
with open(alpine_conf, "r") as f:
contents = f.read()
content = re.findall(r"chorus *{(.*?)}", contents, re.DOTALL)[0]
replace = "active = true\n" + "scheme = HTTPS\n" + "port = %d\n" % port
contents = contents.replace(content, replace)
with open(alpine_conf, "w") as f:
f.write(contents)
logger.info("https has been configured successfully on port %d" % port)
def d_check_disk_space(dir):
@processify(msg=text.get("step_msg", "check_disk"))
def check():
stat = os.statvfs(dir)
free_space = stat.f_frsize*stat.f_bavail / (1024 * 1024 * 1024)
logger.debug("free space in %s: %d" % (dir, free_space))
if free_space < 6:
raise Exception("Not enough space left in %s" % dir)
check()
def d_check_disk_space(dir):
@processify(msg=text.get("step_msg", "check_disk"))
def check():
stat= os.statvfs(dir)
free_space = stat.f_frsize*stat.f_bavail / (1024 * 1024 * 1024)
logger.debug("free space in %s: %d" % (dir, free_space))
if free_space < 6:
raise Exception("Not enough space left in %s" % dir)
check()
def a_check_os_system():
@processify(msg=text.get("step_msg", "check_os"))
def check():
os_name, version, release = platform.linux_distribution()
if os_distribution(os_name) == "redhat" and re.match(r"^[5|6|7]", version):
logger.debug("os version %s-%s-%s" % (os_name, version, release))
elif os_distribution(os_name) == "suse" and version.startswith("11"):
logger.debug("os version %s-%s-%s" % (os_name, version, release))
else:
raise Exception("os version %s-%s-%s not supported!" % (os_name, version, release))
check()
def e_check_free_memory():
executor = ChorusExecutor()
@processify(msg=text.get("step_msg", "check_free_memory"))
def check():
ret, stdout, stderr = executor.run("free -m |awk 'FNR == 2 {print $2}'")
stdout = stdout.strip()
if ret == 0 and stdout.isdigit():
logger.debug("total memory is %s" % stdout)
if int(stdout) < 8192:
raise Exception("Not enough free memory, need at least 8192M free memory")
check()
def e_check_free_memory():
executor = ChorusExecutor()
@processify(msg=text.get("step_msg", "check_free_memory"))
def check():
ret, stdout, stderr = executor.run("free -m |awk 'FNR == 2 {print $2}'")
stdout = stdout.strip()
if ret == 0 and stdout.isdigit():
logger.debug("total memory is %s" % stdout)
if int(stdout) < 8192:
raise Exception("Not enough free memory, need at least 8192M free memory")
check()
def a_check_os_system():
@processify(msg=text.get("step_msg", "check_os"))
def check():
os_name, version, release = platform.linux_distribution()
if os_distribution(os_name) == "redhat" and re.match(r"^[5|6|7]", version):
logger.debug("os version %s-%s-%s" % (os_name, version, release))
elif os_distribution(os_name) == "suse" and version.startswith("11"):
logger.debug("os version %s-%s-%s" % (os_name, version, release))
else:
raise Exception("os version %s-%s-%s not supported!" % (os_name, version, release))
check()
def system_checking(install_mode=False, chorus_path=None, user=None):
logger.info(bold(text.get("step_msg", "health_check")))
for func in _load_configure_func().values():
if func[0] == "b_check_running_user":
func[1](install_mode=install_mode)
elif func[0] == "c_check_java_version":
func[1](install_mode=install_mode, user=user)
elif func[0] == "d_check_disk_space":
if install_mode:
func[1](chorus_path)
else:
func[1]()
def system_checking(install_mode=False, chorus_path=None, user=None):
logger.info(bold(text.get("step_msg", "health_check")))
for func in _load_configure_func().values():
if func[0] == "b_check_running_user":
func[1](install_mode=install_mode)
elif func[0] == "c_check_java_version":
func[1](install_mode=install_mode, user=user)
elif func[0] == "d_check_disk_space":
if install_mode:
func[1](chorus_path)
else:
func[1]()
def update_chorus_caching(options):
from log import logger
from text import text
from chorus_executor import ChorusExecutor
from func_executor import processify
@processify(msg=text.get("step_msg", "update_caching"), interval=1.5)
def run():
executor = ChorusExecutor(options.chorus_path)
executor.start_postgres()
executor.rake("chorus:caching")
executor.stop_postgres()
run()
def enable_alpine_agent(options):
from log import logger
from installer_io import InstallerIO
from text import text
from helper import get_agents
io = InstallerIO(options.silent)
alpine_conf = os.path.join(
options.chorus_path,
"shared/ALPINE_DATA_REPOSITORY/configuration/alpine.conf")
agent_dic = get_agents(alpine_conf)
with open(alpine_conf, "r") as f:
contents = f.read()
agents_str = "\n".join(
str(key + 1) + ". " + agent_dic[key][1] + " " + agent_dic[key][2]
for key in range(3, len(agent_dic)))
agents_str += "\n%d. exit" % (len(agent_dic) + 1)
agents = io.require_selection(
text.get("interview_question", "alpine_agent_menu") % agents_str,
range(4,
len(agent_dic) + 2),
default=[4])
if (len(agent_dic) + 1) in agents:
return
for i in range(1, len(agent_dic) + 1):
contents = re.sub("hadoop.version.%s.agents.%s.enabled=[a-z]+" % (agent_dic[i-1][1], agent_dic[i-1][0]),\
"hadoop.version.%s.agents.%s.enabled=%s" % (agent_dic[i-1][1], agent_dic[i-1][0], str(i in agents).lower()),
contents)
with open(alpine_conf, "w") as f:
f.write(contents)
logger.info(
str([agent_dic[agent - 1][1] for agent in agents]) + " is enabled.")
logger.info(text.get("status_msg", "enable_agent_post_step") % alpine_conf)
def update_chorus_caching(options):
from log import logger
from text import text
from chorus_executor import ChorusExecutor
from func_executor import processify
@processify(msg=text.get("step_msg", "update_caching"), interval=1.5)
def run():
executor = ChorusExecutor(options.chorus_path)
executor.start_postgres()
executor.rake("chorus:caching")
executor.stop_postgres()
run()
def enable_alpine_agent(options):
from log import logger
from installer_io import InstallerIO
from text import text
from helper import get_agents
io = InstallerIO(options.silent)
alpine_conf = os.path.join(options.chorus_path, "shared/ALPINE_DATA_REPOSITORY/configuration/alpine.conf")
agent_dic = get_agents(alpine_conf)
with open(alpine_conf, "r") as f:
contents = f.read()
agents_str = "\n".join(
str(key + 1) + ". " + agent_dic[key][1] + " " + agent_dic[key][2] for key in range(0, len(agent_dic))
)
agents_str += "\n%d. exit" % (len(agent_dic) + 1)
agents = io.require_selection(
text.get("interview_question", "alpine_agent_menu") % agents_str, range(1, len(agent_dic) + 2), default=[4]
)
if (len(agent_dic) + 1) in agents:
return
for i in range(1, len(agent_dic) + 1):
contents = re.sub(
"hadoop.version.%s.agents.%s.enabled=[a-z]+" % (agent_dic[i - 1][1], agent_dic[i - 1][0]),
"hadoop.version.%s.agents.%s.enabled=%s"
% (agent_dic[i - 1][1], agent_dic[i - 1][0], str(i in agents).lower()),
contents,
)
with open(alpine_conf, "w") as f:
f.write(contents)
logger.info(str([agent_dic[agent - 1][1] for agent in agents]) + " is enabled.")
logger.info(text.get("status_msg", "enable_agent_post_step") % alpine_conf)
def main():
try:
signal.signal(signal.SIGINT, exit_gracefully)
if (arg == "setup" and not options.disable_spec) \
or (arg == "health_check" and (health_args == [] or "checkos" in health_args)):
system_checking()
if arg == "health_check":
handler[arg](" ".join(health_args))
else:
handler[arg](options, is_upgrade)
except Exception as e:
logger.error(error(str(e) + "\n" + text.get("error_msg", "exception") % log_path))
logger.debug(traceback.format_exc())
failover(options.chorus_path, options.data_path, is_upgrade)
exit(1)
def main():
try:
signal.signal(signal.SIGINT, exit_gracefully)
if (arg == "setup" and not options.disable_spec) \
or (arg == "health_check" and (health_args == [] or "checkos" in health_args)):
system_checking()
if arg == "health_check":
handler[arg](" ".join(health_args))
else:
handler[arg](options, is_upgrade)
except Exception as e:
logger.error(
error(
str(e) + "\n" + text.get("error_msg", "exception") % log_path))
logger.debug(traceback.format_exc())
failover(options.chorus_path, options.data_path, is_upgrade)
exit(1)
def c_check_java_version(install_mode=False, user=None):
executor = ChorusExecutor()
@processify(msg=text.get("step_msg", "check_java"))
def check():
if install_mode and user is not None:
ret, stdout, stderr = executor.run("su - %s -c 'echo $JAVA_HOME'" %
user)
if ret == 0 and stdout.strip("\n") != "":
command = "su - %s -c '$JAVA_HOME/bin/java -version 2>&1'" % user
else:
command = "su - %s -c 'java -version 2>&1'" % user
else:
if os.getenv("JAVA_HOME") == None:
command = "java -version 2>&1"
else:
command = "$JAVA_HOME/bin/java -version 2>&1"
ret, stdout, stderr = executor.run(command)
if "command not found" in stdout:
raise Exception("no java installed, please install oracle jdk")
elif "openjdk" in stdout.lower():
raise Exception(
"openjdk not supported, please install oracle jdk!")
else:
java_version = None
for line in stdout.split("\n"):
if line.startswith("java version"):
java_version = line
break
if java_version is None:
raise Exception(
"java version not detected, please install oracle jdk")
java_version = java_version.split(" ")[2].strip("\"").split(
".")[0:2]
java_version = float(".".join(java_version))
if java_version < 1.6:
raise Exception(
"%s\n only support java version > 1.6, please upgrade" %
stdout)
check()
def setup(self, options, is_upgrade):
self.set_path(options)
# if not io.require_confirmation("Do you want to set up the chorus, "
# + "please make sure you have installed chorus before setting up?"):
# logger.fatal("Setup aborted, Cancelled by user")
# quit()
# if not self.options.disable_spec:
# health_check()
# self.prompt_for_eula()
# pre step:
passphrase = self.get_passphrase()
logger.debug("Construct Chorus and Data Directory:")
self.construct_shared_structure()
self.construct_data_structure()
self.link_shared_config()
self.link_data_folder()
self.generate_paths_file()
self.configure_secret_key(passphrase)
self.configure_secret_token()
if is_upgrade:
logger.info(bold(text.get("step_msg", "update_database")))
else:
logger.info(bold(text.get("step_msg", "create_database")))
self.extract_postgres()
if is_upgrade:
self.validate_data_sources()
self.stop_previous_release()
self.upgrade_database()
else:
self.create_database_config()
self.generate_chorus_psql_files()
self.generate_chorus_rails_console_file()
self.setup_database()
# self.enqueue_solr_reindex()
# self.clean_up_old_releases()
self.link_current_to_release("current", self.release_path)
if self.is_alpine_exits() and self.options.chorus_only is False:
logger.info(bold(text.get("step_msg", "install_alpine")))
self.configure_alpine()
self.link_current_to_release("alpine-current", self.alpine_release_path)
self.source_chorus_path()
print "*" * 60
print text.get("status_msg", "success_install") % (
self.options.chorus_path,
self.options.data_path,
self.chorus_version,
self.alpine_version,
)
print "*" * 60
# disable it, health check is not good
# if self.io.require_confirmation(text.get("interview_question", "do_health_check"), default="no"):
# health_check()
if self.io.require_confirmation(text.get("interview_question", "do_configure"), default="no"):
configure.config(self.options, is_upgrade)
print bold(text.get("status_msg", "setup_complete"))
if is_upgrade:
print text.get("status_msg", "upgrade")
print "*" * 60
print text.get("status_msg", "setup_post_step") % pwd.getpwuid(os.getuid()).pw_name
print "*" * 60
def config(self, options, is_upgrade):
self.options = options
self.io = InstallerIO(options.silent)
self._version_detect()
print "*" * 60
header = ""
if self.chorus_version is None:
print text.get("status_msg", "no_chorus")
else:
print text.get("status_msg", "chorus_status") % (
self.chorus_version, self.get_chorus_state())
if self.alpine_version is None:
print text.get("status_msg", "no_alpine")
else:
print text.get("status_msg", "alpine_status") % (
self.alpine_version, self.get_alpine_state())
print "CHORUS_HOME:\t%s" % os.getenv("CHORUS_HOME",
"not set in ~/.bashrc")
print "*" * 60
if self.chorus_version is None:
return
self.method = self._load_configure_func()
not_experiment_method = [
"enable_alpine_agent", "role_permission_migrate"
]
while True:
menu = ""
lens = len(self.method) + 1
for e in self.method.keys():
if self.method[e][0] not in not_experiment_method:
menu += str(
e) + ". " + self.method[e][0] + " (experimental) \n"
else:
menu += str(e) + ". " + self.method[e][0] + "\n"
menu += "%d. exit" % lens
selection = self.io.require_menu(
text.get("interview_question", "configuration_menu") % menu,
range(1, lens + 1),
default=lens)
if selection == lens:
break
self.method[selection][1](options)
if self.io.require_confirmation(text.get("interview_question",
"back_to_menu"),
default="no"):
continue
else:
break
print "*" * 60
print text.get("status_msg", "configure_post_step")
print "*" * 60
def setup(self, options, is_upgrade):
self.set_path(options)
#if not io.require_confirmation("Do you want to set up the chorus, "
# + "please make sure you have installed chorus before setting up?"):
# logger.fatal("Setup aborted, Cancelled by user")
# quit()
#if not self.options.disable_spec:
# health_check()
#self.prompt_for_eula()
#pre step:
passphrase = self.get_passphrase()
logger.debug("Construct Chorus and Data Directory:")
self.construct_shared_structure()
self.construct_data_structure()
self.link_shared_config()
self.link_data_folder()
self.generate_paths_file()
self.configure_secret_key(passphrase)
self.configure_secret_token()
if is_upgrade:
logger.info(bold(text.get("step_msg", "update_database")))
else:
logger.info(bold(text.get("step_msg", "create_database")))
self.extract_postgres()
if is_upgrade:
self.validate_data_sources()
self.stop_previous_release()
self.upgrade_database()
self.clear_tmp_directory()
else:
self.create_database_config()
self.generate_chorus_psql_files()
self.setup_database()
#self.enqueue_solr_reindex()
self.generate_chorus_rails_console_file()
#self.clean_up_old_releases()
self.link_current_to_release("current", self.release_path)
if self.is_alpine_exits() and self.options.chorus_only is False:
logger.info(bold(text.get("step_msg", "install_alpine")))
self.configure_alpine()
self.link_current_to_release("alpine-current",
self.alpine_release_path)
self.source_chorus_path()
print "*" * 60
print text.get("status_msg", "success_install") % \
(self.options.chorus_path, self.options.data_path, self.chorus_version, self.alpine_version)
print "*" * 60
#disable it, health check is not good
#if self.io.require_confirmation(text.get("interview_question", "do_health_check"), default="no"):
# health_check()
if self.io.require_confirmation(text.get("interview_question",
"do_configure"),
default="no"):
configure.config(self.options, is_upgrade)
print bold(text.get("status_msg", "setup_complete"))
if is_upgrade:
print text.get("status_msg", "upgrade")
print "*" * 60
print text.get("status_msg", "setup_post_step") % pwd.getpwuid(
os.getuid()).pw_name
print "*" * 60
class ChorusSetup:
"""
chorus installer
"""
def __init__(self):
self.database_username = None
self.database_password = None
self.alpine_installer = None
self.alpine_release_path = None
def set_path(self, options):
failover_file = os.path.join(options.chorus_path, ".failover")
if os.path.exists(failover_file):
os.remove(failover_file)
self.options = options
self.io = InstallerIO(self.options.silent)
self.executor = ChorusExecutor(self.options.chorus_path)
self.chorus_version = get_version(self.options.chorus_path)
self.alpine_version = None
self.release_path = os.path.join(self.options.chorus_path,
'releases/%s' % self.chorus_version)
self.shared = os.path.join(self.options.chorus_path, "shared")
def construct_shared_structure(self):
logger.debug("Construct shared structure in %s" % self.shared)
self._mkdir_p(self.shared)
self._mkdir_p(os.path.join(self.shared, "demo_data"))
self._mkdir_p(os.path.join(self.shared, "libraries"))
self._mkdir_p(os.path.join(self.shared, "solr"))
self._mkdir_p(os.path.join(self.shared, "tmp"))
self._cp_if_not_exist(os.path.join(self.release_path, "packaging/database.yml.example"), \
os.path.join(self.shared, "database.yml"))
self._cp_if_not_exist(os.path.join(self.release_path, "config/chorus.defaults.properties"), \
os.path.join(self.shared, "chorus.properties"))
os.chmod(os.path.join(self.shared, "chorus.properties"), 0600)
self._cp_f(os.path.join(self.release_path, "packaging/sunspot.yml.example"), \
os.path.join(self.shared, "sunspot.yml"))
self._cp_f(os.path.join(self.release_path, "config/chorus.properties.example"), \
os.path.join(self.shared, "chorus.properties.example"))
self._cp_f(os.path.join(self.release_path, "config/chorus.license.default"), \
os.path.join(self.shared, "chorus.license.default"))
self._cp_if_not_exist(os.path.join(self.release_path, "config/chorus.license.default"), \
os.path.join(self.shared, "chorus.license"))
os.chmod(os.path.join(self.shared, "chorus.license"), 0600)
self._cp_f(os.path.join(self.release_path, "config/ldap.properties.active_directory"), \
os.path.join(self.shared, "ldap.properties.active_directory"))
self._cp_f(os.path.join(self.release_path, "config/ldap.properties.opensource_ldap"), \
os.path.join(self.shared, "ldap.properties.opensource_ldap"))
self._cp_if_not_exist(os.path.join(self.release_path, "config/ldap.properties.example"), \
os.path.join(self.shared, "ldap.properties"))
os.chmod(os.path.join(self.shared, "ldap.properties"), 0600)
self._cp_if_not_exist(os.path.join(self.release_path, "config/hadoop_config_fetch_rules.yml"), \
os.path.join(self.shared, "hadoop_config_fetch_rules.yml"))
def construct_data_structure(self):
logger.debug("Construct data structure in %s" % self.options.data_path)
for folder in ["db", "system", "log", "solr/data"]:
self.executor.run("mkdir -p %s" %
os.path.join(self.options.data_path, folder))
def link_shared_config(self):
logger.debug("Linking shared configuration files to %s/config" %
self.release_path)
self._ln_sf(os.path.join(self.shared, "chorus.properties"), \
os.path.join(self.release_path, "config/chorus.properties"))
self._ln_sf(os.path.join(self.shared, "chorus.license"), \
os.path.join(self.release_path, "config/chorus.license"))
self._ln_sf(os.path.join(self.shared, "database.yml"), \
os.path.join(self.release_path, "config/database.yml"))
self._ln_sf(os.path.join(self.shared, "sunspot.yml"), \
os.path.join(self.release_path, "config/sunspot.yml"))
self._ln_sf(os.path.join(self.shared, "hadoop_config_fetch_rules.yml"), \
os.path.join(self.release_path, "config/hadoop_config_fetch_rules.yml"))
self._ln_sf(os.path.join(self.shared, "ldap.properties"), \
os.path.join(self.release_path, "config/ldap.properties"))
self._ln_sf(os.path.join(self.shared, "demo_data"), \
os.path.join(self.release_path, "demo_data"))
self._ln_sf(os.path.join(self.shared, "libraries"), \
os.path.join(self.release_path, "lib/libraries"))
def link_data_folder(self):
logger.debug("Linking data folders to %s" % self.options.data_path)
os.chmod(os.path.join(self.options.data_path, "db"), 0700)
self._ln_sf(os.path.join(self.options.data_path, "db"), \
os.path.join(self.shared, "db"))
self._ln_sf(os.path.join(self.options.data_path, "log"), \
os.path.join(self.shared, "log"))
self._ln_sf(os.path.join(self.options.data_path, "solr/data"), \
os.path.join(self.shared, "solr/data"))
self._ln_sf(os.path.join(self.options.data_path, "system"), \
os.path.join(self.shared, "system"))
self._ln_sf(os.path.join(self.shared, "db"), \
os.path.join(self.release_path, "postgres-db"))
self._ln_sf(os.path.join(self.shared, "log"), \
os.path.join(self.release_path, "log"))
self._ln_sf(os.path.join(self.shared, "solr/data"), \
os.path.join(self.release_path, "solr/data"))
self._ln_sf(os.path.join(self.shared, "tmp"), \
os.path.join(self.release_path, "tmp"))
self._ln_sf(os.path.join(self.shared, "system"), \
os.path.join(self.release_path, "system"))
logger.debug(
"Linking nginx logs to %s/vendor/nginx/nginx_dist/nginx_data/logs"
% self.release_path)
self._mkdir_p(os.path.join(self.shared, "log/nginx"))
self._ln_sf(os.path.join(self.shared, "log/nginx"), \
os.path.join(self.release_path, "vendor/nginx/nginx_dist/nginx_data/logs"))
self._ln_sf(os.path.join(self.release_path, "packaging/chorus_control.sh"), \
os.path.join(self.options.chorus_path, "chorus_control.sh"))
self._ln_sf(os.path.join(self.release_path, "packaging/setup/chorus_server"), \
os.path.join(self.options.chorus_path, "chorus_server"))
self.executor.run("chmod -R 0555 %s" %
os.path.join(self.release_path, "public"))
def extract_postgres(self):
logger.debug("Extract postgres database to %s", self.release_path)
os_name, version, release = platform.linux_distribution()
logger.debug(platform.linux_distribution())
if os_distribution(os_name) == "redhat" and version.startswith("5"):
self.executor.extract_postgres("postgres-redhat5.5-9.2.4.tar.gz")
elif os_distribution(os_name) == "redhat" and (
version.startswith("6") or version.startswith("7")):
self.executor.extract_postgres("postgres-redhat6.2-9.2.4.tar.gz")
elif os_distribution(os_name) == "suse" and version.startswith("11"):
self.executor.extract_postgres("postgres-suse11-9.2.4.tar.gz")
else:
raise Exception(
"postgres not installed, no version match the operation system"
)
def _ln_sf(self, src, dst):
logger.debug("Link %s to %s" % (src, dst))
if os.path.lexists(dst):
if os.path.isdir(dst) and not os.path.islink(dst):
shutil.rmtree(dst)
else:
os.remove(dst)
os.symlink(src, dst)
def _mkdir_p(self, path):
logger.debug("mkdir %s" % path)
if os.path.exists(path):
return
os.makedirs(path)
def _cp_if_not_exist(self, src, dst):
logger.debug("cp %s to %s if not exists" % (src, dst))
if os.path.exists(dst):
return
shutil.copyfile(src, dst)
def _cp_f(self, src, dst):
logger.debug("cp -f %s to %s" % (src, dst))
if os.path.exists(dst):
os.remove(dst)
shutil.copyfile(src, dst)
def _cp_rf(self, src, dst):
logger.debug("cp -rf %s to %s" % (src, dst))
if os.path.exists(dst):
shutil.rmtree(dst)
shutil.copytree(src, dst)
def _eula_by_brand(self):
filename = ""
if os.getenv('PIVOTALLABEL') is None:
filename = 'eula_alpine.txt'
else:
filename = 'eula_emc.txt'
filename = os.path.join(os.path.dirname(os.path.abspath(__file__)),
filename)
with open(filename, 'r') as f:
eula = f.read()
return eula
def prompt_for_eula(self):
eula = self._eula_by_brand()
print eula
ret = self.io.require_confirmation("Do you accept the terms above?")
if not ret:
logger.fatal("Setup aborted, Cancelled by user")
quit()
def get_passphrase(self):
key_file_path = os.path.join(self.options.chorus_path, "shared")
key_file = os.path.join(key_file_path, 'secret.key')
if not os.path.exists(key_file):
return self.options.passphrase
elif os.path.exists(key_file) and self.options.passphrase != "":
return self.options.passphrase
return None
#@processify(msg='->Configuring secret key...')
def configure_secret_key(self, passphrase):
key_file_path = os.path.join(self.options.chorus_path, "shared")
key_file = os.path.join(key_file_path, 'secret.key')
if passphrase is not None:
if passphrase.strip() == '':
passphrase = os.urandom(32)
secret_key = base64.b64encode(
hmac.new(passphrase, digestmod=hashlib.sha256).digest())
with open(key_file, 'w') as f:
f.write(secret_key)
else:
logger.debug(key_file + " already existed, skipped")
logger.debug("Secure " + key_file)
os.chmod(key_file, 0600)
symbolic = os.path.join(self.release_path, "config/secret.key")
logger.debug("Create symbolic to " + symbolic)
if os.path.lexists(symbolic):
os.remove(symbolic)
os.symlink(key_file, symbolic)
#@processify(msg="->Configuring secret token...")
def configure_secret_token(self):
token_file_path = os.path.join(self.options.chorus_path, "shared")
token_file = os.path.join(token_file_path, 'secret.token')
if not os.path.exists(token_file):
with open(token_file, 'w') as f:
f.write(os.urandom(64).encode('hex'))
else:
logger.debug(token_file + " already existed, skipped")
logger.debug("Configuring secret token...")
logger.debug("Secure " + token_file)
os.chmod(token_file, 0600)
symbolic = os.path.join(self.release_path, "config/secret.token")
logger.debug("Create symbolic to " + symbolic)
if os.path.lexists(symbolic):
os.remove(symbolic)
os.symlink(token_file, symbolic)
def generate_paths_file(self):
file_path = os.path.join(self.options.chorus_path, "chorus_path.sh")
logger.debug("Generating paths file: " + file_path)
with open(file_path, 'w') as f:
f.write("export CHORUS_HOME=%s\n" % self.options.chorus_path)
f.write("export CHORUS_DATA=%s\n" % self.options.data_path)
f.write("export PATH=$PATH:$CHORUS_HOME\n")
f.write("export PGPASSFILE=$CHORUS_HOME/.pgpass")
def generate_chorus_psql_files(self):
file_path = os.path.join(self.options.chorus_path, ".pgpass")
logger.debug("generating chorus_psql files")
if os.path.exists(file_path):
os.chmod(file_path, 0600)
with open(file_path, "w") as f:
f.write("*:*:*:" + self.database_username + ":" +
self.database_password)
os.chmod(file_path, 0400)
file_path = os.path.join(self.options.chorus_path, "chorus_psql.sh")
if os.path.exists(file_path):
logger.debug(file_path + " existed, skipped")
else:
with open(file_path, "w") as f:
f.write(CHORUS_PSQL)
os.chmod(file_path, 0500)
def generate_chorus_rails_console_file(self):
file_path = os.path.join(self.options.chorus_path,
"chorus_rails_console.sh")
logger.debug("generating chorus_rails_console file")
with open(file_path, "w") as f:
f.write(CHORUS_RAILS_CONSOLE)
os.chmod(file_path, 0700)
def create_database_config(self):
database_config_file = os.path.join(self.options.chorus_path,
"shared/database.yml")
content = ""
with open(database_config_file, 'r') as f:
lines = f.readlines()
for i in xrange(0, len(lines)):
if lines[i].lstrip().startswith("username:"******":")[1].strip()
elif lines[i].lstrip().startswith("password:"******"!binary" in lines[i]:
self.database_password = base64.b64decode(
lines[i + 1].strip())
logger.debug("password generated already, skipped")
return
else:
self.database_password = os.urandom(16).encode('hex')
lines[i] = lines[i].split(":")[0] + ": !binary |-\n"
lines[i] = lines[i] + " " + base64.b64encode(
self.database_password) + "\n"
if not ":" in lines[i] and not lines[i].startswith("---"):
continue
content += lines[i]
with open(database_config_file, 'w') as f:
f.write(content)
@processify(msg=text.get("step_msg", "initialize_db"), interval=1.5)
def setup_database(self):
logger.debug("->Initializing database...")
pwfile = os.path.join(self.release_path, "postgres/pwfile")
if os.path.exists(pwfile):
os.chmod(pwfile, 0600)
with open(pwfile, "w") as f:
f.write(self.database_password)
os.chmod(pwfile, 0400)
self.executor.initdb(self.options.data_path, self.database_username)
self.executor.start_postgres()
db_commands = "db:custom_create_and_migrate"
#db_commands += " db:seed_permissions"
db_commands += " db:seed"
db_commands += " enqueue:refresh_and_reindex"
self.executor.rake(db_commands)
self.executor.stop_postgres()
@processify(msg=text.get("step_msg", "db_migrate"), interval=1.5)
def upgrade_database(self):
self.executor.start_postgres()
logger.debug("->Running database migrations...")
db_commands = "db:migrate"
db_commands += " db:seed_permissions"
db_commands += " enqueue:refresh_and_reindex"
logger.debug("Running rake " + db_commands)
self.executor.rake(db_commands)
self.executor.stop_postgres()
@processify(msg="->Clearing tmp directory", interval=1.5)
def clear_tmp_directory(self):
logger.debug("->Clearing tmp directory ...")
self.executor.rake("tmp:clear")
@processify(msg=text.get("step_msg", "db_validation"), interval=1.5)
def validate_data_sources(self, msg="", interval=1.5):
logger.debug("->Running data validation...")
self.executor.start_postgres()
self.executor.rake("validations:data_source")
@processify(msg=text.get("step_msg", "stop_pre_chorus"), interval=1.5)
def stop_previous_release(self):
logger.debug("->Shutting down previous Chorus install...")
self.executor.stop_previous_release()
def is_alpine_exits(self):
alpine_dir = os.path.join(self.release_path, "vendor/alpine")
alpine_sources = [
f for f in glob.glob(os.path.join(alpine_dir, "alpine*.sh"))
]
if len(alpine_sources) <= 0:
return False
else:
alpine_sources.sort(key=lambda x: os.path.getmtime(x),
reverse=True)
self.alpine_installer = alpine_sources[0]
self.alpine_version = os.path.basename(
self.alpine_installer.rstrip(".sh"))
self.alpine_release_path = os.path.join(
self.options.chorus_path,
"alpine-releases/%s" % self.alpine_version)
return True
def configure_alpine(self):
@processify(msg=text.get("step_msg", "setting_up_alpine") %
self.alpine_version,
interval=1.5)
def configure():
logger.debug("Extracting %s to %s" %
(self.alpine_installer, self.alpine_release_path))
ret, stdout, stderr = self.executor.run(
"sh %s --target %s --noexec" %
(self.alpine_installer, self.alpine_release_path))
if ret != 0:
raise Exception(stderr)
logger.debug("Preparing Alpine Data Repository")
alpine_data_repo = os.path.join(self.options.chorus_path,
"shared/ALPINE_DATA_REPOSITORY")
if os.path.exists(alpine_data_repo):
logger.debug("Alpine Data Repository existed, skipped")
#if not os.path.exists(os.path.join(alpine_data_repo, "configuration/hadoop_version.properties")):
self._cp_f(os.path.join(self.alpine_release_path, "ALPINE_DATA_REPOSITORY/configuration/hadoop_version.properties"),\
os.path.join(alpine_data_repo, "configuration/hadoop_version.properties"))
self._mkdir_p(os.path.join(alpine_data_repo, "plugins10"))
self._cp_f(os.path.join(self.alpine_release_path, "ALPINE_DATA_REPOSITORY/plugins10/default_operators.jar"),\
os.path.join(alpine_data_repo, "plugins10/default_operators.jar"))
if os.path.exists(
os.path.join(self.alpine_release_path,
"ALPINE_DATA_REPOSITORY/libjars")):
self._cp_rf(os.path.join(self.alpine_release_path, "ALPINE_DATA_REPOSITORY/libjars"), \
os.path.join(alpine_data_repo, "libjars"))
migrate_alpine_conf(os.path.join(alpine_data_repo, "configuration/alpine.conf"), \
os.path.join(self.alpine_release_path, "ALPINE_DATA_REPOSITORY/configuration/alpine.conf"))
else:
shutil.copytree(
os.path.join(self.alpine_release_path,
"ALPINE_DATA_REPOSITORY"), alpine_data_repo)
configure()
def link_current_to_release(self, link_name, rel_path):
current = os.path.join(self.options.chorus_path, link_name)
self._ln_sf(rel_path, current)
def source_chorus_path(self):
logger.debug("source %s/chorus_path.sh" % self.options.chorus_path)
content = ""
if os.path.exists(os.path.join(os.path.expanduser("~"), ".bashrc")):
with open(os.path.join(os.path.expanduser("~"), ".bashrc"),
"r") as f:
content = f.read()
if not "source %s/chorus_path.sh" % self.options.chorus_path in content:
with open(os.path.join(os.path.expanduser("~"), ".bashrc"),
"a") as f:
f.write("source %s/chorus_path.sh\n" %
self.options.chorus_path)
if os.getenv("CHORUS_HOME") == None:
os.environ["CHORUS_HOME"] = self.options.chorus_path
def setup(self, options, is_upgrade):
self.set_path(options)
#if not io.require_confirmation("Do you want to set up the chorus, "
# + "please make sure you have installed chorus before setting up?"):
# logger.fatal("Setup aborted, Cancelled by user")
# quit()
#if not self.options.disable_spec:
# health_check()
#self.prompt_for_eula()
#pre step:
passphrase = self.get_passphrase()
logger.debug("Construct Chorus and Data Directory:")
self.construct_shared_structure()
self.construct_data_structure()
self.link_shared_config()
self.link_data_folder()
self.generate_paths_file()
self.configure_secret_key(passphrase)
self.configure_secret_token()
if is_upgrade:
logger.info(bold(text.get("step_msg", "update_database")))
else:
logger.info(bold(text.get("step_msg", "create_database")))
self.extract_postgres()
if is_upgrade:
self.validate_data_sources()
self.stop_previous_release()
self.upgrade_database()
self.clear_tmp_directory()
else:
self.create_database_config()
self.generate_chorus_psql_files()
self.setup_database()
#self.enqueue_solr_reindex()
self.generate_chorus_rails_console_file()
#self.clean_up_old_releases()
self.link_current_to_release("current", self.release_path)
if self.is_alpine_exits() and self.options.chorus_only is False:
logger.info(bold(text.get("step_msg", "install_alpine")))
self.configure_alpine()
self.link_current_to_release("alpine-current",
self.alpine_release_path)
self.source_chorus_path()
print "*" * 60
print text.get("status_msg", "success_install") % \
(self.options.chorus_path, self.options.data_path, self.chorus_version, self.alpine_version)
print "*" * 60
#disable it, health check is not good
#if self.io.require_confirmation(text.get("interview_question", "do_health_check"), default="no"):
# health_check()
if self.io.require_confirmation(text.get("interview_question",
"do_configure"),
default="no"):
configure.config(self.options, is_upgrade)
print bold(text.get("status_msg", "setup_complete"))
if is_upgrade:
print text.get("status_msg", "upgrade")
print "*" * 60
print text.get("status_msg", "setup_post_step") % pwd.getpwuid(
os.getuid()).pw_name
print "*" * 60
def config(self, options, is_upgrade):
self.options = options
self.io = InstallerIO(options.silent)
self._version_detect()
print "*" * 60
header = ""
if self.chorus_version is None:
print text.get("status_msg", "no_chorus")
else:
print text.get("status_msg", "chorus_status") % (self.chorus_version, self.get_chorus_state())
if self.alpine_version is None:
print text.get("status_msg", "no_alpine")
else:
print text.get("status_msg", "alpine_status") % (self.alpine_version, self.get_alpine_state())
print "CHORUS_HOME:\t%s" % os.getenv("CHORUS_HOME", "not set in ~/.bashrc")
print "*" * 60
if self.chorus_version is None:
return
self.method = self._load_configure_func()
while True:
menu = ""
lens = len(self.method) + 1
for e in self.method.keys():
if self.method[e][0] != "enable_alpine_agent":
menu += str(e) + ". " + self.method[e][0] + " (experimental) \n"
else:
menu += str(e) + ". " + self.method[e][0] + "\n"
menu += "%d. exit" % lens
selection = self.io.require_menu(text.get("interview_question", "configuration_menu") % menu, range(1, lens + 1), default=lens)
if selection == lens:
break
self.method[selection][1](options)
if self.io.require_confirmation(text.get("interview_question", "back_to_menu"), default="no"):
continue
else:
break
print "*" * 60
print text.get("status_msg", "configure_post_step")
print "*" * 60
def b_check_runing_user(install_mode=False):
@processify(msg=text.get("step_msg", "check_user"))
def check():
if os.getuid() == 0 and install_mode:
raise Exception("Please don't run this program as root")
check()