def _is_managed_db(db_properties, force_install=False):
'''
responds true (returns 0) if this IS intended to be a managed database
is expecting the vars:
---- "db_host"
---- "esgf_host"
to be set
Define: managed - (true|0) this means NOT manipulated by this script but done by external means
(hint prepend "externally" before managed to get the meaning - yes I find it confusing but Stephen likes this term :-))
db_managed=no means that it is a LOCAL database. (I have to change this damn verbiage... what I get for following pasco-speak ;-).
'''
db_managed_default = None
default_selection_output = None
db_managed = esg_property_manager.get_property("db_managed")
if not force_install:
if db_managed == "yes":
return True
else:
return False
if not db_managed:
esgf_host = esg_property_manager.get_property("esgf_host")
logger.debug("esgf_host = %s", esgf_host)
logger.debug("db_host = %s", db_properties["db_host"])
# Try to come up with some "sensible" default value for the user...
if db_properties["db_host"] == esgf_host or db_properties[
"db_host"] == "localhost" or not db_properties["db_host"]:
db_managed_default = "no"
default_selection_output = "[y/N]:"
else:
db_managed_default = "yes"
default_selection_output = "[Y/n]:"
external_db_input = raw_input(
"Is the database external to this node? " +
default_selection_output)
if not external_db_input:
db_managed = db_managed_default
esg_property_manager.write_as_property("db_managed", db_managed)
else:
if external_db_input.lower() == "y" or external_db_input.lower(
) == "yes":
db_managed == "yes"
else:
db_managed == "no"
esg_property_manager.write_as_property("db_managed", db_managed)
else:
logger.info("db_managed = [%s]", db_managed)
if db_managed == "yes":
print "Set to use externally \"managed\" database on host: {db_host}".format(
db_host=db_properties["db_host"])
return True
else:
logger.debug("(hmm... setting db_host to localhost)")
# Note: if not managed and on the local machine... always use
# "localhost"
db_properties["db_host"] = "localhost"
return False
def _choose_node_peer_group(force_install=False):
node_peer_group = esg_property_manager.get_property("node_peer_group")
if node_peer_group:
logger.info("node_peer_group = [%s]", node_peer_group)
return
if not node_peer_group or force_install:
try:
node_peer_group
except NameError:
node_peer_group = "esgf-dev"
while True:
print "Only choose esgf-test for test federation install or esgf-prod for production installation. Otherwise choose esgf-dev."
node_peer_group_input = raw_input(
"What peer group(s) will this node participate in? (esgf-test|esgf-prod|esgf-dev) [{node_peer_group}]: "
.format(node_peer_group=node_peer_group)) or node_peer_group
if node_peer_group_input.strip() not in [
"esgf-test", "esgf-prod", "esgf-dev"
]:
print "Invalid Selection: {node_peer_group_input}".format(
node_peer_group_input=node_peer_group_input)
print "Please choose either esgf-test, esgf-dev, or esgf-prod"
continue
else:
esg_property_manager.write_as_property("node_peer_group",
node_peer_group_input)
break
def test_get_property(self):
test_properties_file = "/usr/local/test_properties.ini"
esg_property_manager.write_as_property("Black Panther", "T'Challa",
test_properties_file)
output = esg_property_manager.get_property("Black Panther",
test_properties_file)
self.assertEqual(output, "T'Challa")
def _choose_node_namespace(force_install=False):
node_namespace = esg_property_manager.get_property("node_namespace")
if not node_namespace or force_install:
try:
top_level_domain = tld.get_tld("http://" + socket.gethostname(),
as_object=True)
domain = top_level_domain.domain
suffix = top_level_domain.suffix
default_node_namespace = suffix + "." + domain
except tld.exceptions.TldDomainNotFound, error:
top_level_domain = None
default_node_namespace = None
while True:
node_namespace_input = raw_input(
"What is the namespace to use for this node? (set to your reverse fqdn - Ex: \"gov.llnl\") [{default_node_namespace}]: "
.format(default_node_namespace=default_node_namespace
)) or default_node_namespace
namespace_pattern_requirement = re.compile("(\w+.{1}\w+)$")
if not namespace_pattern_requirement.match(node_namespace_input):
print "Namespace entered is not in a valid format. Valid format is [suffix].[domain]. Example: gov.llnl"
continue
else:
esg_property_manager.write_as_property("node_namespace",
node_namespace_input)
break
def _choose_node_long_name(force_install=False):
node_long_name = esg_property_manager.get_property("node_long_name")
if not node_long_name or force_install:
while True:
node_long_name_input = raw_input(
"Please give this node a more descriptive \"long\" name [{node_long_name}]: "
.format(node_long_name=node_long_name)) or node_long_name
esg_property_manager.write_as_property("node_long_name",
node_long_name_input)
break
else:
logger.info("node_long_name = [%s]", node_long_name)
def init():
esgf_node_manager_egg_file = "esgf_node_manager-{esgf_node_manager_db_version}-py{python_version}.egg".format(
esgf_node_manager_db_version=config["esgf_node_manager_db_version"],
python_version=config["python_version"])
# get_property node_use_ssl && [ -z "${node_use_ssl}" ] && write_as_property node_use_ssl true
node_use_ssl = esg_property_manager.get_property("node_use_ssl")
esg_property_manager.write_as_property("node_use_ssl", True)
# get_property node_manager_service_app_home ${tomcat_install_dir}/webapps/${node_manager_app_context_root}
# write_as_property node_manager_service_app_home
node_manager_service_app_home = esg_property_manager.get_property(
"node_manager_service_app_home",
"{tomcat_install_dir}/webapps/{node_manager_app_context_root}".format(
tomcat_install_dir=config["tomcat_install_dir"],
node_manager_app_context_root=node_manager_app_context_root))
esg_property_manager.write_as_property("node_manager_service_app_home",
node_manager_service_app_home)
# write_as_property node_manager_service_endpoint "http$([ "${node_use_ssl}" = "true" ] && echo "s" || echo "")://${esgf_host}/${node_manager_app_context_root}/node"
if node_use_ssl:
node_manager_service_endpoint = "https://{esgf_host}/{node_manager_app_context_root}/node".format(
esgf_host=esgf_host,
node_manager_app_context_root=node_manager_app_context_root)
else:
node_manager_service_endpoint = "http://{esgf_host}/{node_manager_app_context_root}/node".format(
esgf_host=esgf_host,
node_manager_app_context_root=node_manager_app_context_root)
esg_property_manager.write_as_property("node_manager_service_endpoint",
node_manager_service_endpoint)
# get_property node_use_ips && [ -z "${node_use_ips}" ] && write_as_property node_use_ips true
node_use_ips = esg_property_manager.get_property("node_use_ips")
esg_property_manager.write_as_property("node_use_ips", True)
# get_property node_poke_timeout && [ -z "${node_poke_timeout}" ] && write_as_property node_poke_timeout 6000
node_poke_timeout = esg_property_manager.get_property("node_poke_timeout")
esg_property_manager.write_as_property("node_poke_timeout", 6000)
# Database information....
node_db_node_manager_schema_name = "esgf_node_manager"
# Notification component information...
# mail_smtp_host=${mail_smtp_host:-smtp.`hostname --domain`} #standard guess.
# Overwrite mail_smtp_host value if already defined in props file
# get_property mail_smtp_host ${mail_smtp_host}
config["mail_smtp_host"] = esg_property_manager.get_property(
"mail_smtp_host")
# Launcher script for the esgf-sh
esgf_shell_launcher = "esgf-sh"
def _choose_node_short_name(force_install=False):
node_short_name = esg_property_manager.get_property("node_short_name")
if not node_short_name or force_install:
while True:
node_short_name_input = raw_input(
"Please give this node a \"short\" name [{node_short_name}]: ".
format(node_short_name=node_short_name)) or node_short_name
node_short_name_input.replace("", "_")
esg_property_manager.write_as_property("node_short_name",
node_short_name_input)
break
else:
logger.info("node_short_name = [%s]", node_short_name)
def _choose_mail_admin_address(force_install=False):
mail_admin_address = esg_property_manager.get_property(
"mail_admin_address")
if not mail_admin_address or force_install:
mail_admin_address_input = raw_input(
"What email address should notifications be sent as? [{mail_admin_address}]: "
.format(mail_admin_address=mail_admin_address))
if mail_admin_address_input:
esg_property_manager.write_as_property("mail_admin_address",
mail_admin_address_input)
else:
print " (The notification system will not be enabled without an email address)"
else:
logger.info("mail_admin_address = [%s]", mail_admin_address)
def test_write_as_property(self):
test_properties_file = "/usr/local/test_properties.ini"
esg_property_manager.write_as_property("Batman", "Bruce Wayne",
test_properties_file)
parser = ConfigParser.SafeConfigParser()
parser.read("/usr/local/test_properties.ini")
self.assertEqual(parser.get('installer_properties', 'Batman'),
"Bruce Wayne")
esg_property_manager.write_as_property("Batman", "Damian Wayne",
test_properties_file)
parser.read("/usr/local/test_properties.ini")
self.assertEqual(parser.get('installer_properties', 'Batman'),
"Damian Wayne")
def _choose_esgf_index_peer(force_install=False):
esgf_index_peer = esg_property_manager.get_property("esgf_index_peer")
esgf_default_peer = esg_property_manager.get_property("esgf_default_peer")
esgf_host = esg_property_manager.get_property("esgf_host")
if not esgf_index_peer or force_install:
default_esgf_index_peer = esgf_default_peer or esgf_host or socket.getfqdn(
)
esgf_index_peer_input = raw_input(
"What is the hostname of the node do you plan to publish to? [{default_esgf_index_peer}]: "
.format(default_esgf_index_peer=default_esgf_index_peer
)) or default_esgf_index_peer
esg_property_manager.write_as_property("esgf_index_peer",
esgf_index_peer_input)
else:
logger.info("esgf_index_peer = [%s]", esgf_index_peer)
def _choose_publisher_db_user(force_install=False):
default_publisher_db_user = None
publisher_db_user = esg_property_manager.get_property("publisher_db_user")
if publisher_db_user:
print "Found existing value for property publisher_db_user: {publisher_db_user}".format(
publisher_db_user=publisher_db_user)
logger.info("publisher_db_user: %s", publisher_db_user)
return
if not publisher_db_user or force_install:
default_publisher_db_user = publisher_db_user or "esgcet"
publisher_db_user_input = raw_input(
"What is the (low privilege) db account for publisher? [{default_publisher_db_user}]: "
.format(default_publisher_db_user=default_publisher_db_user
)) or default_publisher_db_user
esg_property_manager.write_as_property("publisher_db_user",
publisher_db_user_input)
def check_shmmax(min_shmmax = 48):
'''
NOTE: This is another **RedHat/CentOS** specialty thing (sort of)
arg1 - min value of shmmax in MB (see: /etc/sysctl.conf)
'''
kernel_shmmax = esg_property_manager.get_property("kernel_shmmax")
set_value_mb = min_shmmax
set_value_bytes = set_value_mb *1024*1024
cur_value_bytes = call_subprocess("sysctl -q kernel.shmmax")["stdout"].split("=")[1]
print "cur_value_bytes:", cur_value_bytes
cur_value_bytes = cur_value_bytes.strip()
if cur_value_bytes < set_value_bytes:
print "Current system shared mem value too low [{cur_value_bytes} bytes] changing to [{set_value_bytes} bytes]".format(cur_value_bytes = cur_value_bytes, set_value_bytes = set_value_bytes)
call_subprocess("sysctl -w kernel.shmmax={set_value_bytes}".format(set_value_bytes = set_value_bytes))
#TODO: replace with Python to update file
call_subprocess("sed -i.bak 's/\(^[^# ]*[ ]*kernel.shmmax[ ]*=[ ]*\)\(.*\)/\1'${set_value_bytes}'/g' /etc/sysctl.conf")
esg_property_manager.write_as_property("kernal_shmmax", set_value_mb)
def _get_db_conn_str_questionnaire(db_properties, force_install=False):
# postgresql://esgcet@localhost:5432/esgcet
user_ = None
host_ = None
port_ = None
dbname_ = None
connstring_ = None
valid_connection_string = None
# Note the values referenced here should have been set by prior get_property *** calls
# that sets these values in the script scope. (see the call in
# questionnaire function - above)
esgf_host = esg_property_manager.get_property("esgf_host")
if not db_properties["db_user"] or not db_properties[
"db_host"] or not db_properties["db_port"] or not db_properties[
"db_database"]:
if not db_properties["db_host"]:
if db_properties["db_host"] == esgf_host or db_properties[
"db_host"] == "localhost":
connstring_ = "{db_user}@localhost"
else:
connstring_ = "{db_user}@{db_host}:{db_port}/{db_database}"
while True:
print "Please enter the database connection string..."
print " (form: postgresql://[username]@[host]:[port]/esgcet)"
db_managed = esg_property_manager.get_property("db_managed")
#(if it is a not a force install and we are using a LOCAL (NOT MANAGED) database then db_managed == "no")
if not connstring_ and db_managed != "yes" and not force_install:
connstring_ = "dbsuper@localhost:5432/esgcet"
db_connection_input = raw_input(
"What is the database connection string? [postgresql://${connstring_}]: postgresql://"
.format(connstring_=connstring_)) or connstring_
parsed_db_conn_string = urlparse.urlparse(db_connection_input)
# result.path[1:] is database name
if not parsed_db_conn_string.username or not parsed_db_conn_string.hostname or parsed_db_conn_string.port or parsed_db_conn_string.result.path[
1:]:
logger.error("ERROR: Incorrect connection string syntax or values")
valid_connection_string = False
else:
valid_connection_string = True
break
logger.debug("user = %s", user_)
logger.debug("host = %s", host_)
logger.debug("port = %s", port_)
logger.debug("database = %s", dbname_)
# write vars to property file
esg_property_manager.write_as_property("db_user", user_)
esg_property_manager.write_as_property("db_host", host_)
esg_property_manager.write_as_property("db_port", port_)
esg_property_manager.write_as_property("db_database", dbname_)
logger.debug("valid_connection_string: %s", valid_connection_string)
return valid_connection_string
def check_for_my_ip(force_install=False):
logger.debug("Checking for IP address(es)...")
matched = 0
my_ip_address = None
eth0 = netifaces.ifaddresses(netifaces.interfaces()[1])
ip_addresses = [ip["addr"] for ip in eth0[netifaces.AF_INET]]
try:
esgf_host_ip
except NameError:
esgf_host_ip = esg_property_manager.get_property("esgf_host_ip")
if esgf_host_ip and not force_install:
logger.info("Using IP: %s", esgf_host_ip)
return 0
# We want to make sure that the IP address we have in our config
# matches one of the IPs that are associated with this host
for ip in ip_addresses:
if ip == esgf_host_ip:
matched += 1
if matched == 0:
logger.info(
"Configured host IP address does not match available IPs...")
if not esgf_host_ip or force_install or matched == 0:
if len(ip_addresses) > 1:
# ask the user to choose...
while True:
_render_ip_address_menu(ip_addresses)
default = 0
choice = _select_ip_address() or default
my_ip_address = ip_addresses[choice]
logger.info("selected address -> %s", my_ip_address)
break
else:
my_ip_address = ip_addresses[0]
esg_property_manager.write_as_property("esgf_host_ip", my_ip_address)
esgf_host_ip = esg_property_manager.get_property("esgf_host_ip")
return esgf_host_ip
def _choose_fqdn(esgf_host, force_install=False):
if not esgf_host or force_install:
default_host_name = esgf_host or socket.getfqdn()
defaultdomain_regex = r"^\w+-*\w*\W*(.+)"
defaultdomain = re.search(defaultdomain_regex,
default_host_name).group(1)
if not default_host_name:
default_host_name = "localhost.localdomain"
elif not defaultdomain:
default_host_name = default_host_name + ".localdomain"
default_host_name = raw_input(
"What is the fully qualified domain name of this node? [{default_host_name}]: "
.format(default_host_name=default_host_name)) or default_host_name
esgf_host = default_host_name
logger.info("esgf_host = [%s]", esgf_host)
esg_property_manager.write_as_property("esgf_host", esgf_host)
else:
logger.info("esgf_host = [%s]", esgf_host)
esg_property_manager.write_as_property("esgf_host", esgf_host)
def _choose_organization_name(force_install=False):
esg_root_id = esg_property_manager.get_property("esg_root_id")
if esg_root_id:
logger.info("esg_root_id = [%s]", esg_root_id)
return
elif force_install:
try:
default_org_name = tld.get_tld("http://" + socket.gethostname(),
as_object=True).domain
except tld.exceptions.TldDomainNotFound, error:
logger.exception("Could not find top level domain for %s.",
socket.gethostname())
default_org_name = "llnl"
while True:
org_name_input = raw_input(
"What is the name of your organization? [{default_org_name}]: "
.format(default_org_name=default_org_name)) or default_org_name
org_name_input.replace("", "_")
esg_property_manager.write_as_property("esg_root_id",
org_name_input)
break
def write_esgcet_install_log():
""" Write the Publisher install properties to the install manifest"""
with open(config["install_manifest"], "a+") as datafile:
datafile.write(
str(datetime.date.today()) + "python:esgcet=" +
config["esgcet_version"] + "\n")
esg_property_manager.write_as_property("publisher_config",
config["publisher_config"])
esg_property_manager.write_as_property("publisher_home",
config["publisher_home"])
esg_property_manager.write_as_property(
"monitor.esg.ini",
os.path.join(config["publisher_home"], config["publisher_config"]))
return 0
