def _is_managed_db(db_properties, force_install=False): ''' responds true (returns 0) if this IS intended to be a managed database is expecting the vars: ---- "db_host" ---- "esgf_host" to be set Define: managed - (true|0) this means NOT manipulated by this script but done by external means (hint prepend "externally" before managed to get the meaning - yes I find it confusing but Stephen likes this term :-)) db_managed=no means that it is a LOCAL database. (I have to change this damn verbiage... what I get for following pasco-speak ;-). ''' db_managed_default = None default_selection_output = None db_managed = esg_property_manager.get_property("db_managed") if not force_install: if db_managed == "yes": return True else: return False if not db_managed: esgf_host = esg_property_manager.get_property("esgf_host") logger.debug("esgf_host = %s", esgf_host) logger.debug("db_host = %s", db_properties["db_host"]) # Try to come up with some "sensible" default value for the user... if db_properties["db_host"] == esgf_host or db_properties[ "db_host"] == "localhost" or not db_properties["db_host"]: db_managed_default = "no" default_selection_output = "[y/N]:" else: db_managed_default = "yes" default_selection_output = "[Y/n]:" external_db_input = raw_input( "Is the database external to this node? " + default_selection_output) if not external_db_input: db_managed = db_managed_default esg_property_manager.write_as_property("db_managed", db_managed) else: if external_db_input.lower() == "y" or external_db_input.lower( ) == "yes": db_managed == "yes" else: db_managed == "no" esg_property_manager.write_as_property("db_managed", db_managed) else: logger.info("db_managed = [%s]", db_managed) if db_managed == "yes": print "Set to use externally \"managed\" database on host: {db_host}".format( db_host=db_properties["db_host"]) return True else: logger.debug("(hmm... setting db_host to localhost)") # Note: if not managed and on the local machine... always use # "localhost" db_properties["db_host"] = "localhost" return False
def _choose_node_peer_group(force_install=False): node_peer_group = esg_property_manager.get_property("node_peer_group") if node_peer_group: logger.info("node_peer_group = [%s]", node_peer_group) return if not node_peer_group or force_install: try: node_peer_group except NameError: node_peer_group = "esgf-dev" while True: print "Only choose esgf-test for test federation install or esgf-prod for production installation. Otherwise choose esgf-dev." node_peer_group_input = raw_input( "What peer group(s) will this node participate in? (esgf-test|esgf-prod|esgf-dev) [{node_peer_group}]: " .format(node_peer_group=node_peer_group)) or node_peer_group if node_peer_group_input.strip() not in [ "esgf-test", "esgf-prod", "esgf-dev" ]: print "Invalid Selection: {node_peer_group_input}".format( node_peer_group_input=node_peer_group_input) print "Please choose either esgf-test, esgf-dev, or esgf-prod" continue else: esg_property_manager.write_as_property("node_peer_group", node_peer_group_input) break
def test_get_property(self): test_properties_file = "/usr/local/test_properties.ini" esg_property_manager.write_as_property("Black Panther", "T'Challa", test_properties_file) output = esg_property_manager.get_property("Black Panther", test_properties_file) self.assertEqual(output, "T'Challa")
def _choose_node_namespace(force_install=False): node_namespace = esg_property_manager.get_property("node_namespace") if not node_namespace or force_install: try: top_level_domain = tld.get_tld("http://" + socket.gethostname(), as_object=True) domain = top_level_domain.domain suffix = top_level_domain.suffix default_node_namespace = suffix + "." + domain except tld.exceptions.TldDomainNotFound, error: top_level_domain = None default_node_namespace = None while True: node_namespace_input = raw_input( "What is the namespace to use for this node? (set to your reverse fqdn - Ex: \"gov.llnl\") [{default_node_namespace}]: " .format(default_node_namespace=default_node_namespace )) or default_node_namespace namespace_pattern_requirement = re.compile("(\w+.{1}\w+)$") if not namespace_pattern_requirement.match(node_namespace_input): print "Namespace entered is not in a valid format. Valid format is [suffix].[domain]. Example: gov.llnl" continue else: esg_property_manager.write_as_property("node_namespace", node_namespace_input) break
def _choose_node_long_name(force_install=False): node_long_name = esg_property_manager.get_property("node_long_name") if not node_long_name or force_install: while True: node_long_name_input = raw_input( "Please give this node a more descriptive \"long\" name [{node_long_name}]: " .format(node_long_name=node_long_name)) or node_long_name esg_property_manager.write_as_property("node_long_name", node_long_name_input) break else: logger.info("node_long_name = [%s]", node_long_name)
def init(): esgf_node_manager_egg_file = "esgf_node_manager-{esgf_node_manager_db_version}-py{python_version}.egg".format( esgf_node_manager_db_version=config["esgf_node_manager_db_version"], python_version=config["python_version"]) # get_property node_use_ssl && [ -z "${node_use_ssl}" ] && write_as_property node_use_ssl true node_use_ssl = esg_property_manager.get_property("node_use_ssl") esg_property_manager.write_as_property("node_use_ssl", True) # get_property node_manager_service_app_home ${tomcat_install_dir}/webapps/${node_manager_app_context_root} # write_as_property node_manager_service_app_home node_manager_service_app_home = esg_property_manager.get_property( "node_manager_service_app_home", "{tomcat_install_dir}/webapps/{node_manager_app_context_root}".format( tomcat_install_dir=config["tomcat_install_dir"], node_manager_app_context_root=node_manager_app_context_root)) esg_property_manager.write_as_property("node_manager_service_app_home", node_manager_service_app_home) # write_as_property node_manager_service_endpoint "http$([ "${node_use_ssl}" = "true" ] && echo "s" || echo "")://${esgf_host}/${node_manager_app_context_root}/node" if node_use_ssl: node_manager_service_endpoint = "https://{esgf_host}/{node_manager_app_context_root}/node".format( esgf_host=esgf_host, node_manager_app_context_root=node_manager_app_context_root) else: node_manager_service_endpoint = "http://{esgf_host}/{node_manager_app_context_root}/node".format( esgf_host=esgf_host, node_manager_app_context_root=node_manager_app_context_root) esg_property_manager.write_as_property("node_manager_service_endpoint", node_manager_service_endpoint) # get_property node_use_ips && [ -z "${node_use_ips}" ] && write_as_property node_use_ips true node_use_ips = esg_property_manager.get_property("node_use_ips") esg_property_manager.write_as_property("node_use_ips", True) # get_property node_poke_timeout && [ -z "${node_poke_timeout}" ] && write_as_property node_poke_timeout 6000 node_poke_timeout = esg_property_manager.get_property("node_poke_timeout") esg_property_manager.write_as_property("node_poke_timeout", 6000) # Database information.... node_db_node_manager_schema_name = "esgf_node_manager" # Notification component information... # mail_smtp_host=${mail_smtp_host:-smtp.`hostname --domain`} #standard guess. # Overwrite mail_smtp_host value if already defined in props file # get_property mail_smtp_host ${mail_smtp_host} config["mail_smtp_host"] = esg_property_manager.get_property( "mail_smtp_host") # Launcher script for the esgf-sh esgf_shell_launcher = "esgf-sh"
def _choose_node_short_name(force_install=False): node_short_name = esg_property_manager.get_property("node_short_name") if not node_short_name or force_install: while True: node_short_name_input = raw_input( "Please give this node a \"short\" name [{node_short_name}]: ". format(node_short_name=node_short_name)) or node_short_name node_short_name_input.replace("", "_") esg_property_manager.write_as_property("node_short_name", node_short_name_input) break else: logger.info("node_short_name = [%s]", node_short_name)
def _choose_mail_admin_address(force_install=False): mail_admin_address = esg_property_manager.get_property( "mail_admin_address") if not mail_admin_address or force_install: mail_admin_address_input = raw_input( "What email address should notifications be sent as? [{mail_admin_address}]: " .format(mail_admin_address=mail_admin_address)) if mail_admin_address_input: esg_property_manager.write_as_property("mail_admin_address", mail_admin_address_input) else: print " (The notification system will not be enabled without an email address)" else: logger.info("mail_admin_address = [%s]", mail_admin_address)
def test_write_as_property(self): test_properties_file = "/usr/local/test_properties.ini" esg_property_manager.write_as_property("Batman", "Bruce Wayne", test_properties_file) parser = ConfigParser.SafeConfigParser() parser.read("/usr/local/test_properties.ini") self.assertEqual(parser.get('installer_properties', 'Batman'), "Bruce Wayne") esg_property_manager.write_as_property("Batman", "Damian Wayne", test_properties_file) parser.read("/usr/local/test_properties.ini") self.assertEqual(parser.get('installer_properties', 'Batman'), "Damian Wayne")
def _choose_esgf_index_peer(force_install=False): esgf_index_peer = esg_property_manager.get_property("esgf_index_peer") esgf_default_peer = esg_property_manager.get_property("esgf_default_peer") esgf_host = esg_property_manager.get_property("esgf_host") if not esgf_index_peer or force_install: default_esgf_index_peer = esgf_default_peer or esgf_host or socket.getfqdn( ) esgf_index_peer_input = raw_input( "What is the hostname of the node do you plan to publish to? [{default_esgf_index_peer}]: " .format(default_esgf_index_peer=default_esgf_index_peer )) or default_esgf_index_peer esg_property_manager.write_as_property("esgf_index_peer", esgf_index_peer_input) else: logger.info("esgf_index_peer = [%s]", esgf_index_peer)
def _choose_publisher_db_user(force_install=False): default_publisher_db_user = None publisher_db_user = esg_property_manager.get_property("publisher_db_user") if publisher_db_user: print "Found existing value for property publisher_db_user: {publisher_db_user}".format( publisher_db_user=publisher_db_user) logger.info("publisher_db_user: %s", publisher_db_user) return if not publisher_db_user or force_install: default_publisher_db_user = publisher_db_user or "esgcet" publisher_db_user_input = raw_input( "What is the (low privilege) db account for publisher? [{default_publisher_db_user}]: " .format(default_publisher_db_user=default_publisher_db_user )) or default_publisher_db_user esg_property_manager.write_as_property("publisher_db_user", publisher_db_user_input)
def check_shmmax(min_shmmax = 48): ''' NOTE: This is another **RedHat/CentOS** specialty thing (sort of) arg1 - min value of shmmax in MB (see: /etc/sysctl.conf) ''' kernel_shmmax = esg_property_manager.get_property("kernel_shmmax") set_value_mb = min_shmmax set_value_bytes = set_value_mb *1024*1024 cur_value_bytes = call_subprocess("sysctl -q kernel.shmmax")["stdout"].split("=")[1] print "cur_value_bytes:", cur_value_bytes cur_value_bytes = cur_value_bytes.strip() if cur_value_bytes < set_value_bytes: print "Current system shared mem value too low [{cur_value_bytes} bytes] changing to [{set_value_bytes} bytes]".format(cur_value_bytes = cur_value_bytes, set_value_bytes = set_value_bytes) call_subprocess("sysctl -w kernel.shmmax={set_value_bytes}".format(set_value_bytes = set_value_bytes)) #TODO: replace with Python to update file call_subprocess("sed -i.bak 's/\(^[^# ]*[ ]*kernel.shmmax[ ]*=[ ]*\)\(.*\)/\1'${set_value_bytes}'/g' /etc/sysctl.conf") esg_property_manager.write_as_property("kernal_shmmax", set_value_mb)
def _get_db_conn_str_questionnaire(db_properties, force_install=False): # postgresql://esgcet@localhost:5432/esgcet user_ = None host_ = None port_ = None dbname_ = None connstring_ = None valid_connection_string = None # Note the values referenced here should have been set by prior get_property *** calls # that sets these values in the script scope. (see the call in # questionnaire function - above) esgf_host = esg_property_manager.get_property("esgf_host") if not db_properties["db_user"] or not db_properties[ "db_host"] or not db_properties["db_port"] or not db_properties[ "db_database"]: if not db_properties["db_host"]: if db_properties["db_host"] == esgf_host or db_properties[ "db_host"] == "localhost": connstring_ = "{db_user}@localhost" else: connstring_ = "{db_user}@{db_host}:{db_port}/{db_database}" while True: print "Please enter the database connection string..." print " (form: postgresql://[username]@[host]:[port]/esgcet)" db_managed = esg_property_manager.get_property("db_managed") #(if it is a not a force install and we are using a LOCAL (NOT MANAGED) database then db_managed == "no") if not connstring_ and db_managed != "yes" and not force_install: connstring_ = "dbsuper@localhost:5432/esgcet" db_connection_input = raw_input( "What is the database connection string? [postgresql://${connstring_}]: postgresql://" .format(connstring_=connstring_)) or connstring_ parsed_db_conn_string = urlparse.urlparse(db_connection_input) # result.path[1:] is database name if not parsed_db_conn_string.username or not parsed_db_conn_string.hostname or parsed_db_conn_string.port or parsed_db_conn_string.result.path[ 1:]: logger.error("ERROR: Incorrect connection string syntax or values") valid_connection_string = False else: valid_connection_string = True break logger.debug("user = %s", user_) logger.debug("host = %s", host_) logger.debug("port = %s", port_) logger.debug("database = %s", dbname_) # write vars to property file esg_property_manager.write_as_property("db_user", user_) esg_property_manager.write_as_property("db_host", host_) esg_property_manager.write_as_property("db_port", port_) esg_property_manager.write_as_property("db_database", dbname_) logger.debug("valid_connection_string: %s", valid_connection_string) return valid_connection_string
def check_for_my_ip(force_install=False): logger.debug("Checking for IP address(es)...") matched = 0 my_ip_address = None eth0 = netifaces.ifaddresses(netifaces.interfaces()[1]) ip_addresses = [ip["addr"] for ip in eth0[netifaces.AF_INET]] try: esgf_host_ip except NameError: esgf_host_ip = esg_property_manager.get_property("esgf_host_ip") if esgf_host_ip and not force_install: logger.info("Using IP: %s", esgf_host_ip) return 0 # We want to make sure that the IP address we have in our config # matches one of the IPs that are associated with this host for ip in ip_addresses: if ip == esgf_host_ip: matched += 1 if matched == 0: logger.info( "Configured host IP address does not match available IPs...") if not esgf_host_ip or force_install or matched == 0: if len(ip_addresses) > 1: # ask the user to choose... while True: _render_ip_address_menu(ip_addresses) default = 0 choice = _select_ip_address() or default my_ip_address = ip_addresses[choice] logger.info("selected address -> %s", my_ip_address) break else: my_ip_address = ip_addresses[0] esg_property_manager.write_as_property("esgf_host_ip", my_ip_address) esgf_host_ip = esg_property_manager.get_property("esgf_host_ip") return esgf_host_ip
def _choose_fqdn(esgf_host, force_install=False): if not esgf_host or force_install: default_host_name = esgf_host or socket.getfqdn() defaultdomain_regex = r"^\w+-*\w*\W*(.+)" defaultdomain = re.search(defaultdomain_regex, default_host_name).group(1) if not default_host_name: default_host_name = "localhost.localdomain" elif not defaultdomain: default_host_name = default_host_name + ".localdomain" default_host_name = raw_input( "What is the fully qualified domain name of this node? [{default_host_name}]: " .format(default_host_name=default_host_name)) or default_host_name esgf_host = default_host_name logger.info("esgf_host = [%s]", esgf_host) esg_property_manager.write_as_property("esgf_host", esgf_host) else: logger.info("esgf_host = [%s]", esgf_host) esg_property_manager.write_as_property("esgf_host", esgf_host)
def _choose_organization_name(force_install=False): esg_root_id = esg_property_manager.get_property("esg_root_id") if esg_root_id: logger.info("esg_root_id = [%s]", esg_root_id) return elif force_install: try: default_org_name = tld.get_tld("http://" + socket.gethostname(), as_object=True).domain except tld.exceptions.TldDomainNotFound, error: logger.exception("Could not find top level domain for %s.", socket.gethostname()) default_org_name = "llnl" while True: org_name_input = raw_input( "What is the name of your organization? [{default_org_name}]: " .format(default_org_name=default_org_name)) or default_org_name org_name_input.replace("", "_") esg_property_manager.write_as_property("esg_root_id", org_name_input) break
def write_esgcet_install_log(): """ Write the Publisher install properties to the install manifest""" with open(config["install_manifest"], "a+") as datafile: datafile.write( str(datetime.date.today()) + "python:esgcet=" + config["esgcet_version"] + "\n") esg_property_manager.write_as_property("publisher_config", config["publisher_config"]) esg_property_manager.write_as_property("publisher_home", config["publisher_home"]) esg_property_manager.write_as_property( "monitor.esg.ini", os.path.join(config["publisher_home"], config["publisher_config"])) return 0