def run_esgsetup(): '''generate esg.ini file using esgsetup script; #Makes call to esgsetup - > Setup the ESG publication configuration''' print "\n*******************************" print "Creating config file (esg.ini) with esgsetup" print "******************************* \n" os.environ["UVCDAT_ANONYMOUS_LOG"] = "no" #Create an initial ESG configuration file (esg.ini); TODO: make break into separate function generate_esg_ini_command = '''esgsetup --config --minimal-setup --rootid {esg_root_id} --db-admin-password password'''.format( esg_root_id=esg_functions.get_esg_root_id()) try: esg_functions.stream_subprocess_output(generate_esg_ini_command) edit_esg_ini() except Exception: logger.exception("Could not finish esgsetup") esg_functions.exit_with_error(1) print "\n*******************************" print "Initializing database with esgsetup" print "******************************* \n" #Initialize the database db_setup_command = generate_esgsetup_options() print "db_setup_command:", db_setup_command try: esg_functions.stream_subprocess_output(db_setup_command) except Exception: logger.exception("Could not initialize database.") esg_functions.exit_with_error(1)
def sync_with_java_truststore(truststore_file): jssecacerts_path = "{java_install_dir}/jre/lib/security/jssecacerts".format( java_install_dir=config["java_install_dir"]) cacerts_path = "{java_install_dir}/jre/lib/security/cacerts".format( java_install_dir=config["java_install_dir"]) if not os.path.isfile(jssecacerts_path) and os.path.isfile(cacerts_path): shutil.copyfile(cacerts_path, jssecacerts_path) if not os.path.join(truststore_file): print "{truststore_file} does not exist. Exiting." esg_functions.exit_with_error(1) print "Syncing {truststore_file} with {java_truststore} ... ".format( truststore_file=truststore_file, java_truststore=jssecacerts_path) if filecmp.cmp(truststore_file, jssecacerts_path): print "Files already in sync" return try: shutil.copyfile(jssecacerts_path, jssecacerts_path + ".bak") except OSError: logger.exception("Could not back up java truststore file.") try: shutil.copyfile(truststore_file, jssecacerts_path) except OSError: logger.exception("Could not sync truststore files.") os.chmod(jssecacerts_path, 0644) os.chown(jssecacerts_path, esg_functions.get_user_id("root"), esg_functions.get_group_id("root"))
def create_postgres_group(): groupadd_command = "/usr/sbin/groupadd -r %s" % ( config["pg_sys_acct_group"]) groupadd_output = esg_functions.call_subprocess(groupadd_command) if groupadd_output["returncode"] != 0 or groupadd_output["returncode"] != 9: print "ERROR: *Could not add postgres system group: %s" % (config["pg_sys_acct_group"]) esg_functions.exit_with_error(1) else: print "Created postgres group with group id: {postgres_group_id}".format(postgres_group_id=grp.getgrnam(config["pg_sys_acct_group"]).gr_gid)
def check_if_root(): '''Check to see if the user is root''' print "Checking that you have root privileges on %s... " % (socket.gethostname()) root_check = os.geteuid() try: if root_check != 0: raise UnprivilegedUserError logger.debug("Root user found.") except UnprivilegedUserError: logger.exception("\nMust run this program with root's effective UID\n\n") esg_functions.exit_with_error(1)
def check_java_version(java_path): print "Checking Java version" try: java_version_output = esg_functions.call_subprocess("{java_path} -version".format(java_path=java_path))["stderr"] except KeyError: logger.exception("Could not check the Java version") esg_functions.exit_with_error(1) installed_java_version = re.search("1.8.0_\w+", java_version_output).group() if esg_version_manager.compare_versions(installed_java_version, config["java_version"]): print "Installed java version meets the minimum requirement " return java_version_output
def download_config_files(force_install): ''' Download config files ''' # #Get files esg_dist_url = "http://distrib-coffee.ipsl.jussieu.fr/pub/esgf/dist" hba_conf_file = "pg_hba.conf" if esg_functions.download_update(hba_conf_file, os.path.join(esg_dist_url,"externals", "bootstrap",hba_conf_file), force_install) > 1: esg_functions.exit_with_error(1) os.chmod(hba_conf_file, 0600) postgres_conf_file = "postgresql.conf" if esg_functions.download_update(postgres_conf_file, os.path.join(esg_dist_url,"externals", "bootstrap",postgres_conf_file), force_install) > 1: esg_functions.exit_with_error(1) os.chmod(postgres_conf_file, 0600)
def convert_per_to_dem(private_key, key_output_dir): '''Convert your private key into from PEM to DER format that java likes''' print "\n*******************************" print "converting private key from PEM to DER... " print "******************************* \n" derkey = os.path.join(key_output_dir, "key.der") convert_to_der = esg_functions.call_subprocess( "openssl pkcs8 -topk8 -nocrypt -inform PEM -in {private_key} -outform DER -out {derkey}" .format(private_key=private_key, derkey=derkey)) if convert_to_der["returncode"] != 0: print "Problem with preparing initial keystore...Exiting." esg_functions.exit_with_error(1) return derkey
def install_tomcat_keypair(private_key="/etc/esgfcerts/hostkey.pem", public_cert="/etc/esgfcerts/hostcert.pem", keystore_name=config["keystore_file"], keystore_alias=config["keystore_alias"]): '''If you want to install a commercial CA issued certificate: esg-node --install-keypair <certificate file> <key file> When prompted for the cachain file, specify the chain file provided by your CA''' #Exit if public_cert(signed CSR isn't found) if not os.path.isfile(public_cert): print "{public_cert} not found. Exiting.".format( public_cert=public_cert) esg_functions.exit_with_error(1) if not os.path.isfile(private_key): print "{private_key} not found. Exiting.".format( private_key=private_key) esg_functions.exit_with_error(1) print "private key = ", private_key print "public cert = ", public_cert print "keystore name = ", keystore_name print "keystore alias = ", keystore_alias #Copy and rename private_key and cert try: shutil.copyfile(private_key, "/etc/certs/hostkey.pem") shutil.copyfile(public_cert, "/etc/certs/hostcert.pem") except shutil.Error: logger.exception("Error copying cert files.") cert_files = create_certificate_chain_list() create_certificate_chain(cert_files) os.chmod("/etc/certs/hostkey.pem", 0400) os.chmod("/etc/certs/hostcert.pem", 0644) os.chmod("/etc/certs/cachain.pem", 0644) generate_tomcat_keystore(keystore_name, keystore_alias, private_key, public_cert, cert_files) copy_cert_to_tomcat_conf(public_cert) if os.path.isfile(config["truststore_file"]): shutil.move(config["truststore_file"], config["truststore_file"] + ".bak") #(In order for ORP or any other local service to trust eachother put your own cert into the truststore) rebuild_truststore(config["truststore_file"]) add_my_cert_to_truststore(config["truststore_file"], keystore_name, keystore_alias)
def create_empty_java_keystore(keystore_name, keystore_alias, keystore_password, distinguished_name): '''Create a new empty Java Keystore using the JDK's keytool''' java_keytool_executable = "{java_install_dir}/bin/keytool".format( java_install_dir=config["java_install_dir"]) generate_keystore_string = "{java_keytool_executable} -genkey -keyalg RSA -alias {keystore_alias} -keystore {keystore_name} -storepass {keystore_password} -keypass {keystore_password} -validity 360 -dname {distinguished_name} -noprompt".format( java_keytool_executable=java_keytool_executable, keystore_alias=keystore_alias, keystore_name=keystore_name, keystore_password=keystore_password, distinguished_name=distinguished_name) keystore_output = esg_functions.call_subprocess(generate_keystore_string) if keystore_output["returncode"] != 0: print "Problem with generating initial keystore...Exiting." esg_functions.exit_with_error(1)
def run_esginitialize(): '''Run the esginitialize script to initialize the ESG node database.''' print "\n*******************************" print "Running esginitialize" print "******************************* \n" esginitialize_process = esg_functions.call_subprocess("esginitialize -c") if esginitialize_process["returncode"] != 0: logger.exception("esginitialize failed") logger.error(esginitialize_process["stderr"]) print esginitialize_process["stderr"] esg_functions.exit_with_error(1) else: print esginitialize_process["stdout"] print esginitialize_process["stderr"]
def check_os(): '''Check if the operating system on server is Redhat or CentOS; returns False Otherwise''' print "Checking operating system....." release_version = re.search( "(centos|redhat)-(\S*)-", platform.platform()).groups() logger.debug("Release Version: %s", release_version) try: if "6" not in release_version[1]: raise WrongOSError except WrongOSError: logger.exception("ESGF can only be installed on versions 6 of Red Hat, CentOS or Scientific Linux x86_64 systems") esg_functions.exit_with_error(1) else: print "Operating System = {OS} {version}".format(OS=release_version[0], version=release_version[1])
def add_my_cert_to_truststore(truststore_file, keystore_file, keystore_alias): #---------------------------------------------------------------- #Re-integrate my public key (I mean, my "certificate") from my keystore into the truststore (the place housing all public keys I allow to talk to me) #---------------------------------------------------------------- print "\n*******************************" print "Adding public key to truststore file {truststore_file}".format( truststore_file=truststore_file) print "******************************* \n" if os.path.isfile(truststore_file): print "Re-Integrating keystore's certificate into truststore.... " print "Extracting keystore's certificate... " keystore_password = esg_functions.get_java_keystore_password() extract_cert_output = esg_functions.call_subprocess( "{java_install_dir}/bin/keytool -export -alias {keystore_alias} -file {keystore_file}.cer -keystore {keystore_file} -storepass {keystore_password}" .format(java_install_dir=config["java_install_dir"], keystore_alias=keystore_alias, keystore_file=keystore_file, keystore_password=keystore_password)) if extract_cert_output["returncode"] != 0: print "Could not extract certificate from keystore" esg_functions.exit_with_error(1) print "Importing keystore's certificate into truststore... " import_to_truststore_output = esg_functions.call_subprocess( "{java_install_dir}/bin/keytool -import -v -trustcacerts -alias {keystore_alias} -keypass {keystore_password} -file {keystore_file}.cer -keystore {truststore_file} -storepass {truststore_password} -noprompt" .format(java_install_dir=config["java_install_dir"], keystore_alias=keystore_alias, keystore_file=keystore_file, keystore_password=keystore_password, truststore_file=config["truststore_file"], truststore_password=config["truststore_password"])) if import_to_truststore_output["returncode"] != 0: print "Could not import the certificate into the truststore" esg_functions.exit_with_error(1) sync_with_java_truststore(truststore_file) try: os.remove(keystore_file + ".cer") except OSError: logger.exception("Could not delete extracted cert file") os.chown(truststore_file, esg_functions.get_user_id("tomcat"), esg_functions.get_group_id("tomcat"))
def create_certificate_chain(cert_files): '''Concatenate the certificates in the chain and copy them to /etc/certs''' print "\n*******************************" print "Creating Certificate Chain" print "******************************* \n" #Copy the tmpchain and rename to cachain with open("/etc/certs/tmpchain", "w") as tmpchain_file: for cert in cert_files: if not os.path.isfile(cert): print "{cert} not found. Exiting.".format(cert=cert) esg_functions.exit_with_error(1) with open(cert, "r") as cert_file_handle: cert_file_contents = cert_file_handle.read() tmpchain_file.write(cert_file_contents + "\n") shutil.copyfile("/etc/certs/tmpchain", "/etc/certs/cachain.pem")
def check_module_version(module_name, min_version): ''' Checks the version of a given python module. Arguments: module_name: a string containing the name of a module that will have it's version checked min_version: the minimum acceptable version string ''' try: module_version = __import__(module_name).__version__ except (AttributeError, ImportError): logger.exception("Couldn't check module version") esg_functions.exit_with_error(1) else: if semver.compare(module_version, min_version) > 0: return True else: print "\nThe detected version of %s %s is less than %s \n" % (module_name, module_version, min_version) return False
def create_postgres_system_user(pg_sys_acct_homedir): print "Creating account..." useradd_command = '''/usr/sbin/useradd -r -c'PostgreSQL Service ESGF' -d {pg_sys_acct_homedir} -g {pg_sys_acct_group} -p {pg_sys_acct_passwd} -s /bin/bash {pg_sys_acct}'''.format(pg_sys_acct_homedir = pg_sys_acct_homedir, pg_sys_acct_group = config["pg_sys_acct_group"], pg_sys_acct_passwd = config["pg_sys_acct_passwd"], pg_sys_acct = config["pg_sys_acct"] ) useradd_output = esg_functions.call_subprocess(useradd_command) postgres_user_id = pwd.getpwnam(config["pg_sys_acct"]).pw_uid if useradd_output["returncode"] != 0 or useradd_output["returncode"] != 9: print "ERROR: Could not add postgres system account user" esg_functions.exit_with_error(1) elif not postgres_user_id: print " ERROR: Problem with {pg_sys_acct} creation!!!".format(pg_sys_acct=config["pg_sys_acct"]) esg_functions.exit_with_error(1) else: print "Created postgres user with group id: {postgres_user_id}".format(postgres_user_id=pwd.getpwnam(config["pg_sys_acct"]).pw_uid)
def connect_to_db(user, db_name=None, host=None, password=None): ''' Connect to database ''' #Using password auth currently; #if the user is postgres, the effective user id (euid) needs to be postgres' user id. #Essentially change user from root to postgres root_id = pwd.getpwnam("root").pw_uid if user == "postgres": postgres_id = pwd.getpwnam("postgres").pw_uid os.seteuid(postgres_id) db_connection_string = build_connection_string(user, db_name, host, password) print "db_connection_string: ", db_connection_string try: conn = psycopg2.connect(db_connection_string) print "Connected to {db_name} database as user '{user}'".format(db_name=db_name, user=user) #Set effective user id (euid) back to root if os.geteuid() != root_id: os.seteuid(root_id) return conn except Exception: logger.exception("Unable to connect to the database.") esg_functions.exit_with_error(1)
os.chmod(file_name, 0600) except OSError, error: logger.error(error) print "\nExpanding war {node_war_file} in {current_directory}".format( node_war_file=node_war_file, current_directory=os.getcwd()) # $JAVA_HOME/bin/jar xf ${node_war_file} try: tar = tarfile.open(node_war_file) tar.extractall() tar.close() except Exception, error: logger.error(error) print "ERROR: Could not extract the ESG Node: {node_war_file}".format( node_war_file=node_war_file) esg_functions.exit_with_error(1) #---------------------------- # Property file fetching and token replacement... #---------------------------- # pushd WEB-INF/classes >& /dev/null with esg_bash2py.pushd("WEB-INF/classes"): # cat ${fetch_file}.tmpl >> ${config_file} with open(download_file_name + ".tmpl", "r") as download_file: with open(config["config_file"], "w") as config_file: download_file_contents = download_file.read() config_file.write(download_file_contents) # chown -R ${tomcat_user} ${node_manager_service_app_home} # chgrp -R ${tomcat_group} ${node_manager_service_app_home} os.chown(
def generate_tomcat_keystore(keystore_name, keystore_alias, private_key, public_cert, intermediate_certs): '''The following helper function creates a new keystore for your tomcat installation''' provider = "org.bouncycastle.jce.provider.BouncyCastleProvider" idptools_install_dir = os.path.join(config["esg_tools_dir"], "idptools") if len(intermediate_certs) < 1: print "No intermediate_certs files given" esg_functions.exit_with_error(1) if not os.path.isfile(private_key): print "Private key file {private_key} does not exist".format( private_key=private_key) keystore_password = esg_functions.get_java_keystore_password() #------------- #Display values #------------- print "Keystore name : {keystore_name}".format(keystore_name=keystore_name) print "Keystore alias: {keystore_alias}".format( keystore_alias=keystore_alias) print "Keystore password: {keystore_password}".format( keystore_password=keystore_password) print "Private key : {private_key}".format(private_key=private_key) print "Public cert : {public_cert}".format(public_cert=public_cert) print "Certificates..." esg_bash2py.mkdir_p(idptools_install_dir) cert_bundle = os.path.join(idptools_install_dir, "cert.bundle") ca_chain_bundle = os.path.join(idptools_install_dir, "ca_chain.bundle") cert_bundle, ca_chain_bundle = bundle_certificates(public_cert, intermediate_certs, idptools_install_dir) print "checking that key pair is congruent... " if check_associate_cert_with_private_key(public_cert, private_key): print "The keypair was congruent" else: print "The keypair was not congruent" esg_functions.exit_with_error(1) print "creating keystore... " #create a keystore with a self-signed cert distinguished_name = "CN={esgf_host}".format( esgf_host=esg_functions.get_esgf_host()) #if previous keystore is found; backup backup_previous_keystore(keystore_name) #------------- #Make empty keystore... #------------- create_empty_java_keystore(keystore_name, keystore_alias, keystore_password, distinguished_name) #------------- #Convert your private key into from PEM to DER format that java likes #------------- derkey = convert_per_to_dem(private_key, idptools_install_dir) #------------- #Now we gather up all the other keys in the key chain... #------------- check_cachain_validity(ca_chain_bundle) print "Constructing new keystore content... " import_cert_into_keystore(keystore_name, keystore_alias, keystore_password, derkey, cert_bundle, provider) #Check keystore output java_keytool_executable = "{java_install_dir}/bin/keytool".format( java_install_dir=config["java_install_dir"]) check_keystore_command = "{java_keytool_executable} -v -list -keystore {keystore_name} -storepass {store_password} | egrep '(Owner|Issuer|MD5|SHA1|Serial number):'".format( java_keytool_executable=java_keytool_executable, keystore_name=keystore_name, store_password=keystore_password) keystore_output = esg_functions.call_subprocess(check_keystore_command) if keystore_output["returncode"] == 0: print "Mmmm, freshly baked keystore!" print "If Everything looks good... then replace your current tomcat keystore with {keystore_name}, if necessary.".format( keystore_name=keystore_name) print "Don't forget to change your tomcat's server.xml entry accordingly :-)" print "Remember: Keep your private key {private_key} and signed cert {public_cert} in a safe place!!!".format( private_key=private_key, public_cert=public_cert) else: print "Failed to check keystore" esg_functions.exit_with_error(1)
def setup_node_manager(mode="install"): ##### # Install The Node Manager ##### # - Takes boolean arg: 0 = setup / install mode (default) # 1 = updated mode # # In setup mode it is an idempotent install (default) # In update mode it will always pull down latest after archiving old # print "Checking for node manager {esgf_node_manager_version}".format( esgf_node_manager_version=config["esgf_node_manager_version"]) if esg_version_manager.check_webapp_version( "esgf-node-manager", config["esgf_node_manager_version"]) == 0 and not force_install: print "\n Found existing version of the node-manager [OK]" return True init() print "*******************************" print "Setting up The ESGF Node Manager..." print "*******************************" # local upgrade=${1:-0} db_set = 0 if force_install: default_answer = "N" else: default_answer = "Y" # local dosetup node_manager_service_app_home = esg_property_manager.get_property( "node_manager_service_app_home") if os.path.isdir(node_manager_service_app_home): db_set = 1 print "Detected an existing node manager installation..." if default_answer == "Y": installation_answer = raw_input( "Do you want to continue with node manager installation and setup? [Y/n]" ) or default_answer else: installation_answer = raw_input( "Do you want to continue with node manager installation and setup? [y/N]" ) or default_answer if installation_answer.lower() not in ["y", "yes"]: print "Skipping node manager installation and setup - will assume it's setup properly" # resetting node manager version to what it is already, not what we prescribed in the script # this way downstream processes will use the *actual* version in play, namely the (access logging) filter(s) esgf_node_manager_version = esg_version_manager.get_current_webapp_version( "esgf_node_manager") return True backup_default_answer = "Y" backup_answer = raw_input( "Do you want to make a back up of the existing distribution [{node_manager_app_context_root}]? [Y/n] " .format(node_manager_app_context_root=node_manager_app_context_root )) or backup_default_answer if backup_answer.lower in ["yes", "y"]: print "Creating a backup archive of this web application [{node_manager_service_app_home}]".format( node_manager_service_app_home=node_manager_service_app_home) esg_functions.backup(node_manager_service_app_home) backup_db_default_answer = "Y" backup_db_answer = raw_input( "Do you want to make a back up of the existing database [{node_db_name}:esgf_node_manager]?? [Y/n] " .format(node_db_name=config["node_db_name"] )) or backup_db_default_answer if backup_db_answer.lower() in ["yes", "y"]: print "Creating a backup archive of the manager database schema [{node_db_name}:esgf_node_manager]".format( node_db_name=config["node_db_name"]) # TODO: Implement this # esg_postgres.backup_db() -db ${node_db_name} -s node_manager esg_bash2py.mkdir_p(config["workdir"]) with esg_bash2py.pushd(config["workdir"]): logger.debug("changed directory to : %s", os.getcwd()) # strip off .tar.gz at the end #(Ex: esgf-node-manager-0.9.0.tar.gz -> esgf-node-manager-0.9.0) node_dist_file = esg_bash2py.trim_string_from_head(node_dist_url) logger.debug("node_dist_file: %s", node_dist_file) # Should just be esgf-node-manager-x.x.x node_dist_dir = node_dist_file # checked_get ${node_dist_file} ${node_dist_url} $((force_install)) if not esg_functions.download_update( node_dist_file, node_dist_url, force_download=force_install): print "ERROR: Could not download {node_dist_url} :-(".format( node_dist_url=node_dist_url) esg_functions.exit_with_error(1) # make room for new install if force_install: print "Removing Previous Installation of the ESGF Node Manager... ({node_dist_dir})".format( node_dist_dir=node_dist_dir) try: shutil.rmtree(node_dist_dir) logger.info("Deleted directory: %s", node_dist_dir) except IOError, error: logger.error(error) logger.error("Could not delete directory: %s", node_dist_dir) esg_functions.exit_with_error(1) clean_node_manager_webapp_subsystem() print "\nunpacking {node_dist_file}...".format( node_dist_file=node_dist_file) # This probably won't work, because the extension has already been stripped, no idea how this even worked in the bash code smh try: tar = tarfile.open(node_dist_file) tar.extractall() tar.close() except Exception, error: logger.error(error) print "ERROR: Could not extract the ESG Node: {node_dist_file}".format( node_dist_file=node_dist_file) esg_functions.exit_with_error(1)
def download_java(java_tarfile): print "Downloading Java from ", config["java_dist_url"] if not esg_functions.download_update(java_tarfile, config["java_dist_url"], force_install): logger.error("ERROR: Could not download Java") esg_functions.exit_with_error(1)
def setup_postgres(force_install=False, backup_existing_db=None, default_continue_install = "N"): print "\n*******************************" print "Setting up Postgres" print "******************************* \n" psql_path = find_executable("psql") if check_existing_pg_version(psql_path): if default_continue_install == "N": continue_install = raw_input("Valid existing Postgres installation found. Do you want to continue with the setup [y/N]: ") or default_continue_install if continue_install.lower() in ["no", 'n']: print "Skipping installation." return True else: force_install = True if not backup_existing_db or backup_existing_db.lower() not in ["yes", "y", "n", "no"]: backup_db_input = raw_input("Do you want to backup the current database? [Y/n]: ") if backup_db_input.lower() in ["y", "yes"]: backup_db("postgres", "postgres") download_postgres() '''Create system account (postgres) if it doesn't exist ''' ######## #Create the system account for postgress to run as. ######## pg_sys_acct_homedir = psql_path if not check_for_postgres_sys_acct(): #NOTE: "useradd/groupadd" are a RedHat/CentOS thing... to make this cross distro compatible clean this up. create_postgres_group() set_pg_sys_account_password() create_postgres_system_user(pg_sys_acct_homedir) else: postgress_user_shell = get_postgres_user_shell() if postgress_user_shell != "/bin/bash": set_postgres_user_shell() change_pg_install_dir_ownership() initialize_postgres() #start the postgres server start_postgres() # check_for_postgres_db_user() # connect_to_db() # with esg_bash2py.pushd(os.path.join(config["postgress_install_dir"], "9.6", "data")): # download_config_files(force_install) # update_port_in_config_file() # update_log_dir_in_config_file() shutil.copyfile("postgres_conf/postgresql.conf", "/var/lib/pgsql/9.6/data/postgresql.conf") postgres_user_id = pwd.getpwnam(config["pg_sys_acct"]).pw_uid postgres_group_id = grp.getgrnam(config["pg_sys_acct_group"]).gr_gid os.chown("/var/lib/pgsql/9.6/data/postgresql.conf", postgres_user_id, postgres_group_id) setup_hba_conf_file() restart_postgres() #TODO: Set password for postgres user setup_db_schemas(force_install) create_pg_pass_file() ''' function calls ''' esg_functions.check_shmmax() write_postgress_env() write_postgress_install_log() esg_functions.exit_with_error(0)