def _insert_cert_into_truststore(cert_file, truststore_file, tmp_dir): '''Takes full path to a pem certificate file and incorporates it into the given truststore''' print "{cert_file} ->".format(cert_file=cert_file) cert_hash = cert_file.split(".")[0] der_file = os.path.join(tmp_dir, cert_hash + ".der") #-------------- # Convert from PEM format to DER format - for ingest into keystore cert_pem = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, open(cert_file).read()) with open(der_file, "w") as der_file_handle: der_file_handle.write( OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, cert_pem)) #-------------- if os.path.isfile(truststore_file): output = esg_functions.call_subprocess( "/usr/local/java/bin/keytool -delete -alias {cert_hash} -keystore {truststore_file} -storepass {truststore_password}" .format(cert_hash=cert_hash, truststore_file=truststore_file, truststore_password=config["truststore_password"])) if output["returncode"] == 0: print "Deleted cert hash" output = esg_functions.call_subprocess( "/usr/local/java/bin/keytool -import -alias {cert_hash} -file {der_file} -keystore {truststore_file} -storepass {truststore_password} -noprompt" .format(cert_hash=cert_hash, der_file=der_file, truststore_file=truststore_file, truststore_password=config["truststore_password"])) if output["returncode"] == 0: print "added {der_file} to {truststore_file}".format( der_file=der_file, truststore_file=truststore_file) os.remove(der_file)
def setup_solr(): '''Setup Apache Solr for faceted search''' print "\n*******************************" print "Setting up Solr" print "******************************* \n" # # Solr/Jetty web application SOLR_VERSION = "5.5.4" SOLR_INSTALL_DIR = "/usr/local/solr" SOLR_HOME = "/usr/local/solr-home" SOLR_DATA_DIR = "/esg/solr-index" SOLR_INCLUDE = "{SOLR_HOME}/solr.in.sh".format(SOLR_HOME=SOLR_HOME) #Download solr tarball solr_tarball_url = "http://archive.apache.org/dist/lucene/solr/{SOLR_VERSION}/solr-{SOLR_VERSION}.tgz".format( SOLR_VERSION=SOLR_VERSION) download_solr_tarball(solr_tarball_url, SOLR_VERSION) #Extract solr tarball extract_solr_tarball( '/tmp/solr-{SOLR_VERSION}.tgz'.format(SOLR_VERSION=SOLR_VERSION), SOLR_VERSION) esg_bash2py.mkdir_p(SOLR_DATA_DIR) # download template directory structure for shards home download_template_directory() esg_bash2py.mkdir_p(SOLR_HOME) # create non-privilged user to run Solr server esg_functions.stream_subprocess_output("groupadd solr") esg_functions.stream_subprocess_output( "useradd -s /sbin/nologin -g solr -d /usr/local/solr solr") SOLR_USER_ID = pwd.getpwnam("solr").pw_uid SOLR_GROUP_ID = grp.getgrnam("solr").gr_gid esg_functions.change_permissions_recursive( "/usr/local/solr-{SOLR_VERSION}".format(SOLR_VERSION=SOLR_VERSION), SOLR_USER_ID, SOLR_GROUP_ID) esg_functions.change_permissions_recursive(SOLR_HOME, SOLR_USER_ID, SOLR_GROUP_ID) esg_functions.change_permissions_recursive(SOLR_DATA_DIR, SOLR_USER_ID, SOLR_GROUP_ID) # #Copy shard files shutil.copyfile("solr_scripts/add_shard.sh", "/usr/local/bin/add_shard.sh") shutil.copyfile("solr_scripts/remove_shard.sh", "/usr/local/bin/remove_shard.sh") # add shards esg_functions.call_subprocess("/usr/local/bin/add_shard.sh master 8984") esg_functions.call_subprocess("/usr/local/bin/add_shard.sh master 8983") # custom logging properties shutil.copyfile( "solr_scripts/log4j.properties", "/{SOLR_INSTALL_DIR}/server/resources/log4j.properties".format( SOLR_INSTALL_DIR=SOLR_INSTALL_DIR))
def set_postgres_user_shell(): print "Noticed that the existing postgres user [{pg_sys_acct}] does not have the bash shell... Hmmm... making it so ".format(pg_sys_acct = config["pg_sys_acct"]) change_shell_command = "sed -i 's#\('{pg_sys_acct}'.*:\)\(.*\)$#\1/\bin/\bash#' /etc/passwd".format(pg_sys_acct = config["pg_sys_acct"]) esg_functions.call_subprocess(change_shell_command) if get_postgres_user_shell() == "/bin/bash": print "[OK]" print "Postgres user shell is properly configured" else: print "[FAIL]" print "Postgres user shell is not properly configured. The shell is {postgres_shell}. It should be /bin/bash".format(postgres_shell=get_postgres_user_shell())
def purge_tomcat(): print "\n*******************************" print "Purging Tomcat" print "******************************* \n" # esg-node --stop may not actually cause Tomcat to exit properly, # so force-kill all remaining instances esg_functions.call_subprocess("pkill -9 -u tomcat") try: shutil.rmtree("/etc/logrotate.d/esgf_tomcat") except OSError, error: logger.exception("Couldn't delete %s", "/etc/logrotate.d/esgf_tomcat")
def postgres_status(): '''Checks the status of the postgres server''' status = esg_functions.call_subprocess("service postgresql-9.6 status") print "Postgres server status:", status["stdout"] if "running" in status["stdout"]: return True else: return False
def create_tomcat_user(): '''Create the Tomcat system user and user group''' print "\n*******************************" print "Creating Tomcat User" print "******************************* \n" esg_functions.call_subprocess("groupadd tomcat") esg_functions.call_subprocess( "useradd -s /sbin/nologin -g tomcat -d /usr/local/tomcat tomcat") tomcat_directory = "/usr/local/apache-tomcat-{TOMCAT_VERSION}".format( TOMCAT_VERSION=TOMCAT_VERSION) tomcat_user_id = pwd.getpwnam("tomcat").pw_uid tomcat_group_id = grp.getgrnam("tomcat").gr_gid esg_functions.change_permissions_recursive(tomcat_directory, tomcat_user_id, tomcat_group_id) os.chmod("/usr/local/tomcat/webapps", 0775)
def add_my_cert_to_truststore(truststore_file, keystore_file, keystore_alias): #---------------------------------------------------------------- #Re-integrate my public key (I mean, my "certificate") from my keystore into the truststore (the place housing all public keys I allow to talk to me) #---------------------------------------------------------------- print "\n*******************************" print "Adding public key to truststore file {truststore_file}".format( truststore_file=truststore_file) print "******************************* \n" if os.path.isfile(truststore_file): print "Re-Integrating keystore's certificate into truststore.... " print "Extracting keystore's certificate... " keystore_password = esg_functions.get_java_keystore_password() extract_cert_output = esg_functions.call_subprocess( "{java_install_dir}/bin/keytool -export -alias {keystore_alias} -file {keystore_file}.cer -keystore {keystore_file} -storepass {keystore_password}" .format(java_install_dir=config["java_install_dir"], keystore_alias=keystore_alias, keystore_file=keystore_file, keystore_password=keystore_password)) if extract_cert_output["returncode"] != 0: print "Could not extract certificate from keystore" esg_functions.exit_with_error(1) print "Importing keystore's certificate into truststore... " import_to_truststore_output = esg_functions.call_subprocess( "{java_install_dir}/bin/keytool -import -v -trustcacerts -alias {keystore_alias} -keypass {keystore_password} -file {keystore_file}.cer -keystore {truststore_file} -storepass {truststore_password} -noprompt" .format(java_install_dir=config["java_install_dir"], keystore_alias=keystore_alias, keystore_file=keystore_file, keystore_password=keystore_password, truststore_file=config["truststore_file"], truststore_password=config["truststore_password"])) if import_to_truststore_output["returncode"] != 0: print "Could not import the certificate into the truststore" esg_functions.exit_with_error(1) sync_with_java_truststore(truststore_file) try: os.remove(keystore_file + ".cer") except OSError: logger.exception("Could not delete extracted cert file") os.chown(truststore_file, esg_functions.get_user_id("tomcat"), esg_functions.get_group_id("tomcat"))
def create_postgres_group(): groupadd_command = "/usr/sbin/groupadd -r %s" % ( config["pg_sys_acct_group"]) groupadd_output = esg_functions.call_subprocess(groupadd_command) if groupadd_output["returncode"] != 0 or groupadd_output["returncode"] != 9: print "ERROR: *Could not add postgres system group: %s" % (config["pg_sys_acct_group"]) esg_functions.exit_with_error(1) else: print "Created postgres group with group id: {postgres_group_id}".format(postgres_group_id=grp.getgrnam(config["pg_sys_acct_group"]).gr_gid)
def main(): print "\n*******************************" print "Setting up Apache (httpd) Web Server" print "******************************* \n" if check_for_apache_installation(): print "Found existing Apache installation." esg_functions.call_subprocess("httpd -version") continue_install = raw_input( "Would you like to continue the Apache installation anyway? [y/N]: " ) or "N" if continue_install.lower() in ["no", "n"]: return install_apache_httpd() install_mod_wsgi() make_python_eggs_dir() copy_apache_conf_files() start_apache()
def check_for_postgres_db_user(): ''' Need to be able to run psql as the postgres system user instead of root to avoid the peer authencation error ''' check_for_pg_user_command = '''sudo -u postgres psql -U postgres -c "select count(*) from pg_roles where rolname='{postgress_user}'" postgres'''.format(postgress_user=config["postgress_user"]) check_for_pg_user_output = esg_functions.call_subprocess(check_for_pg_user_command) print "check_for_postgres_db_user: "******"stdout"].strip().split("\n")[-2].strip()) print "count: ", count print "count type: ", type(count) if count > 0: print "{postgress_user} user found".format(postgress_user=config["postgress_user"]) else: create_postgres_db_user()
def check_java_version(java_path): print "Checking Java version" try: java_version_output = esg_functions.call_subprocess("{java_path} -version".format(java_path=java_path))["stderr"] except KeyError: logger.exception("Could not check the Java version") esg_functions.exit_with_error(1) installed_java_version = re.search("1.8.0_\w+", java_version_output).group() if esg_version_manager.compare_versions(installed_java_version, config["java_version"]): print "Installed java version meets the minimum requirement " return java_version_output
def test_create_empty_java_keystore(self): esg_cert_manager.create_empty_java_keystore("/tmp/test-keystore", "testing", "password", "CN=ESGF") test_keystore_output = esg_functions.call_subprocess( "/usr/local/java/bin/keytool -list -keystore /tmp/test-keystore") print "test_keystore_output:", test_keystore_output["stdout"] self.assertTrue(test_keystore_output["returncode"] == 0) keystore_output = esg_cert_manager.check_keystore( "/tmp/test-keystore", "password") self.assertTrue("testing" in keystore_output.private_keys.keys())
def convert_per_to_dem(private_key, key_output_dir): '''Convert your private key into from PEM to DER format that java likes''' print "\n*******************************" print "converting private key from PEM to DER... " print "******************************* \n" derkey = os.path.join(key_output_dir, "key.der") convert_to_der = esg_functions.call_subprocess( "openssl pkcs8 -topk8 -nocrypt -inform PEM -in {private_key} -outform DER -out {derkey}" .format(private_key=private_key, derkey=derkey)) if convert_to_der["returncode"] != 0: print "Problem with preparing initial keystore...Exiting." esg_functions.exit_with_error(1) return derkey
def check_cachain_validity(ca_chain_bundle): '''Verify that the CA chain is valid''' print "checking that chain is valid... " if os.path.isfile(ca_chain_bundle): valid_chain = esg_functions.call_subprocess( "openssl verify -CAfile {ca_chain_bundle} {ca_chain_bundle}". format(ca_chain_bundle=ca_chain_bundle)) if "error" in valid_chain['stdout'] or "error" in valid_chain['stderr']: print "The chain is not valid. (hint: did you include the root cert for the chain?)" else: print "{ca_chain_bundle} is valid.".format( ca_chain_bundle=ca_chain_bundle) else: print "Hmmm... no chain provided [{ca_chain_bundle}], skipping this check..."
def check_existing_pg_version(psql_path, force_install=False): print "Checking for postgresql >= {postgress_min_version} ".format(postgress_min_version = config["postgress_min_version"]) if not psql_path: print "Postgres not found on system" else: try: postgres_version_found = esg_functions.call_subprocess("psql --version")["stdout"] postgres_version_number = re.search("\d.*", postgres_version_found).group() print "postgres version number: ", postgres_version_number if semver.compare(postgres_version_number, config["postgress_min_version"]) >= 1 and not force_install: return True except OSError: logger.exception("Unable to check existing Postgres version \n")
def create_empty_java_keystore(keystore_name, keystore_alias, keystore_password, distinguished_name): '''Create a new empty Java Keystore using the JDK's keytool''' java_keytool_executable = "{java_install_dir}/bin/keytool".format( java_install_dir=config["java_install_dir"]) generate_keystore_string = "{java_keytool_executable} -genkey -keyalg RSA -alias {keystore_alias} -keystore {keystore_name} -storepass {keystore_password} -keypass {keystore_password} -validity 360 -dname {distinguished_name} -noprompt".format( java_keytool_executable=java_keytool_executable, keystore_alias=keystore_alias, keystore_name=keystore_name, keystore_password=keystore_password, distinguished_name=distinguished_name) keystore_output = esg_functions.call_subprocess(generate_keystore_string) if keystore_output["returncode"] != 0: print "Problem with generating initial keystore...Exiting." esg_functions.exit_with_error(1)
def run_esginitialize(): '''Run the esginitialize script to initialize the ESG node database.''' print "\n*******************************" print "Running esginitialize" print "******************************* \n" esginitialize_process = esg_functions.call_subprocess("esginitialize -c") if esginitialize_process["returncode"] != 0: logger.exception("esginitialize failed") logger.error(esginitialize_process["stderr"]) print esginitialize_process["stderr"] esg_functions.exit_with_error(1) else: print esginitialize_process["stdout"] print esginitialize_process["stderr"]
def setup_temp_ca(): esg_bash2py.mkdir_p("/etc/tempcerts") #Copy CA perl script and openssl conf file that it uses. The CA perl script #is used to create a temporary root CA shutil.copyfile("apache_certs/CA.pl", "/etc/tempcerts/CA.pl") shutil.copyfile("apache_certs/openssl.cnf", "/etc/tempcerts/openssl.cnf") shutil.copyfile("apache_certs/myproxy-server.config", "/etc/tempcerts/myproxy-server.config") os.chmod("/etc/tempcerts/CA.pl", 0755) os.chmod("/etc/tempcerts/openssl.cnf", 0755) with esg_bash2py.pushd("/etc/tempcerts"): esg_bash2py.mkdir_p("CA") ca_answer = "{fqdn}-CA".format(fqdn=esg_functions.get_esgf_host()) print "ca_answer:", ca_answer new_ca_output = esg_functions.call_subprocess("perl CA.pl -newca", command_stdin=ca_answer) print "new_ca_output:", new_ca_output
def create_postgres_system_user(pg_sys_acct_homedir): print "Creating account..." useradd_command = '''/usr/sbin/useradd -r -c'PostgreSQL Service ESGF' -d {pg_sys_acct_homedir} -g {pg_sys_acct_group} -p {pg_sys_acct_passwd} -s /bin/bash {pg_sys_acct}'''.format(pg_sys_acct_homedir = pg_sys_acct_homedir, pg_sys_acct_group = config["pg_sys_acct_group"], pg_sys_acct_passwd = config["pg_sys_acct_passwd"], pg_sys_acct = config["pg_sys_acct"] ) useradd_output = esg_functions.call_subprocess(useradd_command) postgres_user_id = pwd.getpwnam(config["pg_sys_acct"]).pw_uid if useradd_output["returncode"] != 0 or useradd_output["returncode"] != 9: print "ERROR: Could not add postgres system account user" esg_functions.exit_with_error(1) elif not postgres_user_id: print " ERROR: Problem with {pg_sys_acct} creation!!!".format(pg_sys_acct=config["pg_sys_acct"]) esg_functions.exit_with_error(1) else: print "Created postgres user with group id: {postgres_user_id}".format(postgres_user_id=pwd.getpwnam(config["pg_sys_acct"]).pw_uid)
def edit_esg_ini(node_short_name="test_node"): '''Edit placeholder values in the generated esg.ini file''' esg_ini_path = os.path.join(config["publisher_home"], config["publisher_config"]) print "esg_ini_path:", esg_ini_path esg_functions.call_subprocess( 'sed -i s/esgcetpass/password/g {esg_ini_path}'.format( esg_ini_path=esg_ini_path)) esg_functions.call_subprocess( 'sed -i s/"host\.sample\.gov"/{esgf_host}/g {esg_ini_path}'.format( esg_ini_path=esg_ini_path, esgf_host=esg_functions.get_esgf_host())) esg_functions.call_subprocess( 'sed -i s/"LASatYourHost"/LASat{node_short_name}/g {esg_ini_path}'. format(esg_ini_path=esg_ini_path, node_short_name=node_short_name))
def test_install_apache_httpd(self): esg_apache_manager.install_apache_httpd() output = esg_functions.call_subprocess("httpd -version")["stdout"] print "output:", output self.assertIsNotNone(output) esg_apache_manager.install_mod_wsgi() self.assertTrue(os.path.isfile("/etc/httpd/modules/mod_wsgi-py27.so")) esg_apache_manager.make_python_eggs_dir() self.assertTrue(os.path.isdir("/var/www/.python-eggs")) owner, group = esg_functions.get_dir_owner_and_group("/var/www/.python-eggs") self.assertEqual(owner, "apache") self.assertEqual(group, "apache") esg_apache_manager.copy_apache_conf_files() self.assertTrue(os.path.isfile("/etc/httpd/conf.d/httpd.conf")) self.assertTrue(os.path.isfile("/etc/httpd/conf.d/ssl.conf")) self.assertTrue(os.listdir("/var/www/html/")) self.assertTrue(os.listdir("/etc/certs"))
def import_cert_into_keystore(keystore_name, keystore_alias, keystore_password, derkey, cert_bundle, provider): '''Imports a signed Certificate into the keystore''' idptools_install_dir = os.path.join(config["esg_tools_dir"], "idptools") extkeytool_executable = os.path.join(idptools_install_dir, "bin", "extkeytool") if not os.path.isfile(extkeytool_executable): install_extkeytool() command = "{extkeytool} -importkey -keystore {keystore_name} -alias {keystore_alias} -storepass {keystore_password} -keypass {keystore_password} -keyfile {derkey} -certfile {cert_bundle} -provider {provider}".format( extkeytool=extkeytool_executable, keystore_name=keystore_name, keystore_alias=keystore_alias, keystore_password=keystore_password, derkey=derkey, cert_bundle=cert_bundle, provider=provider) construct_keystore_output = esg_functions.call_subprocess(command) if construct_keystore_output["returncode"] != 0: print "Could not import cert %s into keystore %s" % (cert_bundle, keystore_name)
def setup_temp_certs(): '''Setup temporary certs for testing a standalone node installation''' temp_ca_name = "{hostname}-CA".format( hostname=esg_functions.get_esgf_host()) new_ca_output = esg_functions.call_subprocess("perl CA.pl -newca")
def restart_postgres(): '''Restarts the postgres server''' print "Restarting postgres server" esg_functions.call_subprocess("service postgresql-9.6 restart") sleep(7) postgres_status()
def start_apache(): return esg_functions.call_subprocess("service esgf-httpd start")
def check_tomcat_status(): return esg_functions.call_subprocess( "ps -aux | grep tomcat | grep -v grep")
def generate_tomcat_keystore(keystore_name, keystore_alias, private_key, public_cert, intermediate_certs): '''The following helper function creates a new keystore for your tomcat installation''' provider = "org.bouncycastle.jce.provider.BouncyCastleProvider" idptools_install_dir = os.path.join(config["esg_tools_dir"], "idptools") if len(intermediate_certs) < 1: print "No intermediate_certs files given" esg_functions.exit_with_error(1) if not os.path.isfile(private_key): print "Private key file {private_key} does not exist".format( private_key=private_key) keystore_password = esg_functions.get_java_keystore_password() #------------- #Display values #------------- print "Keystore name : {keystore_name}".format(keystore_name=keystore_name) print "Keystore alias: {keystore_alias}".format( keystore_alias=keystore_alias) print "Keystore password: {keystore_password}".format( keystore_password=keystore_password) print "Private key : {private_key}".format(private_key=private_key) print "Public cert : {public_cert}".format(public_cert=public_cert) print "Certificates..." esg_bash2py.mkdir_p(idptools_install_dir) cert_bundle = os.path.join(idptools_install_dir, "cert.bundle") ca_chain_bundle = os.path.join(idptools_install_dir, "ca_chain.bundle") cert_bundle, ca_chain_bundle = bundle_certificates(public_cert, intermediate_certs, idptools_install_dir) print "checking that key pair is congruent... " if check_associate_cert_with_private_key(public_cert, private_key): print "The keypair was congruent" else: print "The keypair was not congruent" esg_functions.exit_with_error(1) print "creating keystore... " #create a keystore with a self-signed cert distinguished_name = "CN={esgf_host}".format( esgf_host=esg_functions.get_esgf_host()) #if previous keystore is found; backup backup_previous_keystore(keystore_name) #------------- #Make empty keystore... #------------- create_empty_java_keystore(keystore_name, keystore_alias, keystore_password, distinguished_name) #------------- #Convert your private key into from PEM to DER format that java likes #------------- derkey = convert_per_to_dem(private_key, idptools_install_dir) #------------- #Now we gather up all the other keys in the key chain... #------------- check_cachain_validity(ca_chain_bundle) print "Constructing new keystore content... " import_cert_into_keystore(keystore_name, keystore_alias, keystore_password, derkey, cert_bundle, provider) #Check keystore output java_keytool_executable = "{java_install_dir}/bin/keytool".format( java_install_dir=config["java_install_dir"]) check_keystore_command = "{java_keytool_executable} -v -list -keystore {keystore_name} -storepass {store_password} | egrep '(Owner|Issuer|MD5|SHA1|Serial number):'".format( java_keytool_executable=java_keytool_executable, keystore_name=keystore_name, store_password=keystore_password) keystore_output = esg_functions.call_subprocess(check_keystore_command) if keystore_output["returncode"] == 0: print "Mmmm, freshly baked keystore!" print "If Everything looks good... then replace your current tomcat keystore with {keystore_name}, if necessary.".format( keystore_name=keystore_name) print "Don't forget to change your tomcat's server.xml entry accordingly :-)" print "Remember: Keep your private key {private_key} and signed cert {public_cert} in a safe place!!!".format( private_key=private_key, public_cert=public_cert) else: print "Failed to check keystore" esg_functions.exit_with_error(1)
def create_postgres_db_user(): postgres_db_user_password = _choose_postgres_user_password() create_pg_db_user_command = '''sudo -u postgres psql -c "create user {postgress_user} with superuser password '{postgres_db_user_password}';" '''.format(postgress_user=config["postgress_user"], postgres_db_user_password=postgres_db_user_password) create_pg_db_user_output = esg_functions.call_subprocess(create_pg_db_user_command) if create_pg_db_user_output["returncode"] == 0: print "Successfully created {postgress_user} db user".format(postgress_user=config["postgress_user"])
def check_apache_status(): return esg_functions.call_subprocess("service esgf-httpd status")
def start_tomcat(): print "\n*******************************" print "Attempting to start Tomcat" print "******************************* \n" return esg_functions.call_subprocess( "/usr/local/tomcat/bin/catalina.sh start")