def _insert_cert_into_truststore(cert_file, truststore_file, tmp_dir):
'''Takes full path to a pem certificate file and incorporates it into the given truststore'''
print "{cert_file} ->".format(cert_file=cert_file)
cert_hash = cert_file.split(".")[0]
der_file = os.path.join(tmp_dir, cert_hash + ".der")
#--------------
# Convert from PEM format to DER format - for ingest into keystore
cert_pem = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
open(cert_file).read())
with open(der_file, "w") as der_file_handle:
der_file_handle.write(
OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1,
cert_pem))
#--------------
if os.path.isfile(truststore_file):
output = esg_functions.call_subprocess(
"/usr/local/java/bin/keytool -delete -alias {cert_hash} -keystore {truststore_file} -storepass {truststore_password}"
.format(cert_hash=cert_hash,
truststore_file=truststore_file,
truststore_password=config["truststore_password"]))
if output["returncode"] == 0:
print "Deleted cert hash"
output = esg_functions.call_subprocess(
"/usr/local/java/bin/keytool -import -alias {cert_hash} -file {der_file} -keystore {truststore_file} -storepass {truststore_password} -noprompt"
.format(cert_hash=cert_hash,
der_file=der_file,
truststore_file=truststore_file,
truststore_password=config["truststore_password"]))
if output["returncode"] == 0:
print "added {der_file} to {truststore_file}".format(
der_file=der_file, truststore_file=truststore_file)
os.remove(der_file)
def setup_solr():
'''Setup Apache Solr for faceted search'''
print "\n*******************************"
print "Setting up Solr"
print "******************************* \n"
# # Solr/Jetty web application
SOLR_VERSION = "5.5.4"
SOLR_INSTALL_DIR = "/usr/local/solr"
SOLR_HOME = "/usr/local/solr-home"
SOLR_DATA_DIR = "/esg/solr-index"
SOLR_INCLUDE = "{SOLR_HOME}/solr.in.sh".format(SOLR_HOME=SOLR_HOME)
#Download solr tarball
solr_tarball_url = "http://archive.apache.org/dist/lucene/solr/{SOLR_VERSION}/solr-{SOLR_VERSION}.tgz".format(
SOLR_VERSION=SOLR_VERSION)
download_solr_tarball(solr_tarball_url, SOLR_VERSION)
#Extract solr tarball
extract_solr_tarball(
'/tmp/solr-{SOLR_VERSION}.tgz'.format(SOLR_VERSION=SOLR_VERSION),
SOLR_VERSION)
esg_bash2py.mkdir_p(SOLR_DATA_DIR)
# download template directory structure for shards home
download_template_directory()
esg_bash2py.mkdir_p(SOLR_HOME)
# create non-privilged user to run Solr server
esg_functions.stream_subprocess_output("groupadd solr")
esg_functions.stream_subprocess_output(
"useradd -s /sbin/nologin -g solr -d /usr/local/solr solr")
SOLR_USER_ID = pwd.getpwnam("solr").pw_uid
SOLR_GROUP_ID = grp.getgrnam("solr").gr_gid
esg_functions.change_permissions_recursive(
"/usr/local/solr-{SOLR_VERSION}".format(SOLR_VERSION=SOLR_VERSION),
SOLR_USER_ID, SOLR_GROUP_ID)
esg_functions.change_permissions_recursive(SOLR_HOME, SOLR_USER_ID,
SOLR_GROUP_ID)
esg_functions.change_permissions_recursive(SOLR_DATA_DIR, SOLR_USER_ID,
SOLR_GROUP_ID)
#
#Copy shard files
shutil.copyfile("solr_scripts/add_shard.sh", "/usr/local/bin/add_shard.sh")
shutil.copyfile("solr_scripts/remove_shard.sh",
"/usr/local/bin/remove_shard.sh")
# add shards
esg_functions.call_subprocess("/usr/local/bin/add_shard.sh master 8984")
esg_functions.call_subprocess("/usr/local/bin/add_shard.sh master 8983")
# custom logging properties
shutil.copyfile(
"solr_scripts/log4j.properties",
"/{SOLR_INSTALL_DIR}/server/resources/log4j.properties".format(
SOLR_INSTALL_DIR=SOLR_INSTALL_DIR))
def set_postgres_user_shell():
print "Noticed that the existing postgres user [{pg_sys_acct}] does not have the bash shell... Hmmm... making it so ".format(pg_sys_acct = config["pg_sys_acct"])
change_shell_command = "sed -i 's#\('{pg_sys_acct}'.*:\)\(.*\)$#\1/\bin/\bash#' /etc/passwd".format(pg_sys_acct = config["pg_sys_acct"])
esg_functions.call_subprocess(change_shell_command)
if get_postgres_user_shell() == "/bin/bash":
print "[OK]"
print "Postgres user shell is properly configured"
else:
print "[FAIL]"
print "Postgres user shell is not properly configured. The shell is {postgres_shell}. It should be /bin/bash".format(postgres_shell=get_postgres_user_shell())
def purge_tomcat():
print "\n*******************************"
print "Purging Tomcat"
print "******************************* \n"
# esg-node --stop may not actually cause Tomcat to exit properly,
# so force-kill all remaining instances
esg_functions.call_subprocess("pkill -9 -u tomcat")
try:
shutil.rmtree("/etc/logrotate.d/esgf_tomcat")
except OSError, error:
logger.exception("Couldn't delete %s", "/etc/logrotate.d/esgf_tomcat")
def postgres_status():
'''Checks the status of the postgres server'''
status = esg_functions.call_subprocess("service postgresql-9.6 status")
print "Postgres server status:", status["stdout"]
if "running" in status["stdout"]:
return True
else:
return False
def create_tomcat_user():
'''Create the Tomcat system user and user group'''
print "\n*******************************"
print "Creating Tomcat User"
print "******************************* \n"
esg_functions.call_subprocess("groupadd tomcat")
esg_functions.call_subprocess(
"useradd -s /sbin/nologin -g tomcat -d /usr/local/tomcat tomcat")
tomcat_directory = "/usr/local/apache-tomcat-{TOMCAT_VERSION}".format(
TOMCAT_VERSION=TOMCAT_VERSION)
tomcat_user_id = pwd.getpwnam("tomcat").pw_uid
tomcat_group_id = grp.getgrnam("tomcat").gr_gid
esg_functions.change_permissions_recursive(tomcat_directory,
tomcat_user_id, tomcat_group_id)
os.chmod("/usr/local/tomcat/webapps", 0775)
def add_my_cert_to_truststore(truststore_file, keystore_file, keystore_alias):
#----------------------------------------------------------------
#Re-integrate my public key (I mean, my "certificate") from my keystore into the truststore (the place housing all public keys I allow to talk to me)
#----------------------------------------------------------------
print "\n*******************************"
print "Adding public key to truststore file {truststore_file}".format(
truststore_file=truststore_file)
print "******************************* \n"
if os.path.isfile(truststore_file):
print "Re-Integrating keystore's certificate into truststore.... "
print "Extracting keystore's certificate... "
keystore_password = esg_functions.get_java_keystore_password()
extract_cert_output = esg_functions.call_subprocess(
"{java_install_dir}/bin/keytool -export -alias {keystore_alias} -file {keystore_file}.cer -keystore {keystore_file} -storepass {keystore_password}"
.format(java_install_dir=config["java_install_dir"],
keystore_alias=keystore_alias,
keystore_file=keystore_file,
keystore_password=keystore_password))
if extract_cert_output["returncode"] != 0:
print "Could not extract certificate from keystore"
esg_functions.exit_with_error(1)
print "Importing keystore's certificate into truststore... "
import_to_truststore_output = esg_functions.call_subprocess(
"{java_install_dir}/bin/keytool -import -v -trustcacerts -alias {keystore_alias} -keypass {keystore_password} -file {keystore_file}.cer -keystore {truststore_file} -storepass {truststore_password} -noprompt"
.format(java_install_dir=config["java_install_dir"],
keystore_alias=keystore_alias,
keystore_file=keystore_file,
keystore_password=keystore_password,
truststore_file=config["truststore_file"],
truststore_password=config["truststore_password"]))
if import_to_truststore_output["returncode"] != 0:
print "Could not import the certificate into the truststore"
esg_functions.exit_with_error(1)
sync_with_java_truststore(truststore_file)
try:
os.remove(keystore_file + ".cer")
except OSError:
logger.exception("Could not delete extracted cert file")
os.chown(truststore_file, esg_functions.get_user_id("tomcat"),
esg_functions.get_group_id("tomcat"))
def create_postgres_group():
groupadd_command = "/usr/sbin/groupadd -r %s" % (
config["pg_sys_acct_group"])
groupadd_output = esg_functions.call_subprocess(groupadd_command)
if groupadd_output["returncode"] != 0 or groupadd_output["returncode"] != 9:
print "ERROR: *Could not add postgres system group: %s" % (config["pg_sys_acct_group"])
esg_functions.exit_with_error(1)
else:
print "Created postgres group with group id: {postgres_group_id}".format(postgres_group_id=grp.getgrnam(config["pg_sys_acct_group"]).gr_gid)
def main():
print "\n*******************************"
print "Setting up Apache (httpd) Web Server"
print "******************************* \n"
if check_for_apache_installation():
print "Found existing Apache installation."
esg_functions.call_subprocess("httpd -version")
continue_install = raw_input(
"Would you like to continue the Apache installation anyway? [y/N]: "
) or "N"
if continue_install.lower() in ["no", "n"]:
return
install_apache_httpd()
install_mod_wsgi()
make_python_eggs_dir()
copy_apache_conf_files()
start_apache()
def check_for_postgres_db_user():
''' Need to be able to run psql as the postgres system user instead of root to avoid the peer authencation error '''
check_for_pg_user_command = '''sudo -u postgres psql -U postgres -c "select count(*) from pg_roles where rolname='{postgress_user}'" postgres'''.format(postgress_user=config["postgress_user"])
check_for_pg_user_output = esg_functions.call_subprocess(check_for_pg_user_command)
print "check_for_postgres_db_user: "******"stdout"].strip().split("\n")[-2].strip())
print "count: ", count
print "count type: ", type(count)
if count > 0:
print "{postgress_user} user found".format(postgress_user=config["postgress_user"])
else:
create_postgres_db_user()
def check_java_version(java_path):
print "Checking Java version"
try:
java_version_output = esg_functions.call_subprocess("{java_path} -version".format(java_path=java_path))["stderr"]
except KeyError:
logger.exception("Could not check the Java version")
esg_functions.exit_with_error(1)
installed_java_version = re.search("1.8.0_\w+", java_version_output).group()
if esg_version_manager.compare_versions(installed_java_version, config["java_version"]):
print "Installed java version meets the minimum requirement "
return java_version_output
def test_create_empty_java_keystore(self):
esg_cert_manager.create_empty_java_keystore("/tmp/test-keystore",
"testing", "password",
"CN=ESGF")
test_keystore_output = esg_functions.call_subprocess(
"/usr/local/java/bin/keytool -list -keystore /tmp/test-keystore")
print "test_keystore_output:", test_keystore_output["stdout"]
self.assertTrue(test_keystore_output["returncode"] == 0)
keystore_output = esg_cert_manager.check_keystore(
"/tmp/test-keystore", "password")
self.assertTrue("testing" in keystore_output.private_keys.keys())
def convert_per_to_dem(private_key, key_output_dir):
'''Convert your private key into from PEM to DER format that java likes'''
print "\n*******************************"
print "converting private key from PEM to DER... "
print "******************************* \n"
derkey = os.path.join(key_output_dir, "key.der")
convert_to_der = esg_functions.call_subprocess(
"openssl pkcs8 -topk8 -nocrypt -inform PEM -in {private_key} -outform DER -out {derkey}"
.format(private_key=private_key, derkey=derkey))
if convert_to_der["returncode"] != 0:
print "Problem with preparing initial keystore...Exiting."
esg_functions.exit_with_error(1)
return derkey
def check_cachain_validity(ca_chain_bundle):
'''Verify that the CA chain is valid'''
print "checking that chain is valid... "
if os.path.isfile(ca_chain_bundle):
valid_chain = esg_functions.call_subprocess(
"openssl verify -CAfile {ca_chain_bundle} {ca_chain_bundle}".
format(ca_chain_bundle=ca_chain_bundle))
if "error" in valid_chain['stdout'] or "error" in valid_chain['stderr']:
print "The chain is not valid. (hint: did you include the root cert for the chain?)"
else:
print "{ca_chain_bundle} is valid.".format(
ca_chain_bundle=ca_chain_bundle)
else:
print "Hmmm... no chain provided [{ca_chain_bundle}], skipping this check..."
def check_existing_pg_version(psql_path, force_install=False):
print "Checking for postgresql >= {postgress_min_version} ".format(postgress_min_version = config["postgress_min_version"])
if not psql_path:
print "Postgres not found on system"
else:
try:
postgres_version_found = esg_functions.call_subprocess("psql --version")["stdout"]
postgres_version_number = re.search("\d.*", postgres_version_found).group()
print "postgres version number: ", postgres_version_number
if semver.compare(postgres_version_number, config["postgress_min_version"]) >= 1 and not force_install:
return True
except OSError:
logger.exception("Unable to check existing Postgres version \n")
def create_empty_java_keystore(keystore_name, keystore_alias,
keystore_password, distinguished_name):
'''Create a new empty Java Keystore using the JDK's keytool'''
java_keytool_executable = "{java_install_dir}/bin/keytool".format(
java_install_dir=config["java_install_dir"])
generate_keystore_string = "{java_keytool_executable} -genkey -keyalg RSA -alias {keystore_alias} -keystore {keystore_name} -storepass {keystore_password} -keypass {keystore_password} -validity 360 -dname {distinguished_name} -noprompt".format(
java_keytool_executable=java_keytool_executable,
keystore_alias=keystore_alias,
keystore_name=keystore_name,
keystore_password=keystore_password,
distinguished_name=distinguished_name)
keystore_output = esg_functions.call_subprocess(generate_keystore_string)
if keystore_output["returncode"] != 0:
print "Problem with generating initial keystore...Exiting."
esg_functions.exit_with_error(1)
def run_esginitialize():
'''Run the esginitialize script to initialize the ESG node database.'''
print "\n*******************************"
print "Running esginitialize"
print "******************************* \n"
esginitialize_process = esg_functions.call_subprocess("esginitialize -c")
if esginitialize_process["returncode"] != 0:
logger.exception("esginitialize failed")
logger.error(esginitialize_process["stderr"])
print esginitialize_process["stderr"]
esg_functions.exit_with_error(1)
else:
print esginitialize_process["stdout"]
print esginitialize_process["stderr"]
def setup_temp_ca():
esg_bash2py.mkdir_p("/etc/tempcerts")
#Copy CA perl script and openssl conf file that it uses. The CA perl script
#is used to create a temporary root CA
shutil.copyfile("apache_certs/CA.pl", "/etc/tempcerts/CA.pl")
shutil.copyfile("apache_certs/openssl.cnf", "/etc/tempcerts/openssl.cnf")
shutil.copyfile("apache_certs/myproxy-server.config",
"/etc/tempcerts/myproxy-server.config")
os.chmod("/etc/tempcerts/CA.pl", 0755)
os.chmod("/etc/tempcerts/openssl.cnf", 0755)
with esg_bash2py.pushd("/etc/tempcerts"):
esg_bash2py.mkdir_p("CA")
ca_answer = "{fqdn}-CA".format(fqdn=esg_functions.get_esgf_host())
print "ca_answer:", ca_answer
new_ca_output = esg_functions.call_subprocess("perl CA.pl -newca",
command_stdin=ca_answer)
print "new_ca_output:", new_ca_output
def create_postgres_system_user(pg_sys_acct_homedir):
print "Creating account..."
useradd_command = '''/usr/sbin/useradd -r -c'PostgreSQL Service ESGF'
-d {pg_sys_acct_homedir} -g {pg_sys_acct_group} -p
{pg_sys_acct_passwd} -s /bin/bash {pg_sys_acct}'''.format(pg_sys_acct_homedir = pg_sys_acct_homedir,
pg_sys_acct_group = config["pg_sys_acct_group"],
pg_sys_acct_passwd = config["pg_sys_acct_passwd"],
pg_sys_acct = config["pg_sys_acct"] )
useradd_output = esg_functions.call_subprocess(useradd_command)
postgres_user_id = pwd.getpwnam(config["pg_sys_acct"]).pw_uid
if useradd_output["returncode"] != 0 or useradd_output["returncode"] != 9:
print "ERROR: Could not add postgres system account user"
esg_functions.exit_with_error(1)
elif not postgres_user_id:
print " ERROR: Problem with {pg_sys_acct} creation!!!".format(pg_sys_acct=config["pg_sys_acct"])
esg_functions.exit_with_error(1)
else:
print "Created postgres user with group id: {postgres_user_id}".format(postgres_user_id=pwd.getpwnam(config["pg_sys_acct"]).pw_uid)
def edit_esg_ini(node_short_name="test_node"):
'''Edit placeholder values in the generated esg.ini file'''
esg_ini_path = os.path.join(config["publisher_home"],
config["publisher_config"])
print "esg_ini_path:", esg_ini_path
esg_functions.call_subprocess(
'sed -i s/esgcetpass/password/g {esg_ini_path}'.format(
esg_ini_path=esg_ini_path))
esg_functions.call_subprocess(
'sed -i s/"host\.sample\.gov"/{esgf_host}/g {esg_ini_path}'.format(
esg_ini_path=esg_ini_path,
esgf_host=esg_functions.get_esgf_host()))
esg_functions.call_subprocess(
'sed -i s/"LASatYourHost"/LASat{node_short_name}/g {esg_ini_path}'.
format(esg_ini_path=esg_ini_path, node_short_name=node_short_name))
def test_install_apache_httpd(self):
esg_apache_manager.install_apache_httpd()
output = esg_functions.call_subprocess("httpd -version")["stdout"]
print "output:", output
self.assertIsNotNone(output)
esg_apache_manager.install_mod_wsgi()
self.assertTrue(os.path.isfile("/etc/httpd/modules/mod_wsgi-py27.so"))
esg_apache_manager.make_python_eggs_dir()
self.assertTrue(os.path.isdir("/var/www/.python-eggs"))
owner, group = esg_functions.get_dir_owner_and_group("/var/www/.python-eggs")
self.assertEqual(owner, "apache")
self.assertEqual(group, "apache")
esg_apache_manager.copy_apache_conf_files()
self.assertTrue(os.path.isfile("/etc/httpd/conf.d/httpd.conf"))
self.assertTrue(os.path.isfile("/etc/httpd/conf.d/ssl.conf"))
self.assertTrue(os.listdir("/var/www/html/"))
self.assertTrue(os.listdir("/etc/certs"))
def import_cert_into_keystore(keystore_name, keystore_alias, keystore_password,
derkey, cert_bundle, provider):
'''Imports a signed Certificate into the keystore'''
idptools_install_dir = os.path.join(config["esg_tools_dir"], "idptools")
extkeytool_executable = os.path.join(idptools_install_dir, "bin",
"extkeytool")
if not os.path.isfile(extkeytool_executable):
install_extkeytool()
command = "{extkeytool} -importkey -keystore {keystore_name} -alias {keystore_alias} -storepass {keystore_password} -keypass {keystore_password} -keyfile {derkey} -certfile {cert_bundle} -provider {provider}".format(
extkeytool=extkeytool_executable,
keystore_name=keystore_name,
keystore_alias=keystore_alias,
keystore_password=keystore_password,
derkey=derkey,
cert_bundle=cert_bundle,
provider=provider)
construct_keystore_output = esg_functions.call_subprocess(command)
if construct_keystore_output["returncode"] != 0:
print "Could not import cert %s into keystore %s" % (cert_bundle,
keystore_name)
def setup_temp_certs():
'''Setup temporary certs for testing a standalone node installation'''
temp_ca_name = "{hostname}-CA".format(
hostname=esg_functions.get_esgf_host())
new_ca_output = esg_functions.call_subprocess("perl CA.pl -newca")
def restart_postgres():
'''Restarts the postgres server'''
print "Restarting postgres server"
esg_functions.call_subprocess("service postgresql-9.6 restart")
sleep(7)
postgres_status()
def start_apache():
return esg_functions.call_subprocess("service esgf-httpd start")
def check_tomcat_status():
return esg_functions.call_subprocess(
"ps -aux | grep tomcat | grep -v grep")
def generate_tomcat_keystore(keystore_name, keystore_alias, private_key,
public_cert, intermediate_certs):
'''The following helper function creates a new keystore for your tomcat installation'''
provider = "org.bouncycastle.jce.provider.BouncyCastleProvider"
idptools_install_dir = os.path.join(config["esg_tools_dir"], "idptools")
if len(intermediate_certs) < 1:
print "No intermediate_certs files given"
esg_functions.exit_with_error(1)
if not os.path.isfile(private_key):
print "Private key file {private_key} does not exist".format(
private_key=private_key)
keystore_password = esg_functions.get_java_keystore_password()
#-------------
#Display values
#-------------
print "Keystore name : {keystore_name}".format(keystore_name=keystore_name)
print "Keystore alias: {keystore_alias}".format(
keystore_alias=keystore_alias)
print "Keystore password: {keystore_password}".format(
keystore_password=keystore_password)
print "Private key : {private_key}".format(private_key=private_key)
print "Public cert : {public_cert}".format(public_cert=public_cert)
print "Certificates..."
esg_bash2py.mkdir_p(idptools_install_dir)
cert_bundle = os.path.join(idptools_install_dir, "cert.bundle")
ca_chain_bundle = os.path.join(idptools_install_dir, "ca_chain.bundle")
cert_bundle, ca_chain_bundle = bundle_certificates(public_cert,
intermediate_certs,
idptools_install_dir)
print "checking that key pair is congruent... "
if check_associate_cert_with_private_key(public_cert, private_key):
print "The keypair was congruent"
else:
print "The keypair was not congruent"
esg_functions.exit_with_error(1)
print "creating keystore... "
#create a keystore with a self-signed cert
distinguished_name = "CN={esgf_host}".format(
esgf_host=esg_functions.get_esgf_host())
#if previous keystore is found; backup
backup_previous_keystore(keystore_name)
#-------------
#Make empty keystore...
#-------------
create_empty_java_keystore(keystore_name, keystore_alias,
keystore_password, distinguished_name)
#-------------
#Convert your private key into from PEM to DER format that java likes
#-------------
derkey = convert_per_to_dem(private_key, idptools_install_dir)
#-------------
#Now we gather up all the other keys in the key chain...
#-------------
check_cachain_validity(ca_chain_bundle)
print "Constructing new keystore content... "
import_cert_into_keystore(keystore_name, keystore_alias, keystore_password,
derkey, cert_bundle, provider)
#Check keystore output
java_keytool_executable = "{java_install_dir}/bin/keytool".format(
java_install_dir=config["java_install_dir"])
check_keystore_command = "{java_keytool_executable} -v -list -keystore {keystore_name} -storepass {store_password} | egrep '(Owner|Issuer|MD5|SHA1|Serial number):'".format(
java_keytool_executable=java_keytool_executable,
keystore_name=keystore_name,
store_password=keystore_password)
keystore_output = esg_functions.call_subprocess(check_keystore_command)
if keystore_output["returncode"] == 0:
print "Mmmm, freshly baked keystore!"
print "If Everything looks good... then replace your current tomcat keystore with {keystore_name}, if necessary.".format(
keystore_name=keystore_name)
print "Don't forget to change your tomcat's server.xml entry accordingly :-)"
print "Remember: Keep your private key {private_key} and signed cert {public_cert} in a safe place!!!".format(
private_key=private_key, public_cert=public_cert)
else:
print "Failed to check keystore"
esg_functions.exit_with_error(1)
def create_postgres_db_user():
postgres_db_user_password = _choose_postgres_user_password()
create_pg_db_user_command = '''sudo -u postgres psql -c "create user {postgress_user} with superuser password '{postgres_db_user_password}';" '''.format(postgress_user=config["postgress_user"], postgres_db_user_password=postgres_db_user_password)
create_pg_db_user_output = esg_functions.call_subprocess(create_pg_db_user_command)
if create_pg_db_user_output["returncode"] == 0:
print "Successfully created {postgress_user} db user".format(postgress_user=config["postgress_user"])
def check_apache_status():
return esg_functions.call_subprocess("service esgf-httpd status")
def start_tomcat():
print "\n*******************************"
print "Attempting to start Tomcat"
print "******************************* \n"
return esg_functions.call_subprocess(
"/usr/local/tomcat/bin/catalina.sh start")
