def rebuild_truststore(truststore_file,
certs_dir=config["globus_global_certs_dir"]):
'''Converts ESG certificates (that can be fetch by above function) into a truststore'''
print "(Re)building truststore from esg certificates... [{truststore_file}]".format(
truststore_file=truststore_file)
if not os.path.isdir(certs_dir):
print "Sorry, No esg certificates found... in {certs_dir}".format(
certs_dir=certs_dir)
fetch_esgf_certificates(certs_dir)
#If you don't already have a truststore to build on....
#Start building from a solid foundation i.e. Java's set of ca certs...
if not os.path.isfile(truststore_file):
create_new_truststore(truststore_file)
tmp_dir = "/tmp/esg_scratch"
esg_bash2py.mkdir_p(tmp_dir)
cert_files = glob.glob('{certs_dir}/*.0'.format(certs_dir=certs_dir))
for cert in cert_files:
_insert_cert_into_truststore(cert, truststore_file, tmp_dir)
shutil.rmtree(tmp_dir)
sync_with_java_truststore(truststore_file)
os.chown(truststore_file, esg_functions.get_user_id("tomcat"),
esg_functions.get_group_id("tomcat"))
def setup_solr():
'''Setup Apache Solr for faceted search'''
print "\n*******************************"
print "Setting up Solr"
print "******************************* \n"
# # Solr/Jetty web application
SOLR_VERSION = "5.5.4"
SOLR_INSTALL_DIR = "/usr/local/solr"
SOLR_HOME = "/usr/local/solr-home"
SOLR_DATA_DIR = "/esg/solr-index"
SOLR_INCLUDE = "{SOLR_HOME}/solr.in.sh".format(SOLR_HOME=SOLR_HOME)
#Download solr tarball
solr_tarball_url = "http://archive.apache.org/dist/lucene/solr/{SOLR_VERSION}/solr-{SOLR_VERSION}.tgz".format(
SOLR_VERSION=SOLR_VERSION)
download_solr_tarball(solr_tarball_url, SOLR_VERSION)
#Extract solr tarball
extract_solr_tarball(
'/tmp/solr-{SOLR_VERSION}.tgz'.format(SOLR_VERSION=SOLR_VERSION),
SOLR_VERSION)
esg_bash2py.mkdir_p(SOLR_DATA_DIR)
# download template directory structure for shards home
download_template_directory()
esg_bash2py.mkdir_p(SOLR_HOME)
# create non-privilged user to run Solr server
esg_functions.stream_subprocess_output("groupadd solr")
esg_functions.stream_subprocess_output(
"useradd -s /sbin/nologin -g solr -d /usr/local/solr solr")
SOLR_USER_ID = pwd.getpwnam("solr").pw_uid
SOLR_GROUP_ID = grp.getgrnam("solr").gr_gid
esg_functions.change_permissions_recursive(
"/usr/local/solr-{SOLR_VERSION}".format(SOLR_VERSION=SOLR_VERSION),
SOLR_USER_ID, SOLR_GROUP_ID)
esg_functions.change_permissions_recursive(SOLR_HOME, SOLR_USER_ID,
SOLR_GROUP_ID)
esg_functions.change_permissions_recursive(SOLR_DATA_DIR, SOLR_USER_ID,
SOLR_GROUP_ID)
#
#Copy shard files
shutil.copyfile("solr_scripts/add_shard.sh", "/usr/local/bin/add_shard.sh")
shutil.copyfile("solr_scripts/remove_shard.sh",
"/usr/local/bin/remove_shard.sh")
# add shards
esg_functions.call_subprocess("/usr/local/bin/add_shard.sh master 8984")
esg_functions.call_subprocess("/usr/local/bin/add_shard.sh master 8983")
# custom logging properties
shutil.copyfile(
"solr_scripts/log4j.properties",
"/{SOLR_INSTALL_DIR}/server/resources/log4j.properties".format(
SOLR_INSTALL_DIR=SOLR_INSTALL_DIR))
def copy_config_files():
'''copy custom configuration'''
'''server.xml: includes references to keystore, truststore in /esg/config/tomcat'''
'''context.xml: increases the Tomcat cache to avoid flood of warning messages'''
print "\n*******************************"
print "Copying custom Tomcat config files"
print "******************************* \n"
try:
shutil.copyfile("tomcat_conf/server.xml",
"/usr/local/tomcat/conf/server.xml")
shutil.copyfile("tomcat_conf/context.xml",
"/usr/local/tomcat/conf/context.xml")
# shutil.copyfile("certs/esg-truststore.ts", "/esg/config/tomcat/esg-truststore.ts")
# shutil.copyfile("certs/esg-truststore.ts-orig", "/esg/config/tomcat/esg-truststore.ts-orig")
# shutil.copyfile("certs/keystore-tomcat", "/esg/config/tomcat/keystore-tomcat")
esg_bash2py.mkdir_p("/esg/config/tomcat")
shutil.copyfile("certs/tomcat-users.xml",
"/esg/config/tomcat/tomcat-users.xml")
#
shutil.copy("tomcat_conf/setenv.sh",
os.path.join(CATALINA_HOME, "bin"))
except OSError, error:
# if error.errno == errno.EEXIST:
# pass
# else:
logger.exception()
def setup_orp():
'''Setup the ORP subsystem'''
if os.path.isdir("/usr/local/tomcat/webapps/esg-orp"):
orp_install = raw_input(
"Existing ORP installation found. Do you want to continue with the ORP installation [y/N]: "
) or "no"
if orp_install.lower() in ["no", "n"]:
return
print "\n*******************************"
print "Setting up ORP"
print "******************************* \n"
esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esg-orp")
#COPY esgf-orp/esg-orp.war /usr/local/tomcat/webapps/esg-orp/esg-orp.war
orp_url = os.path.join("http://", config["esgf_dist_mirror"], "dist",
"devel", "esg-orp", "esg-orp.war")
print "orp_url:", orp_url
download_orp_war(orp_url)
with esg_bash2py.pushd("/usr/local/tomcat/webapps/esg-orp"):
with zipfile.ZipFile("/usr/local/tomcat/webapps/esg-orp/esg-orp.war",
'r') as zf:
zf.extractall()
os.remove("esg-orp.war")
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive(
"/usr/local/tomcat/webapps/esg-orp", TOMCAT_USER_ID,
TOMCAT_GROUP_ID)
# properties to read the Tomcat keystore, used to sign the authentication cookie
# these values are the same for all ESGF nodes
shutil.copyfile(
"esgf_orp_conf/esg-orp.properties",
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/classes/esg-orp.properties")
def migrate_tomcat_credentials_to_esgf(esg_dist_url, tomcat_config_dir):
'''
Move selected config files into esgf tomcat's config dir (certificate et al)
Ex: /esg/config/tomcat
-rw-r--r-- 1 tomcat tomcat 181779 Apr 22 19:44 esg-truststore.ts
-r-------- 1 tomcat tomcat 887 Apr 22 19:32 hostkey.pem
-rw-r--r-- 1 tomcat tomcat 1276 Apr 22 19:32 keystore-tomcat
-rw-r--r-- 1 tomcat tomcat 590 Apr 22 19:32 pcmdi11.llnl.gov-esg-node.csr
-rw-r--r-- 1 tomcat tomcat 733 Apr 22 19:32 pcmdi11.llnl.gov-esg-node.pem
-rw-r--r-- 1 tomcat tomcat 295 Apr 22 19:42 tomcat-users.xml
Only called when migration conditions are present.
'''
tomcat_install_config_dir = os.path.join(config["tomcat_install_dir"],
"conf")
if tomcat_install_config_dir != config["tomcat_conf_dir"]:
if not os.path.exists(config["tomcat_conf_dir"]):
esg_bash2py.mkdir_p(config["tomcat_conf_dir"])
esg_functions.backup(tomcat_install_config_dir)
copy_credential_files(tomcat_install_config_dir)
os.chown(config["tomcat_conf_dir"],
esg_functions.get_user_id("tomcat"),
esg_functions.get_group_id("tomcat"))
if not check_server_xml():
download_server_config_file(esg_dist_url)
#SET the server.xml variables to contain proper values
logger.debug("Editing %s/conf/server.xml accordingly...",
config["tomcat_install_dir"])
edit_tomcat_server_xml(config["keystore_password"])
def create_postgres_log_dir():
''' Create log directory '''
esg_bash2py.mkdir_p(os.path.join(config["postgress_install_dir"], "log"))
postgres_user_id = pwd.getpwnam(config["pg_sys_acct"]).pw_uid
try:
os.chown(os.path.join(config["postgress_install_dir"], "log"), postgres_user_id, -1)
except OSError, error:
print " ERROR: Could not change ownership of postgres' log to \"{pg_sys_acct}\" user".format(pg_sys_acct = config["pg_sys_acct"])
def test_extract_tomcat_tarball(self):
esg_bash2py.mkdir_p("/usr/local/tomcat_test")
esg_tomcat_manager.download_tomcat()
esg_tomcat_manager.extract_tomcat_tarball("/usr/local/tomcat_test")
self.assertTrue(
os.path.isdir("/usr/local/tomcat_test/apache-tomcat-8.5.20"))
self.assertTrue(os.path.isdir("/usr/local/tomcat_test/"))
esg_tomcat_manager.copy_config_files()
self.assertTrue(os.path.isfile("/usr/local/tomcat/conf/server.xml"))
def create_esg_directories():
'''Create directories to hold ESGF scripts, config files, and logs'''
directories_to_check = [config["scripts_dir"], config["esg_backup_dir"], config["esg_tools_dir"],
config[
"esg_log_dir"], config["esg_config_dir"], config["esg_etc_dir"],
config["tomcat_conf_dir"]]
for directory in directories_to_check:
if not os.path.isdir(directory):
esg_bash2py.mkdir_p(directory)
os.chmod(config["esg_etc_dir"], 0777)
def setup_dashboard():
if os.path.isdir("/usr/local/tomcat/webapps/esgf-stats-api"):
stats_api_install = raw_input(
"Existing Stats API installation found. Do you want to continue with the Stats API installation [y/N]: "
) or "no"
if stats_api_install.lower() in ["no", "n"]:
return
print "\n*******************************"
print "Setting up ESGF Stats API (dashboard)"
print "******************************* \n"
esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esgf-stats-api")
stats_api_url = os.path.join("http://", config["esgf_dist_mirror"], "dist",
"devel", "esgf-stats-api",
"esgf-stats-api.war")
download_stats_api_war(stats_api_url)
with esg_bash2py.pushd("/usr/local/tomcat/webapps/esgf-stats-api"):
with zipfile.ZipFile(
"/usr/local/tomcat/webapps/esgf-stats-api/esgf-stats-api.war",
'r') as zf:
zf.extractall()
os.remove("esgf-stats-api.war")
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive(
"/usr/local/tomcat/webapps/esgf-stats-api", TOMCAT_USER_ID,
TOMCAT_GROUP_ID)
# execute dashboard installation script (without the postgres schema)
run_dashboard_script()
# create non-privileged user to run the dashboard application
# RUN groupadd dashboard && \
# useradd -s /sbin/nologin -g dashboard -d /usr/local/dashboard dashboard && \
# chown -R dashboard:dashboard /usr/local/esgf-dashboard-ip
# RUN chmod a+w /var/run
esg_functions.stream_subprocess_output("groupadd dashboard")
esg_functions.stream_subprocess_output(
"useradd -s /sbin/nologin -g dashboard -d /usr/local/dashboard dashboard"
)
DASHBOARD_USER_ID = pwd.getpwnam("dashboard").pw_uid
DASHBOARD_GROUP_ID = grp.getgrnam("dashboard").gr_gid
esg_functions.change_permissions_recursive("/usr/local/esgf-dashboard-ip",
DASHBOARD_USER_ID,
DASHBOARD_GROUP_ID)
os.chmod("/var/run", stat.S_IWRITE)
os.chmod("/var/run", stat.S_IWGRP)
os.chmod("/var/run", stat.S_IWOTH)
start_dashboard_service()
def copy_apache_conf_files():
''' Copy custom apache conf files '''
esg_bash2py.mkdir_p("/etc/certs")
#TODO: Generate certs from esg_cert_manager and copy here
# esg_cert_manager.create_self_signed_cert("/etc/certs")
# esg_cert_manager.create_certificate_chain("/etc/certs/hostcert.pem")
# shutil.copyfile("apache_certs/hostcert.pem", "/etc/certs/hostcert.pem")
# shutil.copyfile("apache_certs/hostkey.pem", "/etc/certs/hostkey.pem")
shutil.copyfile("apache_certs/esgf-ca-bundle.crt",
"/etc/certs/esgf-ca-bundle.crt")
# shutil.copyfile("apache_certs/temp_cachain.pem", "/etc/certs/cachain.pem")
shutil.copyfile("apache_html/index.html", "/var/www/html/index.html")
# shutil.copyfile("apache_conf/httpd.conf", "/etc/httpd/conf.d/httpd.conf")
shutil.copyfile("apache_conf/ssl.conf", "/etc/httpd/conf.d/ssl.conf")
def setup_temp_ca():
esg_bash2py.mkdir_p("/etc/tempcerts")
#Copy CA perl script and openssl conf file that it uses. The CA perl script
#is used to create a temporary root CA
shutil.copyfile("apache_certs/CA.pl", "/etc/tempcerts/CA.pl")
shutil.copyfile("apache_certs/openssl.cnf", "/etc/tempcerts/openssl.cnf")
shutil.copyfile("apache_certs/myproxy-server.config",
"/etc/tempcerts/myproxy-server.config")
os.chmod("/etc/tempcerts/CA.pl", 0755)
os.chmod("/etc/tempcerts/openssl.cnf", 0755)
with esg_bash2py.pushd("/etc/tempcerts"):
esg_bash2py.mkdir_p("CA")
ca_answer = "{fqdn}-CA".format(fqdn=esg_functions.get_esgf_host())
print "ca_answer:", ca_answer
new_ca_output = esg_functions.call_subprocess("perl CA.pl -newca",
command_stdin=ca_answer)
print "new_ca_output:", new_ca_output
def backup(path, backup_dir = config["esg_backup_dir"], num_of_backups=config["num_backups_to_keep"]):
'''
Given a directory the contents of the directory is backed up as a tar.gz file in
path - a filesystem path
backup_dir - destination directory for putting backup archive (default esg_backup_dir:-/esg/backups)
num_of_backups - the number of backup files you wish to have present in destination directory (default num_backups_to_keep:-7)
'''
source_directory = readlinkf(path)
print "Backup - Creating a backup archive of %s" % (source_directory)
current_directory = os.getcwd()
os.chdir(source_directory)
esg_bash2py.mkdir_p(source_directory)
source_backup_name = re.search("\w+$", source_directory).group()
backup_filename=readlinkf(backup_dir)+"/"+source_backup_name + "." + str(datetime.date.today())+".tgz"
try:
with tarfile.open(backup_filename, "w:gz") as tar:
tar.add(source_directory)
except:
print "ERROR: Problem with creating backup archive: {backup_filename}".format(backup_filename = backup_filename)
os.chdir(current_directory)
return 1
if os.path.isfile(backup_filename):
print "Created backup: %s" % (backup_filename)
else:
"Could not locate backup file %s" % (backup_filename)
os.chdir(current_directory)
return 1
# Cleanup
if os.getcwd() != backup_dir:
os.chdir(backup_dir)
files= subprocess.Popen('ls -t | grep %s.\*.tgz | tail -n +$((%i+1)) | xargs' %(source_backup_name,int(num_of_backups)), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if len(files.stdout.readlines()) > 0:
print "Tidying up a bit..."
print "old backup files to remove: %s" % (''.join(files.stdout.readlines()))
for file in files.stdout.readlines():
os.remove(file)
os.chdir(current_directory)
return 0
def setup_java():
'''
Installs Oracle Java from rpm using yum localinstall. Does nothing if an acceptible Java install is found.
'''
print "*******************************"
print "Setting up Java {java_version}".format(java_version=config["java_version"])
print "******************************* \n"
if force_install:
pass
if check_for_existing_java():
setup_java_answer = raw_input("Do you want to continue with Java installation and setup? [y/N]: ") or "N"
if setup_java_answer.lower().strip() not in ["y", "yes"]:
print "Skipping Java installation"
return
last_java_truststore_file = esg_functions.readlinkf(config["truststore_file"])
esg_bash2py.mkdir_p(config["workdir"])
with esg_bash2py.pushd(config["workdir"]):
java_tarfile = esg_bash2py.trim_string_from_head(config["java_dist_url"])
jdk_directory = java_tarfile.split("-")[0]
java_install_dir_parent = config["java_install_dir"].rsplit("/",1)[0]
#Check for Java tar file
if not os.path.isfile(java_tarfile):
print "Don't see java distribution file {java_dist_file_path} either".format(java_dist_file_path=os.path.join(os.getcwd(),java_tarfile))
download_java(java_tarfile)
print "Extracting Java tarfile", java_tarfile
esg_functions.extract_tarball(java_tarfile, java_install_dir_parent)
#Create symlink to Java install directory (/usr/local/java)
esg_bash2py.symlink_force(os.path.join(java_install_dir_parent, jdk_directory), config["java_install_dir"])
os.chown(config["java_install_dir"], config["installer_uid"], config["installer_gid"])
#recursively change permissions
esg_functions.change_permissions_recursive(config["java_install_dir"], config["installer_uid"], config["installer_gid"])
set_default_java()
print check_java_version("java")
def create_self_signed_cert(cert_dir):
"""
If datacard.crt and datacard.key don't exist in cert_dir, create a new
self-signed cert and keypair and write them into that directory.
Source: https://skippylovesmalorie.wordpress.com/2010/02/12/how-to-generate-a-self-signed-certificate-using-pyopenssl/
"""
CERT_FILE = "hostcert.pem"
KEY_FILE = "hostkey.pem"
if not os.path.exists(os.path.join(cert_dir, CERT_FILE)) \
or not os.path.exists(os.path.join(cert_dir, KEY_FILE)):
# create a key pair
k = OpenSSL.crypto.PKey()
k.generate_key(OpenSSL.crypto.TYPE_RSA, 4096)
# create a self-signed cert
cert = OpenSSL.crypto.X509()
cert.get_subject().C = "US"
cert.get_subject().ST = "California"
cert.get_subject().L = "Livermore"
cert.get_subject().O = "LLNL"
cert.get_subject().OU = "ESGF"
cert.get_subject().CN = socket.gethostname()
cert.set_serial_number(1000)
cert.gmtime_adj_notBefore(0)
cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)
cert.set_issuer(cert.get_subject())
cert.set_pubkey(k)
cert.sign(k, 'sha1')
esg_bash2py.mkdir_p(cert_dir)
with open(os.path.join(cert_dir, CERT_FILE), "wt") as cert_file_handle:
cert_file_handle.write(
OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM,
cert))
with open(os.path.join(cert_dir, KEY_FILE), "wt") as key_file_handle:
key_file_handle.write(
OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, k))
def create_certificate_chain_list():
'''Create a list of the certificates that will be a part of the certificate
chain file'''
default_cachain = "/etc/esgfcerts/cachain.pem"
cert_files = []
#Enter ca_chain file into list
print "Please enter your Certificate Authority's certificate chain file(s)"
print "[enter each cert file/url press return, press return with blank entry when done]"
while True:
certfile_entry = raw_input("Enter certificate chain file name: ")
if not certfile_entry:
if not cert_files:
print "Adding default certificate chain file {default_cachain}".format(
default_cachain=default_cachain)
if os.path.isfile(default_cachain):
cert_files.append(default_cachain)
break
else:
print "{default_cachain} does not exist".format(
default_cachain=default_cachain)
print "Creating {default_cachain}".format(
default_cachain=default_cachain)
esg_bash2py.mkdir_p("/etc/esgfcerts")
esg_bash2py.touch(default_cachain)
cert_files.append(default_cachain)
break
# esg_functions.exit_with_error(1)
else:
break
else:
if os.path.isfile(certfile_entry):
cert_files.append(certfile_entry)
else:
print "{certfile_entry} does not exist".format(
certfile_entry=certfile_entry)
return cert_files
def setup_node_manager_old():
if os.path.isdir("/usr/local/tomcat/webapps/esgf-node-manager"):
node_manager_install = raw_input(
"Existing Node Manager installation found. Do you want to continue with the Node Manager installation [y/N]: "
) or "no"
if node_manager_install.lower() in ["no", "n"]:
return
print "\n*******************************"
print "Setting up ESGF Node Manager (old)"
print "******************************* \n"
esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esgf-node-manager")
node_manager_url = os.path.join("http://", config["esgf_dist_mirror"],
"dist", "devel", "esgf-node-manager",
"esgf-node-manager.war")
download_node_manager_war(node_manager_url)
with esg_bash2py.pushd("/usr/local/tomcat/webapps/esgf-node-manager/"):
with zipfile.ZipFile(
"/usr/local/tomcat/webapps/esgf-node-manager/esgf-node-manager.war",
'r') as zf:
zf.extractall()
os.remove("esgf-node-manager.war")
def setup_cog():
# choose CoG version
COG_TAG = "v3.9.7"
# # env variable to execute CoG initialization
# # may be overridden from command line after first container startup
INIT = True
# setup CoG environment
COG_DIR = "/usr/local/cog"
esg_bash2py.mkdir_p(COG_DIR)
COG_CONFIG_DIR = "{COG_DIR}/cog_config".format(COG_DIR=COG_DIR)
esg_bash2py.mkdir_p(COG_CONFIG_DIR)
COG_INSTALL_DIR = "{COG_DIR}/cog_install".format(COG_DIR=COG_DIR)
esg_bash2py.mkdir_p(COG_INSTALL_DIR)
# ENV LD_LIBRARY_PATH=/usr/local/lib
#
# # install Python virtual environment
# RUN cd $COG_DIR && \
# virtualenv venv
#
# # download CoG specific tag or branch
clone_cog_repo(COG_INSTALL_DIR)
#(Ex: esgf-node-manager-0.9.0 -> esgf-node-manager.war)
# local trimmed_name=$(pwd)/${node_dist_dir%-*}
split_dir_name_list = node_dist_dir.split("-")
versionless_name = '-'.join(split_dir_name_list[:3])
trimmed_name = os.path.join(os.getcwd(), versionless_name)
node_war_file = trimmed_name + ".war"
logger.debug("node_war_file: %s", node_war_file)
#----------------------------
# make room for new INSTALL
# ((upgrade == 0)) && set_aside_web_app ${node_manager_service_app_home}
if mode != "upgrade":
set_aside_web_app(node_manager_service_app_home)
#----------------------------
# mkdir -p ${node_manager_service_app_home}
esg_bash2py.mkdir_p(node_manager_service_app_home)
# cd ${node_manager_service_app_home}
os.chdir(node_manager_service_app_home)
logger.debug("changed directory to : %s", os.getcwd())
#----------------------------
# fetch_file=esgf-node-manager.properties
download_file_name = "esgf-node-manager.properties"
# NOTE: The saving of the last config file must be done *BEFORE* we untar the new distro!
# if ((upgrade)) && [ -e WEB-INF/classes/${fetch_file} ]; then
if mode == "upgrade" and os.path.isfile(
"WEB-INF/classes/{download_file_name}".format(
download_file_name=download_file_name)):
# cp WEB-INF/classes/${fetch_file} WEB-INF/classes/${fetch_file}.saved
esg_functions.create_backup_file(
def generate_tomcat_keystore(keystore_name, keystore_alias, private_key,
public_cert, intermediate_certs):
'''The following helper function creates a new keystore for your tomcat installation'''
provider = "org.bouncycastle.jce.provider.BouncyCastleProvider"
idptools_install_dir = os.path.join(config["esg_tools_dir"], "idptools")
if len(intermediate_certs) < 1:
print "No intermediate_certs files given"
esg_functions.exit_with_error(1)
if not os.path.isfile(private_key):
print "Private key file {private_key} does not exist".format(
private_key=private_key)
keystore_password = esg_functions.get_java_keystore_password()
#-------------
#Display values
#-------------
print "Keystore name : {keystore_name}".format(keystore_name=keystore_name)
print "Keystore alias: {keystore_alias}".format(
keystore_alias=keystore_alias)
print "Keystore password: {keystore_password}".format(
keystore_password=keystore_password)
print "Private key : {private_key}".format(private_key=private_key)
print "Public cert : {public_cert}".format(public_cert=public_cert)
print "Certificates..."
esg_bash2py.mkdir_p(idptools_install_dir)
cert_bundle = os.path.join(idptools_install_dir, "cert.bundle")
ca_chain_bundle = os.path.join(idptools_install_dir, "ca_chain.bundle")
cert_bundle, ca_chain_bundle = bundle_certificates(public_cert,
intermediate_certs,
idptools_install_dir)
print "checking that key pair is congruent... "
if check_associate_cert_with_private_key(public_cert, private_key):
print "The keypair was congruent"
else:
print "The keypair was not congruent"
esg_functions.exit_with_error(1)
print "creating keystore... "
#create a keystore with a self-signed cert
distinguished_name = "CN={esgf_host}".format(
esgf_host=esg_functions.get_esgf_host())
#if previous keystore is found; backup
backup_previous_keystore(keystore_name)
#-------------
#Make empty keystore...
#-------------
create_empty_java_keystore(keystore_name, keystore_alias,
keystore_password, distinguished_name)
#-------------
#Convert your private key into from PEM to DER format that java likes
#-------------
derkey = convert_per_to_dem(private_key, idptools_install_dir)
#-------------
#Now we gather up all the other keys in the key chain...
#-------------
check_cachain_validity(ca_chain_bundle)
print "Constructing new keystore content... "
import_cert_into_keystore(keystore_name, keystore_alias, keystore_password,
derkey, cert_bundle, provider)
#Check keystore output
java_keytool_executable = "{java_install_dir}/bin/keytool".format(
java_install_dir=config["java_install_dir"])
check_keystore_command = "{java_keytool_executable} -v -list -keystore {keystore_name} -storepass {store_password} | egrep '(Owner|Issuer|MD5|SHA1|Serial number):'".format(
java_keytool_executable=java_keytool_executable,
keystore_name=keystore_name,
store_password=keystore_password)
keystore_output = esg_functions.call_subprocess(check_keystore_command)
if keystore_output["returncode"] == 0:
print "Mmmm, freshly baked keystore!"
print "If Everything looks good... then replace your current tomcat keystore with {keystore_name}, if necessary.".format(
keystore_name=keystore_name)
print "Don't forget to change your tomcat's server.xml entry accordingly :-)"
print "Remember: Keep your private key {private_key} and signed cert {public_cert} in a safe place!!!".format(
private_key=private_key, public_cert=public_cert)
else:
print "Failed to check keystore"
esg_functions.exit_with_error(1)
def fetch_esgf_certificates(
globus_certs_dir=config["globus_global_certs_dir"]):
'''Goes to ESG distribution server and pulls down all certificates for the federation.
(suitable for crontabbing)'''
print "\n*******************************"
print "Fetching freshest ESG Federation Certificates..."
print "******************************* \n"
#if globus_global_certs_dir already exists, backup and delete, then recreate empty directory
if os.path.isdir(config["globus_global_certs_dir"]):
esg_functions.backup(
config["globus_global_certs_dir"],
os.path.join(config["globus_global_certs_dir"], ".bak.tz"))
shutil.rmtree(config["globus_global_certs_dir"])
esg_bash2py.mkdir_p(config["globus_global_certs_dir"])
#Download trusted certs tarball
esg_trusted_certs_file = "esg_trusted_certificates.tar"
esg_trusted_certs_file_url = "https://aims1.llnl.gov/esgf/dist/certs/{esg_trusted_certs_file}".format(
esg_trusted_certs_file=esg_trusted_certs_file)
esg_functions.download_update(
os.path.join(globus_certs_dir, esg_trusted_certs_file),
esg_trusted_certs_file_url)
#untar the esg_trusted_certs_file
esg_functions.extract_tarball(
os.path.join(globus_certs_dir, esg_trusted_certs_file),
globus_certs_dir)
os.remove(os.path.join(globus_certs_dir, esg_trusted_certs_file))
#certificate_issuer_cert "/var/lib/globus-connect-server/myproxy-ca/cacert.pem"
simpleCA_cert = "/var/lib/globus-connect-server/myproxy-ca/cacert.pem"
if os.path.isfile(simpleCA_cert):
simpleCA_cert_hash = esg_functions.get_md5sum(simpleCA_cert)
print "checking for MY cert: {globus_global_certs_dir}/{simpleCA_cert_hash}.0".format(
globus_global_certs_dir=config["globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash)
if os.path.isfile(
"{globus_global_certs_dir}/{simpleCA_cert_hash}.0".format(
globus_global_certs_dir=config["globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash)):
print "Local CA cert file detected...."
print "Integrating in local simpleCA_cert... "
print "Local SimpleCA Root Cert: {simpleCA_cert}".format(
simpleCA_cert=simpleCA_cert)
print "Extracting Signing policy"
#Copy simple CA cert to globus cert directory
shutil.copyfile(
simpleCA_cert,
"{globus_global_certs_dir}/{simpleCA_cert_hash}.0".format(
globus_global_certs_dir=config["globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash))
#extract simple CA cert tarball and copy to globus cert directory
simpleCA_cert_parent_dir = esg_functions.get_parent_directory(
simpleCA_cert)
simpleCA_setup_tar_file = os.path.join(
simpleCA_cert_parent_dir,
"globus_simple_ca_{simpleCA_cert_hash}_setup-0.tar.gz".format(
simpleCA_cert_hash=simpleCA_cert_hash))
esg_functions.extract_tarball(simpleCA_setup_tar_file)
with esg_bash2py.pushd(
"globus_simple_ca_{simpleCA_cert_hash}_setup-0".format(
simpleCA_cert_hash=simpleCA_cert_hash)):
shutil.copyfile(
"{simpleCA_cert_hash}.signing_policy".format(
simpleCA_cert_hash=simpleCA_cert_hash),
"{globus_global_certs_dir}/{simpleCA_cert_hash}.signing_policy"
.format(globus_global_certs_dir=config[
"globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash))
if os.path.isdir("/usr/local/tomcat/webapps/ROOT"):
esg_functions.stream_subprocess_output(
"openssl x509 -text -hash -in {simpleCA_cert} > {tomcat_install_dir}/webapps/ROOT/cacert.pem"
.format(simpleCA_cert=simpleCA_cert,
tomcat_install_dir="/usr/loca/tomcat"))
print " My CA Cert now posted @ http://{fqdn}/cacert.pem ".format(
fqdn=socket.getfqdn())
os.chmod("/usr/local/tomcat/webapps/ROOT/cacert.pem", 0644)
os.chmod(config["globus_global_certs_dir"], 0755)
esg_functions.change_permissions_recursive(
config["globus_global_certs_dir"], 0644)
def initial_setup_questionnaire(force_install=False):
print "-------------------------------------------------------"
print 'Welcome to the ESGF Node installation program! :-)'
print "-------------------------------------------------------"
esg_bash2py.mkdir_p(config['esg_config_dir'])
starting_directory = os.getcwd()
os.chdir(config['esg_config_dir'])
esgf_host = esg_property_manager.get_property("esgf_host")
_choose_fqdn(esgf_host)
if not esg_functions.get_security_admin_password() or force_install:
_choose_admin_password()
else:
logger.info("Previously set password found.")
_choose_organization_name()
_choose_node_short_name()
_choose_node_long_name()
_choose_node_namespace()
_choose_node_peer_group()
_choose_esgf_index_peer()
_choose_mail_admin_address()
#TODO:Extract constructring DB string into separate function
db_properties = get_db_properties()
if not all(db_properties) or force_install:
_is_managed_db(db_properties)
_get_db_conn_str_questionnaire(db_properties)
else:
if db_properties["db_host"] == esgf_host or db_properties[
"db_host"] == "localhost":
print "db_connection_string = {db_user}@localhost".format(
db_user=db_properties["db_user"])
else:
connstring_ = "{db_user}@{db_host}:{db_port}/{db_database} [external = ${db_managed}]".format(
db_user=db_properties["db_user"],
db_host=db_properties["db_host"],
db_port=db_properties["db_port"],
db_database=db_properties["db_database"],
db_managed=db_properties["db_managed"])
_choose_publisher_db_user()
_choose_publisher_db_user_passwd()
os.chmod(config['pub_secret_file'], 0640)
if "tomcat" not in esg_functions.get_group_list():
esg_functions.add_unix_group(config["tomcat_group"])
os.chown(config['esgf_secret_file'], config["installer_uid"],
esg_functions.get_tomcat_group_id())
if db_properties["db_host"] == esgf_host or db_properties[
"db_host"] == "localhost":
logger.info("db publisher connection string %s@localhost",
db_properties["db_user"])
else:
logger.info("db publisher connection string %s@%s:%s/%s",
db_properties["db_user"], db_properties["db_host"],
db_properties["db_port"], db_properties["db_database"])
os.chdir(starting_directory)
return True
def setup_thredds():
if os.path.isdir("/usr/local/tomcat/webapps/thredds"):
thredds_install = raw_input(
"Existing Thredds installation found. Do you want to continue with the Thredds installation [y/N]: "
) or "no"
if thredds_install.lower() in ["no", "n"]:
return
print "\n*******************************"
print "Setting up Thredds"
print "******************************* \n"
esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/thredds")
thredds_url = os.path.join("http://", config["esgf_dist_mirror"], "dist",
"devel", "thredds", "5.0", "5.0.1",
"thredds.war")
download_thredds_war(thredds_url)
with esg_bash2py.pushd("/usr/local/tomcat/webapps/thredds"):
with zipfile.ZipFile("/usr/local/tomcat/webapps/thredds/thredds.war",
'r') as zf:
zf.extractall()
os.remove("thredds.war")
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive(
"/usr/local/tomcat/webapps/thredds", TOMCAT_USER_ID,
TOMCAT_GROUP_ID)
# TDS configuration root
esg_bash2py.mkdir_p(os.path.join(config["thredds_content_dir"], "thredds"))
# TDS memory configuration
shutil.copyfile("thredds_conf/threddsConfig.xml",
"/esg/content/thredds/threddsConfig.xml")
# ESGF root catalog
shutil.copyfile("thredds_conf/catalog.xml",
"/esg/content/thredds/catalog.xml-esgcet")
esg_bash2py.mkdir_p("/esg/content/thredds/esgcet")
# TDS customized applicationContext.xml file with ESGF authorizer
shutil.copyfile(
"thredds_conf/applicationContext.xml",
"/usr/local/tomcat/webapps/thredds/WEB-INF/applicationContext.xml")
# TDS jars necessary to support ESGF security filters
# some jars are retrieved from the ESGF repository
# other jars are copied from the unpacked ORP or NM distributions
esgf_devel_url = os.path.join("http://", config["esgf_dist_mirror"],
"dist", "devel")
urllib.urlretrieve(
"{esgf_devel_url}/filters/XSGroupRole-1.0.0.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/XSGroupRole-1.0.0.jar")
urllib.urlretrieve(
"{esgf_devel_url}/filters/commons-httpclient-3.1.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-httpclient-3.1.jar"
)
urllib.urlretrieve(
"{esgf_devel_url}/filters/commons-lang-2.6.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-lang-2.6.jar")
urllib.urlretrieve(
"{esgf_devel_url}/esg-orp/esg-orp-2.9.3.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esg-orp-2.9.3.jar")
urllib.urlretrieve(
"{esgf_devel_url}/esgf-node-manager/esgf-node-manager-common-1.0.0.jar"
.format(esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-node-manager-common-1.0.0.jar"
)
urllib.urlretrieve(
"{esgf_devel_url}/esgf-node-manager/esgf-node-manager-filters-1.0.0.jar"
.format(esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-node-manager-filters-1.0.0.jar"
)
urllib.urlretrieve(
"{esgf_devel_url}/esgf-security/esgf-security-2.7.10.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-security-2.7.10.jar"
)
urllib.urlretrieve(
"{esgf_devel_url}/filters/jdom-legacy-1.1.3.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/jdom-legacy-1.1.3.jar")
urllib.urlretrieve(
"{esgf_devel_url}/filters/opensaml-2.3.2.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/opensaml-2.3.2.jar")
urllib.urlretrieve(
"{esgf_devel_url}/filters/openws-1.3.1.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/openws-1.3.1.jar")
urllib.urlretrieve(
"{esgf_devel_url}/filters/xmltooling-1.2.2.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xmltooling-1.2.2.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/serializer-2.9.1.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/serializer-2.9.1.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/velocity-1.5.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/velocity-1.5.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xalan-2.7.2.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xalan-2.7.2.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xercesImpl-2.10.0.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xercesImpl-2.10.0.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xml-apis-1.4.01.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xml-apis-1.4.01.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xmlsec-1.4.2.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xmlsec-1.4.2.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/log4j-1.2.17.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/log4j-1.2.17.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/commons-io-2.4.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-io-2.4.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-dbcp-1.4.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-dbcp-1.4.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-dbutils-1.3.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-dbutils-1.3.jar"
)
shutil.copyfile(
"/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-pool-1.5.4.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-pool-1.5.4.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/postgresql-8.4-703.jdbc3.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/postgresql-8.4-703.jdbc3.jar"
)
# TDS customized logging (uses DEBUG)
shutil.copyfile(
"thredds_conf/log4j2.xml",
"/usr/local/tomcat/webapps/thredds/WEB-INF/classes/log4j2.xml")
# data node scripts
#TODO: Convert data node scripts to Python
# change ownership of content directory
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive("/esg/content/thredds/",
TOMCAT_USER_ID, TOMCAT_GROUP_ID)
# change ownership of source directory
esg_functions.change_permissions_recursive("/usr/local/webapps/thredds",
TOMCAT_USER_ID, TOMCAT_GROUP_ID)
# cleanup
shutil.rmtree("/usr/local/tomcat/webapps/esgf-node-manager/")
def test_download_orp_war(self):
esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esg-orp")
esg_subsystem.download_orp_war("http://aims1.llnl.gov/esgf/dist/devel/esg-orp/esg-orp.war")
self.assertTrue(os.path.isfile("/usr/local/tomcat/webapps/esg-orp/esg-orp.war"))
st = os.stat("/usr/local/tomcat/webapps/esg-orp/esg-orp.war")
print "war file size:", st.st_size
def setup_subsystem(subsystem,
distribution_directory,
esg_dist_url,
force_install=False):
'''
arg (1) - name of installation script root name. Ex:security which resolves to script file esg-security
arg (2) - directory on the distribution site where script is fetched from Ex: orp
usage: setup_subsystem security orp - looks for the script esg-security in the distriubtion dir orp
'''
subsystem_install_script_path = os.path.join(
config["scripts_dir"], "esg-{subsystem}".format(subsystem=subsystem))
#---
#check that you have at one point in time fetched the subsystem's installation script
#if indeed you have we will assume you would like to proceed with setting up the latest...
#Otherwise we just ask you first before you pull down new code to your machine...
#---
if force_install:
default = "y"
else:
default = "n"
if os.path.exists(subsystem_install_script_path) or force_install:
if default.lower() in ["y", "yes"]:
run_installation = raw_input(
"Would you like to set up {subsystem} services? [Y/n]: ".
format(subsystem=subsystem)) or "y"
else:
run_installation = raw_input(
"Would you like to set up {subsystem} services? [y/N]: ".
format(subsystem=subsystem)) or "n"
if run_installation.lower() in ["n", "no"]:
print "Skipping installation of {subsystem}".format(
subsystem=subsystem)
return True
print "-------------------------------"
print "LOADING installer for {subsystem}... ".format(subsystem=subsystem)
esg_bash2py.mkdir_p(config["workdir"])
with esg_bash2py.pushd(config["workdir"]):
logger.debug("Changed directory to %s", os.getcwd())
with esg_bash2py.pushd(config["scripts_dir"]):
logger.debug("Changed directory to %s", os.getcwd())
subsystem_full_name = "esg-{subsystem}".format(subsystem=subsystem)
subsystem_remote_url = "{esg_dist_url}/{distribution_directory}/{subsystem_full_name}".format(
esg_dist_url=esg_dist_url,
distribution_directory=distribution_directory,
subsystem_full_name=subsystem_full_name)
if not esg_functions.download_update(
"{subsystem_full_name}".format(
subsystem_full_name=subsystem_full_name),
subsystem_remote_url):
logger.error("Could not download %s", subsystem_full_name)
return False
try:
os.chmod(subsystem_full_name, 0755)
except OSError:
logger.exception("Unable to change permissions on %s",
subsystem_full_name)
logger.info("script_dir contents: %s", os.listdir(config["scripts_dir"]))
subsystem_underscore = subsystem.replace("-", "_")
execute_subsystem_command = ". {scripts_dir}/{subsystem_full_name}; setup_{subsystem_underscore}".format(
scripts_dir=config["scripts_dir"],
subsystem_full_name=subsystem_full_name,
subsystem_underscore=subsystem_underscore)
setup_subsystem_process = subprocess.Popen(
['bash', '-c', execute_subsystem_command])
setup_subsystem_stdout, setup_subsystem_stderr = setup_subsystem_process.communicate(
)
logger.debug("setup_subsystem_stdout: %s", setup_subsystem_stdout)
logger.debug("setup_subsystem_stderr: %s", setup_subsystem_stderr)
#!/usr/local/bin/python2.7
import glob
import sys
import os
import datetime
import time
from pylint.lint import Run
from contextlib import contextmanager
import esg_bash2py
import esg_logging_manager
logger = esg_logging_manager.create_rotating_log(__name__)
# create folder pylint_score_reports
esg_bash2py.mkdir_p("pylint_score_reports")
# get all files that contain the esg*_**.py pattern
esgf_python_scripts = glob.glob("esg*_**.py")
#TODO: Might do this set difference operation to remove this script (esg_pylint_script) from the linting
# esgf_python_scripts = set(esgf_python_scripts) - set(glob("esg_pylint_script"))
@contextmanager
def suppress_stdout():
'''Source: http://thesmithfam.org/blog/2012/10/25/temporarily-suppress-console-output-in-python/'''
with open(os.devnull, "w") as devnull:
old_stdout = sys.stdout
sys.stdout = devnull
try:
def process_arguments(node_type_list, devel, esg_dist_url):
logger.debug("node_type_list at start of process_arguments: %s", node_type_list)
logger.info("node_type_list at start of process_arguments: %s", node_type_list)
print "node_type_list at start of process_arguments: %s", node_type_list
args, parser = _define_acceptable_arguments()
logging.debug(pprint.pformat(args))
logger.info("args: %s", args)
print "args:", args
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
if args.install:
if args.type:
for arg in args.type:
node_type_list = set_node_type_value(arg, node_type_list, True)
installer_mode_dictionary["upgrade_mode"] = False
installer_mode_dictionary["install_mode"] = True
set_node_type_value("install", node_type_list, True)
print "node_type_list before returning from args.install:", node_type_list
return node_type_list
logger.debug("Install Services")
if args.update or args.upgrade:
installer_mode_dictionary["upgrade_mode"] = True
installer_mode_dictionary["install_mode"] = False
set_node_type_value("install", node_type_list, True)
logger.debug("Update Services")
esg_functions.verify_esg_node_script("esg_node.py", esg_dist_url, script_version, script_maj_version, devel,"update")
if args.fixperms:
logger.debug("fixing permissions")
setup_sensible_confs()
sys.exit(0)
if args.installlocalcerts:
logger.debug("installing local certs")
get_previous_node_type_config(config["esg_config_type_file"])
install_local_certs()
sys.exit(0)
if args.generateesgfcsrs:
logger.debug("generating esgf csrs")
get_previous_node_type_config(config["esg_config_type_file"])
generate_esgf_csrs()
sys.exit(0)
if args.generateesgfcsrsext:
logger.debug("generating esgf csrs for other node")
get_previous_node_type_config(config["esg_config_type_file"])
generate_esgf_csrs_ext()
sys.exit(0)
if args.certhowto:
logger.debug("cert howto")
cert_howto()
sys.exit(0)
elif args.type:
logger.debug("selecting type")
logger.debug("args.type: %s", args.type)
print "args.type:", args.type
for arg in args.type:
set_node_type_value(arg, node_type_list, True)
sys.exit(0)
elif args.settype:
logger.debug("Selecting type for next start up")
for arg in args.settype:
logger.debug("arg: %s", arg)
node_type_list = []
node_type_list = set_node_type_value(arg, node_type_list, True)
esg_bash2py.mkdir_p(config["esg_config_dir"])
set_node_type_config(node_type_list, config["esg_config_type_file"])
sys.exit(0)
elif args.gettype:
get_previous_node_type_config(config["esg_config_type_file"])
show_type()
sys.exit(0)
elif args.start:
logger.debug("args: %s", args)
# if check_prerequisites() is not 0:
# logger.error("Prerequisites for startup not satisfied. Exiting.")
# sys.exit(1)
logger.debug("START SERVICES: %s", node_type_list)
esg_setup.init_structure()
start(node_type_list)
sys.exit(0)
elif args.stop:
# if check_prerequisites() is not 0:
# logger.error("Prerequisites for startup not satisfied. Exiting.")
# sys.exit(1)
logger.debug("STOP SERVICES")
esg_setup.init_structure()
stop(node_type_list)
sys.exit(0)
elif args.restart:
# if check_prerequisites() is not 0:
# logger.error("Prerequisites for startup not satisfied. Exiting.")
# sys.exit(1)
logger.debug("RESTARTING SERVICES")
esg_setup.init_structure()
stop(node_type_list)
sleep(2)
start(node_type_list)
sys.exit(0)
elif args.status:
# if check_prerequisites() is not 0:
# logger.error("Prerequisites for startup not satisfied. Exiting.")
# sys.exit(1)
get_node_status()
#TODO: Exit with status code dependent on what is returned from get_node_status()
sys.exit(0)
elif args.updatesubinstaller:
esg_functions.verify_esg_node_script("esg_node.py", esg_dist_url, script_version, script_maj_version, devel,"update")
# if check_prerequisites() is not 0:
# logger.error("Prerequisites for startup not satisfied. Exiting.")
# sys.exit(1)
esg_setup.init_structure()
update_script(args[1], args[2])
sys.exit(0)
# elif args.updateapacheconf:
# logger.debug("checking for updated apache frontend configuration")
# esg_apache_manager.update_apache_conf()
# sys.exit(0)
elif args.version:
logger.info("Version: %s", script_version)
logger.info("Release: %s", script_release)
logger.info("Earth Systems Grid Federation (http://esgf.llnl.gov)")
logger.info("ESGF Node Installation Script")
sys.exit(0)
elif args.recommendedsetup:
recommended_setup = True
custom_setup = False
elif args.customsetup:
recommended_setup = False
custom_setup = True
elif args.uselocalfiles:
use_local_files = True
elif args.devel:
devel = True
elif args.prod:
devel = False
def setup_node_manager(mode="install"):
#####
# Install The Node Manager
#####
# - Takes boolean arg: 0 = setup / install mode (default)
# 1 = updated mode
#
# In setup mode it is an idempotent install (default)
# In update mode it will always pull down latest after archiving old
#
print "Checking for node manager {esgf_node_manager_version}".format(
esgf_node_manager_version=config["esgf_node_manager_version"])
if esg_version_manager.check_webapp_version(
"esgf-node-manager",
config["esgf_node_manager_version"]) == 0 and not force_install:
print "\n Found existing version of the node-manager [OK]"
return True
init()
print "*******************************"
print "Setting up The ESGF Node Manager..."
print "*******************************"
# local upgrade=${1:-0}
db_set = 0
if force_install:
default_answer = "N"
else:
default_answer = "Y"
# local dosetup
node_manager_service_app_home = esg_property_manager.get_property(
"node_manager_service_app_home")
if os.path.isdir(node_manager_service_app_home):
db_set = 1
print "Detected an existing node manager installation..."
if default_answer == "Y":
installation_answer = raw_input(
"Do you want to continue with node manager installation and setup? [Y/n]"
) or default_answer
else:
installation_answer = raw_input(
"Do you want to continue with node manager installation and setup? [y/N]"
) or default_answer
if installation_answer.lower() not in ["y", "yes"]:
print "Skipping node manager installation and setup - will assume it's setup properly"
# resetting node manager version to what it is already, not what we prescribed in the script
# this way downstream processes will use the *actual* version in play, namely the (access logging) filter(s)
esgf_node_manager_version = esg_version_manager.get_current_webapp_version(
"esgf_node_manager")
return True
backup_default_answer = "Y"
backup_answer = raw_input(
"Do you want to make a back up of the existing distribution [{node_manager_app_context_root}]? [Y/n] "
.format(node_manager_app_context_root=node_manager_app_context_root
)) or backup_default_answer
if backup_answer.lower in ["yes", "y"]:
print "Creating a backup archive of this web application [{node_manager_service_app_home}]".format(
node_manager_service_app_home=node_manager_service_app_home)
esg_functions.backup(node_manager_service_app_home)
backup_db_default_answer = "Y"
backup_db_answer = raw_input(
"Do you want to make a back up of the existing database [{node_db_name}:esgf_node_manager]?? [Y/n] "
.format(node_db_name=config["node_db_name"]
)) or backup_db_default_answer
if backup_db_answer.lower() in ["yes", "y"]:
print "Creating a backup archive of the manager database schema [{node_db_name}:esgf_node_manager]".format(
node_db_name=config["node_db_name"])
# TODO: Implement this
# esg_postgres.backup_db() -db ${node_db_name} -s node_manager
esg_bash2py.mkdir_p(config["workdir"])
with esg_bash2py.pushd(config["workdir"]):
logger.debug("changed directory to : %s", os.getcwd())
# strip off .tar.gz at the end
#(Ex: esgf-node-manager-0.9.0.tar.gz -> esgf-node-manager-0.9.0)
node_dist_file = esg_bash2py.trim_string_from_head(node_dist_url)
logger.debug("node_dist_file: %s", node_dist_file)
# Should just be esgf-node-manager-x.x.x
node_dist_dir = node_dist_file
# checked_get ${node_dist_file} ${node_dist_url} $((force_install))
if not esg_functions.download_update(
node_dist_file, node_dist_url, force_download=force_install):
print "ERROR: Could not download {node_dist_url} :-(".format(
node_dist_url=node_dist_url)
esg_functions.exit_with_error(1)
# make room for new install
if force_install:
print "Removing Previous Installation of the ESGF Node Manager... ({node_dist_dir})".format(
node_dist_dir=node_dist_dir)
try:
shutil.rmtree(node_dist_dir)
logger.info("Deleted directory: %s", node_dist_dir)
except IOError, error:
logger.error(error)
logger.error("Could not delete directory: %s", node_dist_dir)
esg_functions.exit_with_error(1)
clean_node_manager_webapp_subsystem()
print "\nunpacking {node_dist_file}...".format(
node_dist_file=node_dist_file)
# This probably won't work, because the extension has already been stripped, no idea how this even worked in the bash code smh
try:
tar = tarfile.open(node_dist_file)
tar.extractall()
tar.close()
except Exception, error:
logger.error(error)
print "ERROR: Could not extract the ESG Node: {node_dist_file}".format(
node_dist_file=node_dist_file)
esg_functions.exit_with_error(1)
def make_python_eggs_dir():
esg_bash2py.mkdir_p("/var/www/.python-eggs")
apache_user_id = esg_functions.get_user_id("apache")
apache_group_id = esg_functions.get_group_id("apache")
os.chown("/var/www/.python-eggs", apache_user_id, apache_group_id)
def test_backup(self):
test_backup_dir = "/esg/test_backup"
esg_bash2py.mkdir_p(test_backup_dir)
output = esg_functions.backup(os.getcwd(), backup_dir=test_backup_dir)
self.assertEqual(output, 0)
