def setup_orp():
'''Setup the ORP subsystem'''
if os.path.isdir("/usr/local/tomcat/webapps/esg-orp"):
orp_install = raw_input(
"Existing ORP installation found. Do you want to continue with the ORP installation [y/N]: "
) or "no"
if orp_install.lower() in ["no", "n"]:
return
print "\n*******************************"
print "Setting up ORP"
print "******************************* \n"
esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esg-orp")
#COPY esgf-orp/esg-orp.war /usr/local/tomcat/webapps/esg-orp/esg-orp.war
orp_url = os.path.join("http://", config["esgf_dist_mirror"], "dist",
"devel", "esg-orp", "esg-orp.war")
print "orp_url:", orp_url
download_orp_war(orp_url)
with esg_bash2py.pushd("/usr/local/tomcat/webapps/esg-orp"):
with zipfile.ZipFile("/usr/local/tomcat/webapps/esg-orp/esg-orp.war",
'r') as zf:
zf.extractall()
os.remove("esg-orp.war")
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive(
"/usr/local/tomcat/webapps/esg-orp", TOMCAT_USER_ID,
TOMCAT_GROUP_ID)
# properties to read the Tomcat keystore, used to sign the authentication cookie
# these values are the same for all ESGF nodes
shutil.copyfile(
"esgf_orp_conf/esg-orp.properties",
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/classes/esg-orp.properties")
def setup_dashboard():
if os.path.isdir("/usr/local/tomcat/webapps/esgf-stats-api"):
stats_api_install = raw_input(
"Existing Stats API installation found. Do you want to continue with the Stats API installation [y/N]: "
) or "no"
if stats_api_install.lower() in ["no", "n"]:
return
print "\n*******************************"
print "Setting up ESGF Stats API (dashboard)"
print "******************************* \n"
esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esgf-stats-api")
stats_api_url = os.path.join("http://", config["esgf_dist_mirror"], "dist",
"devel", "esgf-stats-api",
"esgf-stats-api.war")
download_stats_api_war(stats_api_url)
with esg_bash2py.pushd("/usr/local/tomcat/webapps/esgf-stats-api"):
with zipfile.ZipFile(
"/usr/local/tomcat/webapps/esgf-stats-api/esgf-stats-api.war",
'r') as zf:
zf.extractall()
os.remove("esgf-stats-api.war")
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive(
"/usr/local/tomcat/webapps/esgf-stats-api", TOMCAT_USER_ID,
TOMCAT_GROUP_ID)
# execute dashboard installation script (without the postgres schema)
run_dashboard_script()
# create non-privileged user to run the dashboard application
# RUN groupadd dashboard && \
# useradd -s /sbin/nologin -g dashboard -d /usr/local/dashboard dashboard && \
# chown -R dashboard:dashboard /usr/local/esgf-dashboard-ip
# RUN chmod a+w /var/run
esg_functions.stream_subprocess_output("groupadd dashboard")
esg_functions.stream_subprocess_output(
"useradd -s /sbin/nologin -g dashboard -d /usr/local/dashboard dashboard"
)
DASHBOARD_USER_ID = pwd.getpwnam("dashboard").pw_uid
DASHBOARD_GROUP_ID = grp.getgrnam("dashboard").gr_gid
esg_functions.change_permissions_recursive("/usr/local/esgf-dashboard-ip",
DASHBOARD_USER_ID,
DASHBOARD_GROUP_ID)
os.chmod("/var/run", stat.S_IWRITE)
os.chmod("/var/run", stat.S_IWGRP)
os.chmod("/var/run", stat.S_IWOTH)
start_dashboard_service()
def create_tomcat_user():
'''Create the Tomcat system user and user group'''
print "\n*******************************"
print "Creating Tomcat User"
print "******************************* \n"
esg_functions.call_subprocess("groupadd tomcat")
esg_functions.call_subprocess(
"useradd -s /sbin/nologin -g tomcat -d /usr/local/tomcat tomcat")
tomcat_directory = "/usr/local/apache-tomcat-{TOMCAT_VERSION}".format(
TOMCAT_VERSION=TOMCAT_VERSION)
tomcat_user_id = pwd.getpwnam("tomcat").pw_uid
tomcat_group_id = grp.getgrnam("tomcat").gr_gid
esg_functions.change_permissions_recursive(tomcat_directory,
tomcat_user_id, tomcat_group_id)
os.chmod("/usr/local/tomcat/webapps", 0775)
def setup_java():
'''
Installs Oracle Java from rpm using yum localinstall. Does nothing if an acceptible Java install is found.
'''
print "*******************************"
print "Setting up Java {java_version}".format(java_version=config["java_version"])
print "******************************* \n"
if force_install:
pass
if check_for_existing_java():
setup_java_answer = raw_input("Do you want to continue with Java installation and setup? [y/N]: ") or "N"
if setup_java_answer.lower().strip() not in ["y", "yes"]:
print "Skipping Java installation"
return
last_java_truststore_file = esg_functions.readlinkf(config["truststore_file"])
esg_bash2py.mkdir_p(config["workdir"])
with esg_bash2py.pushd(config["workdir"]):
java_tarfile = esg_bash2py.trim_string_from_head(config["java_dist_url"])
jdk_directory = java_tarfile.split("-")[0]
java_install_dir_parent = config["java_install_dir"].rsplit("/",1)[0]
#Check for Java tar file
if not os.path.isfile(java_tarfile):
print "Don't see java distribution file {java_dist_file_path} either".format(java_dist_file_path=os.path.join(os.getcwd(),java_tarfile))
download_java(java_tarfile)
print "Extracting Java tarfile", java_tarfile
esg_functions.extract_tarball(java_tarfile, java_install_dir_parent)
#Create symlink to Java install directory (/usr/local/java)
esg_bash2py.symlink_force(os.path.join(java_install_dir_parent, jdk_directory), config["java_install_dir"])
os.chown(config["java_install_dir"], config["installer_uid"], config["installer_gid"])
#recursively change permissions
esg_functions.change_permissions_recursive(config["java_install_dir"], config["installer_uid"], config["installer_gid"])
set_default_java()
print check_java_version("java")
def setup_solr():
'''Setup Apache Solr for faceted search'''
print "\n*******************************"
print "Setting up Solr"
print "******************************* \n"
# # Solr/Jetty web application
SOLR_VERSION = "5.5.4"
SOLR_INSTALL_DIR = "/usr/local/solr"
SOLR_HOME = "/usr/local/solr-home"
SOLR_DATA_DIR = "/esg/solr-index"
SOLR_INCLUDE = "{SOLR_HOME}/solr.in.sh".format(SOLR_HOME=SOLR_HOME)
#Download solr tarball
solr_tarball_url = "http://archive.apache.org/dist/lucene/solr/{SOLR_VERSION}/solr-{SOLR_VERSION}.tgz".format(
SOLR_VERSION=SOLR_VERSION)
download_solr_tarball(solr_tarball_url, SOLR_VERSION)
#Extract solr tarball
extract_solr_tarball(
'/tmp/solr-{SOLR_VERSION}.tgz'.format(SOLR_VERSION=SOLR_VERSION),
SOLR_VERSION)
esg_bash2py.mkdir_p(SOLR_DATA_DIR)
# download template directory structure for shards home
download_template_directory()
esg_bash2py.mkdir_p(SOLR_HOME)
# create non-privilged user to run Solr server
esg_functions.stream_subprocess_output("groupadd solr")
esg_functions.stream_subprocess_output(
"useradd -s /sbin/nologin -g solr -d /usr/local/solr solr")
SOLR_USER_ID = pwd.getpwnam("solr").pw_uid
SOLR_GROUP_ID = grp.getgrnam("solr").gr_gid
esg_functions.change_permissions_recursive(
"/usr/local/solr-{SOLR_VERSION}".format(SOLR_VERSION=SOLR_VERSION),
SOLR_USER_ID, SOLR_GROUP_ID)
esg_functions.change_permissions_recursive(SOLR_HOME, SOLR_USER_ID,
SOLR_GROUP_ID)
esg_functions.change_permissions_recursive(SOLR_DATA_DIR, SOLR_USER_ID,
SOLR_GROUP_ID)
#
#Copy shard files
shutil.copyfile("solr_scripts/add_shard.sh", "/usr/local/bin/add_shard.sh")
shutil.copyfile("solr_scripts/remove_shard.sh",
"/usr/local/bin/remove_shard.sh")
# add shards
esg_functions.call_subprocess("/usr/local/bin/add_shard.sh master 8984")
esg_functions.call_subprocess("/usr/local/bin/add_shard.sh master 8983")
# custom logging properties
shutil.copyfile(
"solr_scripts/log4j.properties",
"/{SOLR_INSTALL_DIR}/server/resources/log4j.properties".format(
SOLR_INSTALL_DIR=SOLR_INSTALL_DIR))
def setup_esg_search():
'''Setting up the ESG Search application'''
print "\n*******************************"
print "Setting up ESG Search"
print "******************************* \n"
ESGF_REPO = "http://aims1.llnl.gov/esgf"
esg_search_war_url = "{ESGF_REPO}/esg-search/esg-search.war".format(
ESGF_REPO=ESGF_REPO)
download_esg_search_war(esg_search_war_url)
#Extract esg-search war
with esg_bash2py.pushd("/usr/local/tomcat/webapps/esg-search"):
with zipfile.ZipFile(
"/usr/local/tomcat/webapps/esg-search/esg-search.war",
'r') as zf:
zf.extractall()
os.remove("esg-search.war")
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive(
"/usr/local/tomcat/webapps/esg-search", TOMCAT_USER_ID,
TOMCAT_GROUP_ID)
def setup_thredds():
if os.path.isdir("/usr/local/tomcat/webapps/thredds"):
thredds_install = raw_input(
"Existing Thredds installation found. Do you want to continue with the Thredds installation [y/N]: "
) or "no"
if thredds_install.lower() in ["no", "n"]:
return
print "\n*******************************"
print "Setting up Thredds"
print "******************************* \n"
esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/thredds")
thredds_url = os.path.join("http://", config["esgf_dist_mirror"], "dist",
"devel", "thredds", "5.0", "5.0.1",
"thredds.war")
download_thredds_war(thredds_url)
with esg_bash2py.pushd("/usr/local/tomcat/webapps/thredds"):
with zipfile.ZipFile("/usr/local/tomcat/webapps/thredds/thredds.war",
'r') as zf:
zf.extractall()
os.remove("thredds.war")
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive(
"/usr/local/tomcat/webapps/thredds", TOMCAT_USER_ID,
TOMCAT_GROUP_ID)
# TDS configuration root
esg_bash2py.mkdir_p(os.path.join(config["thredds_content_dir"], "thredds"))
# TDS memory configuration
shutil.copyfile("thredds_conf/threddsConfig.xml",
"/esg/content/thredds/threddsConfig.xml")
# ESGF root catalog
shutil.copyfile("thredds_conf/catalog.xml",
"/esg/content/thredds/catalog.xml-esgcet")
esg_bash2py.mkdir_p("/esg/content/thredds/esgcet")
# TDS customized applicationContext.xml file with ESGF authorizer
shutil.copyfile(
"thredds_conf/applicationContext.xml",
"/usr/local/tomcat/webapps/thredds/WEB-INF/applicationContext.xml")
# TDS jars necessary to support ESGF security filters
# some jars are retrieved from the ESGF repository
# other jars are copied from the unpacked ORP or NM distributions
esgf_devel_url = os.path.join("http://", config["esgf_dist_mirror"],
"dist", "devel")
urllib.urlretrieve(
"{esgf_devel_url}/filters/XSGroupRole-1.0.0.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/XSGroupRole-1.0.0.jar")
urllib.urlretrieve(
"{esgf_devel_url}/filters/commons-httpclient-3.1.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-httpclient-3.1.jar"
)
urllib.urlretrieve(
"{esgf_devel_url}/filters/commons-lang-2.6.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-lang-2.6.jar")
urllib.urlretrieve(
"{esgf_devel_url}/esg-orp/esg-orp-2.9.3.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esg-orp-2.9.3.jar")
urllib.urlretrieve(
"{esgf_devel_url}/esgf-node-manager/esgf-node-manager-common-1.0.0.jar"
.format(esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-node-manager-common-1.0.0.jar"
)
urllib.urlretrieve(
"{esgf_devel_url}/esgf-node-manager/esgf-node-manager-filters-1.0.0.jar"
.format(esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-node-manager-filters-1.0.0.jar"
)
urllib.urlretrieve(
"{esgf_devel_url}/esgf-security/esgf-security-2.7.10.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-security-2.7.10.jar"
)
urllib.urlretrieve(
"{esgf_devel_url}/filters/jdom-legacy-1.1.3.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/jdom-legacy-1.1.3.jar")
urllib.urlretrieve(
"{esgf_devel_url}/filters/opensaml-2.3.2.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/opensaml-2.3.2.jar")
urllib.urlretrieve(
"{esgf_devel_url}/filters/openws-1.3.1.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/openws-1.3.1.jar")
urllib.urlretrieve(
"{esgf_devel_url}/filters/xmltooling-1.2.2.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xmltooling-1.2.2.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/serializer-2.9.1.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/serializer-2.9.1.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/velocity-1.5.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/velocity-1.5.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xalan-2.7.2.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xalan-2.7.2.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xercesImpl-2.10.0.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xercesImpl-2.10.0.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xml-apis-1.4.01.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xml-apis-1.4.01.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xmlsec-1.4.2.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xmlsec-1.4.2.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/log4j-1.2.17.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/log4j-1.2.17.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/commons-io-2.4.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-io-2.4.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-dbcp-1.4.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-dbcp-1.4.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-dbutils-1.3.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-dbutils-1.3.jar"
)
shutil.copyfile(
"/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-pool-1.5.4.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-pool-1.5.4.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/postgresql-8.4-703.jdbc3.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/postgresql-8.4-703.jdbc3.jar"
)
# TDS customized logging (uses DEBUG)
shutil.copyfile(
"thredds_conf/log4j2.xml",
"/usr/local/tomcat/webapps/thredds/WEB-INF/classes/log4j2.xml")
# data node scripts
#TODO: Convert data node scripts to Python
# change ownership of content directory
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive("/esg/content/thredds/",
TOMCAT_USER_ID, TOMCAT_GROUP_ID)
# change ownership of source directory
esg_functions.change_permissions_recursive("/usr/local/webapps/thredds",
TOMCAT_USER_ID, TOMCAT_GROUP_ID)
# cleanup
shutil.rmtree("/usr/local/tomcat/webapps/esgf-node-manager/")