def setup_orp():
'''Setup the ORP subsystem'''
if os.path.isdir("/usr/local/tomcat/webapps/esg-orp"):
orp_install = raw_input(
"Existing ORP installation found. Do you want to continue with the ORP installation [y/N]: "
) or "no"
if orp_install.lower() in ["no", "n"]:
return
print "\n*******************************"
print "Setting up ORP"
print "******************************* \n"
esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esg-orp")
#COPY esgf-orp/esg-orp.war /usr/local/tomcat/webapps/esg-orp/esg-orp.war
orp_url = os.path.join("http://", config["esgf_dist_mirror"], "dist",
"devel", "esg-orp", "esg-orp.war")
print "orp_url:", orp_url
download_orp_war(orp_url)
with esg_bash2py.pushd("/usr/local/tomcat/webapps/esg-orp"):
with zipfile.ZipFile("/usr/local/tomcat/webapps/esg-orp/esg-orp.war",
'r') as zf:
zf.extractall()
os.remove("esg-orp.war")
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive(
"/usr/local/tomcat/webapps/esg-orp", TOMCAT_USER_ID,
TOMCAT_GROUP_ID)
# properties to read the Tomcat keystore, used to sign the authentication cookie
# these values are the same for all ESGF nodes
shutil.copyfile(
"esgf_orp_conf/esg-orp.properties",
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/classes/esg-orp.properties")
def extract_solr_tarball(solr_tarball_path, SOLR_VERSION):
'''Extract the solr tarball to /usr/local and symlink it to /usr/local/solr'''
print "\n*******************************"
print "Extracting Solr"
print "******************************* \n"
with esg_bash2py.pushd("/usr/local"):
esg_functions.extract_tarball(solr_tarball_path)
os.remove(solr_tarball_path)
esg_bash2py.symlink_force(
"solr-{SOLR_VERSION}".format(SOLR_VERSION=SOLR_VERSION), "solr")
def setup_dashboard():
if os.path.isdir("/usr/local/tomcat/webapps/esgf-stats-api"):
stats_api_install = raw_input(
"Existing Stats API installation found. Do you want to continue with the Stats API installation [y/N]: "
) or "no"
if stats_api_install.lower() in ["no", "n"]:
return
print "\n*******************************"
print "Setting up ESGF Stats API (dashboard)"
print "******************************* \n"
esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esgf-stats-api")
stats_api_url = os.path.join("http://", config["esgf_dist_mirror"], "dist",
"devel", "esgf-stats-api",
"esgf-stats-api.war")
download_stats_api_war(stats_api_url)
with esg_bash2py.pushd("/usr/local/tomcat/webapps/esgf-stats-api"):
with zipfile.ZipFile(
"/usr/local/tomcat/webapps/esgf-stats-api/esgf-stats-api.war",
'r') as zf:
zf.extractall()
os.remove("esgf-stats-api.war")
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive(
"/usr/local/tomcat/webapps/esgf-stats-api", TOMCAT_USER_ID,
TOMCAT_GROUP_ID)
# execute dashboard installation script (without the postgres schema)
run_dashboard_script()
# create non-privileged user to run the dashboard application
# RUN groupadd dashboard && \
# useradd -s /sbin/nologin -g dashboard -d /usr/local/dashboard dashboard && \
# chown -R dashboard:dashboard /usr/local/esgf-dashboard-ip
# RUN chmod a+w /var/run
esg_functions.stream_subprocess_output("groupadd dashboard")
esg_functions.stream_subprocess_output(
"useradd -s /sbin/nologin -g dashboard -d /usr/local/dashboard dashboard"
)
DASHBOARD_USER_ID = pwd.getpwnam("dashboard").pw_uid
DASHBOARD_GROUP_ID = grp.getgrnam("dashboard").gr_gid
esg_functions.change_permissions_recursive("/usr/local/esgf-dashboard-ip",
DASHBOARD_USER_ID,
DASHBOARD_GROUP_ID)
os.chmod("/var/run", stat.S_IWRITE)
os.chmod("/var/run", stat.S_IWGRP)
os.chmod("/var/run", stat.S_IWOTH)
start_dashboard_service()
def setup_publisher():
'''Install ESGF publisher'''
if os.path.isfile(
os.path.join(config["publisher_home"],
config["publisher_config"])):
publisher_install = raw_input(
"Detected an existing esgcet installation. Do you want to continue with the Publisher installation [y/N]: "
) or "no"
if publisher_install.lower() in ["no", "n"]:
return
print "\n*******************************"
print "Setting up ESGCET Package...(%s)" % (config["esgcet_egg_file"])
print "******************************* \n"
ESG_PUBLISHER_VERSION = "v3.2.7"
with esg_bash2py.pushd("/tmp"):
clone_publisher_repo("/tmp/esg-publisher")
with esg_bash2py.pushd("esg-publisher"):
checkout_publisher_branch("/tmp/esg-publisher", "devel")
with esg_bash2py.pushd("src/python/esgcet"):
install_publisher()
def remove_example_webapps():
'''remove Tomcat example applications'''
with esg_bash2py.pushd("/usr/local/tomcat/webapps"):
try:
shutil.rmtree("docs")
shutil.rmtree("examples")
shutil.rmtree("host-manager")
# shutil.rmtree("manager")
except OSError, error:
if error.errno == errno.ENOENT:
pass
else:
logger.exception()
def extract_tomcat_tarball(dest_dir="/usr/local"):
with esg_bash2py.pushd(dest_dir):
esg_functions.extract_tarball(
"/tmp/apache-tomcat-{TOMCAT_VERSION}.tar.gz".format(
TOMCAT_VERSION=TOMCAT_VERSION))
# Create symlink
create_symlink(TOMCAT_VERSION)
try:
os.remove("/tmp/apache-tomcat-{TOMCAT_VERSION}.tar.gz".format(
TOMCAT_VERSION=TOMCAT_VERSION))
except OSError, error:
print "error:", error
pass
def download_template_directory():
'''download template directory structure for shards home'''
ESGF_REPO = "http://distrib-coffee.ipsl.jussieu.fr/pub/esgf"
with esg_bash2py.pushd("/usr/local/src"):
r = requests.get("{ESGF_REPO}/dist/esg-search/solr-home.tar".format(
ESGF_REPO=ESGF_REPO))
path = 'solr-home.tar'
with open(path, 'wb') as f:
total_length = int(r.headers.get('content-length'))
for chunk in progress.bar(r.iter_content(chunk_size=1024),
expected_size=(total_length / 1024) + 1):
if chunk:
f.write(chunk)
f.flush()
esg_functions.extract_tarball("/usr/local/src/solr-home.tar")
def install_mod_wsgi():
'''Have to ensure python is install properly with the shared library for mod_wsgi installation to work'''
print "\n*******************************"
print "Setting mod_wsgi"
print "******************************* \n"
pip.main(['install', "mod_wsgi==4.5.3"])
with esg_bash2py.pushd("/etc/httpd/modules"):
#If installer running in a conda env
if "conda" in find_executable("python"):
esg_bash2py.symlink_force(
"/usr/local/conda/envs/esgf-pub/lib/python2.7/site-packages/mod_wsgi/server/mod_wsgi-py27.so",
"/etc/httpd/modules/mod_wsgi-py27.so")
else:
esg_bash2py.symlink_force(
"/usr/local/lib/python2.7/site-packages/mod_wsgi/server/mod_wsgi-py27.so",
"/etc/httpd/modules/mod_wsgi-py27.so")
def setup_temp_ca():
esg_bash2py.mkdir_p("/etc/tempcerts")
#Copy CA perl script and openssl conf file that it uses. The CA perl script
#is used to create a temporary root CA
shutil.copyfile("apache_certs/CA.pl", "/etc/tempcerts/CA.pl")
shutil.copyfile("apache_certs/openssl.cnf", "/etc/tempcerts/openssl.cnf")
shutil.copyfile("apache_certs/myproxy-server.config",
"/etc/tempcerts/myproxy-server.config")
os.chmod("/etc/tempcerts/CA.pl", 0755)
os.chmod("/etc/tempcerts/openssl.cnf", 0755)
with esg_bash2py.pushd("/etc/tempcerts"):
esg_bash2py.mkdir_p("CA")
ca_answer = "{fqdn}-CA".format(fqdn=esg_functions.get_esgf_host())
print "ca_answer:", ca_answer
new_ca_output = esg_functions.call_subprocess("perl CA.pl -newca",
command_stdin=ca_answer)
print "new_ca_output:", new_ca_output
def setup_java():
'''
Installs Oracle Java from rpm using yum localinstall. Does nothing if an acceptible Java install is found.
'''
print "*******************************"
print "Setting up Java {java_version}".format(java_version=config["java_version"])
print "******************************* \n"
if force_install:
pass
if check_for_existing_java():
setup_java_answer = raw_input("Do you want to continue with Java installation and setup? [y/N]: ") or "N"
if setup_java_answer.lower().strip() not in ["y", "yes"]:
print "Skipping Java installation"
return
last_java_truststore_file = esg_functions.readlinkf(config["truststore_file"])
esg_bash2py.mkdir_p(config["workdir"])
with esg_bash2py.pushd(config["workdir"]):
java_tarfile = esg_bash2py.trim_string_from_head(config["java_dist_url"])
jdk_directory = java_tarfile.split("-")[0]
java_install_dir_parent = config["java_install_dir"].rsplit("/",1)[0]
#Check for Java tar file
if not os.path.isfile(java_tarfile):
print "Don't see java distribution file {java_dist_file_path} either".format(java_dist_file_path=os.path.join(os.getcwd(),java_tarfile))
download_java(java_tarfile)
print "Extracting Java tarfile", java_tarfile
esg_functions.extract_tarball(java_tarfile, java_install_dir_parent)
#Create symlink to Java install directory (/usr/local/java)
esg_bash2py.symlink_force(os.path.join(java_install_dir_parent, jdk_directory), config["java_install_dir"])
os.chown(config["java_install_dir"], config["installer_uid"], config["installer_gid"])
#recursively change permissions
esg_functions.change_permissions_recursive(config["java_install_dir"], config["installer_uid"], config["installer_gid"])
set_default_java()
print check_java_version("java")
def run_dashboard_script():
#default values
DashDir = "/usr/local/esgf-dashboard-ip"
GeoipDir = "/usr/local/geoip"
Fed = "no"
with esg_bash2py.pushd("/usr/local"):
clone_dashboard_repo()
os.chdir("esgf-dashboard")
dashboard_repo_local = Repo(".")
dashboard_repo_local.git.checkout("work_plana")
os.chdir("src/c/esgf-dashboard-ip")
print "\n*******************************"
print "Running ESGF Dashboard Script"
print "******************************* \n"
esg_functions.stream_subprocess_output(
"./configure --prefix={DashDir} --with-geoip-prefix-path={GeoipDir} --with-allow-federation={Fed}"
.format(DashDir=DashDir, GeoipDir=GeoipDir, Fed=Fed))
esg_functions.stream_subprocess_output("make")
esg_functions.stream_subprocess_output("make install")
def setup_esg_search():
'''Setting up the ESG Search application'''
print "\n*******************************"
print "Setting up ESG Search"
print "******************************* \n"
ESGF_REPO = "http://aims1.llnl.gov/esgf"
esg_search_war_url = "{ESGF_REPO}/esg-search/esg-search.war".format(
ESGF_REPO=ESGF_REPO)
download_esg_search_war(esg_search_war_url)
#Extract esg-search war
with esg_bash2py.pushd("/usr/local/tomcat/webapps/esg-search"):
with zipfile.ZipFile(
"/usr/local/tomcat/webapps/esg-search/esg-search.war",
'r') as zf:
zf.extractall()
os.remove("esg-search.war")
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive(
"/usr/local/tomcat/webapps/esg-search", TOMCAT_USER_ID,
TOMCAT_GROUP_ID)
def setup_node_manager_old():
if os.path.isdir("/usr/local/tomcat/webapps/esgf-node-manager"):
node_manager_install = raw_input(
"Existing Node Manager installation found. Do you want to continue with the Node Manager installation [y/N]: "
) or "no"
if node_manager_install.lower() in ["no", "n"]:
return
print "\n*******************************"
print "Setting up ESGF Node Manager (old)"
print "******************************* \n"
esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esgf-node-manager")
node_manager_url = os.path.join("http://", config["esgf_dist_mirror"],
"dist", "devel", "esgf-node-manager",
"esgf-node-manager.war")
download_node_manager_war(node_manager_url)
with esg_bash2py.pushd("/usr/local/tomcat/webapps/esgf-node-manager/"):
with zipfile.ZipFile(
"/usr/local/tomcat/webapps/esgf-node-manager/esgf-node-manager.war",
'r') as zf:
zf.extractall()
os.remove("esgf-node-manager.war")
def install_python27():
'''Install python with shared library '''
with esg_bash2py.pushd("/tmp"):
python_download_url = "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz"
def fetch_esgf_certificates(
globus_certs_dir=config["globus_global_certs_dir"]):
'''Goes to ESG distribution server and pulls down all certificates for the federation.
(suitable for crontabbing)'''
print "\n*******************************"
print "Fetching freshest ESG Federation Certificates..."
print "******************************* \n"
#if globus_global_certs_dir already exists, backup and delete, then recreate empty directory
if os.path.isdir(config["globus_global_certs_dir"]):
esg_functions.backup(
config["globus_global_certs_dir"],
os.path.join(config["globus_global_certs_dir"], ".bak.tz"))
shutil.rmtree(config["globus_global_certs_dir"])
esg_bash2py.mkdir_p(config["globus_global_certs_dir"])
#Download trusted certs tarball
esg_trusted_certs_file = "esg_trusted_certificates.tar"
esg_trusted_certs_file_url = "https://aims1.llnl.gov/esgf/dist/certs/{esg_trusted_certs_file}".format(
esg_trusted_certs_file=esg_trusted_certs_file)
esg_functions.download_update(
os.path.join(globus_certs_dir, esg_trusted_certs_file),
esg_trusted_certs_file_url)
#untar the esg_trusted_certs_file
esg_functions.extract_tarball(
os.path.join(globus_certs_dir, esg_trusted_certs_file),
globus_certs_dir)
os.remove(os.path.join(globus_certs_dir, esg_trusted_certs_file))
#certificate_issuer_cert "/var/lib/globus-connect-server/myproxy-ca/cacert.pem"
simpleCA_cert = "/var/lib/globus-connect-server/myproxy-ca/cacert.pem"
if os.path.isfile(simpleCA_cert):
simpleCA_cert_hash = esg_functions.get_md5sum(simpleCA_cert)
print "checking for MY cert: {globus_global_certs_dir}/{simpleCA_cert_hash}.0".format(
globus_global_certs_dir=config["globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash)
if os.path.isfile(
"{globus_global_certs_dir}/{simpleCA_cert_hash}.0".format(
globus_global_certs_dir=config["globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash)):
print "Local CA cert file detected...."
print "Integrating in local simpleCA_cert... "
print "Local SimpleCA Root Cert: {simpleCA_cert}".format(
simpleCA_cert=simpleCA_cert)
print "Extracting Signing policy"
#Copy simple CA cert to globus cert directory
shutil.copyfile(
simpleCA_cert,
"{globus_global_certs_dir}/{simpleCA_cert_hash}.0".format(
globus_global_certs_dir=config["globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash))
#extract simple CA cert tarball and copy to globus cert directory
simpleCA_cert_parent_dir = esg_functions.get_parent_directory(
simpleCA_cert)
simpleCA_setup_tar_file = os.path.join(
simpleCA_cert_parent_dir,
"globus_simple_ca_{simpleCA_cert_hash}_setup-0.tar.gz".format(
simpleCA_cert_hash=simpleCA_cert_hash))
esg_functions.extract_tarball(simpleCA_setup_tar_file)
with esg_bash2py.pushd(
"globus_simple_ca_{simpleCA_cert_hash}_setup-0".format(
simpleCA_cert_hash=simpleCA_cert_hash)):
shutil.copyfile(
"{simpleCA_cert_hash}.signing_policy".format(
simpleCA_cert_hash=simpleCA_cert_hash),
"{globus_global_certs_dir}/{simpleCA_cert_hash}.signing_policy"
.format(globus_global_certs_dir=config[
"globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash))
if os.path.isdir("/usr/local/tomcat/webapps/ROOT"):
esg_functions.stream_subprocess_output(
"openssl x509 -text -hash -in {simpleCA_cert} > {tomcat_install_dir}/webapps/ROOT/cacert.pem"
.format(simpleCA_cert=simpleCA_cert,
tomcat_install_dir="/usr/loca/tomcat"))
print " My CA Cert now posted @ http://{fqdn}/cacert.pem ".format(
fqdn=socket.getfqdn())
os.chmod("/usr/local/tomcat/webapps/ROOT/cacert.pem", 0644)
os.chmod(config["globus_global_certs_dir"], 0755)
esg_functions.change_permissions_recursive(
config["globus_global_certs_dir"], 0644)
def setup_thredds():
if os.path.isdir("/usr/local/tomcat/webapps/thredds"):
thredds_install = raw_input(
"Existing Thredds installation found. Do you want to continue with the Thredds installation [y/N]: "
) or "no"
if thredds_install.lower() in ["no", "n"]:
return
print "\n*******************************"
print "Setting up Thredds"
print "******************************* \n"
esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/thredds")
thredds_url = os.path.join("http://", config["esgf_dist_mirror"], "dist",
"devel", "thredds", "5.0", "5.0.1",
"thredds.war")
download_thredds_war(thredds_url)
with esg_bash2py.pushd("/usr/local/tomcat/webapps/thredds"):
with zipfile.ZipFile("/usr/local/tomcat/webapps/thredds/thredds.war",
'r') as zf:
zf.extractall()
os.remove("thredds.war")
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive(
"/usr/local/tomcat/webapps/thredds", TOMCAT_USER_ID,
TOMCAT_GROUP_ID)
# TDS configuration root
esg_bash2py.mkdir_p(os.path.join(config["thredds_content_dir"], "thredds"))
# TDS memory configuration
shutil.copyfile("thredds_conf/threddsConfig.xml",
"/esg/content/thredds/threddsConfig.xml")
# ESGF root catalog
shutil.copyfile("thredds_conf/catalog.xml",
"/esg/content/thredds/catalog.xml-esgcet")
esg_bash2py.mkdir_p("/esg/content/thredds/esgcet")
# TDS customized applicationContext.xml file with ESGF authorizer
shutil.copyfile(
"thredds_conf/applicationContext.xml",
"/usr/local/tomcat/webapps/thredds/WEB-INF/applicationContext.xml")
# TDS jars necessary to support ESGF security filters
# some jars are retrieved from the ESGF repository
# other jars are copied from the unpacked ORP or NM distributions
esgf_devel_url = os.path.join("http://", config["esgf_dist_mirror"],
"dist", "devel")
urllib.urlretrieve(
"{esgf_devel_url}/filters/XSGroupRole-1.0.0.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/XSGroupRole-1.0.0.jar")
urllib.urlretrieve(
"{esgf_devel_url}/filters/commons-httpclient-3.1.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-httpclient-3.1.jar"
)
urllib.urlretrieve(
"{esgf_devel_url}/filters/commons-lang-2.6.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-lang-2.6.jar")
urllib.urlretrieve(
"{esgf_devel_url}/esg-orp/esg-orp-2.9.3.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esg-orp-2.9.3.jar")
urllib.urlretrieve(
"{esgf_devel_url}/esgf-node-manager/esgf-node-manager-common-1.0.0.jar"
.format(esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-node-manager-common-1.0.0.jar"
)
urllib.urlretrieve(
"{esgf_devel_url}/esgf-node-manager/esgf-node-manager-filters-1.0.0.jar"
.format(esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-node-manager-filters-1.0.0.jar"
)
urllib.urlretrieve(
"{esgf_devel_url}/esgf-security/esgf-security-2.7.10.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-security-2.7.10.jar"
)
urllib.urlretrieve(
"{esgf_devel_url}/filters/jdom-legacy-1.1.3.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/jdom-legacy-1.1.3.jar")
urllib.urlretrieve(
"{esgf_devel_url}/filters/opensaml-2.3.2.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/opensaml-2.3.2.jar")
urllib.urlretrieve(
"{esgf_devel_url}/filters/openws-1.3.1.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/openws-1.3.1.jar")
urllib.urlretrieve(
"{esgf_devel_url}/filters/xmltooling-1.2.2.jar".format(
esgf_devel_url=esgf_devel_url),
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xmltooling-1.2.2.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/serializer-2.9.1.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/serializer-2.9.1.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/velocity-1.5.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/velocity-1.5.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xalan-2.7.2.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xalan-2.7.2.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xercesImpl-2.10.0.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xercesImpl-2.10.0.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xml-apis-1.4.01.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xml-apis-1.4.01.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xmlsec-1.4.2.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xmlsec-1.4.2.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/log4j-1.2.17.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/log4j-1.2.17.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/commons-io-2.4.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-io-2.4.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-dbcp-1.4.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-dbcp-1.4.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-dbutils-1.3.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-dbutils-1.3.jar"
)
shutil.copyfile(
"/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-pool-1.5.4.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-pool-1.5.4.jar")
shutil.copyfile(
"/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/postgresql-8.4-703.jdbc3.jar",
"/usr/local/tomcat/webapps/thredds/WEB-INF/lib/postgresql-8.4-703.jdbc3.jar"
)
# TDS customized logging (uses DEBUG)
shutil.copyfile(
"thredds_conf/log4j2.xml",
"/usr/local/tomcat/webapps/thredds/WEB-INF/classes/log4j2.xml")
# data node scripts
#TODO: Convert data node scripts to Python
# change ownership of content directory
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive("/esg/content/thredds/",
TOMCAT_USER_ID, TOMCAT_GROUP_ID)
# change ownership of source directory
esg_functions.change_permissions_recursive("/usr/local/webapps/thredds",
TOMCAT_USER_ID, TOMCAT_GROUP_ID)
# cleanup
shutil.rmtree("/usr/local/tomcat/webapps/esgf-node-manager/")
def setup_subsystem(subsystem,
distribution_directory,
esg_dist_url,
force_install=False):
'''
arg (1) - name of installation script root name. Ex:security which resolves to script file esg-security
arg (2) - directory on the distribution site where script is fetched from Ex: orp
usage: setup_subsystem security orp - looks for the script esg-security in the distriubtion dir orp
'''
subsystem_install_script_path = os.path.join(
config["scripts_dir"], "esg-{subsystem}".format(subsystem=subsystem))
#---
#check that you have at one point in time fetched the subsystem's installation script
#if indeed you have we will assume you would like to proceed with setting up the latest...
#Otherwise we just ask you first before you pull down new code to your machine...
#---
if force_install:
default = "y"
else:
default = "n"
if os.path.exists(subsystem_install_script_path) or force_install:
if default.lower() in ["y", "yes"]:
run_installation = raw_input(
"Would you like to set up {subsystem} services? [Y/n]: ".
format(subsystem=subsystem)) or "y"
else:
run_installation = raw_input(
"Would you like to set up {subsystem} services? [y/N]: ".
format(subsystem=subsystem)) or "n"
if run_installation.lower() in ["n", "no"]:
print "Skipping installation of {subsystem}".format(
subsystem=subsystem)
return True
print "-------------------------------"
print "LOADING installer for {subsystem}... ".format(subsystem=subsystem)
esg_bash2py.mkdir_p(config["workdir"])
with esg_bash2py.pushd(config["workdir"]):
logger.debug("Changed directory to %s", os.getcwd())
with esg_bash2py.pushd(config["scripts_dir"]):
logger.debug("Changed directory to %s", os.getcwd())
subsystem_full_name = "esg-{subsystem}".format(subsystem=subsystem)
subsystem_remote_url = "{esg_dist_url}/{distribution_directory}/{subsystem_full_name}".format(
esg_dist_url=esg_dist_url,
distribution_directory=distribution_directory,
subsystem_full_name=subsystem_full_name)
if not esg_functions.download_update(
"{subsystem_full_name}".format(
subsystem_full_name=subsystem_full_name),
subsystem_remote_url):
logger.error("Could not download %s", subsystem_full_name)
return False
try:
os.chmod(subsystem_full_name, 0755)
except OSError:
logger.exception("Unable to change permissions on %s",
subsystem_full_name)
logger.info("script_dir contents: %s", os.listdir(config["scripts_dir"]))
subsystem_underscore = subsystem.replace("-", "_")
execute_subsystem_command = ". {scripts_dir}/{subsystem_full_name}; setup_{subsystem_underscore}".format(
scripts_dir=config["scripts_dir"],
subsystem_full_name=subsystem_full_name,
subsystem_underscore=subsystem_underscore)
setup_subsystem_process = subprocess.Popen(
['bash', '-c', execute_subsystem_command])
setup_subsystem_stdout, setup_subsystem_stderr = setup_subsystem_process.communicate(
)
logger.debug("setup_subsystem_stdout: %s", setup_subsystem_stdout)
logger.debug("setup_subsystem_stderr: %s", setup_subsystem_stderr)
def setup_node_manager(mode="install"):
#####
# Install The Node Manager
#####
# - Takes boolean arg: 0 = setup / install mode (default)
# 1 = updated mode
#
# In setup mode it is an idempotent install (default)
# In update mode it will always pull down latest after archiving old
#
print "Checking for node manager {esgf_node_manager_version}".format(
esgf_node_manager_version=config["esgf_node_manager_version"])
if esg_version_manager.check_webapp_version(
"esgf-node-manager",
config["esgf_node_manager_version"]) == 0 and not force_install:
print "\n Found existing version of the node-manager [OK]"
return True
init()
print "*******************************"
print "Setting up The ESGF Node Manager..."
print "*******************************"
# local upgrade=${1:-0}
db_set = 0
if force_install:
default_answer = "N"
else:
default_answer = "Y"
# local dosetup
node_manager_service_app_home = esg_property_manager.get_property(
"node_manager_service_app_home")
if os.path.isdir(node_manager_service_app_home):
db_set = 1
print "Detected an existing node manager installation..."
if default_answer == "Y":
installation_answer = raw_input(
"Do you want to continue with node manager installation and setup? [Y/n]"
) or default_answer
else:
installation_answer = raw_input(
"Do you want to continue with node manager installation and setup? [y/N]"
) or default_answer
if installation_answer.lower() not in ["y", "yes"]:
print "Skipping node manager installation and setup - will assume it's setup properly"
# resetting node manager version to what it is already, not what we prescribed in the script
# this way downstream processes will use the *actual* version in play, namely the (access logging) filter(s)
esgf_node_manager_version = esg_version_manager.get_current_webapp_version(
"esgf_node_manager")
return True
backup_default_answer = "Y"
backup_answer = raw_input(
"Do you want to make a back up of the existing distribution [{node_manager_app_context_root}]? [Y/n] "
.format(node_manager_app_context_root=node_manager_app_context_root
)) or backup_default_answer
if backup_answer.lower in ["yes", "y"]:
print "Creating a backup archive of this web application [{node_manager_service_app_home}]".format(
node_manager_service_app_home=node_manager_service_app_home)
esg_functions.backup(node_manager_service_app_home)
backup_db_default_answer = "Y"
backup_db_answer = raw_input(
"Do you want to make a back up of the existing database [{node_db_name}:esgf_node_manager]?? [Y/n] "
.format(node_db_name=config["node_db_name"]
)) or backup_db_default_answer
if backup_db_answer.lower() in ["yes", "y"]:
print "Creating a backup archive of the manager database schema [{node_db_name}:esgf_node_manager]".format(
node_db_name=config["node_db_name"])
# TODO: Implement this
# esg_postgres.backup_db() -db ${node_db_name} -s node_manager
esg_bash2py.mkdir_p(config["workdir"])
with esg_bash2py.pushd(config["workdir"]):
logger.debug("changed directory to : %s", os.getcwd())
# strip off .tar.gz at the end
#(Ex: esgf-node-manager-0.9.0.tar.gz -> esgf-node-manager-0.9.0)
node_dist_file = esg_bash2py.trim_string_from_head(node_dist_url)
logger.debug("node_dist_file: %s", node_dist_file)
# Should just be esgf-node-manager-x.x.x
node_dist_dir = node_dist_file
# checked_get ${node_dist_file} ${node_dist_url} $((force_install))
if not esg_functions.download_update(
node_dist_file, node_dist_url, force_download=force_install):
print "ERROR: Could not download {node_dist_url} :-(".format(
node_dist_url=node_dist_url)
esg_functions.exit_with_error(1)
# make room for new install
if force_install:
print "Removing Previous Installation of the ESGF Node Manager... ({node_dist_dir})".format(
node_dist_dir=node_dist_dir)
try:
shutil.rmtree(node_dist_dir)
logger.info("Deleted directory: %s", node_dist_dir)
except IOError, error:
logger.error(error)
logger.error("Could not delete directory: %s", node_dist_dir)
esg_functions.exit_with_error(1)
clean_node_manager_webapp_subsystem()
print "\nunpacking {node_dist_file}...".format(
node_dist_file=node_dist_file)
# This probably won't work, because the extension has already been stripped, no idea how this even worked in the bash code smh
try:
tar = tarfile.open(node_dist_file)
tar.extractall()
tar.close()
except Exception, error:
logger.error(error)
print "ERROR: Could not extract the ESG Node: {node_dist_file}".format(
node_dist_file=node_dist_file)
esg_functions.exit_with_error(1)
print "\nunpacking {node_dist_file}...".format(
node_dist_file=node_dist_file)
# This probably won't work, because the extension has already been stripped, no idea how this even worked in the bash code smh
try:
tar = tarfile.open(node_dist_file)
tar.extractall()
tar.close()
except Exception, error:
logger.error(error)
print "ERROR: Could not extract the ESG Node: {node_dist_file}".format(
node_dist_file=node_dist_file)
esg_functions.exit_with_error(1)
# pushd ${node_dist_dir} >& /dev/null
with esg_bash2py.pushd(node_dist_dir):
logger.debug("changed directory to : %s", os.getcwd())
stop_tomcat()
# strip the version number off(#.#.#) the dir and append .war to get the name of war file
#(Ex: esgf-node-manager-0.9.0 -> esgf-node-manager.war)
# local trimmed_name=$(pwd)/${node_dist_dir%-*}
split_dir_name_list = node_dist_dir.split("-")
versionless_name = '-'.join(split_dir_name_list[:3])
trimmed_name = os.path.join(os.getcwd(), versionless_name)
node_war_file = trimmed_name + ".war"
logger.debug("node_war_file: %s", node_war_file)
#----------------------------
# make room for new INSTALL
# ((upgrade == 0)) && set_aside_web_app ${node_manager_service_app_home}