def setup_orp(): '''Setup the ORP subsystem''' if os.path.isdir("/usr/local/tomcat/webapps/esg-orp"): orp_install = raw_input( "Existing ORP installation found. Do you want to continue with the ORP installation [y/N]: " ) or "no" if orp_install.lower() in ["no", "n"]: return print "\n*******************************" print "Setting up ORP" print "******************************* \n" esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esg-orp") #COPY esgf-orp/esg-orp.war /usr/local/tomcat/webapps/esg-orp/esg-orp.war orp_url = os.path.join("http://", config["esgf_dist_mirror"], "dist", "devel", "esg-orp", "esg-orp.war") print "orp_url:", orp_url download_orp_war(orp_url) with esg_bash2py.pushd("/usr/local/tomcat/webapps/esg-orp"): with zipfile.ZipFile("/usr/local/tomcat/webapps/esg-orp/esg-orp.war", 'r') as zf: zf.extractall() os.remove("esg-orp.war") TOMCAT_USER_ID = esg_functions.get_tomcat_user_id() TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id() esg_functions.change_permissions_recursive( "/usr/local/tomcat/webapps/esg-orp", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # properties to read the Tomcat keystore, used to sign the authentication cookie # these values are the same for all ESGF nodes shutil.copyfile( "esgf_orp_conf/esg-orp.properties", "/usr/local/tomcat/webapps/esg-orp/WEB-INF/classes/esg-orp.properties")
def setup_dashboard(): if os.path.isdir("/usr/local/tomcat/webapps/esgf-stats-api"): stats_api_install = raw_input( "Existing Stats API installation found. Do you want to continue with the Stats API installation [y/N]: " ) or "no" if stats_api_install.lower() in ["no", "n"]: return print "\n*******************************" print "Setting up ESGF Stats API (dashboard)" print "******************************* \n" esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esgf-stats-api") stats_api_url = os.path.join("http://", config["esgf_dist_mirror"], "dist", "devel", "esgf-stats-api", "esgf-stats-api.war") download_stats_api_war(stats_api_url) with esg_bash2py.pushd("/usr/local/tomcat/webapps/esgf-stats-api"): with zipfile.ZipFile( "/usr/local/tomcat/webapps/esgf-stats-api/esgf-stats-api.war", 'r') as zf: zf.extractall() os.remove("esgf-stats-api.war") TOMCAT_USER_ID = esg_functions.get_tomcat_user_id() TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id() esg_functions.change_permissions_recursive( "/usr/local/tomcat/webapps/esgf-stats-api", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # execute dashboard installation script (without the postgres schema) run_dashboard_script() # create non-privileged user to run the dashboard application # RUN groupadd dashboard && \ # useradd -s /sbin/nologin -g dashboard -d /usr/local/dashboard dashboard && \ # chown -R dashboard:dashboard /usr/local/esgf-dashboard-ip # RUN chmod a+w /var/run esg_functions.stream_subprocess_output("groupadd dashboard") esg_functions.stream_subprocess_output( "useradd -s /sbin/nologin -g dashboard -d /usr/local/dashboard dashboard" ) DASHBOARD_USER_ID = pwd.getpwnam("dashboard").pw_uid DASHBOARD_GROUP_ID = grp.getgrnam("dashboard").gr_gid esg_functions.change_permissions_recursive("/usr/local/esgf-dashboard-ip", DASHBOARD_USER_ID, DASHBOARD_GROUP_ID) os.chmod("/var/run", stat.S_IWRITE) os.chmod("/var/run", stat.S_IWGRP) os.chmod("/var/run", stat.S_IWOTH) start_dashboard_service()
def create_tomcat_user(): '''Create the Tomcat system user and user group''' print "\n*******************************" print "Creating Tomcat User" print "******************************* \n" esg_functions.call_subprocess("groupadd tomcat") esg_functions.call_subprocess( "useradd -s /sbin/nologin -g tomcat -d /usr/local/tomcat tomcat") tomcat_directory = "/usr/local/apache-tomcat-{TOMCAT_VERSION}".format( TOMCAT_VERSION=TOMCAT_VERSION) tomcat_user_id = pwd.getpwnam("tomcat").pw_uid tomcat_group_id = grp.getgrnam("tomcat").gr_gid esg_functions.change_permissions_recursive(tomcat_directory, tomcat_user_id, tomcat_group_id) os.chmod("/usr/local/tomcat/webapps", 0775)
def setup_java(): ''' Installs Oracle Java from rpm using yum localinstall. Does nothing if an acceptible Java install is found. ''' print "*******************************" print "Setting up Java {java_version}".format(java_version=config["java_version"]) print "******************************* \n" if force_install: pass if check_for_existing_java(): setup_java_answer = raw_input("Do you want to continue with Java installation and setup? [y/N]: ") or "N" if setup_java_answer.lower().strip() not in ["y", "yes"]: print "Skipping Java installation" return last_java_truststore_file = esg_functions.readlinkf(config["truststore_file"]) esg_bash2py.mkdir_p(config["workdir"]) with esg_bash2py.pushd(config["workdir"]): java_tarfile = esg_bash2py.trim_string_from_head(config["java_dist_url"]) jdk_directory = java_tarfile.split("-")[0] java_install_dir_parent = config["java_install_dir"].rsplit("/",1)[0] #Check for Java tar file if not os.path.isfile(java_tarfile): print "Don't see java distribution file {java_dist_file_path} either".format(java_dist_file_path=os.path.join(os.getcwd(),java_tarfile)) download_java(java_tarfile) print "Extracting Java tarfile", java_tarfile esg_functions.extract_tarball(java_tarfile, java_install_dir_parent) #Create symlink to Java install directory (/usr/local/java) esg_bash2py.symlink_force(os.path.join(java_install_dir_parent, jdk_directory), config["java_install_dir"]) os.chown(config["java_install_dir"], config["installer_uid"], config["installer_gid"]) #recursively change permissions esg_functions.change_permissions_recursive(config["java_install_dir"], config["installer_uid"], config["installer_gid"]) set_default_java() print check_java_version("java")
def setup_solr(): '''Setup Apache Solr for faceted search''' print "\n*******************************" print "Setting up Solr" print "******************************* \n" # # Solr/Jetty web application SOLR_VERSION = "5.5.4" SOLR_INSTALL_DIR = "/usr/local/solr" SOLR_HOME = "/usr/local/solr-home" SOLR_DATA_DIR = "/esg/solr-index" SOLR_INCLUDE = "{SOLR_HOME}/solr.in.sh".format(SOLR_HOME=SOLR_HOME) #Download solr tarball solr_tarball_url = "http://archive.apache.org/dist/lucene/solr/{SOLR_VERSION}/solr-{SOLR_VERSION}.tgz".format( SOLR_VERSION=SOLR_VERSION) download_solr_tarball(solr_tarball_url, SOLR_VERSION) #Extract solr tarball extract_solr_tarball( '/tmp/solr-{SOLR_VERSION}.tgz'.format(SOLR_VERSION=SOLR_VERSION), SOLR_VERSION) esg_bash2py.mkdir_p(SOLR_DATA_DIR) # download template directory structure for shards home download_template_directory() esg_bash2py.mkdir_p(SOLR_HOME) # create non-privilged user to run Solr server esg_functions.stream_subprocess_output("groupadd solr") esg_functions.stream_subprocess_output( "useradd -s /sbin/nologin -g solr -d /usr/local/solr solr") SOLR_USER_ID = pwd.getpwnam("solr").pw_uid SOLR_GROUP_ID = grp.getgrnam("solr").gr_gid esg_functions.change_permissions_recursive( "/usr/local/solr-{SOLR_VERSION}".format(SOLR_VERSION=SOLR_VERSION), SOLR_USER_ID, SOLR_GROUP_ID) esg_functions.change_permissions_recursive(SOLR_HOME, SOLR_USER_ID, SOLR_GROUP_ID) esg_functions.change_permissions_recursive(SOLR_DATA_DIR, SOLR_USER_ID, SOLR_GROUP_ID) # #Copy shard files shutil.copyfile("solr_scripts/add_shard.sh", "/usr/local/bin/add_shard.sh") shutil.copyfile("solr_scripts/remove_shard.sh", "/usr/local/bin/remove_shard.sh") # add shards esg_functions.call_subprocess("/usr/local/bin/add_shard.sh master 8984") esg_functions.call_subprocess("/usr/local/bin/add_shard.sh master 8983") # custom logging properties shutil.copyfile( "solr_scripts/log4j.properties", "/{SOLR_INSTALL_DIR}/server/resources/log4j.properties".format( SOLR_INSTALL_DIR=SOLR_INSTALL_DIR))
def setup_esg_search(): '''Setting up the ESG Search application''' print "\n*******************************" print "Setting up ESG Search" print "******************************* \n" ESGF_REPO = "http://aims1.llnl.gov/esgf" esg_search_war_url = "{ESGF_REPO}/esg-search/esg-search.war".format( ESGF_REPO=ESGF_REPO) download_esg_search_war(esg_search_war_url) #Extract esg-search war with esg_bash2py.pushd("/usr/local/tomcat/webapps/esg-search"): with zipfile.ZipFile( "/usr/local/tomcat/webapps/esg-search/esg-search.war", 'r') as zf: zf.extractall() os.remove("esg-search.war") TOMCAT_USER_ID = esg_functions.get_tomcat_user_id() TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id() esg_functions.change_permissions_recursive( "/usr/local/tomcat/webapps/esg-search", TOMCAT_USER_ID, TOMCAT_GROUP_ID)
def setup_thredds(): if os.path.isdir("/usr/local/tomcat/webapps/thredds"): thredds_install = raw_input( "Existing Thredds installation found. Do you want to continue with the Thredds installation [y/N]: " ) or "no" if thredds_install.lower() in ["no", "n"]: return print "\n*******************************" print "Setting up Thredds" print "******************************* \n" esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/thredds") thredds_url = os.path.join("http://", config["esgf_dist_mirror"], "dist", "devel", "thredds", "5.0", "5.0.1", "thredds.war") download_thredds_war(thredds_url) with esg_bash2py.pushd("/usr/local/tomcat/webapps/thredds"): with zipfile.ZipFile("/usr/local/tomcat/webapps/thredds/thredds.war", 'r') as zf: zf.extractall() os.remove("thredds.war") TOMCAT_USER_ID = esg_functions.get_tomcat_user_id() TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id() esg_functions.change_permissions_recursive( "/usr/local/tomcat/webapps/thredds", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # TDS configuration root esg_bash2py.mkdir_p(os.path.join(config["thredds_content_dir"], "thredds")) # TDS memory configuration shutil.copyfile("thredds_conf/threddsConfig.xml", "/esg/content/thredds/threddsConfig.xml") # ESGF root catalog shutil.copyfile("thredds_conf/catalog.xml", "/esg/content/thredds/catalog.xml-esgcet") esg_bash2py.mkdir_p("/esg/content/thredds/esgcet") # TDS customized applicationContext.xml file with ESGF authorizer shutil.copyfile( "thredds_conf/applicationContext.xml", "/usr/local/tomcat/webapps/thredds/WEB-INF/applicationContext.xml") # TDS jars necessary to support ESGF security filters # some jars are retrieved from the ESGF repository # other jars are copied from the unpacked ORP or NM distributions esgf_devel_url = os.path.join("http://", config["esgf_dist_mirror"], "dist", "devel") urllib.urlretrieve( "{esgf_devel_url}/filters/XSGroupRole-1.0.0.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/XSGroupRole-1.0.0.jar") urllib.urlretrieve( "{esgf_devel_url}/filters/commons-httpclient-3.1.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-httpclient-3.1.jar" ) urllib.urlretrieve( "{esgf_devel_url}/filters/commons-lang-2.6.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-lang-2.6.jar") urllib.urlretrieve( "{esgf_devel_url}/esg-orp/esg-orp-2.9.3.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esg-orp-2.9.3.jar") urllib.urlretrieve( "{esgf_devel_url}/esgf-node-manager/esgf-node-manager-common-1.0.0.jar" .format(esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-node-manager-common-1.0.0.jar" ) urllib.urlretrieve( "{esgf_devel_url}/esgf-node-manager/esgf-node-manager-filters-1.0.0.jar" .format(esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-node-manager-filters-1.0.0.jar" ) urllib.urlretrieve( "{esgf_devel_url}/esgf-security/esgf-security-2.7.10.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-security-2.7.10.jar" ) urllib.urlretrieve( "{esgf_devel_url}/filters/jdom-legacy-1.1.3.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/jdom-legacy-1.1.3.jar") urllib.urlretrieve( "{esgf_devel_url}/filters/opensaml-2.3.2.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/opensaml-2.3.2.jar") urllib.urlretrieve( "{esgf_devel_url}/filters/openws-1.3.1.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/openws-1.3.1.jar") urllib.urlretrieve( "{esgf_devel_url}/filters/xmltooling-1.2.2.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xmltooling-1.2.2.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/serializer-2.9.1.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/serializer-2.9.1.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/velocity-1.5.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/velocity-1.5.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xalan-2.7.2.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xalan-2.7.2.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xercesImpl-2.10.0.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xercesImpl-2.10.0.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xml-apis-1.4.01.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xml-apis-1.4.01.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xmlsec-1.4.2.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xmlsec-1.4.2.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/log4j-1.2.17.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/log4j-1.2.17.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/commons-io-2.4.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-io-2.4.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-dbcp-1.4.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-dbcp-1.4.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-dbutils-1.3.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-dbutils-1.3.jar" ) shutil.copyfile( "/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-pool-1.5.4.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-pool-1.5.4.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/postgresql-8.4-703.jdbc3.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/postgresql-8.4-703.jdbc3.jar" ) # TDS customized logging (uses DEBUG) shutil.copyfile( "thredds_conf/log4j2.xml", "/usr/local/tomcat/webapps/thredds/WEB-INF/classes/log4j2.xml") # data node scripts #TODO: Convert data node scripts to Python # change ownership of content directory TOMCAT_USER_ID = esg_functions.get_tomcat_user_id() TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id() esg_functions.change_permissions_recursive("/esg/content/thredds/", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # change ownership of source directory esg_functions.change_permissions_recursive("/usr/local/webapps/thredds", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # cleanup shutil.rmtree("/usr/local/tomcat/webapps/esgf-node-manager/")