def setup_orp(): '''Setup the ORP subsystem''' if os.path.isdir("/usr/local/tomcat/webapps/esg-orp"): orp_install = raw_input( "Existing ORP installation found. Do you want to continue with the ORP installation [y/N]: " ) or "no" if orp_install.lower() in ["no", "n"]: return print "\n*******************************" print "Setting up ORP" print "******************************* \n" esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esg-orp") #COPY esgf-orp/esg-orp.war /usr/local/tomcat/webapps/esg-orp/esg-orp.war orp_url = os.path.join("http://", config["esgf_dist_mirror"], "dist", "devel", "esg-orp", "esg-orp.war") print "orp_url:", orp_url download_orp_war(orp_url) with esg_bash2py.pushd("/usr/local/tomcat/webapps/esg-orp"): with zipfile.ZipFile("/usr/local/tomcat/webapps/esg-orp/esg-orp.war", 'r') as zf: zf.extractall() os.remove("esg-orp.war") TOMCAT_USER_ID = esg_functions.get_tomcat_user_id() TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id() esg_functions.change_permissions_recursive( "/usr/local/tomcat/webapps/esg-orp", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # properties to read the Tomcat keystore, used to sign the authentication cookie # these values are the same for all ESGF nodes shutil.copyfile( "esgf_orp_conf/esg-orp.properties", "/usr/local/tomcat/webapps/esg-orp/WEB-INF/classes/esg-orp.properties")
def _update_postgres_password(): '''Updates the Postgres system account password; gets saved to /esg/config/.esg_pg_pass''' if not esg_functions.get_tomcat_group_id(): esg_functions.add_unix_group(config["tomcat_group"]) tomcat_group_id = esg_functions.get_tomcat_group_id() try: with open(config['pg_secret_file'], "w") as secret_file: secret_file.write(config["pg_sys_acct_passwd"]) except IOError: logger.exception("Could not open %s", config['pg_secret_file']) os.chmod(config['pg_secret_file'], 0640) try: os.chown(config['pg_secret_file'], config["installer_uid"], tomcat_group_id) except OSError: logger.exception("Unable to change ownership of %s", config["pg_secret_file"])
def setup_dashboard(): if os.path.isdir("/usr/local/tomcat/webapps/esgf-stats-api"): stats_api_install = raw_input( "Existing Stats API installation found. Do you want to continue with the Stats API installation [y/N]: " ) or "no" if stats_api_install.lower() in ["no", "n"]: return print "\n*******************************" print "Setting up ESGF Stats API (dashboard)" print "******************************* \n" esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esgf-stats-api") stats_api_url = os.path.join("http://", config["esgf_dist_mirror"], "dist", "devel", "esgf-stats-api", "esgf-stats-api.war") download_stats_api_war(stats_api_url) with esg_bash2py.pushd("/usr/local/tomcat/webapps/esgf-stats-api"): with zipfile.ZipFile( "/usr/local/tomcat/webapps/esgf-stats-api/esgf-stats-api.war", 'r') as zf: zf.extractall() os.remove("esgf-stats-api.war") TOMCAT_USER_ID = esg_functions.get_tomcat_user_id() TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id() esg_functions.change_permissions_recursive( "/usr/local/tomcat/webapps/esgf-stats-api", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # execute dashboard installation script (without the postgres schema) run_dashboard_script() # create non-privileged user to run the dashboard application # RUN groupadd dashboard && \ # useradd -s /sbin/nologin -g dashboard -d /usr/local/dashboard dashboard && \ # chown -R dashboard:dashboard /usr/local/esgf-dashboard-ip # RUN chmod a+w /var/run esg_functions.stream_subprocess_output("groupadd dashboard") esg_functions.stream_subprocess_output( "useradd -s /sbin/nologin -g dashboard -d /usr/local/dashboard dashboard" ) DASHBOARD_USER_ID = pwd.getpwnam("dashboard").pw_uid DASHBOARD_GROUP_ID = grp.getgrnam("dashboard").gr_gid esg_functions.change_permissions_recursive("/usr/local/esgf-dashboard-ip", DASHBOARD_USER_ID, DASHBOARD_GROUP_ID) os.chmod("/var/run", stat.S_IWRITE) os.chmod("/var/run", stat.S_IWGRP) os.chmod("/var/run", stat.S_IWOTH) start_dashboard_service()
def setup_esg_search(): '''Setting up the ESG Search application''' print "\n*******************************" print "Setting up ESG Search" print "******************************* \n" ESGF_REPO = "http://aims1.llnl.gov/esgf" esg_search_war_url = "{ESGF_REPO}/esg-search/esg-search.war".format( ESGF_REPO=ESGF_REPO) download_esg_search_war(esg_search_war_url) #Extract esg-search war with esg_bash2py.pushd("/usr/local/tomcat/webapps/esg-search"): with zipfile.ZipFile( "/usr/local/tomcat/webapps/esg-search/esg-search.war", 'r') as zf: zf.extractall() os.remove("esg-search.war") TOMCAT_USER_ID = esg_functions.get_tomcat_user_id() TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id() esg_functions.change_permissions_recursive( "/usr/local/tomcat/webapps/esg-search", TOMCAT_USER_ID, TOMCAT_GROUP_ID)
def initial_setup_questionnaire(force_install=False): print "-------------------------------------------------------" print 'Welcome to the ESGF Node installation program! :-)' print "-------------------------------------------------------" esg_bash2py.mkdir_p(config['esg_config_dir']) starting_directory = os.getcwd() os.chdir(config['esg_config_dir']) esgf_host = esg_property_manager.get_property("esgf_host") _choose_fqdn(esgf_host) if not esg_functions.get_security_admin_password() or force_install: _choose_admin_password() else: logger.info("Previously set password found.") _choose_organization_name() _choose_node_short_name() _choose_node_long_name() _choose_node_namespace() _choose_node_peer_group() _choose_esgf_index_peer() _choose_mail_admin_address() #TODO:Extract constructring DB string into separate function db_properties = get_db_properties() if not all(db_properties) or force_install: _is_managed_db(db_properties) _get_db_conn_str_questionnaire(db_properties) else: if db_properties["db_host"] == esgf_host or db_properties[ "db_host"] == "localhost": print "db_connection_string = {db_user}@localhost".format( db_user=db_properties["db_user"]) else: connstring_ = "{db_user}@{db_host}:{db_port}/{db_database} [external = ${db_managed}]".format( db_user=db_properties["db_user"], db_host=db_properties["db_host"], db_port=db_properties["db_port"], db_database=db_properties["db_database"], db_managed=db_properties["db_managed"]) _choose_publisher_db_user() _choose_publisher_db_user_passwd() os.chmod(config['pub_secret_file'], 0640) if "tomcat" not in esg_functions.get_group_list(): esg_functions.add_unix_group(config["tomcat_group"]) os.chown(config['esgf_secret_file'], config["installer_uid"], esg_functions.get_tomcat_group_id()) if db_properties["db_host"] == esgf_host or db_properties[ "db_host"] == "localhost": logger.info("db publisher connection string %s@localhost", db_properties["db_user"]) else: logger.info("db publisher connection string %s@%s:%s/%s", db_properties["db_user"], db_properties["db_host"], db_properties["db_port"], db_properties["db_database"]) os.chdir(starting_directory) return True
def setup_thredds(): if os.path.isdir("/usr/local/tomcat/webapps/thredds"): thredds_install = raw_input( "Existing Thredds installation found. Do you want to continue with the Thredds installation [y/N]: " ) or "no" if thredds_install.lower() in ["no", "n"]: return print "\n*******************************" print "Setting up Thredds" print "******************************* \n" esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/thredds") thredds_url = os.path.join("http://", config["esgf_dist_mirror"], "dist", "devel", "thredds", "5.0", "5.0.1", "thredds.war") download_thredds_war(thredds_url) with esg_bash2py.pushd("/usr/local/tomcat/webapps/thredds"): with zipfile.ZipFile("/usr/local/tomcat/webapps/thredds/thredds.war", 'r') as zf: zf.extractall() os.remove("thredds.war") TOMCAT_USER_ID = esg_functions.get_tomcat_user_id() TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id() esg_functions.change_permissions_recursive( "/usr/local/tomcat/webapps/thredds", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # TDS configuration root esg_bash2py.mkdir_p(os.path.join(config["thredds_content_dir"], "thredds")) # TDS memory configuration shutil.copyfile("thredds_conf/threddsConfig.xml", "/esg/content/thredds/threddsConfig.xml") # ESGF root catalog shutil.copyfile("thredds_conf/catalog.xml", "/esg/content/thredds/catalog.xml-esgcet") esg_bash2py.mkdir_p("/esg/content/thredds/esgcet") # TDS customized applicationContext.xml file with ESGF authorizer shutil.copyfile( "thredds_conf/applicationContext.xml", "/usr/local/tomcat/webapps/thredds/WEB-INF/applicationContext.xml") # TDS jars necessary to support ESGF security filters # some jars are retrieved from the ESGF repository # other jars are copied from the unpacked ORP or NM distributions esgf_devel_url = os.path.join("http://", config["esgf_dist_mirror"], "dist", "devel") urllib.urlretrieve( "{esgf_devel_url}/filters/XSGroupRole-1.0.0.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/XSGroupRole-1.0.0.jar") urllib.urlretrieve( "{esgf_devel_url}/filters/commons-httpclient-3.1.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-httpclient-3.1.jar" ) urllib.urlretrieve( "{esgf_devel_url}/filters/commons-lang-2.6.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-lang-2.6.jar") urllib.urlretrieve( "{esgf_devel_url}/esg-orp/esg-orp-2.9.3.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esg-orp-2.9.3.jar") urllib.urlretrieve( "{esgf_devel_url}/esgf-node-manager/esgf-node-manager-common-1.0.0.jar" .format(esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-node-manager-common-1.0.0.jar" ) urllib.urlretrieve( "{esgf_devel_url}/esgf-node-manager/esgf-node-manager-filters-1.0.0.jar" .format(esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-node-manager-filters-1.0.0.jar" ) urllib.urlretrieve( "{esgf_devel_url}/esgf-security/esgf-security-2.7.10.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-security-2.7.10.jar" ) urllib.urlretrieve( "{esgf_devel_url}/filters/jdom-legacy-1.1.3.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/jdom-legacy-1.1.3.jar") urllib.urlretrieve( "{esgf_devel_url}/filters/opensaml-2.3.2.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/opensaml-2.3.2.jar") urllib.urlretrieve( "{esgf_devel_url}/filters/openws-1.3.1.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/openws-1.3.1.jar") urllib.urlretrieve( "{esgf_devel_url}/filters/xmltooling-1.2.2.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xmltooling-1.2.2.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/serializer-2.9.1.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/serializer-2.9.1.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/velocity-1.5.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/velocity-1.5.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xalan-2.7.2.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xalan-2.7.2.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xercesImpl-2.10.0.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xercesImpl-2.10.0.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xml-apis-1.4.01.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xml-apis-1.4.01.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xmlsec-1.4.2.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xmlsec-1.4.2.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/log4j-1.2.17.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/log4j-1.2.17.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/commons-io-2.4.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-io-2.4.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-dbcp-1.4.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-dbcp-1.4.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-dbutils-1.3.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-dbutils-1.3.jar" ) shutil.copyfile( "/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-pool-1.5.4.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-pool-1.5.4.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/postgresql-8.4-703.jdbc3.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/postgresql-8.4-703.jdbc3.jar" ) # TDS customized logging (uses DEBUG) shutil.copyfile( "thredds_conf/log4j2.xml", "/usr/local/tomcat/webapps/thredds/WEB-INF/classes/log4j2.xml") # data node scripts #TODO: Convert data node scripts to Python # change ownership of content directory TOMCAT_USER_ID = esg_functions.get_tomcat_user_id() TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id() esg_functions.change_permissions_recursive("/esg/content/thredds/", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # change ownership of source directory esg_functions.change_permissions_recursive("/usr/local/webapps/thredds", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # cleanup shutil.rmtree("/usr/local/tomcat/webapps/esgf-node-manager/")