def rebuild_truststore(truststore_file, certs_dir=config["globus_global_certs_dir"]): '''Converts ESG certificates (that can be fetch by above function) into a truststore''' print "(Re)building truststore from esg certificates... [{truststore_file}]".format( truststore_file=truststore_file) if not os.path.isdir(certs_dir): print "Sorry, No esg certificates found... in {certs_dir}".format( certs_dir=certs_dir) fetch_esgf_certificates(certs_dir) #If you don't already have a truststore to build on.... #Start building from a solid foundation i.e. Java's set of ca certs... if not os.path.isfile(truststore_file): create_new_truststore(truststore_file) tmp_dir = "/tmp/esg_scratch" esg_bash2py.mkdir_p(tmp_dir) cert_files = glob.glob('{certs_dir}/*.0'.format(certs_dir=certs_dir)) for cert in cert_files: _insert_cert_into_truststore(cert, truststore_file, tmp_dir) shutil.rmtree(tmp_dir) sync_with_java_truststore(truststore_file) os.chown(truststore_file, esg_functions.get_user_id("tomcat"), esg_functions.get_group_id("tomcat"))
def setup_solr(): '''Setup Apache Solr for faceted search''' print "\n*******************************" print "Setting up Solr" print "******************************* \n" # # Solr/Jetty web application SOLR_VERSION = "5.5.4" SOLR_INSTALL_DIR = "/usr/local/solr" SOLR_HOME = "/usr/local/solr-home" SOLR_DATA_DIR = "/esg/solr-index" SOLR_INCLUDE = "{SOLR_HOME}/solr.in.sh".format(SOLR_HOME=SOLR_HOME) #Download solr tarball solr_tarball_url = "http://archive.apache.org/dist/lucene/solr/{SOLR_VERSION}/solr-{SOLR_VERSION}.tgz".format( SOLR_VERSION=SOLR_VERSION) download_solr_tarball(solr_tarball_url, SOLR_VERSION) #Extract solr tarball extract_solr_tarball( '/tmp/solr-{SOLR_VERSION}.tgz'.format(SOLR_VERSION=SOLR_VERSION), SOLR_VERSION) esg_bash2py.mkdir_p(SOLR_DATA_DIR) # download template directory structure for shards home download_template_directory() esg_bash2py.mkdir_p(SOLR_HOME) # create non-privilged user to run Solr server esg_functions.stream_subprocess_output("groupadd solr") esg_functions.stream_subprocess_output( "useradd -s /sbin/nologin -g solr -d /usr/local/solr solr") SOLR_USER_ID = pwd.getpwnam("solr").pw_uid SOLR_GROUP_ID = grp.getgrnam("solr").gr_gid esg_functions.change_permissions_recursive( "/usr/local/solr-{SOLR_VERSION}".format(SOLR_VERSION=SOLR_VERSION), SOLR_USER_ID, SOLR_GROUP_ID) esg_functions.change_permissions_recursive(SOLR_HOME, SOLR_USER_ID, SOLR_GROUP_ID) esg_functions.change_permissions_recursive(SOLR_DATA_DIR, SOLR_USER_ID, SOLR_GROUP_ID) # #Copy shard files shutil.copyfile("solr_scripts/add_shard.sh", "/usr/local/bin/add_shard.sh") shutil.copyfile("solr_scripts/remove_shard.sh", "/usr/local/bin/remove_shard.sh") # add shards esg_functions.call_subprocess("/usr/local/bin/add_shard.sh master 8984") esg_functions.call_subprocess("/usr/local/bin/add_shard.sh master 8983") # custom logging properties shutil.copyfile( "solr_scripts/log4j.properties", "/{SOLR_INSTALL_DIR}/server/resources/log4j.properties".format( SOLR_INSTALL_DIR=SOLR_INSTALL_DIR))
def copy_config_files(): '''copy custom configuration''' '''server.xml: includes references to keystore, truststore in /esg/config/tomcat''' '''context.xml: increases the Tomcat cache to avoid flood of warning messages''' print "\n*******************************" print "Copying custom Tomcat config files" print "******************************* \n" try: shutil.copyfile("tomcat_conf/server.xml", "/usr/local/tomcat/conf/server.xml") shutil.copyfile("tomcat_conf/context.xml", "/usr/local/tomcat/conf/context.xml") # shutil.copyfile("certs/esg-truststore.ts", "/esg/config/tomcat/esg-truststore.ts") # shutil.copyfile("certs/esg-truststore.ts-orig", "/esg/config/tomcat/esg-truststore.ts-orig") # shutil.copyfile("certs/keystore-tomcat", "/esg/config/tomcat/keystore-tomcat") esg_bash2py.mkdir_p("/esg/config/tomcat") shutil.copyfile("certs/tomcat-users.xml", "/esg/config/tomcat/tomcat-users.xml") # shutil.copy("tomcat_conf/setenv.sh", os.path.join(CATALINA_HOME, "bin")) except OSError, error: # if error.errno == errno.EEXIST: # pass # else: logger.exception()
def setup_orp(): '''Setup the ORP subsystem''' if os.path.isdir("/usr/local/tomcat/webapps/esg-orp"): orp_install = raw_input( "Existing ORP installation found. Do you want to continue with the ORP installation [y/N]: " ) or "no" if orp_install.lower() in ["no", "n"]: return print "\n*******************************" print "Setting up ORP" print "******************************* \n" esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esg-orp") #COPY esgf-orp/esg-orp.war /usr/local/tomcat/webapps/esg-orp/esg-orp.war orp_url = os.path.join("http://", config["esgf_dist_mirror"], "dist", "devel", "esg-orp", "esg-orp.war") print "orp_url:", orp_url download_orp_war(orp_url) with esg_bash2py.pushd("/usr/local/tomcat/webapps/esg-orp"): with zipfile.ZipFile("/usr/local/tomcat/webapps/esg-orp/esg-orp.war", 'r') as zf: zf.extractall() os.remove("esg-orp.war") TOMCAT_USER_ID = esg_functions.get_tomcat_user_id() TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id() esg_functions.change_permissions_recursive( "/usr/local/tomcat/webapps/esg-orp", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # properties to read the Tomcat keystore, used to sign the authentication cookie # these values are the same for all ESGF nodes shutil.copyfile( "esgf_orp_conf/esg-orp.properties", "/usr/local/tomcat/webapps/esg-orp/WEB-INF/classes/esg-orp.properties")
def migrate_tomcat_credentials_to_esgf(esg_dist_url, tomcat_config_dir): ''' Move selected config files into esgf tomcat's config dir (certificate et al) Ex: /esg/config/tomcat -rw-r--r-- 1 tomcat tomcat 181779 Apr 22 19:44 esg-truststore.ts -r-------- 1 tomcat tomcat 887 Apr 22 19:32 hostkey.pem -rw-r--r-- 1 tomcat tomcat 1276 Apr 22 19:32 keystore-tomcat -rw-r--r-- 1 tomcat tomcat 590 Apr 22 19:32 pcmdi11.llnl.gov-esg-node.csr -rw-r--r-- 1 tomcat tomcat 733 Apr 22 19:32 pcmdi11.llnl.gov-esg-node.pem -rw-r--r-- 1 tomcat tomcat 295 Apr 22 19:42 tomcat-users.xml Only called when migration conditions are present. ''' tomcat_install_config_dir = os.path.join(config["tomcat_install_dir"], "conf") if tomcat_install_config_dir != config["tomcat_conf_dir"]: if not os.path.exists(config["tomcat_conf_dir"]): esg_bash2py.mkdir_p(config["tomcat_conf_dir"]) esg_functions.backup(tomcat_install_config_dir) copy_credential_files(tomcat_install_config_dir) os.chown(config["tomcat_conf_dir"], esg_functions.get_user_id("tomcat"), esg_functions.get_group_id("tomcat")) if not check_server_xml(): download_server_config_file(esg_dist_url) #SET the server.xml variables to contain proper values logger.debug("Editing %s/conf/server.xml accordingly...", config["tomcat_install_dir"]) edit_tomcat_server_xml(config["keystore_password"])
def create_postgres_log_dir(): ''' Create log directory ''' esg_bash2py.mkdir_p(os.path.join(config["postgress_install_dir"], "log")) postgres_user_id = pwd.getpwnam(config["pg_sys_acct"]).pw_uid try: os.chown(os.path.join(config["postgress_install_dir"], "log"), postgres_user_id, -1) except OSError, error: print " ERROR: Could not change ownership of postgres' log to \"{pg_sys_acct}\" user".format(pg_sys_acct = config["pg_sys_acct"])
def test_extract_tomcat_tarball(self): esg_bash2py.mkdir_p("/usr/local/tomcat_test") esg_tomcat_manager.download_tomcat() esg_tomcat_manager.extract_tomcat_tarball("/usr/local/tomcat_test") self.assertTrue( os.path.isdir("/usr/local/tomcat_test/apache-tomcat-8.5.20")) self.assertTrue(os.path.isdir("/usr/local/tomcat_test/")) esg_tomcat_manager.copy_config_files() self.assertTrue(os.path.isfile("/usr/local/tomcat/conf/server.xml"))
def create_esg_directories(): '''Create directories to hold ESGF scripts, config files, and logs''' directories_to_check = [config["scripts_dir"], config["esg_backup_dir"], config["esg_tools_dir"], config[ "esg_log_dir"], config["esg_config_dir"], config["esg_etc_dir"], config["tomcat_conf_dir"]] for directory in directories_to_check: if not os.path.isdir(directory): esg_bash2py.mkdir_p(directory) os.chmod(config["esg_etc_dir"], 0777)
def setup_dashboard(): if os.path.isdir("/usr/local/tomcat/webapps/esgf-stats-api"): stats_api_install = raw_input( "Existing Stats API installation found. Do you want to continue with the Stats API installation [y/N]: " ) or "no" if stats_api_install.lower() in ["no", "n"]: return print "\n*******************************" print "Setting up ESGF Stats API (dashboard)" print "******************************* \n" esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esgf-stats-api") stats_api_url = os.path.join("http://", config["esgf_dist_mirror"], "dist", "devel", "esgf-stats-api", "esgf-stats-api.war") download_stats_api_war(stats_api_url) with esg_bash2py.pushd("/usr/local/tomcat/webapps/esgf-stats-api"): with zipfile.ZipFile( "/usr/local/tomcat/webapps/esgf-stats-api/esgf-stats-api.war", 'r') as zf: zf.extractall() os.remove("esgf-stats-api.war") TOMCAT_USER_ID = esg_functions.get_tomcat_user_id() TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id() esg_functions.change_permissions_recursive( "/usr/local/tomcat/webapps/esgf-stats-api", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # execute dashboard installation script (without the postgres schema) run_dashboard_script() # create non-privileged user to run the dashboard application # RUN groupadd dashboard && \ # useradd -s /sbin/nologin -g dashboard -d /usr/local/dashboard dashboard && \ # chown -R dashboard:dashboard /usr/local/esgf-dashboard-ip # RUN chmod a+w /var/run esg_functions.stream_subprocess_output("groupadd dashboard") esg_functions.stream_subprocess_output( "useradd -s /sbin/nologin -g dashboard -d /usr/local/dashboard dashboard" ) DASHBOARD_USER_ID = pwd.getpwnam("dashboard").pw_uid DASHBOARD_GROUP_ID = grp.getgrnam("dashboard").gr_gid esg_functions.change_permissions_recursive("/usr/local/esgf-dashboard-ip", DASHBOARD_USER_ID, DASHBOARD_GROUP_ID) os.chmod("/var/run", stat.S_IWRITE) os.chmod("/var/run", stat.S_IWGRP) os.chmod("/var/run", stat.S_IWOTH) start_dashboard_service()
def copy_apache_conf_files(): ''' Copy custom apache conf files ''' esg_bash2py.mkdir_p("/etc/certs") #TODO: Generate certs from esg_cert_manager and copy here # esg_cert_manager.create_self_signed_cert("/etc/certs") # esg_cert_manager.create_certificate_chain("/etc/certs/hostcert.pem") # shutil.copyfile("apache_certs/hostcert.pem", "/etc/certs/hostcert.pem") # shutil.copyfile("apache_certs/hostkey.pem", "/etc/certs/hostkey.pem") shutil.copyfile("apache_certs/esgf-ca-bundle.crt", "/etc/certs/esgf-ca-bundle.crt") # shutil.copyfile("apache_certs/temp_cachain.pem", "/etc/certs/cachain.pem") shutil.copyfile("apache_html/index.html", "/var/www/html/index.html") # shutil.copyfile("apache_conf/httpd.conf", "/etc/httpd/conf.d/httpd.conf") shutil.copyfile("apache_conf/ssl.conf", "/etc/httpd/conf.d/ssl.conf")
def setup_temp_ca(): esg_bash2py.mkdir_p("/etc/tempcerts") #Copy CA perl script and openssl conf file that it uses. The CA perl script #is used to create a temporary root CA shutil.copyfile("apache_certs/CA.pl", "/etc/tempcerts/CA.pl") shutil.copyfile("apache_certs/openssl.cnf", "/etc/tempcerts/openssl.cnf") shutil.copyfile("apache_certs/myproxy-server.config", "/etc/tempcerts/myproxy-server.config") os.chmod("/etc/tempcerts/CA.pl", 0755) os.chmod("/etc/tempcerts/openssl.cnf", 0755) with esg_bash2py.pushd("/etc/tempcerts"): esg_bash2py.mkdir_p("CA") ca_answer = "{fqdn}-CA".format(fqdn=esg_functions.get_esgf_host()) print "ca_answer:", ca_answer new_ca_output = esg_functions.call_subprocess("perl CA.pl -newca", command_stdin=ca_answer) print "new_ca_output:", new_ca_output
def backup(path, backup_dir = config["esg_backup_dir"], num_of_backups=config["num_backups_to_keep"]): ''' Given a directory the contents of the directory is backed up as a tar.gz file in path - a filesystem path backup_dir - destination directory for putting backup archive (default esg_backup_dir:-/esg/backups) num_of_backups - the number of backup files you wish to have present in destination directory (default num_backups_to_keep:-7) ''' source_directory = readlinkf(path) print "Backup - Creating a backup archive of %s" % (source_directory) current_directory = os.getcwd() os.chdir(source_directory) esg_bash2py.mkdir_p(source_directory) source_backup_name = re.search("\w+$", source_directory).group() backup_filename=readlinkf(backup_dir)+"/"+source_backup_name + "." + str(datetime.date.today())+".tgz" try: with tarfile.open(backup_filename, "w:gz") as tar: tar.add(source_directory) except: print "ERROR: Problem with creating backup archive: {backup_filename}".format(backup_filename = backup_filename) os.chdir(current_directory) return 1 if os.path.isfile(backup_filename): print "Created backup: %s" % (backup_filename) else: "Could not locate backup file %s" % (backup_filename) os.chdir(current_directory) return 1 # Cleanup if os.getcwd() != backup_dir: os.chdir(backup_dir) files= subprocess.Popen('ls -t | grep %s.\*.tgz | tail -n +$((%i+1)) | xargs' %(source_backup_name,int(num_of_backups)), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if len(files.stdout.readlines()) > 0: print "Tidying up a bit..." print "old backup files to remove: %s" % (''.join(files.stdout.readlines())) for file in files.stdout.readlines(): os.remove(file) os.chdir(current_directory) return 0
def setup_java(): ''' Installs Oracle Java from rpm using yum localinstall. Does nothing if an acceptible Java install is found. ''' print "*******************************" print "Setting up Java {java_version}".format(java_version=config["java_version"]) print "******************************* \n" if force_install: pass if check_for_existing_java(): setup_java_answer = raw_input("Do you want to continue with Java installation and setup? [y/N]: ") or "N" if setup_java_answer.lower().strip() not in ["y", "yes"]: print "Skipping Java installation" return last_java_truststore_file = esg_functions.readlinkf(config["truststore_file"]) esg_bash2py.mkdir_p(config["workdir"]) with esg_bash2py.pushd(config["workdir"]): java_tarfile = esg_bash2py.trim_string_from_head(config["java_dist_url"]) jdk_directory = java_tarfile.split("-")[0] java_install_dir_parent = config["java_install_dir"].rsplit("/",1)[0] #Check for Java tar file if not os.path.isfile(java_tarfile): print "Don't see java distribution file {java_dist_file_path} either".format(java_dist_file_path=os.path.join(os.getcwd(),java_tarfile)) download_java(java_tarfile) print "Extracting Java tarfile", java_tarfile esg_functions.extract_tarball(java_tarfile, java_install_dir_parent) #Create symlink to Java install directory (/usr/local/java) esg_bash2py.symlink_force(os.path.join(java_install_dir_parent, jdk_directory), config["java_install_dir"]) os.chown(config["java_install_dir"], config["installer_uid"], config["installer_gid"]) #recursively change permissions esg_functions.change_permissions_recursive(config["java_install_dir"], config["installer_uid"], config["installer_gid"]) set_default_java() print check_java_version("java")
def create_self_signed_cert(cert_dir): """ If datacard.crt and datacard.key don't exist in cert_dir, create a new self-signed cert and keypair and write them into that directory. Source: https://skippylovesmalorie.wordpress.com/2010/02/12/how-to-generate-a-self-signed-certificate-using-pyopenssl/ """ CERT_FILE = "hostcert.pem" KEY_FILE = "hostkey.pem" if not os.path.exists(os.path.join(cert_dir, CERT_FILE)) \ or not os.path.exists(os.path.join(cert_dir, KEY_FILE)): # create a key pair k = OpenSSL.crypto.PKey() k.generate_key(OpenSSL.crypto.TYPE_RSA, 4096) # create a self-signed cert cert = OpenSSL.crypto.X509() cert.get_subject().C = "US" cert.get_subject().ST = "California" cert.get_subject().L = "Livermore" cert.get_subject().O = "LLNL" cert.get_subject().OU = "ESGF" cert.get_subject().CN = socket.gethostname() cert.set_serial_number(1000) cert.gmtime_adj_notBefore(0) cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) cert.set_issuer(cert.get_subject()) cert.set_pubkey(k) cert.sign(k, 'sha1') esg_bash2py.mkdir_p(cert_dir) with open(os.path.join(cert_dir, CERT_FILE), "wt") as cert_file_handle: cert_file_handle.write( OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)) with open(os.path.join(cert_dir, KEY_FILE), "wt") as key_file_handle: key_file_handle.write( OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, k))
def create_certificate_chain_list(): '''Create a list of the certificates that will be a part of the certificate chain file''' default_cachain = "/etc/esgfcerts/cachain.pem" cert_files = [] #Enter ca_chain file into list print "Please enter your Certificate Authority's certificate chain file(s)" print "[enter each cert file/url press return, press return with blank entry when done]" while True: certfile_entry = raw_input("Enter certificate chain file name: ") if not certfile_entry: if not cert_files: print "Adding default certificate chain file {default_cachain}".format( default_cachain=default_cachain) if os.path.isfile(default_cachain): cert_files.append(default_cachain) break else: print "{default_cachain} does not exist".format( default_cachain=default_cachain) print "Creating {default_cachain}".format( default_cachain=default_cachain) esg_bash2py.mkdir_p("/etc/esgfcerts") esg_bash2py.touch(default_cachain) cert_files.append(default_cachain) break # esg_functions.exit_with_error(1) else: break else: if os.path.isfile(certfile_entry): cert_files.append(certfile_entry) else: print "{certfile_entry} does not exist".format( certfile_entry=certfile_entry) return cert_files
def setup_node_manager_old(): if os.path.isdir("/usr/local/tomcat/webapps/esgf-node-manager"): node_manager_install = raw_input( "Existing Node Manager installation found. Do you want to continue with the Node Manager installation [y/N]: " ) or "no" if node_manager_install.lower() in ["no", "n"]: return print "\n*******************************" print "Setting up ESGF Node Manager (old)" print "******************************* \n" esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esgf-node-manager") node_manager_url = os.path.join("http://", config["esgf_dist_mirror"], "dist", "devel", "esgf-node-manager", "esgf-node-manager.war") download_node_manager_war(node_manager_url) with esg_bash2py.pushd("/usr/local/tomcat/webapps/esgf-node-manager/"): with zipfile.ZipFile( "/usr/local/tomcat/webapps/esgf-node-manager/esgf-node-manager.war", 'r') as zf: zf.extractall() os.remove("esgf-node-manager.war")
def setup_cog(): # choose CoG version COG_TAG = "v3.9.7" # # env variable to execute CoG initialization # # may be overridden from command line after first container startup INIT = True # setup CoG environment COG_DIR = "/usr/local/cog" esg_bash2py.mkdir_p(COG_DIR) COG_CONFIG_DIR = "{COG_DIR}/cog_config".format(COG_DIR=COG_DIR) esg_bash2py.mkdir_p(COG_CONFIG_DIR) COG_INSTALL_DIR = "{COG_DIR}/cog_install".format(COG_DIR=COG_DIR) esg_bash2py.mkdir_p(COG_INSTALL_DIR) # ENV LD_LIBRARY_PATH=/usr/local/lib # # # install Python virtual environment # RUN cd $COG_DIR && \ # virtualenv venv # # # download CoG specific tag or branch clone_cog_repo(COG_INSTALL_DIR)
#(Ex: esgf-node-manager-0.9.0 -> esgf-node-manager.war) # local trimmed_name=$(pwd)/${node_dist_dir%-*} split_dir_name_list = node_dist_dir.split("-") versionless_name = '-'.join(split_dir_name_list[:3]) trimmed_name = os.path.join(os.getcwd(), versionless_name) node_war_file = trimmed_name + ".war" logger.debug("node_war_file: %s", node_war_file) #---------------------------- # make room for new INSTALL # ((upgrade == 0)) && set_aside_web_app ${node_manager_service_app_home} if mode != "upgrade": set_aside_web_app(node_manager_service_app_home) #---------------------------- # mkdir -p ${node_manager_service_app_home} esg_bash2py.mkdir_p(node_manager_service_app_home) # cd ${node_manager_service_app_home} os.chdir(node_manager_service_app_home) logger.debug("changed directory to : %s", os.getcwd()) #---------------------------- # fetch_file=esgf-node-manager.properties download_file_name = "esgf-node-manager.properties" # NOTE: The saving of the last config file must be done *BEFORE* we untar the new distro! # if ((upgrade)) && [ -e WEB-INF/classes/${fetch_file} ]; then if mode == "upgrade" and os.path.isfile( "WEB-INF/classes/{download_file_name}".format( download_file_name=download_file_name)): # cp WEB-INF/classes/${fetch_file} WEB-INF/classes/${fetch_file}.saved esg_functions.create_backup_file(
def generate_tomcat_keystore(keystore_name, keystore_alias, private_key, public_cert, intermediate_certs): '''The following helper function creates a new keystore for your tomcat installation''' provider = "org.bouncycastle.jce.provider.BouncyCastleProvider" idptools_install_dir = os.path.join(config["esg_tools_dir"], "idptools") if len(intermediate_certs) < 1: print "No intermediate_certs files given" esg_functions.exit_with_error(1) if not os.path.isfile(private_key): print "Private key file {private_key} does not exist".format( private_key=private_key) keystore_password = esg_functions.get_java_keystore_password() #------------- #Display values #------------- print "Keystore name : {keystore_name}".format(keystore_name=keystore_name) print "Keystore alias: {keystore_alias}".format( keystore_alias=keystore_alias) print "Keystore password: {keystore_password}".format( keystore_password=keystore_password) print "Private key : {private_key}".format(private_key=private_key) print "Public cert : {public_cert}".format(public_cert=public_cert) print "Certificates..." esg_bash2py.mkdir_p(idptools_install_dir) cert_bundle = os.path.join(idptools_install_dir, "cert.bundle") ca_chain_bundle = os.path.join(idptools_install_dir, "ca_chain.bundle") cert_bundle, ca_chain_bundle = bundle_certificates(public_cert, intermediate_certs, idptools_install_dir) print "checking that key pair is congruent... " if check_associate_cert_with_private_key(public_cert, private_key): print "The keypair was congruent" else: print "The keypair was not congruent" esg_functions.exit_with_error(1) print "creating keystore... " #create a keystore with a self-signed cert distinguished_name = "CN={esgf_host}".format( esgf_host=esg_functions.get_esgf_host()) #if previous keystore is found; backup backup_previous_keystore(keystore_name) #------------- #Make empty keystore... #------------- create_empty_java_keystore(keystore_name, keystore_alias, keystore_password, distinguished_name) #------------- #Convert your private key into from PEM to DER format that java likes #------------- derkey = convert_per_to_dem(private_key, idptools_install_dir) #------------- #Now we gather up all the other keys in the key chain... #------------- check_cachain_validity(ca_chain_bundle) print "Constructing new keystore content... " import_cert_into_keystore(keystore_name, keystore_alias, keystore_password, derkey, cert_bundle, provider) #Check keystore output java_keytool_executable = "{java_install_dir}/bin/keytool".format( java_install_dir=config["java_install_dir"]) check_keystore_command = "{java_keytool_executable} -v -list -keystore {keystore_name} -storepass {store_password} | egrep '(Owner|Issuer|MD5|SHA1|Serial number):'".format( java_keytool_executable=java_keytool_executable, keystore_name=keystore_name, store_password=keystore_password) keystore_output = esg_functions.call_subprocess(check_keystore_command) if keystore_output["returncode"] == 0: print "Mmmm, freshly baked keystore!" print "If Everything looks good... then replace your current tomcat keystore with {keystore_name}, if necessary.".format( keystore_name=keystore_name) print "Don't forget to change your tomcat's server.xml entry accordingly :-)" print "Remember: Keep your private key {private_key} and signed cert {public_cert} in a safe place!!!".format( private_key=private_key, public_cert=public_cert) else: print "Failed to check keystore" esg_functions.exit_with_error(1)
def fetch_esgf_certificates( globus_certs_dir=config["globus_global_certs_dir"]): '''Goes to ESG distribution server and pulls down all certificates for the federation. (suitable for crontabbing)''' print "\n*******************************" print "Fetching freshest ESG Federation Certificates..." print "******************************* \n" #if globus_global_certs_dir already exists, backup and delete, then recreate empty directory if os.path.isdir(config["globus_global_certs_dir"]): esg_functions.backup( config["globus_global_certs_dir"], os.path.join(config["globus_global_certs_dir"], ".bak.tz")) shutil.rmtree(config["globus_global_certs_dir"]) esg_bash2py.mkdir_p(config["globus_global_certs_dir"]) #Download trusted certs tarball esg_trusted_certs_file = "esg_trusted_certificates.tar" esg_trusted_certs_file_url = "https://aims1.llnl.gov/esgf/dist/certs/{esg_trusted_certs_file}".format( esg_trusted_certs_file=esg_trusted_certs_file) esg_functions.download_update( os.path.join(globus_certs_dir, esg_trusted_certs_file), esg_trusted_certs_file_url) #untar the esg_trusted_certs_file esg_functions.extract_tarball( os.path.join(globus_certs_dir, esg_trusted_certs_file), globus_certs_dir) os.remove(os.path.join(globus_certs_dir, esg_trusted_certs_file)) #certificate_issuer_cert "/var/lib/globus-connect-server/myproxy-ca/cacert.pem" simpleCA_cert = "/var/lib/globus-connect-server/myproxy-ca/cacert.pem" if os.path.isfile(simpleCA_cert): simpleCA_cert_hash = esg_functions.get_md5sum(simpleCA_cert) print "checking for MY cert: {globus_global_certs_dir}/{simpleCA_cert_hash}.0".format( globus_global_certs_dir=config["globus_global_certs_dir"], simpleCA_cert_hash=simpleCA_cert_hash) if os.path.isfile( "{globus_global_certs_dir}/{simpleCA_cert_hash}.0".format( globus_global_certs_dir=config["globus_global_certs_dir"], simpleCA_cert_hash=simpleCA_cert_hash)): print "Local CA cert file detected...." print "Integrating in local simpleCA_cert... " print "Local SimpleCA Root Cert: {simpleCA_cert}".format( simpleCA_cert=simpleCA_cert) print "Extracting Signing policy" #Copy simple CA cert to globus cert directory shutil.copyfile( simpleCA_cert, "{globus_global_certs_dir}/{simpleCA_cert_hash}.0".format( globus_global_certs_dir=config["globus_global_certs_dir"], simpleCA_cert_hash=simpleCA_cert_hash)) #extract simple CA cert tarball and copy to globus cert directory simpleCA_cert_parent_dir = esg_functions.get_parent_directory( simpleCA_cert) simpleCA_setup_tar_file = os.path.join( simpleCA_cert_parent_dir, "globus_simple_ca_{simpleCA_cert_hash}_setup-0.tar.gz".format( simpleCA_cert_hash=simpleCA_cert_hash)) esg_functions.extract_tarball(simpleCA_setup_tar_file) with esg_bash2py.pushd( "globus_simple_ca_{simpleCA_cert_hash}_setup-0".format( simpleCA_cert_hash=simpleCA_cert_hash)): shutil.copyfile( "{simpleCA_cert_hash}.signing_policy".format( simpleCA_cert_hash=simpleCA_cert_hash), "{globus_global_certs_dir}/{simpleCA_cert_hash}.signing_policy" .format(globus_global_certs_dir=config[ "globus_global_certs_dir"], simpleCA_cert_hash=simpleCA_cert_hash)) if os.path.isdir("/usr/local/tomcat/webapps/ROOT"): esg_functions.stream_subprocess_output( "openssl x509 -text -hash -in {simpleCA_cert} > {tomcat_install_dir}/webapps/ROOT/cacert.pem" .format(simpleCA_cert=simpleCA_cert, tomcat_install_dir="/usr/loca/tomcat")) print " My CA Cert now posted @ http://{fqdn}/cacert.pem ".format( fqdn=socket.getfqdn()) os.chmod("/usr/local/tomcat/webapps/ROOT/cacert.pem", 0644) os.chmod(config["globus_global_certs_dir"], 0755) esg_functions.change_permissions_recursive( config["globus_global_certs_dir"], 0644)
def initial_setup_questionnaire(force_install=False): print "-------------------------------------------------------" print 'Welcome to the ESGF Node installation program! :-)' print "-------------------------------------------------------" esg_bash2py.mkdir_p(config['esg_config_dir']) starting_directory = os.getcwd() os.chdir(config['esg_config_dir']) esgf_host = esg_property_manager.get_property("esgf_host") _choose_fqdn(esgf_host) if not esg_functions.get_security_admin_password() or force_install: _choose_admin_password() else: logger.info("Previously set password found.") _choose_organization_name() _choose_node_short_name() _choose_node_long_name() _choose_node_namespace() _choose_node_peer_group() _choose_esgf_index_peer() _choose_mail_admin_address() #TODO:Extract constructring DB string into separate function db_properties = get_db_properties() if not all(db_properties) or force_install: _is_managed_db(db_properties) _get_db_conn_str_questionnaire(db_properties) else: if db_properties["db_host"] == esgf_host or db_properties[ "db_host"] == "localhost": print "db_connection_string = {db_user}@localhost".format( db_user=db_properties["db_user"]) else: connstring_ = "{db_user}@{db_host}:{db_port}/{db_database} [external = ${db_managed}]".format( db_user=db_properties["db_user"], db_host=db_properties["db_host"], db_port=db_properties["db_port"], db_database=db_properties["db_database"], db_managed=db_properties["db_managed"]) _choose_publisher_db_user() _choose_publisher_db_user_passwd() os.chmod(config['pub_secret_file'], 0640) if "tomcat" not in esg_functions.get_group_list(): esg_functions.add_unix_group(config["tomcat_group"]) os.chown(config['esgf_secret_file'], config["installer_uid"], esg_functions.get_tomcat_group_id()) if db_properties["db_host"] == esgf_host or db_properties[ "db_host"] == "localhost": logger.info("db publisher connection string %s@localhost", db_properties["db_user"]) else: logger.info("db publisher connection string %s@%s:%s/%s", db_properties["db_user"], db_properties["db_host"], db_properties["db_port"], db_properties["db_database"]) os.chdir(starting_directory) return True
def setup_thredds(): if os.path.isdir("/usr/local/tomcat/webapps/thredds"): thredds_install = raw_input( "Existing Thredds installation found. Do you want to continue with the Thredds installation [y/N]: " ) or "no" if thredds_install.lower() in ["no", "n"]: return print "\n*******************************" print "Setting up Thredds" print "******************************* \n" esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/thredds") thredds_url = os.path.join("http://", config["esgf_dist_mirror"], "dist", "devel", "thredds", "5.0", "5.0.1", "thredds.war") download_thredds_war(thredds_url) with esg_bash2py.pushd("/usr/local/tomcat/webapps/thredds"): with zipfile.ZipFile("/usr/local/tomcat/webapps/thredds/thredds.war", 'r') as zf: zf.extractall() os.remove("thredds.war") TOMCAT_USER_ID = esg_functions.get_tomcat_user_id() TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id() esg_functions.change_permissions_recursive( "/usr/local/tomcat/webapps/thredds", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # TDS configuration root esg_bash2py.mkdir_p(os.path.join(config["thredds_content_dir"], "thredds")) # TDS memory configuration shutil.copyfile("thredds_conf/threddsConfig.xml", "/esg/content/thredds/threddsConfig.xml") # ESGF root catalog shutil.copyfile("thredds_conf/catalog.xml", "/esg/content/thredds/catalog.xml-esgcet") esg_bash2py.mkdir_p("/esg/content/thredds/esgcet") # TDS customized applicationContext.xml file with ESGF authorizer shutil.copyfile( "thredds_conf/applicationContext.xml", "/usr/local/tomcat/webapps/thredds/WEB-INF/applicationContext.xml") # TDS jars necessary to support ESGF security filters # some jars are retrieved from the ESGF repository # other jars are copied from the unpacked ORP or NM distributions esgf_devel_url = os.path.join("http://", config["esgf_dist_mirror"], "dist", "devel") urllib.urlretrieve( "{esgf_devel_url}/filters/XSGroupRole-1.0.0.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/XSGroupRole-1.0.0.jar") urllib.urlretrieve( "{esgf_devel_url}/filters/commons-httpclient-3.1.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-httpclient-3.1.jar" ) urllib.urlretrieve( "{esgf_devel_url}/filters/commons-lang-2.6.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-lang-2.6.jar") urllib.urlretrieve( "{esgf_devel_url}/esg-orp/esg-orp-2.9.3.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esg-orp-2.9.3.jar") urllib.urlretrieve( "{esgf_devel_url}/esgf-node-manager/esgf-node-manager-common-1.0.0.jar" .format(esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-node-manager-common-1.0.0.jar" ) urllib.urlretrieve( "{esgf_devel_url}/esgf-node-manager/esgf-node-manager-filters-1.0.0.jar" .format(esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-node-manager-filters-1.0.0.jar" ) urllib.urlretrieve( "{esgf_devel_url}/esgf-security/esgf-security-2.7.10.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/esgf-security-2.7.10.jar" ) urllib.urlretrieve( "{esgf_devel_url}/filters/jdom-legacy-1.1.3.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/jdom-legacy-1.1.3.jar") urllib.urlretrieve( "{esgf_devel_url}/filters/opensaml-2.3.2.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/opensaml-2.3.2.jar") urllib.urlretrieve( "{esgf_devel_url}/filters/openws-1.3.1.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/openws-1.3.1.jar") urllib.urlretrieve( "{esgf_devel_url}/filters/xmltooling-1.2.2.jar".format( esgf_devel_url=esgf_devel_url), "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xmltooling-1.2.2.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/serializer-2.9.1.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/serializer-2.9.1.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/velocity-1.5.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/velocity-1.5.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xalan-2.7.2.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xalan-2.7.2.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xercesImpl-2.10.0.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xercesImpl-2.10.0.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xml-apis-1.4.01.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xml-apis-1.4.01.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/xmlsec-1.4.2.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/xmlsec-1.4.2.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/log4j-1.2.17.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/log4j-1.2.17.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esg-orp/WEB-INF/lib/commons-io-2.4.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-io-2.4.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-dbcp-1.4.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-dbcp-1.4.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-dbutils-1.3.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-dbutils-1.3.jar" ) shutil.copyfile( "/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/commons-pool-1.5.4.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/commons-pool-1.5.4.jar") shutil.copyfile( "/usr/local/tomcat/webapps/esgf-node-manager/WEB-INF/lib/postgresql-8.4-703.jdbc3.jar", "/usr/local/tomcat/webapps/thredds/WEB-INF/lib/postgresql-8.4-703.jdbc3.jar" ) # TDS customized logging (uses DEBUG) shutil.copyfile( "thredds_conf/log4j2.xml", "/usr/local/tomcat/webapps/thredds/WEB-INF/classes/log4j2.xml") # data node scripts #TODO: Convert data node scripts to Python # change ownership of content directory TOMCAT_USER_ID = esg_functions.get_tomcat_user_id() TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id() esg_functions.change_permissions_recursive("/esg/content/thredds/", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # change ownership of source directory esg_functions.change_permissions_recursive("/usr/local/webapps/thredds", TOMCAT_USER_ID, TOMCAT_GROUP_ID) # cleanup shutil.rmtree("/usr/local/tomcat/webapps/esgf-node-manager/")
def test_download_orp_war(self): esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esg-orp") esg_subsystem.download_orp_war("http://aims1.llnl.gov/esgf/dist/devel/esg-orp/esg-orp.war") self.assertTrue(os.path.isfile("/usr/local/tomcat/webapps/esg-orp/esg-orp.war")) st = os.stat("/usr/local/tomcat/webapps/esg-orp/esg-orp.war") print "war file size:", st.st_size
def setup_subsystem(subsystem, distribution_directory, esg_dist_url, force_install=False): ''' arg (1) - name of installation script root name. Ex:security which resolves to script file esg-security arg (2) - directory on the distribution site where script is fetched from Ex: orp usage: setup_subsystem security orp - looks for the script esg-security in the distriubtion dir orp ''' subsystem_install_script_path = os.path.join( config["scripts_dir"], "esg-{subsystem}".format(subsystem=subsystem)) #--- #check that you have at one point in time fetched the subsystem's installation script #if indeed you have we will assume you would like to proceed with setting up the latest... #Otherwise we just ask you first before you pull down new code to your machine... #--- if force_install: default = "y" else: default = "n" if os.path.exists(subsystem_install_script_path) or force_install: if default.lower() in ["y", "yes"]: run_installation = raw_input( "Would you like to set up {subsystem} services? [Y/n]: ". format(subsystem=subsystem)) or "y" else: run_installation = raw_input( "Would you like to set up {subsystem} services? [y/N]: ". format(subsystem=subsystem)) or "n" if run_installation.lower() in ["n", "no"]: print "Skipping installation of {subsystem}".format( subsystem=subsystem) return True print "-------------------------------" print "LOADING installer for {subsystem}... ".format(subsystem=subsystem) esg_bash2py.mkdir_p(config["workdir"]) with esg_bash2py.pushd(config["workdir"]): logger.debug("Changed directory to %s", os.getcwd()) with esg_bash2py.pushd(config["scripts_dir"]): logger.debug("Changed directory to %s", os.getcwd()) subsystem_full_name = "esg-{subsystem}".format(subsystem=subsystem) subsystem_remote_url = "{esg_dist_url}/{distribution_directory}/{subsystem_full_name}".format( esg_dist_url=esg_dist_url, distribution_directory=distribution_directory, subsystem_full_name=subsystem_full_name) if not esg_functions.download_update( "{subsystem_full_name}".format( subsystem_full_name=subsystem_full_name), subsystem_remote_url): logger.error("Could not download %s", subsystem_full_name) return False try: os.chmod(subsystem_full_name, 0755) except OSError: logger.exception("Unable to change permissions on %s", subsystem_full_name) logger.info("script_dir contents: %s", os.listdir(config["scripts_dir"])) subsystem_underscore = subsystem.replace("-", "_") execute_subsystem_command = ". {scripts_dir}/{subsystem_full_name}; setup_{subsystem_underscore}".format( scripts_dir=config["scripts_dir"], subsystem_full_name=subsystem_full_name, subsystem_underscore=subsystem_underscore) setup_subsystem_process = subprocess.Popen( ['bash', '-c', execute_subsystem_command]) setup_subsystem_stdout, setup_subsystem_stderr = setup_subsystem_process.communicate( ) logger.debug("setup_subsystem_stdout: %s", setup_subsystem_stdout) logger.debug("setup_subsystem_stderr: %s", setup_subsystem_stderr)
#!/usr/local/bin/python2.7 import glob import sys import os import datetime import time from pylint.lint import Run from contextlib import contextmanager import esg_bash2py import esg_logging_manager logger = esg_logging_manager.create_rotating_log(__name__) # create folder pylint_score_reports esg_bash2py.mkdir_p("pylint_score_reports") # get all files that contain the esg*_**.py pattern esgf_python_scripts = glob.glob("esg*_**.py") #TODO: Might do this set difference operation to remove this script (esg_pylint_script) from the linting # esgf_python_scripts = set(esgf_python_scripts) - set(glob("esg_pylint_script")) @contextmanager def suppress_stdout(): '''Source: http://thesmithfam.org/blog/2012/10/25/temporarily-suppress-console-output-in-python/''' with open(os.devnull, "w") as devnull: old_stdout = sys.stdout sys.stdout = devnull try:
def process_arguments(node_type_list, devel, esg_dist_url): logger.debug("node_type_list at start of process_arguments: %s", node_type_list) logger.info("node_type_list at start of process_arguments: %s", node_type_list) print "node_type_list at start of process_arguments: %s", node_type_list args, parser = _define_acceptable_arguments() logging.debug(pprint.pformat(args)) logger.info("args: %s", args) print "args:", args if len(sys.argv) == 1: parser.print_help() sys.exit(0) if args.install: if args.type: for arg in args.type: node_type_list = set_node_type_value(arg, node_type_list, True) installer_mode_dictionary["upgrade_mode"] = False installer_mode_dictionary["install_mode"] = True set_node_type_value("install", node_type_list, True) print "node_type_list before returning from args.install:", node_type_list return node_type_list logger.debug("Install Services") if args.update or args.upgrade: installer_mode_dictionary["upgrade_mode"] = True installer_mode_dictionary["install_mode"] = False set_node_type_value("install", node_type_list, True) logger.debug("Update Services") esg_functions.verify_esg_node_script("esg_node.py", esg_dist_url, script_version, script_maj_version, devel,"update") if args.fixperms: logger.debug("fixing permissions") setup_sensible_confs() sys.exit(0) if args.installlocalcerts: logger.debug("installing local certs") get_previous_node_type_config(config["esg_config_type_file"]) install_local_certs() sys.exit(0) if args.generateesgfcsrs: logger.debug("generating esgf csrs") get_previous_node_type_config(config["esg_config_type_file"]) generate_esgf_csrs() sys.exit(0) if args.generateesgfcsrsext: logger.debug("generating esgf csrs for other node") get_previous_node_type_config(config["esg_config_type_file"]) generate_esgf_csrs_ext() sys.exit(0) if args.certhowto: logger.debug("cert howto") cert_howto() sys.exit(0) elif args.type: logger.debug("selecting type") logger.debug("args.type: %s", args.type) print "args.type:", args.type for arg in args.type: set_node_type_value(arg, node_type_list, True) sys.exit(0) elif args.settype: logger.debug("Selecting type for next start up") for arg in args.settype: logger.debug("arg: %s", arg) node_type_list = [] node_type_list = set_node_type_value(arg, node_type_list, True) esg_bash2py.mkdir_p(config["esg_config_dir"]) set_node_type_config(node_type_list, config["esg_config_type_file"]) sys.exit(0) elif args.gettype: get_previous_node_type_config(config["esg_config_type_file"]) show_type() sys.exit(0) elif args.start: logger.debug("args: %s", args) # if check_prerequisites() is not 0: # logger.error("Prerequisites for startup not satisfied. Exiting.") # sys.exit(1) logger.debug("START SERVICES: %s", node_type_list) esg_setup.init_structure() start(node_type_list) sys.exit(0) elif args.stop: # if check_prerequisites() is not 0: # logger.error("Prerequisites for startup not satisfied. Exiting.") # sys.exit(1) logger.debug("STOP SERVICES") esg_setup.init_structure() stop(node_type_list) sys.exit(0) elif args.restart: # if check_prerequisites() is not 0: # logger.error("Prerequisites for startup not satisfied. Exiting.") # sys.exit(1) logger.debug("RESTARTING SERVICES") esg_setup.init_structure() stop(node_type_list) sleep(2) start(node_type_list) sys.exit(0) elif args.status: # if check_prerequisites() is not 0: # logger.error("Prerequisites for startup not satisfied. Exiting.") # sys.exit(1) get_node_status() #TODO: Exit with status code dependent on what is returned from get_node_status() sys.exit(0) elif args.updatesubinstaller: esg_functions.verify_esg_node_script("esg_node.py", esg_dist_url, script_version, script_maj_version, devel,"update") # if check_prerequisites() is not 0: # logger.error("Prerequisites for startup not satisfied. Exiting.") # sys.exit(1) esg_setup.init_structure() update_script(args[1], args[2]) sys.exit(0) # elif args.updateapacheconf: # logger.debug("checking for updated apache frontend configuration") # esg_apache_manager.update_apache_conf() # sys.exit(0) elif args.version: logger.info("Version: %s", script_version) logger.info("Release: %s", script_release) logger.info("Earth Systems Grid Federation (http://esgf.llnl.gov)") logger.info("ESGF Node Installation Script") sys.exit(0) elif args.recommendedsetup: recommended_setup = True custom_setup = False elif args.customsetup: recommended_setup = False custom_setup = True elif args.uselocalfiles: use_local_files = True elif args.devel: devel = True elif args.prod: devel = False
def setup_node_manager(mode="install"): ##### # Install The Node Manager ##### # - Takes boolean arg: 0 = setup / install mode (default) # 1 = updated mode # # In setup mode it is an idempotent install (default) # In update mode it will always pull down latest after archiving old # print "Checking for node manager {esgf_node_manager_version}".format( esgf_node_manager_version=config["esgf_node_manager_version"]) if esg_version_manager.check_webapp_version( "esgf-node-manager", config["esgf_node_manager_version"]) == 0 and not force_install: print "\n Found existing version of the node-manager [OK]" return True init() print "*******************************" print "Setting up The ESGF Node Manager..." print "*******************************" # local upgrade=${1:-0} db_set = 0 if force_install: default_answer = "N" else: default_answer = "Y" # local dosetup node_manager_service_app_home = esg_property_manager.get_property( "node_manager_service_app_home") if os.path.isdir(node_manager_service_app_home): db_set = 1 print "Detected an existing node manager installation..." if default_answer == "Y": installation_answer = raw_input( "Do you want to continue with node manager installation and setup? [Y/n]" ) or default_answer else: installation_answer = raw_input( "Do you want to continue with node manager installation and setup? [y/N]" ) or default_answer if installation_answer.lower() not in ["y", "yes"]: print "Skipping node manager installation and setup - will assume it's setup properly" # resetting node manager version to what it is already, not what we prescribed in the script # this way downstream processes will use the *actual* version in play, namely the (access logging) filter(s) esgf_node_manager_version = esg_version_manager.get_current_webapp_version( "esgf_node_manager") return True backup_default_answer = "Y" backup_answer = raw_input( "Do you want to make a back up of the existing distribution [{node_manager_app_context_root}]? [Y/n] " .format(node_manager_app_context_root=node_manager_app_context_root )) or backup_default_answer if backup_answer.lower in ["yes", "y"]: print "Creating a backup archive of this web application [{node_manager_service_app_home}]".format( node_manager_service_app_home=node_manager_service_app_home) esg_functions.backup(node_manager_service_app_home) backup_db_default_answer = "Y" backup_db_answer = raw_input( "Do you want to make a back up of the existing database [{node_db_name}:esgf_node_manager]?? [Y/n] " .format(node_db_name=config["node_db_name"] )) or backup_db_default_answer if backup_db_answer.lower() in ["yes", "y"]: print "Creating a backup archive of the manager database schema [{node_db_name}:esgf_node_manager]".format( node_db_name=config["node_db_name"]) # TODO: Implement this # esg_postgres.backup_db() -db ${node_db_name} -s node_manager esg_bash2py.mkdir_p(config["workdir"]) with esg_bash2py.pushd(config["workdir"]): logger.debug("changed directory to : %s", os.getcwd()) # strip off .tar.gz at the end #(Ex: esgf-node-manager-0.9.0.tar.gz -> esgf-node-manager-0.9.0) node_dist_file = esg_bash2py.trim_string_from_head(node_dist_url) logger.debug("node_dist_file: %s", node_dist_file) # Should just be esgf-node-manager-x.x.x node_dist_dir = node_dist_file # checked_get ${node_dist_file} ${node_dist_url} $((force_install)) if not esg_functions.download_update( node_dist_file, node_dist_url, force_download=force_install): print "ERROR: Could not download {node_dist_url} :-(".format( node_dist_url=node_dist_url) esg_functions.exit_with_error(1) # make room for new install if force_install: print "Removing Previous Installation of the ESGF Node Manager... ({node_dist_dir})".format( node_dist_dir=node_dist_dir) try: shutil.rmtree(node_dist_dir) logger.info("Deleted directory: %s", node_dist_dir) except IOError, error: logger.error(error) logger.error("Could not delete directory: %s", node_dist_dir) esg_functions.exit_with_error(1) clean_node_manager_webapp_subsystem() print "\nunpacking {node_dist_file}...".format( node_dist_file=node_dist_file) # This probably won't work, because the extension has already been stripped, no idea how this even worked in the bash code smh try: tar = tarfile.open(node_dist_file) tar.extractall() tar.close() except Exception, error: logger.error(error) print "ERROR: Could not extract the ESG Node: {node_dist_file}".format( node_dist_file=node_dist_file) esg_functions.exit_with_error(1)
def make_python_eggs_dir(): esg_bash2py.mkdir_p("/var/www/.python-eggs") apache_user_id = esg_functions.get_user_id("apache") apache_group_id = esg_functions.get_group_id("apache") os.chown("/var/www/.python-eggs", apache_user_id, apache_group_id)
def test_backup(self): test_backup_dir = "/esg/test_backup" esg_bash2py.mkdir_p(test_backup_dir) output = esg_functions.backup(os.getcwd(), backup_dir=test_backup_dir) self.assertEqual(output, 0)