def install_extkeytool(): '''Install the Extkeytool from the distribution mirror''' print "\n*******************************" print "Installing Extkeytool" print "******************************* \n" extkeytool_tarfile = esg_bash2py.trim_string_from_head( config["extkeytool_download_url"]) esg_functions.download_update(extkeytool_tarfile, config["extkeytool_download_url"]) esg_functions.extract_tarball(extkeytool_tarfile, "/esg/tools/idptools")
def download_esg_installarg(esg_dist_url): ''' Downloading esg-installarg file ''' if not os.path.isfile(config.esg_installarg_file) or force_install or os.path.getmtime(config.esg_installarg_file) < os.path.getmtime(os.path.realpath(__file__)): esg_installarg_file_name = esg_bash2py.trim_string_from_head( config.esg_installarg_file) esg_functions.download_update(config.esg_installarg_file, os.path.join( esg_dist_url, "esgf-installer", esg_installarg_file_name), force_download=force_install) try: if not os.path.getsize(config.esg_installarg_file) > 0: os.remove(config.esg_installarg_file) esg_bash2py.touch(config.esg_installarg_file) except IOError: logger.exception("Unable to access esg-installarg file")
def download_config_files(force_install): ''' Download config files ''' # #Get files esg_dist_url = "http://distrib-coffee.ipsl.jussieu.fr/pub/esgf/dist" hba_conf_file = "pg_hba.conf" if esg_functions.download_update(hba_conf_file, os.path.join(esg_dist_url,"externals", "bootstrap",hba_conf_file), force_install) > 1: esg_functions.exit_with_error(1) os.chmod(hba_conf_file, 0600) postgres_conf_file = "postgresql.conf" if esg_functions.download_update(postgres_conf_file, os.path.join(esg_dist_url,"externals", "bootstrap",postgres_conf_file), force_install) > 1: esg_functions.exit_with_error(1) os.chmod(postgres_conf_file, 0600)
def fetch_esgf_certificates( globus_certs_dir=config["globus_global_certs_dir"]): '''Goes to ESG distribution server and pulls down all certificates for the federation. (suitable for crontabbing)''' print "\n*******************************" print "Fetching freshest ESG Federation Certificates..." print "******************************* \n" #if globus_global_certs_dir already exists, backup and delete, then recreate empty directory if os.path.isdir(config["globus_global_certs_dir"]): esg_functions.backup( config["globus_global_certs_dir"], os.path.join(config["globus_global_certs_dir"], ".bak.tz")) shutil.rmtree(config["globus_global_certs_dir"]) esg_bash2py.mkdir_p(config["globus_global_certs_dir"]) #Download trusted certs tarball esg_trusted_certs_file = "esg_trusted_certificates.tar" esg_trusted_certs_file_url = "https://aims1.llnl.gov/esgf/dist/certs/{esg_trusted_certs_file}".format( esg_trusted_certs_file=esg_trusted_certs_file) esg_functions.download_update( os.path.join(globus_certs_dir, esg_trusted_certs_file), esg_trusted_certs_file_url) #untar the esg_trusted_certs_file esg_functions.extract_tarball( os.path.join(globus_certs_dir, esg_trusted_certs_file), globus_certs_dir) os.remove(os.path.join(globus_certs_dir, esg_trusted_certs_file)) #certificate_issuer_cert "/var/lib/globus-connect-server/myproxy-ca/cacert.pem" simpleCA_cert = "/var/lib/globus-connect-server/myproxy-ca/cacert.pem" if os.path.isfile(simpleCA_cert): simpleCA_cert_hash = esg_functions.get_md5sum(simpleCA_cert) print "checking for MY cert: {globus_global_certs_dir}/{simpleCA_cert_hash}.0".format( globus_global_certs_dir=config["globus_global_certs_dir"], simpleCA_cert_hash=simpleCA_cert_hash) if os.path.isfile( "{globus_global_certs_dir}/{simpleCA_cert_hash}.0".format( globus_global_certs_dir=config["globus_global_certs_dir"], simpleCA_cert_hash=simpleCA_cert_hash)): print "Local CA cert file detected...." print "Integrating in local simpleCA_cert... " print "Local SimpleCA Root Cert: {simpleCA_cert}".format( simpleCA_cert=simpleCA_cert) print "Extracting Signing policy" #Copy simple CA cert to globus cert directory shutil.copyfile( simpleCA_cert, "{globus_global_certs_dir}/{simpleCA_cert_hash}.0".format( globus_global_certs_dir=config["globus_global_certs_dir"], simpleCA_cert_hash=simpleCA_cert_hash)) #extract simple CA cert tarball and copy to globus cert directory simpleCA_cert_parent_dir = esg_functions.get_parent_directory( simpleCA_cert) simpleCA_setup_tar_file = os.path.join( simpleCA_cert_parent_dir, "globus_simple_ca_{simpleCA_cert_hash}_setup-0.tar.gz".format( simpleCA_cert_hash=simpleCA_cert_hash)) esg_functions.extract_tarball(simpleCA_setup_tar_file) with esg_bash2py.pushd( "globus_simple_ca_{simpleCA_cert_hash}_setup-0".format( simpleCA_cert_hash=simpleCA_cert_hash)): shutil.copyfile( "{simpleCA_cert_hash}.signing_policy".format( simpleCA_cert_hash=simpleCA_cert_hash), "{globus_global_certs_dir}/{simpleCA_cert_hash}.signing_policy" .format(globus_global_certs_dir=config[ "globus_global_certs_dir"], simpleCA_cert_hash=simpleCA_cert_hash)) if os.path.isdir("/usr/local/tomcat/webapps/ROOT"): esg_functions.stream_subprocess_output( "openssl x509 -text -hash -in {simpleCA_cert} > {tomcat_install_dir}/webapps/ROOT/cacert.pem" .format(simpleCA_cert=simpleCA_cert, tomcat_install_dir="/usr/loca/tomcat")) print " My CA Cert now posted @ http://{fqdn}/cacert.pem ".format( fqdn=socket.getfqdn()) os.chmod("/usr/local/tomcat/webapps/ROOT/cacert.pem", 0644) os.chmod(config["globus_global_certs_dir"], 0755) esg_functions.change_permissions_recursive( config["globus_global_certs_dir"], 0644)
def setup_node_manager(mode="install"): ##### # Install The Node Manager ##### # - Takes boolean arg: 0 = setup / install mode (default) # 1 = updated mode # # In setup mode it is an idempotent install (default) # In update mode it will always pull down latest after archiving old # print "Checking for node manager {esgf_node_manager_version}".format( esgf_node_manager_version=config["esgf_node_manager_version"]) if esg_version_manager.check_webapp_version( "esgf-node-manager", config["esgf_node_manager_version"]) == 0 and not force_install: print "\n Found existing version of the node-manager [OK]" return True init() print "*******************************" print "Setting up The ESGF Node Manager..." print "*******************************" # local upgrade=${1:-0} db_set = 0 if force_install: default_answer = "N" else: default_answer = "Y" # local dosetup node_manager_service_app_home = esg_property_manager.get_property( "node_manager_service_app_home") if os.path.isdir(node_manager_service_app_home): db_set = 1 print "Detected an existing node manager installation..." if default_answer == "Y": installation_answer = raw_input( "Do you want to continue with node manager installation and setup? [Y/n]" ) or default_answer else: installation_answer = raw_input( "Do you want to continue with node manager installation and setup? [y/N]" ) or default_answer if installation_answer.lower() not in ["y", "yes"]: print "Skipping node manager installation and setup - will assume it's setup properly" # resetting node manager version to what it is already, not what we prescribed in the script # this way downstream processes will use the *actual* version in play, namely the (access logging) filter(s) esgf_node_manager_version = esg_version_manager.get_current_webapp_version( "esgf_node_manager") return True backup_default_answer = "Y" backup_answer = raw_input( "Do you want to make a back up of the existing distribution [{node_manager_app_context_root}]? [Y/n] " .format(node_manager_app_context_root=node_manager_app_context_root )) or backup_default_answer if backup_answer.lower in ["yes", "y"]: print "Creating a backup archive of this web application [{node_manager_service_app_home}]".format( node_manager_service_app_home=node_manager_service_app_home) esg_functions.backup(node_manager_service_app_home) backup_db_default_answer = "Y" backup_db_answer = raw_input( "Do you want to make a back up of the existing database [{node_db_name}:esgf_node_manager]?? [Y/n] " .format(node_db_name=config["node_db_name"] )) or backup_db_default_answer if backup_db_answer.lower() in ["yes", "y"]: print "Creating a backup archive of the manager database schema [{node_db_name}:esgf_node_manager]".format( node_db_name=config["node_db_name"]) # TODO: Implement this # esg_postgres.backup_db() -db ${node_db_name} -s node_manager esg_bash2py.mkdir_p(config["workdir"]) with esg_bash2py.pushd(config["workdir"]): logger.debug("changed directory to : %s", os.getcwd()) # strip off .tar.gz at the end #(Ex: esgf-node-manager-0.9.0.tar.gz -> esgf-node-manager-0.9.0) node_dist_file = esg_bash2py.trim_string_from_head(node_dist_url) logger.debug("node_dist_file: %s", node_dist_file) # Should just be esgf-node-manager-x.x.x node_dist_dir = node_dist_file # checked_get ${node_dist_file} ${node_dist_url} $((force_install)) if not esg_functions.download_update( node_dist_file, node_dist_url, force_download=force_install): print "ERROR: Could not download {node_dist_url} :-(".format( node_dist_url=node_dist_url) esg_functions.exit_with_error(1) # make room for new install if force_install: print "Removing Previous Installation of the ESGF Node Manager... ({node_dist_dir})".format( node_dist_dir=node_dist_dir) try: shutil.rmtree(node_dist_dir) logger.info("Deleted directory: %s", node_dist_dir) except IOError, error: logger.error(error) logger.error("Could not delete directory: %s", node_dist_dir) esg_functions.exit_with_error(1) clean_node_manager_webapp_subsystem() print "\nunpacking {node_dist_file}...".format( node_dist_file=node_dist_file) # This probably won't work, because the extension has already been stripped, no idea how this even worked in the bash code smh try: tar = tarfile.open(node_dist_file) tar.extractall() tar.close() except Exception, error: logger.error(error) print "ERROR: Could not extract the ESG Node: {node_dist_file}".format( node_dist_file=node_dist_file) esg_functions.exit_with_error(1)
def setup_subsystem(subsystem, distribution_directory, esg_dist_url, force_install=False): ''' arg (1) - name of installation script root name. Ex:security which resolves to script file esg-security arg (2) - directory on the distribution site where script is fetched from Ex: orp usage: setup_subsystem security orp - looks for the script esg-security in the distriubtion dir orp ''' subsystem_install_script_path = os.path.join( config["scripts_dir"], "esg-{subsystem}".format(subsystem=subsystem)) #--- #check that you have at one point in time fetched the subsystem's installation script #if indeed you have we will assume you would like to proceed with setting up the latest... #Otherwise we just ask you first before you pull down new code to your machine... #--- if force_install: default = "y" else: default = "n" if os.path.exists(subsystem_install_script_path) or force_install: if default.lower() in ["y", "yes"]: run_installation = raw_input( "Would you like to set up {subsystem} services? [Y/n]: ". format(subsystem=subsystem)) or "y" else: run_installation = raw_input( "Would you like to set up {subsystem} services? [y/N]: ". format(subsystem=subsystem)) or "n" if run_installation.lower() in ["n", "no"]: print "Skipping installation of {subsystem}".format( subsystem=subsystem) return True print "-------------------------------" print "LOADING installer for {subsystem}... ".format(subsystem=subsystem) esg_bash2py.mkdir_p(config["workdir"]) with esg_bash2py.pushd(config["workdir"]): logger.debug("Changed directory to %s", os.getcwd()) with esg_bash2py.pushd(config["scripts_dir"]): logger.debug("Changed directory to %s", os.getcwd()) subsystem_full_name = "esg-{subsystem}".format(subsystem=subsystem) subsystem_remote_url = "{esg_dist_url}/{distribution_directory}/{subsystem_full_name}".format( esg_dist_url=esg_dist_url, distribution_directory=distribution_directory, subsystem_full_name=subsystem_full_name) if not esg_functions.download_update( "{subsystem_full_name}".format( subsystem_full_name=subsystem_full_name), subsystem_remote_url): logger.error("Could not download %s", subsystem_full_name) return False try: os.chmod(subsystem_full_name, 0755) except OSError: logger.exception("Unable to change permissions on %s", subsystem_full_name) logger.info("script_dir contents: %s", os.listdir(config["scripts_dir"])) subsystem_underscore = subsystem.replace("-", "_") execute_subsystem_command = ". {scripts_dir}/{subsystem_full_name}; setup_{subsystem_underscore}".format( scripts_dir=config["scripts_dir"], subsystem_full_name=subsystem_full_name, subsystem_underscore=subsystem_underscore) setup_subsystem_process = subprocess.Popen( ['bash', '-c', execute_subsystem_command]) setup_subsystem_stdout, setup_subsystem_stderr = setup_subsystem_process.communicate( ) logger.debug("setup_subsystem_stdout: %s", setup_subsystem_stdout) logger.debug("setup_subsystem_stderr: %s", setup_subsystem_stderr)
def download_java(java_tarfile): print "Downloading Java from ", config["java_dist_url"] if not esg_functions.download_update(java_tarfile, config["java_dist_url"], force_install): logger.error("ERROR: Could not download Java") esg_functions.exit_with_error(1)