def install_extkeytool():
'''Install the Extkeytool from the distribution mirror'''
print "\n*******************************"
print "Installing Extkeytool"
print "******************************* \n"
extkeytool_tarfile = esg_bash2py.trim_string_from_head(
config["extkeytool_download_url"])
esg_functions.download_update(extkeytool_tarfile,
config["extkeytool_download_url"])
esg_functions.extract_tarball(extkeytool_tarfile, "/esg/tools/idptools")
def download_esg_installarg(esg_dist_url):
''' Downloading esg-installarg file '''
if not os.path.isfile(config.esg_installarg_file) or force_install or os.path.getmtime(config.esg_installarg_file) < os.path.getmtime(os.path.realpath(__file__)):
esg_installarg_file_name = esg_bash2py.trim_string_from_head(
config.esg_installarg_file)
esg_functions.download_update(config.esg_installarg_file, os.path.join(
esg_dist_url, "esgf-installer", esg_installarg_file_name), force_download=force_install)
try:
if not os.path.getsize(config.esg_installarg_file) > 0:
os.remove(config.esg_installarg_file)
esg_bash2py.touch(config.esg_installarg_file)
except IOError:
logger.exception("Unable to access esg-installarg file")
def download_config_files(force_install):
''' Download config files '''
# #Get files
esg_dist_url = "http://distrib-coffee.ipsl.jussieu.fr/pub/esgf/dist"
hba_conf_file = "pg_hba.conf"
if esg_functions.download_update(hba_conf_file, os.path.join(esg_dist_url,"externals", "bootstrap",hba_conf_file), force_install) > 1:
esg_functions.exit_with_error(1)
os.chmod(hba_conf_file, 0600)
postgres_conf_file = "postgresql.conf"
if esg_functions.download_update(postgres_conf_file, os.path.join(esg_dist_url,"externals", "bootstrap",postgres_conf_file), force_install) > 1:
esg_functions.exit_with_error(1)
os.chmod(postgres_conf_file, 0600)
def fetch_esgf_certificates(
globus_certs_dir=config["globus_global_certs_dir"]):
'''Goes to ESG distribution server and pulls down all certificates for the federation.
(suitable for crontabbing)'''
print "\n*******************************"
print "Fetching freshest ESG Federation Certificates..."
print "******************************* \n"
#if globus_global_certs_dir already exists, backup and delete, then recreate empty directory
if os.path.isdir(config["globus_global_certs_dir"]):
esg_functions.backup(
config["globus_global_certs_dir"],
os.path.join(config["globus_global_certs_dir"], ".bak.tz"))
shutil.rmtree(config["globus_global_certs_dir"])
esg_bash2py.mkdir_p(config["globus_global_certs_dir"])
#Download trusted certs tarball
esg_trusted_certs_file = "esg_trusted_certificates.tar"
esg_trusted_certs_file_url = "https://aims1.llnl.gov/esgf/dist/certs/{esg_trusted_certs_file}".format(
esg_trusted_certs_file=esg_trusted_certs_file)
esg_functions.download_update(
os.path.join(globus_certs_dir, esg_trusted_certs_file),
esg_trusted_certs_file_url)
#untar the esg_trusted_certs_file
esg_functions.extract_tarball(
os.path.join(globus_certs_dir, esg_trusted_certs_file),
globus_certs_dir)
os.remove(os.path.join(globus_certs_dir, esg_trusted_certs_file))
#certificate_issuer_cert "/var/lib/globus-connect-server/myproxy-ca/cacert.pem"
simpleCA_cert = "/var/lib/globus-connect-server/myproxy-ca/cacert.pem"
if os.path.isfile(simpleCA_cert):
simpleCA_cert_hash = esg_functions.get_md5sum(simpleCA_cert)
print "checking for MY cert: {globus_global_certs_dir}/{simpleCA_cert_hash}.0".format(
globus_global_certs_dir=config["globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash)
if os.path.isfile(
"{globus_global_certs_dir}/{simpleCA_cert_hash}.0".format(
globus_global_certs_dir=config["globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash)):
print "Local CA cert file detected...."
print "Integrating in local simpleCA_cert... "
print "Local SimpleCA Root Cert: {simpleCA_cert}".format(
simpleCA_cert=simpleCA_cert)
print "Extracting Signing policy"
#Copy simple CA cert to globus cert directory
shutil.copyfile(
simpleCA_cert,
"{globus_global_certs_dir}/{simpleCA_cert_hash}.0".format(
globus_global_certs_dir=config["globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash))
#extract simple CA cert tarball and copy to globus cert directory
simpleCA_cert_parent_dir = esg_functions.get_parent_directory(
simpleCA_cert)
simpleCA_setup_tar_file = os.path.join(
simpleCA_cert_parent_dir,
"globus_simple_ca_{simpleCA_cert_hash}_setup-0.tar.gz".format(
simpleCA_cert_hash=simpleCA_cert_hash))
esg_functions.extract_tarball(simpleCA_setup_tar_file)
with esg_bash2py.pushd(
"globus_simple_ca_{simpleCA_cert_hash}_setup-0".format(
simpleCA_cert_hash=simpleCA_cert_hash)):
shutil.copyfile(
"{simpleCA_cert_hash}.signing_policy".format(
simpleCA_cert_hash=simpleCA_cert_hash),
"{globus_global_certs_dir}/{simpleCA_cert_hash}.signing_policy"
.format(globus_global_certs_dir=config[
"globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash))
if os.path.isdir("/usr/local/tomcat/webapps/ROOT"):
esg_functions.stream_subprocess_output(
"openssl x509 -text -hash -in {simpleCA_cert} > {tomcat_install_dir}/webapps/ROOT/cacert.pem"
.format(simpleCA_cert=simpleCA_cert,
tomcat_install_dir="/usr/loca/tomcat"))
print " My CA Cert now posted @ http://{fqdn}/cacert.pem ".format(
fqdn=socket.getfqdn())
os.chmod("/usr/local/tomcat/webapps/ROOT/cacert.pem", 0644)
os.chmod(config["globus_global_certs_dir"], 0755)
esg_functions.change_permissions_recursive(
config["globus_global_certs_dir"], 0644)
def setup_node_manager(mode="install"):
#####
# Install The Node Manager
#####
# - Takes boolean arg: 0 = setup / install mode (default)
# 1 = updated mode
#
# In setup mode it is an idempotent install (default)
# In update mode it will always pull down latest after archiving old
#
print "Checking for node manager {esgf_node_manager_version}".format(
esgf_node_manager_version=config["esgf_node_manager_version"])
if esg_version_manager.check_webapp_version(
"esgf-node-manager",
config["esgf_node_manager_version"]) == 0 and not force_install:
print "\n Found existing version of the node-manager [OK]"
return True
init()
print "*******************************"
print "Setting up The ESGF Node Manager..."
print "*******************************"
# local upgrade=${1:-0}
db_set = 0
if force_install:
default_answer = "N"
else:
default_answer = "Y"
# local dosetup
node_manager_service_app_home = esg_property_manager.get_property(
"node_manager_service_app_home")
if os.path.isdir(node_manager_service_app_home):
db_set = 1
print "Detected an existing node manager installation..."
if default_answer == "Y":
installation_answer = raw_input(
"Do you want to continue with node manager installation and setup? [Y/n]"
) or default_answer
else:
installation_answer = raw_input(
"Do you want to continue with node manager installation and setup? [y/N]"
) or default_answer
if installation_answer.lower() not in ["y", "yes"]:
print "Skipping node manager installation and setup - will assume it's setup properly"
# resetting node manager version to what it is already, not what we prescribed in the script
# this way downstream processes will use the *actual* version in play, namely the (access logging) filter(s)
esgf_node_manager_version = esg_version_manager.get_current_webapp_version(
"esgf_node_manager")
return True
backup_default_answer = "Y"
backup_answer = raw_input(
"Do you want to make a back up of the existing distribution [{node_manager_app_context_root}]? [Y/n] "
.format(node_manager_app_context_root=node_manager_app_context_root
)) or backup_default_answer
if backup_answer.lower in ["yes", "y"]:
print "Creating a backup archive of this web application [{node_manager_service_app_home}]".format(
node_manager_service_app_home=node_manager_service_app_home)
esg_functions.backup(node_manager_service_app_home)
backup_db_default_answer = "Y"
backup_db_answer = raw_input(
"Do you want to make a back up of the existing database [{node_db_name}:esgf_node_manager]?? [Y/n] "
.format(node_db_name=config["node_db_name"]
)) or backup_db_default_answer
if backup_db_answer.lower() in ["yes", "y"]:
print "Creating a backup archive of the manager database schema [{node_db_name}:esgf_node_manager]".format(
node_db_name=config["node_db_name"])
# TODO: Implement this
# esg_postgres.backup_db() -db ${node_db_name} -s node_manager
esg_bash2py.mkdir_p(config["workdir"])
with esg_bash2py.pushd(config["workdir"]):
logger.debug("changed directory to : %s", os.getcwd())
# strip off .tar.gz at the end
#(Ex: esgf-node-manager-0.9.0.tar.gz -> esgf-node-manager-0.9.0)
node_dist_file = esg_bash2py.trim_string_from_head(node_dist_url)
logger.debug("node_dist_file: %s", node_dist_file)
# Should just be esgf-node-manager-x.x.x
node_dist_dir = node_dist_file
# checked_get ${node_dist_file} ${node_dist_url} $((force_install))
if not esg_functions.download_update(
node_dist_file, node_dist_url, force_download=force_install):
print "ERROR: Could not download {node_dist_url} :-(".format(
node_dist_url=node_dist_url)
esg_functions.exit_with_error(1)
# make room for new install
if force_install:
print "Removing Previous Installation of the ESGF Node Manager... ({node_dist_dir})".format(
node_dist_dir=node_dist_dir)
try:
shutil.rmtree(node_dist_dir)
logger.info("Deleted directory: %s", node_dist_dir)
except IOError, error:
logger.error(error)
logger.error("Could not delete directory: %s", node_dist_dir)
esg_functions.exit_with_error(1)
clean_node_manager_webapp_subsystem()
print "\nunpacking {node_dist_file}...".format(
node_dist_file=node_dist_file)
# This probably won't work, because the extension has already been stripped, no idea how this even worked in the bash code smh
try:
tar = tarfile.open(node_dist_file)
tar.extractall()
tar.close()
except Exception, error:
logger.error(error)
print "ERROR: Could not extract the ESG Node: {node_dist_file}".format(
node_dist_file=node_dist_file)
esg_functions.exit_with_error(1)
def setup_subsystem(subsystem,
distribution_directory,
esg_dist_url,
force_install=False):
'''
arg (1) - name of installation script root name. Ex:security which resolves to script file esg-security
arg (2) - directory on the distribution site where script is fetched from Ex: orp
usage: setup_subsystem security orp - looks for the script esg-security in the distriubtion dir orp
'''
subsystem_install_script_path = os.path.join(
config["scripts_dir"], "esg-{subsystem}".format(subsystem=subsystem))
#---
#check that you have at one point in time fetched the subsystem's installation script
#if indeed you have we will assume you would like to proceed with setting up the latest...
#Otherwise we just ask you first before you pull down new code to your machine...
#---
if force_install:
default = "y"
else:
default = "n"
if os.path.exists(subsystem_install_script_path) or force_install:
if default.lower() in ["y", "yes"]:
run_installation = raw_input(
"Would you like to set up {subsystem} services? [Y/n]: ".
format(subsystem=subsystem)) or "y"
else:
run_installation = raw_input(
"Would you like to set up {subsystem} services? [y/N]: ".
format(subsystem=subsystem)) or "n"
if run_installation.lower() in ["n", "no"]:
print "Skipping installation of {subsystem}".format(
subsystem=subsystem)
return True
print "-------------------------------"
print "LOADING installer for {subsystem}... ".format(subsystem=subsystem)
esg_bash2py.mkdir_p(config["workdir"])
with esg_bash2py.pushd(config["workdir"]):
logger.debug("Changed directory to %s", os.getcwd())
with esg_bash2py.pushd(config["scripts_dir"]):
logger.debug("Changed directory to %s", os.getcwd())
subsystem_full_name = "esg-{subsystem}".format(subsystem=subsystem)
subsystem_remote_url = "{esg_dist_url}/{distribution_directory}/{subsystem_full_name}".format(
esg_dist_url=esg_dist_url,
distribution_directory=distribution_directory,
subsystem_full_name=subsystem_full_name)
if not esg_functions.download_update(
"{subsystem_full_name}".format(
subsystem_full_name=subsystem_full_name),
subsystem_remote_url):
logger.error("Could not download %s", subsystem_full_name)
return False
try:
os.chmod(subsystem_full_name, 0755)
except OSError:
logger.exception("Unable to change permissions on %s",
subsystem_full_name)
logger.info("script_dir contents: %s", os.listdir(config["scripts_dir"]))
subsystem_underscore = subsystem.replace("-", "_")
execute_subsystem_command = ". {scripts_dir}/{subsystem_full_name}; setup_{subsystem_underscore}".format(
scripts_dir=config["scripts_dir"],
subsystem_full_name=subsystem_full_name,
subsystem_underscore=subsystem_underscore)
setup_subsystem_process = subprocess.Popen(
['bash', '-c', execute_subsystem_command])
setup_subsystem_stdout, setup_subsystem_stderr = setup_subsystem_process.communicate(
)
logger.debug("setup_subsystem_stdout: %s", setup_subsystem_stdout)
logger.debug("setup_subsystem_stderr: %s", setup_subsystem_stderr)
def download_java(java_tarfile):
print "Downloading Java from ", config["java_dist_url"]
if not esg_functions.download_update(java_tarfile, config["java_dist_url"], force_install):
logger.error("ERROR: Could not download Java")
esg_functions.exit_with_error(1)