def run_esgsetup():
'''generate esg.ini file using esgsetup script; #Makes call to esgsetup - > Setup the ESG publication configuration'''
print "\n*******************************"
print "Creating config file (esg.ini) with esgsetup"
print "******************************* \n"
os.environ["UVCDAT_ANONYMOUS_LOG"] = "no"
#Create an initial ESG configuration file (esg.ini); TODO: make break into separate function
generate_esg_ini_command = '''esgsetup --config --minimal-setup --rootid {esg_root_id} --db-admin-password password'''.format(
esg_root_id=esg_functions.get_esg_root_id())
try:
esg_functions.stream_subprocess_output(generate_esg_ini_command)
edit_esg_ini()
except Exception:
logger.exception("Could not finish esgsetup")
esg_functions.exit_with_error(1)
print "\n*******************************"
print "Initializing database with esgsetup"
print "******************************* \n"
#Initialize the database
db_setup_command = generate_esgsetup_options()
print "db_setup_command:", db_setup_command
try:
esg_functions.stream_subprocess_output(db_setup_command)
except Exception:
logger.exception("Could not initialize database.")
esg_functions.exit_with_error(1)
def setup_solr():
'''Setup Apache Solr for faceted search'''
print "\n*******************************"
print "Setting up Solr"
print "******************************* \n"
# # Solr/Jetty web application
SOLR_VERSION = "5.5.4"
SOLR_INSTALL_DIR = "/usr/local/solr"
SOLR_HOME = "/usr/local/solr-home"
SOLR_DATA_DIR = "/esg/solr-index"
SOLR_INCLUDE = "{SOLR_HOME}/solr.in.sh".format(SOLR_HOME=SOLR_HOME)
#Download solr tarball
solr_tarball_url = "http://archive.apache.org/dist/lucene/solr/{SOLR_VERSION}/solr-{SOLR_VERSION}.tgz".format(
SOLR_VERSION=SOLR_VERSION)
download_solr_tarball(solr_tarball_url, SOLR_VERSION)
#Extract solr tarball
extract_solr_tarball(
'/tmp/solr-{SOLR_VERSION}.tgz'.format(SOLR_VERSION=SOLR_VERSION),
SOLR_VERSION)
esg_bash2py.mkdir_p(SOLR_DATA_DIR)
# download template directory structure for shards home
download_template_directory()
esg_bash2py.mkdir_p(SOLR_HOME)
# create non-privilged user to run Solr server
esg_functions.stream_subprocess_output("groupadd solr")
esg_functions.stream_subprocess_output(
"useradd -s /sbin/nologin -g solr -d /usr/local/solr solr")
SOLR_USER_ID = pwd.getpwnam("solr").pw_uid
SOLR_GROUP_ID = grp.getgrnam("solr").gr_gid
esg_functions.change_permissions_recursive(
"/usr/local/solr-{SOLR_VERSION}".format(SOLR_VERSION=SOLR_VERSION),
SOLR_USER_ID, SOLR_GROUP_ID)
esg_functions.change_permissions_recursive(SOLR_HOME, SOLR_USER_ID,
SOLR_GROUP_ID)
esg_functions.change_permissions_recursive(SOLR_DATA_DIR, SOLR_USER_ID,
SOLR_GROUP_ID)
#
#Copy shard files
shutil.copyfile("solr_scripts/add_shard.sh", "/usr/local/bin/add_shard.sh")
shutil.copyfile("solr_scripts/remove_shard.sh",
"/usr/local/bin/remove_shard.sh")
# add shards
esg_functions.call_subprocess("/usr/local/bin/add_shard.sh master 8984")
esg_functions.call_subprocess("/usr/local/bin/add_shard.sh master 8983")
# custom logging properties
shutil.copyfile(
"solr_scripts/log4j.properties",
"/{SOLR_INSTALL_DIR}/server/resources/log4j.properties".format(
SOLR_INSTALL_DIR=SOLR_INSTALL_DIR))
def start_postgres():
''' Start db '''
#if the data directory doesn't exist or is empty
if not os.path.isdir("/var/lib/pgsql/9.6/data/") or not os.listdir("/var/lib/pgsql/9.6/data/"):
initialize_postgres()
esg_functions.stream_subprocess_output("service postgresql-9.6 start")
esg_functions.stream_subprocess_output("chkconfig postgresql-9.6 on")
sleep(3)
if postgres_status():
return True
def tearDownClass(cls):
esg_functions.stream_subprocess_output("yum remove -y httpd")
pip.main(["uninstall", "mod_wsgi"])
try:
shutil.rmtree("/var/www/.python-eggs")
shutil.rmtree('/var/www/html/')
shutil.rmtree('/etc/certs/')
except OSError, error:
if error.errno == errno.ENOENT:
pass
else:
print "error:", error
def setup_dashboard():
if os.path.isdir("/usr/local/tomcat/webapps/esgf-stats-api"):
stats_api_install = raw_input(
"Existing Stats API installation found. Do you want to continue with the Stats API installation [y/N]: "
) or "no"
if stats_api_install.lower() in ["no", "n"]:
return
print "\n*******************************"
print "Setting up ESGF Stats API (dashboard)"
print "******************************* \n"
esg_bash2py.mkdir_p("/usr/local/tomcat/webapps/esgf-stats-api")
stats_api_url = os.path.join("http://", config["esgf_dist_mirror"], "dist",
"devel", "esgf-stats-api",
"esgf-stats-api.war")
download_stats_api_war(stats_api_url)
with esg_bash2py.pushd("/usr/local/tomcat/webapps/esgf-stats-api"):
with zipfile.ZipFile(
"/usr/local/tomcat/webapps/esgf-stats-api/esgf-stats-api.war",
'r') as zf:
zf.extractall()
os.remove("esgf-stats-api.war")
TOMCAT_USER_ID = esg_functions.get_tomcat_user_id()
TOMCAT_GROUP_ID = esg_functions.get_tomcat_group_id()
esg_functions.change_permissions_recursive(
"/usr/local/tomcat/webapps/esgf-stats-api", TOMCAT_USER_ID,
TOMCAT_GROUP_ID)
# execute dashboard installation script (without the postgres schema)
run_dashboard_script()
# create non-privileged user to run the dashboard application
# RUN groupadd dashboard && \
# useradd -s /sbin/nologin -g dashboard -d /usr/local/dashboard dashboard && \
# chown -R dashboard:dashboard /usr/local/esgf-dashboard-ip
# RUN chmod a+w /var/run
esg_functions.stream_subprocess_output("groupadd dashboard")
esg_functions.stream_subprocess_output(
"useradd -s /sbin/nologin -g dashboard -d /usr/local/dashboard dashboard"
)
DASHBOARD_USER_ID = pwd.getpwnam("dashboard").pw_uid
DASHBOARD_GROUP_ID = grp.getgrnam("dashboard").gr_gid
esg_functions.change_permissions_recursive("/usr/local/esgf-dashboard-ip",
DASHBOARD_USER_ID,
DASHBOARD_GROUP_ID)
os.chmod("/var/run", stat.S_IWRITE)
os.chmod("/var/run", stat.S_IWGRP)
os.chmod("/var/run", stat.S_IWOTH)
start_dashboard_service()
def download_postgres():
#Edit distributions .repo file per https://wiki.postgresql.org/wiki/YUM_Installation
config = configobj.ConfigObj('/etc/yum.repos.d/CentOS-Base.repo')
try:
config["updates"]["exclude"]
config["base"]["exclude"]
except KeyError:
config["updates"]["exclude"] ="postgresql*"
config["base"]["exclude"] ="postgresql*"
config.write()
esg_functions.stream_subprocess_output("rpm -Uvh https://yum.postgresql.org/9.6/redhat/rhel-6-x86_64/pgdg-redhat96-9.6-3.noarch.rpm")
esg_functions.stream_subprocess_output("yum -y install postgresql96-server postgresql96 postgresql96-devel")
def purge_postgres():
print "\n*******************************"
print "Purging Postgres"
print "******************************* \n"
esg_functions.stream_subprocess_output("service postgresql-9.6 stop")
esg_functions.stream_subprocess_output("yum remove -y postgresql postgresql-libs postgresql-server")
try:
# shutil.rmtree("/usr/local/pgsql")
shutil.rmtree("/var/lib/pgsql")
os.remove(os.path.join(os.environ["HOME"], ".pgpass"))
except OSError, error:
logger.exception("Could not delete /var/lib/pgsql")
def setup_ant():
'''Install ant via yum'''
print "\n*******************************"
print "Setting up Ant"
print "******************************* \n"
if os.path.exists(os.path.join("/usr", "bin", "ant")):
logger.info("Found existing Ant installation. Skipping set up.")
esg_functions.stream_subprocess_output("ant -version")
force_ant_install = raw_input("Do you want to continue with the Ant installation [y/N]: ") or "no"
if force_ant_install.lower() in ["n", "no"]:
return
esg_functions.stream_subprocess_output("yum -y install ant")
class test_ESG_Setup(unittest.TestCase):
@classmethod
def tearDownClass(cls):
try:
os.remove("/tmp/Miniconda2-latest-Linux-x86_64.sh")
except OSError, error:
if error.errno == errno.EEXIST:
pass
esg_functions.stream_subprocess_output(
"/usr/local/testConda/bin/conda install -y anaconda-clean")
esg_functions.stream_subprocess_output("/usr/local/testConda/bin/anaconda-clean --yes")
try:
shutil.rmtree("/usr/local/testConda")
except OSError, error:
if error.errno == errno.EEXIST:
pass
def install_apache_httpd():
esg_functions.stream_subprocess_output("yum -y update")
esg_functions.stream_subprocess_output(
"yum install -y httpd httpd-devel mod_ssl")
esg_functions.stream_subprocess_output("yum clean all")
#Custom ESGF Apache files that setup proxying
shutil.copyfile("apache_conf/esgf-httpd", "/etc/init.d/esgf-httpd")
os.chmod("/etc/init.d/esgf-httpd", 0755)
shutil.copyfile("apache_conf/esgf-httpd.conf",
"/etc/httpd/conf/esgf-httpd.conf")
shutil.copyfile("apache_conf/esgf-httpd-local.conf",
"/etc/httpd/conf/esgf-httpd-local.conf")
shutil.copyfile("apache_conf/esgf-httpd-locals.conf",
"/etc/httpd/conf/esgf-httpd-locals.conf")
def run_dashboard_script():
#default values
DashDir = "/usr/local/esgf-dashboard-ip"
GeoipDir = "/usr/local/geoip"
Fed = "no"
with esg_bash2py.pushd("/usr/local"):
clone_dashboard_repo()
os.chdir("esgf-dashboard")
dashboard_repo_local = Repo(".")
dashboard_repo_local.git.checkout("work_plana")
os.chdir("src/c/esgf-dashboard-ip")
print "\n*******************************"
print "Running ESGF Dashboard Script"
print "******************************* \n"
esg_functions.stream_subprocess_output(
"./configure --prefix={DashDir} --with-geoip-prefix-path={GeoipDir} --with-allow-federation={Fed}"
.format(DashDir=DashDir, GeoipDir=GeoipDir, Fed=Fed))
esg_functions.stream_subprocess_output("make")
esg_functions.stream_subprocess_output("make install")
def postgres_create_db(db_name):
esg_functions.stream_subprocess_output("createdb -U {postgress_user} {db_name}".format(postgress_user=config["postgress_user"], db_name=db_name))
def run_apache_config_test():
esg_functions.stream_subprocess_output("service esgf-httpd configtest")
def restart_apache():
esg_functions.stream_subprocess_output("service esgf-httpd restart")
def stop_apache():
esg_functions.stream_subprocess_output("service esgf-httpd stop")
def fetch_esgf_certificates(
globus_certs_dir=config["globus_global_certs_dir"]):
'''Goes to ESG distribution server and pulls down all certificates for the federation.
(suitable for crontabbing)'''
print "\n*******************************"
print "Fetching freshest ESG Federation Certificates..."
print "******************************* \n"
#if globus_global_certs_dir already exists, backup and delete, then recreate empty directory
if os.path.isdir(config["globus_global_certs_dir"]):
esg_functions.backup(
config["globus_global_certs_dir"],
os.path.join(config["globus_global_certs_dir"], ".bak.tz"))
shutil.rmtree(config["globus_global_certs_dir"])
esg_bash2py.mkdir_p(config["globus_global_certs_dir"])
#Download trusted certs tarball
esg_trusted_certs_file = "esg_trusted_certificates.tar"
esg_trusted_certs_file_url = "https://aims1.llnl.gov/esgf/dist/certs/{esg_trusted_certs_file}".format(
esg_trusted_certs_file=esg_trusted_certs_file)
esg_functions.download_update(
os.path.join(globus_certs_dir, esg_trusted_certs_file),
esg_trusted_certs_file_url)
#untar the esg_trusted_certs_file
esg_functions.extract_tarball(
os.path.join(globus_certs_dir, esg_trusted_certs_file),
globus_certs_dir)
os.remove(os.path.join(globus_certs_dir, esg_trusted_certs_file))
#certificate_issuer_cert "/var/lib/globus-connect-server/myproxy-ca/cacert.pem"
simpleCA_cert = "/var/lib/globus-connect-server/myproxy-ca/cacert.pem"
if os.path.isfile(simpleCA_cert):
simpleCA_cert_hash = esg_functions.get_md5sum(simpleCA_cert)
print "checking for MY cert: {globus_global_certs_dir}/{simpleCA_cert_hash}.0".format(
globus_global_certs_dir=config["globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash)
if os.path.isfile(
"{globus_global_certs_dir}/{simpleCA_cert_hash}.0".format(
globus_global_certs_dir=config["globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash)):
print "Local CA cert file detected...."
print "Integrating in local simpleCA_cert... "
print "Local SimpleCA Root Cert: {simpleCA_cert}".format(
simpleCA_cert=simpleCA_cert)
print "Extracting Signing policy"
#Copy simple CA cert to globus cert directory
shutil.copyfile(
simpleCA_cert,
"{globus_global_certs_dir}/{simpleCA_cert_hash}.0".format(
globus_global_certs_dir=config["globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash))
#extract simple CA cert tarball and copy to globus cert directory
simpleCA_cert_parent_dir = esg_functions.get_parent_directory(
simpleCA_cert)
simpleCA_setup_tar_file = os.path.join(
simpleCA_cert_parent_dir,
"globus_simple_ca_{simpleCA_cert_hash}_setup-0.tar.gz".format(
simpleCA_cert_hash=simpleCA_cert_hash))
esg_functions.extract_tarball(simpleCA_setup_tar_file)
with esg_bash2py.pushd(
"globus_simple_ca_{simpleCA_cert_hash}_setup-0".format(
simpleCA_cert_hash=simpleCA_cert_hash)):
shutil.copyfile(
"{simpleCA_cert_hash}.signing_policy".format(
simpleCA_cert_hash=simpleCA_cert_hash),
"{globus_global_certs_dir}/{simpleCA_cert_hash}.signing_policy"
.format(globus_global_certs_dir=config[
"globus_global_certs_dir"],
simpleCA_cert_hash=simpleCA_cert_hash))
if os.path.isdir("/usr/local/tomcat/webapps/ROOT"):
esg_functions.stream_subprocess_output(
"openssl x509 -text -hash -in {simpleCA_cert} > {tomcat_install_dir}/webapps/ROOT/cacert.pem"
.format(simpleCA_cert=simpleCA_cert,
tomcat_install_dir="/usr/loca/tomcat"))
print " My CA Cert now posted @ http://{fqdn}/cacert.pem ".format(
fqdn=socket.getfqdn())
os.chmod("/usr/local/tomcat/webapps/ROOT/cacert.pem", 0644)
os.chmod(config["globus_global_certs_dir"], 0755)
esg_functions.change_permissions_recursive(
config["globus_global_certs_dir"], 0644)
def purge_ant():
print "\n*******************************"
print "Purging Ant"
print "******************************* \n"
esg_functions.stream_subprocess_output("yum remove -y ant")
def check_tomcat_version():
esg_functions.stream_subprocess_output("/usr/local/tomcat/bin/version.sh")
def run_tomcat_config_test():
esg_functions.stream_subprocess_output(
"/usr/local/tomcat/bin/catalina.sh configtest")
def stop_tomcat():
esg_functions.stream_subprocess_output(
"/usr/local/tomcat/bin/catalina.sh stop")
def set_default_java():
# default_java_version_info = esg_functions.call_subprocess("java -version")["stderr"]
# default_java_version_number = re.search("1.8.0_\w+", default_java_version_info).group()
# if semver.compare(default_java_version_number, config["java_version"]) != 0:
esg_functions.stream_subprocess_output("alternatives --install /usr/bin/java java /usr/local/java/bin/java 3")
esg_functions.stream_subprocess_output("alternatives --set java /usr/local/java/bin/java")
def install_publisher():
symlink_pg_binary()
esg_functions.stream_subprocess_output("python setup.py install")
def start_dashboard_service():
os.chmod("dashboard_conf/ip.service", 0555)
esg_functions.stream_subprocess_output("dashboard_conf/ip.service start")
def purge_apache():
esg_functions.stream_subprocess_output("yum remove -y httpd httpd-devel mod_ssl")
shutil.rmtree("/etc/httpd")
def stop_postgress():
'''Stops the postgres server'''
esg_functions.stream_subprocess_output("service postgresql-9.6 stop")
except KeyError:
config["updates"]["exclude"] ="postgresql*"
config["base"]["exclude"] ="postgresql*"
config.write()
esg_functions.stream_subprocess_output("rpm -Uvh https://yum.postgresql.org/9.6/redhat/rhel-6-x86_64/pgdg-redhat96-9.6-3.noarch.rpm")
esg_functions.stream_subprocess_output("yum -y install postgresql96-server postgresql96 postgresql96-devel")
def initialize_postgres():
try:
if os.listdir("/var/lib/pgsql/9.6/data"):
print "Data directory already exists. Skipping initialize_postgres()."
return
except OSError, error:
print "error:", error
esg_functions.stream_subprocess_output("service postgresql-9.6 initdb")
os.chmod(os.path.join(config["postgress_install_dir"], "9.6", "data"), 0700)
def check_for_postgres_sys_acct():
try:
postgres_user_id = pwd.getpwnam(config["pg_sys_acct"]).pw_uid
except KeyError:
print " Hmmm...: There is no postgres system account user \"{pg_sys_acct}\" present on system, making one...".format(pg_sys_acct = config["pg_sys_acct"])
else:
return True
def create_postgres_group():
groupadd_command = "/usr/sbin/groupadd -r %s" % (
config["pg_sys_acct_group"])
groupadd_output = esg_functions.call_subprocess(groupadd_command)
if groupadd_output["returncode"] != 0 or groupadd_output["returncode"] != 9: