def __install_toil( self ):
# FIXME: consider using a virtualenv for Toil like we do for s3am
# Older versions of pip don't support the 'extra' mechanism used by Toil's setup.py
pip( 'install --upgrade pip', use_sudo=True )
pip( concat( 'install', self._toil_pip_args( ) ), use_sudo=True )
self._lazy_mkdir( '/var/lib', 'toil', persistent=None )
sudo( 'echo "TOIL_WORKDIR=/var/lib/toil" >> /etc/environment' )
def __install_toil(self):
# FIXME: consider using a virtualenv for Toil like we do for s3am
# Older versions of pip don't support the 'extra' mechanism used by Toil's setup.py
pip('install --upgrade pip', use_sudo=True)
pip(concat('install', self._toil_pip_args()), use_sudo=True)
self._lazy_mkdir('/var/lib', 'toil', persistent=None)
sudo('echo "TOIL_WORKDIR=/var/lib/toil" >> /etc/environment')
def _install_mesos_egg( self ):
egg = 'mesos-' + self._mesos_egg_version( ) + '-py2.7-linux-x86_64.egg'
version = self.release( ).version
run( fmt( 'wget http://downloads.mesosphere.io/master/ubuntu/{version}/{egg}' ) )
# We need a newer version of protobuf than what comes default on Ubuntu
pip( 'install --upgrade protobuf', use_sudo=True )
sudo( 'easy_install -a ' + egg )
run( 'rm ' + egg )
def __install_mesos_egg( self ):
egg = 'mesos-' + self._mesos_egg_version( ) + '-py2.7-linux-x86_64.egg'
version = self.release( ).version
run( fmt( 'wget http://downloads.mesosphere.io/master/ubuntu/{version}/{egg}' ) )
# We need a newer version of protobuf than what comes default on Ubuntu
pip( 'install --upgrade protobuf', use_sudo=True )
sudo( 'easy_install -a ' + egg )
run( 'rm ' + egg )
def __install_tools( self ):
"""
Installs the mesos-master-discovery init script and its companion mesos-tools. The latter
is a Python package distribution that's included in cgcloud-mesos as a resource. This is
in contrast to the cgcloud agent, which is a standalone distribution.
"""
tools_dir = install_dir + '/tools'
admin = self.admin_account( )
sudo( fmt( 'mkdir -p {tools_dir}' ) )
sudo( fmt( 'chown {admin}:{admin} {tools_dir}' ) )
run( fmt( 'virtualenv --no-pip {tools_dir}' ) )
run( fmt( '{tools_dir}/bin/easy_install pip==1.5.2' ) )
with settings( forward_agent=True ):
with self._project_artifacts( 'mesos-tools' ) as artifacts:
pip( use_sudo=True,
path=tools_dir + '/bin/pip',
args=concat( 'install', artifacts ) )
sudo( fmt( 'chown -R root:root {tools_dir}' ) )
mesos_tools = "MesosTools(**%r)" % dict( user=user,
shared_dir=self._shared_dir( ),
ephemeral_dir=ephemeral_dir,
persistent_dir=persistent_dir,
lazy_dirs=self.lazy_dirs )
self.lazy_dirs = None # make sure it can't be used anymore once we are done with it
self._register_init_script(
"mesosbox",
heredoc( """
description "Mesos master discovery"
console log
start on (local-filesystems and net-device-up IFACE!=lo)
stop on runlevel [!2345]
pre-start script
for i in 1 2 3; do if {tools_dir}/bin/python2.7 - <<END
import logging
logging.basicConfig( level=logging.INFO )
from cgcloud.mesos_tools import MesosTools
mesos_tools = {mesos_tools}
mesos_tools.start()
END
then exit 0; fi; echo Retrying in 60s; sleep 60; done; exit 1
end script
post-stop script
{tools_dir}/bin/python2.7 - <<END
import logging
logging.basicConfig( level=logging.INFO )
from cgcloud.mesos_tools import MesosTools
mesos_tools = {mesos_tools}
mesos_tools.stop()
END
end script""" ) )
# Explicitly start the mesosbox service to achieve creation of lazy directoriess right
# now. This makes a generic mesosbox useful for adhoc tests that involve Mesos and Toil.
self._run_init_script( 'mesosbox' )
def __install_tools(self):
"""
Installs the mesos-master-discovery init script and its companion mesos-tools. The latter
is a Python package distribution that's included in cgcloud-mesos as a resource. This is
in contrast to the cgcloud agent, which is a standalone distribution.
"""
tools_dir = install_dir + '/tools'
admin = self.admin_account()
sudo(fmt('mkdir -p {tools_dir}'))
sudo(fmt('chown {admin}:{admin} {tools_dir}'))
run(fmt('virtualenv --no-pip {tools_dir}'))
run(fmt('{tools_dir}/bin/easy_install pip==1.5.2'))
with settings(forward_agent=True):
with self._project_artifacts('mesos-tools') as artifacts:
pip(use_sudo=True,
path=tools_dir + '/bin/pip',
args=concat('install', artifacts))
sudo(fmt('chown -R root:root {tools_dir}'))
mesos_tools = "MesosTools(**%r)" % dict(user=user,
shared_dir=self._shared_dir(),
ephemeral_dir=ephemeral_dir,
persistent_dir=persistent_dir,
lazy_dirs=self.lazy_dirs)
self.lazy_dirs = None # make sure it can't be used anymore once we are done with it
self._register_init_script(
"mesosbox",
heredoc("""
description "Mesos master discovery"
console log
start on (local-filesystems and net-device-up IFACE!=lo)
stop on runlevel [!2345]
pre-start script
for i in 1 2 3; do if {tools_dir}/bin/python2.7 - <<END
import logging
logging.basicConfig( level=logging.INFO )
from cgcloud.mesos_tools import MesosTools
mesos_tools = {mesos_tools}
mesos_tools.start()
END
then exit 0; fi; echo Retrying in 60s; sleep 60; done; exit 1
end script
post-stop script
{tools_dir}/bin/python2.7 - <<END
import logging
logging.basicConfig( level=logging.INFO )
from cgcloud.mesos_tools import MesosTools
mesos_tools = {mesos_tools}
mesos_tools.stop()
END
end script"""))
# Explicitly start the mesosbox service to achieve creation of lazy directoriess right
# now. This makes a generic mesosbox useful for adhoc tests that involve Mesos and Toil.
self._run_init_script('mesosbox')
def __setup_agent(self):
availability_zone = self.ctx.availability_zone
namespace = self.ctx.namespace
ec2_keypair_globs = ' '.join(
shell.quote(_) for _ in self.ec2_keypair_globs)
accounts = ' '.join([self.admin_account()] + self.other_accounts())
admin_account = self.admin_account()
run_dir = '/var/run/cgcloudagent'
log_dir = '/var/log'
install_dir = '/opt/cgcloudagent'
# Lucid & CentOS 5 have an ancient pip
pip('install --upgrade pip==1.5.2', use_sudo=True)
pip('install --upgrade virtualenv', use_sudo=True)
sudo(fmt('mkdir -p {install_dir}'))
sudo(fmt('chown {admin_account}:{admin_account} {install_dir}'))
# By default, virtualenv installs the latest version of pip. We want a specific
# version, so we tell virtualenv not to install pip and then install that version of
# pip using easy_install.
run(fmt('virtualenv --no-pip {install_dir}'))
run(fmt('{install_dir}/bin/easy_install pip==1.5.2'))
with settings(forward_agent=True):
venv_pip = install_dir + '/bin/pip'
if self._enable_agent_metrics():
pip(path=venv_pip, args='install psutil==3.4.1')
with self._project_artifacts('agent') as artifacts:
pip(
path=venv_pip,
args=concat(
'install',
'--allow-external',
'argparse', # needed on CentOS 5 and 6
artifacts))
sudo(fmt('mkdir {run_dir}'))
script = self.__gunzip_base64_decode(
run(
fmt('{install_dir}/bin/cgcloudagent'
' --init-script'
' --zone {availability_zone}'
' --namespace {namespace}'
' --accounts {accounts}'
' --keypairs {ec2_keypair_globs}'
' --user root'
' --group root'
' --pid-file {run_dir}/cgcloudagent.pid'
' --log-spill {log_dir}/cgcloudagent.out'
'| gzip -c | base64')))
self._register_init_script('cgcloudagent', script)
self._run_init_script('cgcloudagent')
def __setup_agent( self ):
availability_zone = self.ctx.availability_zone
namespace = self.ctx.namespace
ec2_keypair_globs = ' '.join( shell.quote( _ ) for _ in self.ec2_keypair_globs )
accounts = ' '.join( [ self.admin_account( ) ] + self.other_accounts( ) )
admin_account = self.admin_account( )
run_dir = '/var/run/cgcloudagent'
log_dir = '/var/log'
install_dir = '/opt/cgcloudagent'
# Lucid & CentOS 5 have an ancient pip
pip( 'install --upgrade pip==1.5.2', use_sudo=True )
pip( 'install --upgrade virtualenv', use_sudo=True )
sudo( fmt( 'mkdir -p {install_dir}' ) )
sudo( fmt( 'chown {admin_account}:{admin_account} {install_dir}' ) )
# By default, virtualenv installs the latest version of pip. We want a specific
# version, so we tell virtualenv not to install pip and then install that version of
# pip using easy_install.
run( fmt( 'virtualenv --no-pip {install_dir}' ) )
run( fmt( '{install_dir}/bin/easy_install pip==1.5.2' ) )
with settings( forward_agent=True ):
venv_pip = install_dir + '/bin/pip'
if self._enable_agent_metrics( ):
pip( path=venv_pip, args='install psutil==3.4.1' )
with self._project_artifacts( 'agent' ) as artifacts:
pip( path=venv_pip,
args=concat( 'install',
'--allow-external', 'argparse', # needed on CentOS 5 and 6
artifacts ) )
sudo( fmt( 'mkdir {run_dir}' ) )
script = self.__gunzip_base64_decode( run( fmt(
'{install_dir}/bin/cgcloudagent'
' --init-script'
' --zone {availability_zone}'
' --namespace {namespace}'
' --accounts {accounts}'
' --keypairs {ec2_keypair_globs}'
' --user root'
' --group root'
' --pid-file {run_dir}/cgcloudagent.pid'
' --log-spill {log_dir}/cgcloudagent.out'
'| gzip -c | base64' ) ) )
self._register_init_script( 'cgcloudagent', script )
self._run_init_script( 'cgcloudagent' )
def __install_tools( self ):
"""
Installs the spark-master-discovery init script and its companion spark-tools. The latter
is a Python package distribution that's included in cgcloud-spark as a resource. This is
in contrast to the cgcloud agent, which is a standalone distribution.
"""
tools_dir = install_dir + '/tools'
admin = self.admin_account( )
sudo( fmt( 'mkdir -p {tools_dir}' ) )
sudo( fmt( 'chown {admin}:{admin} {tools_dir}' ) )
run( fmt( 'virtualenv --no-pip {tools_dir}' ) )
run( fmt( '{tools_dir}/bin/easy_install pip==1.5.2' ) )
with settings( forward_agent=True ):
with self._project_artifacts( 'spark-tools' ) as artifacts:
pip( use_sudo=True,
path=tools_dir + '/bin/pip',
args=concat( 'install', artifacts ) )
sudo( fmt( 'chown -R root:root {tools_dir}' ) )
spark_tools = "SparkTools(**%r)" % dict( user=user,
shared_dir=self._shared_dir( ),
install_dir=install_dir,
ephemeral_dir=ephemeral_dir,
persistent_dir=persistent_dir,
lazy_dirs=self.lazy_dirs )
self.lazy_dirs = None # make sure it can't be used anymore once we are done with it
self._register_init_script(
"sparkbox",
heredoc( """
description "Spark/HDFS master discovery"
console log
start on (local-filesystems and net-device-up IFACE!=lo)
stop on runlevel [!2345]
pre-start script
for i in 1 2 3; do if {tools_dir}/bin/python2.7 - <<END
import logging
logging.basicConfig( level=logging.INFO )
from cgcloud.spark_tools import SparkTools
spark_tools = {spark_tools}
spark_tools.start()
END
then exit 0; fi; echo Retrying in 60s; sleep 60; done; exit 1
end script
post-stop script
{tools_dir}/bin/python2.7 - <<END
import logging
logging.basicConfig( level=logging.INFO )
from cgcloud.spark_tools import SparkTools
spark_tools = {spark_tools}
spark_tools.stop()
END
end script""" ) )
script_path = "/usr/local/bin/sparkbox-manage-slaves"
put( remote_path=script_path, use_sudo=True, local_path=StringIO( heredoc( """
#!{tools_dir}/bin/python2.7
import sys
import logging
# Prefix each log line to make it more obvious that it's the master logging when the
# slave calls this script via ssh.
logging.basicConfig( level=logging.INFO,
format="manage_slaves: " + logging.BASIC_FORMAT )
from cgcloud.spark_tools import SparkTools
spark_tools = {spark_tools}
spark_tools.manage_slaves( slaves_to_add=sys.argv[1:] )""" ) ) )
sudo( fmt( "chown root:root {script_path} && chmod 755 {script_path}" ) )
def __install_tools( self ):
"""
Installs the mesos-master-discovery init script and its companion mesos-tools. The latter
is a Python package distribution that's included in cgcloud-mesos as a resource. This is
in contrast to the cgcloud agent, which is a standalone distribution.
"""
tools_dir = install_dir + '/tools'
admin = self.admin_account( )
sudo( fmt( 'mkdir -p {tools_dir}' ) )
sudo( fmt( 'chown {admin}:{admin} {tools_dir}' ) )
run( fmt( 'virtualenv --no-pip {tools_dir}' ) )
run( fmt( '{tools_dir}/bin/easy_install pip==1.5.2' ) )
with settings( forward_agent=True ):
with self._project_artifacts( 'mesos-tools' ) as artifacts:
pip( use_sudo=True,
path=tools_dir + '/bin/pip',
args=concat( 'install', artifacts ) )
sudo( fmt( 'chown -R root:root {tools_dir}' ) )
mesos_tools = "MesosTools(**%r)" % dict( user=user,
shared_dir=self._shared_dir( ),
ephemeral_dir=ephemeral_dir,
persistent_dir=persistent_dir,
lazy_dirs=self.lazy_dirs )
self.lazy_dirs = None # make sure it can't be used anymore once we are done with it
mesosbox_start_path = '/usr/sbin/mesosbox-start.sh'
mesosbox_stop_path = '/usr/sbin/mesosbox-stop.sh'
systemd_heredoc = heredoc( """
[Unit]
Description=Mesos master discovery
Requires=networking.service network-online.target
After=networking.service network-online.target
[Service]
Type=simple
ExecStart={mesosbox_start_path}
RemainAfterExit=true
ExecStop={mesosbox_stop_path}
[Install]
WantedBy=multi-user.target
""" )
mesosbox_setup_start_script = heredoc( """
#!/bin/sh
for i in 1 2 3; do if {tools_dir}/bin/python2.7 - <<END
import logging
logging.basicConfig( level=logging.INFO )
from cgcloud.mesos_tools import MesosTools
mesos_tools = {mesos_tools}
mesos_tools.start()
END
then exit 0; fi; echo Retrying in 60s; sleep 60; done; exit 1""" )
mesosbox_setup_stop_script = heredoc ("""
#!/{tools_dir}/bin/python2.7
import logging
logging.basicConfig( level=logging.INFO )
from cgcloud.mesos_tools import MesosTools
mesos_tools = {mesos_tools}
mesos_tools.stop()""" )
put( local_path=StringIO( mesosbox_setup_start_script ), remote_path=mesosbox_start_path, use_sudo=True )
sudo( "chown root:root '%s'" % mesosbox_start_path )
sudo( "chmod +x '%s'" % mesosbox_start_path )
put( local_path=StringIO( mesosbox_setup_stop_script ), remote_path=mesosbox_stop_path, use_sudo=True )
sudo( "chown root:root '%s'" % mesosbox_stop_path )
sudo( "chmod +x '%s'" % mesosbox_stop_path )
self._register_init_script(
"mesosbox",
systemd_heredoc )
# Enable mesosbox to start on boot
sudo( "systemctl enable mesosbox" )
# Explicitly start the mesosbox service to achieve creation of lazy directoriess right
# now. This makes a generic mesosbox useful for adhoc tests that involve Mesos and Toil.
self._run_init_script( 'mesosbox' )
def __install_toil( self ):
# Older versions of pip don't support the 'extra' mechanism used by Toil's setup.py
pip( 'install --upgrade pip', use_sudo=True )
pip( 'install --pre s3am', use_sudo=True )
pip( concat( 'install', self._toil_pip_args( ) ), use_sudo=True )
self._lazy_mkdir( '/var/lib', 'toil', persistent=None )
def __install_toil(self):
# Older versions of pip don't support the 'extra' mechanism used by Toil's setup.py
pip('install --upgrade pip', use_sudo=True)
pip('install --pre s3am', use_sudo=True)
pip(concat('install', self._toil_pip_args()), use_sudo=True)
self._lazy_mkdir('/var/lib', 'toil', persistent=None)
def __install_tools(self):
"""
Installs the spark-master-discovery init script and its companion spark-tools. The latter
is a Python package distribution that's included in cgcloud-spark as a resource. This is
in contrast to the cgcloud agent, which is a standalone distribution.
"""
tools_dir = install_dir + '/tools'
admin = self.admin_account()
sudo(fmt('mkdir -p {tools_dir}'))
sudo(fmt('chown {admin}:{admin} {tools_dir}'))
run(fmt('virtualenv --no-pip {tools_dir}'))
run(fmt('{tools_dir}/bin/easy_install pip==1.5.2'))
with settings(forward_agent=True):
with self._project_artifacts('spark-tools') as artifacts:
pip(use_sudo=True,
path=tools_dir + '/bin/pip',
args=concat('install', artifacts))
sudo(fmt('chown -R root:root {tools_dir}'))
spark_tools = "SparkTools(**%r)" % dict(user=user,
shared_dir=self._shared_dir(),
install_dir=install_dir,
ephemeral_dir=ephemeral_dir,
persistent_dir=persistent_dir,
lazy_dirs=self.lazy_dirs)
self.lazy_dirs = None # make sure it can't be used anymore once we are done with it
self._register_init_script(
"sparkbox",
heredoc("""
description "Spark/HDFS master discovery"
console log
start on (local-filesystems and net-device-up IFACE!=lo)
stop on runlevel [!2345]
pre-start script
for i in 1 2 3; do if {tools_dir}/bin/python2.7 - <<END
import logging
logging.basicConfig( level=logging.INFO )
from cgcloud.spark_tools import SparkTools
spark_tools = {spark_tools}
spark_tools.start()
END
then exit 0; fi; echo Retrying in 60s; sleep 60; done; exit 1
end script
post-stop script
{tools_dir}/bin/python2.7 - <<END
import logging
logging.basicConfig( level=logging.INFO )
from cgcloud.spark_tools import SparkTools
spark_tools = {spark_tools}
spark_tools.stop()
END
end script"""))
script_path = "/usr/local/bin/sparkbox-manage-slaves"
put(remote_path=script_path,
use_sudo=True,
local_path=StringIO(
heredoc("""
#!{tools_dir}/bin/python2.7
import sys
import logging
# Prefix each log line to make it more obvious that it's the master logging when the
# slave calls this script via ssh.
logging.basicConfig( level=logging.INFO,
format="manage_slaves: " + logging.BASIC_FORMAT )
from cgcloud.spark_tools import SparkTools
spark_tools = {spark_tools}
spark_tools.manage_slaves( slaves_to_add=sys.argv[1:] )""")))
sudo(fmt("chown root:root {script_path} && chmod 755 {script_path}"))
