def __install_toil( self ): # FIXME: consider using a virtualenv for Toil like we do for s3am # Older versions of pip don't support the 'extra' mechanism used by Toil's setup.py pip( 'install --upgrade pip', use_sudo=True ) pip( concat( 'install', self._toil_pip_args( ) ), use_sudo=True ) self._lazy_mkdir( '/var/lib', 'toil', persistent=None ) sudo( 'echo "TOIL_WORKDIR=/var/lib/toil" >> /etc/environment' )
def __install_toil(self): # FIXME: consider using a virtualenv for Toil like we do for s3am # Older versions of pip don't support the 'extra' mechanism used by Toil's setup.py pip('install --upgrade pip', use_sudo=True) pip(concat('install', self._toil_pip_args()), use_sudo=True) self._lazy_mkdir('/var/lib', 'toil', persistent=None) sudo('echo "TOIL_WORKDIR=/var/lib/toil" >> /etc/environment')
def _install_mesos_egg( self ): egg = 'mesos-' + self._mesos_egg_version( ) + '-py2.7-linux-x86_64.egg' version = self.release( ).version run( fmt( 'wget http://downloads.mesosphere.io/master/ubuntu/{version}/{egg}' ) ) # We need a newer version of protobuf than what comes default on Ubuntu pip( 'install --upgrade protobuf', use_sudo=True ) sudo( 'easy_install -a ' + egg ) run( 'rm ' + egg )
def __install_mesos_egg( self ): egg = 'mesos-' + self._mesos_egg_version( ) + '-py2.7-linux-x86_64.egg' version = self.release( ).version run( fmt( 'wget http://downloads.mesosphere.io/master/ubuntu/{version}/{egg}' ) ) # We need a newer version of protobuf than what comes default on Ubuntu pip( 'install --upgrade protobuf', use_sudo=True ) sudo( 'easy_install -a ' + egg ) run( 'rm ' + egg )
def __install_tools( self ): """ Installs the mesos-master-discovery init script and its companion mesos-tools. The latter is a Python package distribution that's included in cgcloud-mesos as a resource. This is in contrast to the cgcloud agent, which is a standalone distribution. """ tools_dir = install_dir + '/tools' admin = self.admin_account( ) sudo( fmt( 'mkdir -p {tools_dir}' ) ) sudo( fmt( 'chown {admin}:{admin} {tools_dir}' ) ) run( fmt( 'virtualenv --no-pip {tools_dir}' ) ) run( fmt( '{tools_dir}/bin/easy_install pip==1.5.2' ) ) with settings( forward_agent=True ): with self._project_artifacts( 'mesos-tools' ) as artifacts: pip( use_sudo=True, path=tools_dir + '/bin/pip', args=concat( 'install', artifacts ) ) sudo( fmt( 'chown -R root:root {tools_dir}' ) ) mesos_tools = "MesosTools(**%r)" % dict( user=user, shared_dir=self._shared_dir( ), ephemeral_dir=ephemeral_dir, persistent_dir=persistent_dir, lazy_dirs=self.lazy_dirs ) self.lazy_dirs = None # make sure it can't be used anymore once we are done with it self._register_init_script( "mesosbox", heredoc( """ description "Mesos master discovery" console log start on (local-filesystems and net-device-up IFACE!=lo) stop on runlevel [!2345] pre-start script for i in 1 2 3; do if {tools_dir}/bin/python2.7 - <<END import logging logging.basicConfig( level=logging.INFO ) from cgcloud.mesos_tools import MesosTools mesos_tools = {mesos_tools} mesos_tools.start() END then exit 0; fi; echo Retrying in 60s; sleep 60; done; exit 1 end script post-stop script {tools_dir}/bin/python2.7 - <<END import logging logging.basicConfig( level=logging.INFO ) from cgcloud.mesos_tools import MesosTools mesos_tools = {mesos_tools} mesos_tools.stop() END end script""" ) ) # Explicitly start the mesosbox service to achieve creation of lazy directoriess right # now. This makes a generic mesosbox useful for adhoc tests that involve Mesos and Toil. self._run_init_script( 'mesosbox' )
def __install_tools(self): """ Installs the mesos-master-discovery init script and its companion mesos-tools. The latter is a Python package distribution that's included in cgcloud-mesos as a resource. This is in contrast to the cgcloud agent, which is a standalone distribution. """ tools_dir = install_dir + '/tools' admin = self.admin_account() sudo(fmt('mkdir -p {tools_dir}')) sudo(fmt('chown {admin}:{admin} {tools_dir}')) run(fmt('virtualenv --no-pip {tools_dir}')) run(fmt('{tools_dir}/bin/easy_install pip==1.5.2')) with settings(forward_agent=True): with self._project_artifacts('mesos-tools') as artifacts: pip(use_sudo=True, path=tools_dir + '/bin/pip', args=concat('install', artifacts)) sudo(fmt('chown -R root:root {tools_dir}')) mesos_tools = "MesosTools(**%r)" % dict(user=user, shared_dir=self._shared_dir(), ephemeral_dir=ephemeral_dir, persistent_dir=persistent_dir, lazy_dirs=self.lazy_dirs) self.lazy_dirs = None # make sure it can't be used anymore once we are done with it self._register_init_script( "mesosbox", heredoc(""" description "Mesos master discovery" console log start on (local-filesystems and net-device-up IFACE!=lo) stop on runlevel [!2345] pre-start script for i in 1 2 3; do if {tools_dir}/bin/python2.7 - <<END import logging logging.basicConfig( level=logging.INFO ) from cgcloud.mesos_tools import MesosTools mesos_tools = {mesos_tools} mesos_tools.start() END then exit 0; fi; echo Retrying in 60s; sleep 60; done; exit 1 end script post-stop script {tools_dir}/bin/python2.7 - <<END import logging logging.basicConfig( level=logging.INFO ) from cgcloud.mesos_tools import MesosTools mesos_tools = {mesos_tools} mesos_tools.stop() END end script""")) # Explicitly start the mesosbox service to achieve creation of lazy directoriess right # now. This makes a generic mesosbox useful for adhoc tests that involve Mesos and Toil. self._run_init_script('mesosbox')
def __setup_agent(self): availability_zone = self.ctx.availability_zone namespace = self.ctx.namespace ec2_keypair_globs = ' '.join( shell.quote(_) for _ in self.ec2_keypair_globs) accounts = ' '.join([self.admin_account()] + self.other_accounts()) admin_account = self.admin_account() run_dir = '/var/run/cgcloudagent' log_dir = '/var/log' install_dir = '/opt/cgcloudagent' # Lucid & CentOS 5 have an ancient pip pip('install --upgrade pip==1.5.2', use_sudo=True) pip('install --upgrade virtualenv', use_sudo=True) sudo(fmt('mkdir -p {install_dir}')) sudo(fmt('chown {admin_account}:{admin_account} {install_dir}')) # By default, virtualenv installs the latest version of pip. We want a specific # version, so we tell virtualenv not to install pip and then install that version of # pip using easy_install. run(fmt('virtualenv --no-pip {install_dir}')) run(fmt('{install_dir}/bin/easy_install pip==1.5.2')) with settings(forward_agent=True): venv_pip = install_dir + '/bin/pip' if self._enable_agent_metrics(): pip(path=venv_pip, args='install psutil==3.4.1') with self._project_artifacts('agent') as artifacts: pip( path=venv_pip, args=concat( 'install', '--allow-external', 'argparse', # needed on CentOS 5 and 6 artifacts)) sudo(fmt('mkdir {run_dir}')) script = self.__gunzip_base64_decode( run( fmt('{install_dir}/bin/cgcloudagent' ' --init-script' ' --zone {availability_zone}' ' --namespace {namespace}' ' --accounts {accounts}' ' --keypairs {ec2_keypair_globs}' ' --user root' ' --group root' ' --pid-file {run_dir}/cgcloudagent.pid' ' --log-spill {log_dir}/cgcloudagent.out' '| gzip -c | base64'))) self._register_init_script('cgcloudagent', script) self._run_init_script('cgcloudagent')
def __setup_agent( self ): availability_zone = self.ctx.availability_zone namespace = self.ctx.namespace ec2_keypair_globs = ' '.join( shell.quote( _ ) for _ in self.ec2_keypair_globs ) accounts = ' '.join( [ self.admin_account( ) ] + self.other_accounts( ) ) admin_account = self.admin_account( ) run_dir = '/var/run/cgcloudagent' log_dir = '/var/log' install_dir = '/opt/cgcloudagent' # Lucid & CentOS 5 have an ancient pip pip( 'install --upgrade pip==1.5.2', use_sudo=True ) pip( 'install --upgrade virtualenv', use_sudo=True ) sudo( fmt( 'mkdir -p {install_dir}' ) ) sudo( fmt( 'chown {admin_account}:{admin_account} {install_dir}' ) ) # By default, virtualenv installs the latest version of pip. We want a specific # version, so we tell virtualenv not to install pip and then install that version of # pip using easy_install. run( fmt( 'virtualenv --no-pip {install_dir}' ) ) run( fmt( '{install_dir}/bin/easy_install pip==1.5.2' ) ) with settings( forward_agent=True ): venv_pip = install_dir + '/bin/pip' if self._enable_agent_metrics( ): pip( path=venv_pip, args='install psutil==3.4.1' ) with self._project_artifacts( 'agent' ) as artifacts: pip( path=venv_pip, args=concat( 'install', '--allow-external', 'argparse', # needed on CentOS 5 and 6 artifacts ) ) sudo( fmt( 'mkdir {run_dir}' ) ) script = self.__gunzip_base64_decode( run( fmt( '{install_dir}/bin/cgcloudagent' ' --init-script' ' --zone {availability_zone}' ' --namespace {namespace}' ' --accounts {accounts}' ' --keypairs {ec2_keypair_globs}' ' --user root' ' --group root' ' --pid-file {run_dir}/cgcloudagent.pid' ' --log-spill {log_dir}/cgcloudagent.out' '| gzip -c | base64' ) ) ) self._register_init_script( 'cgcloudagent', script ) self._run_init_script( 'cgcloudagent' )
def __install_tools( self ): """ Installs the spark-master-discovery init script and its companion spark-tools. The latter is a Python package distribution that's included in cgcloud-spark as a resource. This is in contrast to the cgcloud agent, which is a standalone distribution. """ tools_dir = install_dir + '/tools' admin = self.admin_account( ) sudo( fmt( 'mkdir -p {tools_dir}' ) ) sudo( fmt( 'chown {admin}:{admin} {tools_dir}' ) ) run( fmt( 'virtualenv --no-pip {tools_dir}' ) ) run( fmt( '{tools_dir}/bin/easy_install pip==1.5.2' ) ) with settings( forward_agent=True ): with self._project_artifacts( 'spark-tools' ) as artifacts: pip( use_sudo=True, path=tools_dir + '/bin/pip', args=concat( 'install', artifacts ) ) sudo( fmt( 'chown -R root:root {tools_dir}' ) ) spark_tools = "SparkTools(**%r)" % dict( user=user, shared_dir=self._shared_dir( ), install_dir=install_dir, ephemeral_dir=ephemeral_dir, persistent_dir=persistent_dir, lazy_dirs=self.lazy_dirs ) self.lazy_dirs = None # make sure it can't be used anymore once we are done with it self._register_init_script( "sparkbox", heredoc( """ description "Spark/HDFS master discovery" console log start on (local-filesystems and net-device-up IFACE!=lo) stop on runlevel [!2345] pre-start script for i in 1 2 3; do if {tools_dir}/bin/python2.7 - <<END import logging logging.basicConfig( level=logging.INFO ) from cgcloud.spark_tools import SparkTools spark_tools = {spark_tools} spark_tools.start() END then exit 0; fi; echo Retrying in 60s; sleep 60; done; exit 1 end script post-stop script {tools_dir}/bin/python2.7 - <<END import logging logging.basicConfig( level=logging.INFO ) from cgcloud.spark_tools import SparkTools spark_tools = {spark_tools} spark_tools.stop() END end script""" ) ) script_path = "/usr/local/bin/sparkbox-manage-slaves" put( remote_path=script_path, use_sudo=True, local_path=StringIO( heredoc( """ #!{tools_dir}/bin/python2.7 import sys import logging # Prefix each log line to make it more obvious that it's the master logging when the # slave calls this script via ssh. logging.basicConfig( level=logging.INFO, format="manage_slaves: " + logging.BASIC_FORMAT ) from cgcloud.spark_tools import SparkTools spark_tools = {spark_tools} spark_tools.manage_slaves( slaves_to_add=sys.argv[1:] )""" ) ) ) sudo( fmt( "chown root:root {script_path} && chmod 755 {script_path}" ) )
def __install_tools( self ): """ Installs the mesos-master-discovery init script and its companion mesos-tools. The latter is a Python package distribution that's included in cgcloud-mesos as a resource. This is in contrast to the cgcloud agent, which is a standalone distribution. """ tools_dir = install_dir + '/tools' admin = self.admin_account( ) sudo( fmt( 'mkdir -p {tools_dir}' ) ) sudo( fmt( 'chown {admin}:{admin} {tools_dir}' ) ) run( fmt( 'virtualenv --no-pip {tools_dir}' ) ) run( fmt( '{tools_dir}/bin/easy_install pip==1.5.2' ) ) with settings( forward_agent=True ): with self._project_artifacts( 'mesos-tools' ) as artifacts: pip( use_sudo=True, path=tools_dir + '/bin/pip', args=concat( 'install', artifacts ) ) sudo( fmt( 'chown -R root:root {tools_dir}' ) ) mesos_tools = "MesosTools(**%r)" % dict( user=user, shared_dir=self._shared_dir( ), ephemeral_dir=ephemeral_dir, persistent_dir=persistent_dir, lazy_dirs=self.lazy_dirs ) self.lazy_dirs = None # make sure it can't be used anymore once we are done with it mesosbox_start_path = '/usr/sbin/mesosbox-start.sh' mesosbox_stop_path = '/usr/sbin/mesosbox-stop.sh' systemd_heredoc = heredoc( """ [Unit] Description=Mesos master discovery Requires=networking.service network-online.target After=networking.service network-online.target [Service] Type=simple ExecStart={mesosbox_start_path} RemainAfterExit=true ExecStop={mesosbox_stop_path} [Install] WantedBy=multi-user.target """ ) mesosbox_setup_start_script = heredoc( """ #!/bin/sh for i in 1 2 3; do if {tools_dir}/bin/python2.7 - <<END import logging logging.basicConfig( level=logging.INFO ) from cgcloud.mesos_tools import MesosTools mesos_tools = {mesos_tools} mesos_tools.start() END then exit 0; fi; echo Retrying in 60s; sleep 60; done; exit 1""" ) mesosbox_setup_stop_script = heredoc (""" #!/{tools_dir}/bin/python2.7 import logging logging.basicConfig( level=logging.INFO ) from cgcloud.mesos_tools import MesosTools mesos_tools = {mesos_tools} mesos_tools.stop()""" ) put( local_path=StringIO( mesosbox_setup_start_script ), remote_path=mesosbox_start_path, use_sudo=True ) sudo( "chown root:root '%s'" % mesosbox_start_path ) sudo( "chmod +x '%s'" % mesosbox_start_path ) put( local_path=StringIO( mesosbox_setup_stop_script ), remote_path=mesosbox_stop_path, use_sudo=True ) sudo( "chown root:root '%s'" % mesosbox_stop_path ) sudo( "chmod +x '%s'" % mesosbox_stop_path ) self._register_init_script( "mesosbox", systemd_heredoc ) # Enable mesosbox to start on boot sudo( "systemctl enable mesosbox" ) # Explicitly start the mesosbox service to achieve creation of lazy directoriess right # now. This makes a generic mesosbox useful for adhoc tests that involve Mesos and Toil. self._run_init_script( 'mesosbox' )
def __install_toil( self ): # Older versions of pip don't support the 'extra' mechanism used by Toil's setup.py pip( 'install --upgrade pip', use_sudo=True ) pip( 'install --pre s3am', use_sudo=True ) pip( concat( 'install', self._toil_pip_args( ) ), use_sudo=True ) self._lazy_mkdir( '/var/lib', 'toil', persistent=None )
def __install_toil(self): # Older versions of pip don't support the 'extra' mechanism used by Toil's setup.py pip('install --upgrade pip', use_sudo=True) pip('install --pre s3am', use_sudo=True) pip(concat('install', self._toil_pip_args()), use_sudo=True) self._lazy_mkdir('/var/lib', 'toil', persistent=None)
def __install_tools(self): """ Installs the spark-master-discovery init script and its companion spark-tools. The latter is a Python package distribution that's included in cgcloud-spark as a resource. This is in contrast to the cgcloud agent, which is a standalone distribution. """ tools_dir = install_dir + '/tools' admin = self.admin_account() sudo(fmt('mkdir -p {tools_dir}')) sudo(fmt('chown {admin}:{admin} {tools_dir}')) run(fmt('virtualenv --no-pip {tools_dir}')) run(fmt('{tools_dir}/bin/easy_install pip==1.5.2')) with settings(forward_agent=True): with self._project_artifacts('spark-tools') as artifacts: pip(use_sudo=True, path=tools_dir + '/bin/pip', args=concat('install', artifacts)) sudo(fmt('chown -R root:root {tools_dir}')) spark_tools = "SparkTools(**%r)" % dict(user=user, shared_dir=self._shared_dir(), install_dir=install_dir, ephemeral_dir=ephemeral_dir, persistent_dir=persistent_dir, lazy_dirs=self.lazy_dirs) self.lazy_dirs = None # make sure it can't be used anymore once we are done with it self._register_init_script( "sparkbox", heredoc(""" description "Spark/HDFS master discovery" console log start on (local-filesystems and net-device-up IFACE!=lo) stop on runlevel [!2345] pre-start script for i in 1 2 3; do if {tools_dir}/bin/python2.7 - <<END import logging logging.basicConfig( level=logging.INFO ) from cgcloud.spark_tools import SparkTools spark_tools = {spark_tools} spark_tools.start() END then exit 0; fi; echo Retrying in 60s; sleep 60; done; exit 1 end script post-stop script {tools_dir}/bin/python2.7 - <<END import logging logging.basicConfig( level=logging.INFO ) from cgcloud.spark_tools import SparkTools spark_tools = {spark_tools} spark_tools.stop() END end script""")) script_path = "/usr/local/bin/sparkbox-manage-slaves" put(remote_path=script_path, use_sudo=True, local_path=StringIO( heredoc(""" #!{tools_dir}/bin/python2.7 import sys import logging # Prefix each log line to make it more obvious that it's the master logging when the # slave calls this script via ssh. logging.basicConfig( level=logging.INFO, format="manage_slaves: " + logging.BASIC_FORMAT ) from cgcloud.spark_tools import SparkTools spark_tools = {spark_tools} spark_tools.manage_slaves( slaves_to_add=sys.argv[1:] )"""))) sudo(fmt("chown root:root {script_path} && chmod 755 {script_path}"))