def run_on_host(host, config, conn, funct, outdir, **kwargs):
"""Runs the given funct on the specified host."""
from fabric.api import settings, show, hide
hostdir = os.path.join(outdir, host)
if not os.path.isdir(hostdir):
os.makedirs(hostdir)
os.chdir(hostdir)
orig_stdout = sys.stdout
orig_stderr = sys.stderr
print '<%s>: writing stdout/stderr to %s' % (host,
os.path.join(os.getcwd(), 'run.out'))
sys.stdout = sys.stderr = open('run.out', 'wb')
settings_kwargs = {}
settings_args = []
debug = config.getboolean(host, 'debug')
settings_kwargs['host_string'] = config.get(host, 'addr', None)
if config.has_option(host, 'identity'):
settings_kwargs['key_filename'] = os.path.expanduser(
os.path.expandvars(config.get(host, 'identity')))
if config.has_option(host, 'user'):
settings_kwargs['user'] = config.get(host, 'user')
platform = config.get(host, 'platform')
if platform == 'windows':
settings_kwargs['shell'] = 'cmd /C'
else:
settings_kwargs['shell'] = '/bin/bash -l -c'
if debug:
settings_args.append(show('debug'))
orig_stdout.write("<%s>: calling %s" %
(host, print_funct_call(funct, **kwargs)))
else:
settings_args.append(hide('running'))
with settings(*settings_args, **settings_kwargs):
return funct(**kwargs)
def run_on_host(host, config, conn, funct, outdir, **kwargs):
"""Runs the given funct on the specified host."""
from fabric.api import settings, show, hide
hostdir = os.path.join(outdir, host)
if not os.path.isdir(hostdir):
os.makedirs(hostdir)
os.chdir(hostdir)
orig_stdout = sys.stdout
orig_stderr = sys.stderr
print '<%s>: writing stdout/stderr to %s' % (
host, os.path.join(os.getcwd(), 'run.out'))
sys.stdout = sys.stderr = open('run.out', 'wb')
settings_kwargs = {}
settings_args = []
debug = config.getboolean(host, 'debug')
settings_kwargs['host_string'] = config.get(host, 'addr', None)
if config.has_option(host, 'identity'):
settings_kwargs['key_filename'] = os.path.expanduser(
os.path.expandvars(config.get(host, 'identity')))
if config.has_option(host, 'user'):
settings_kwargs['user'] = config.get(host, 'user')
platform = config.get(host, 'platform')
if platform == 'windows':
settings_kwargs['shell'] = 'cmd /C'
else:
settings_kwargs['shell'] = '/bin/bash -l -c'
if debug:
settings_args.append(show('debug'))
orig_stdout.write("<%s>: calling %s" %
(host, print_funct_call(funct, **kwargs)))
else:
settings_args.append(hide('running'))
with settings(*settings_args, **settings_kwargs):
return funct(**kwargs)
def run_on_ec2(host, config, conn, funct, outdir, **kwargs):
"""Runs the given function on an EC2 instance. The instance may be either
created from an image or may be an existing image that is stopped.
In either case, the instance will be started and the function will run.
If the instance was created from an image, it will be
terminated unless there is an error or keep==True, which will result in
the instance being stopped but not terminated. If the instance was
pre-existing, then it will just be stopped instead of terminated.
"""
# put fabric import here to prevent sphinx doc generation failure
# on windows when win32api isn't installed
from fabric.api import settings, hide, show
hostdir = os.path.join(outdir, host)
if not os.path.isdir(hostdir):
os.makedirs(hostdir)
os.chdir(hostdir)
orig_stdout = sys.stdout
orig_stderr = sys.stderr
sys.stdout = sys.stderr = open('run.out', 'wb')
outf = MultiFile(sys.stdout, orig_stdout)
settings_kwargs = {}
settings_args = []
settings_kwargs['key_filename'] = os.path.expanduser(
os.path.expandvars(config.get(host, 'identity').strip()))
settings_kwargs['user'] = config.get(host, 'user')
debug = config.getboolean(host, 'debug')
if config.has_option(host, 'instance_id'): # start a stopped instance
inst_id = config.get(host, 'instance_id')
outf.write("starting instance %s from stopped instance\n" % host)
inst = start_instance(conn, inst_id, debug=debug)
terminate = False
else: # stand up an instance of the specified image
outf.write("starting instance %s from image\n" % host)
inst = start_instance_from_image(conn, config, host, stream=outf)
terminate = True
settings_kwargs['host_string'] = inst.public_dns_name
settings_kwargs['disable_known_hosts'] = True
outf.write("instance %s was started successfully. dns=%s\n" %
(host, inst.public_dns_name))
if debug:
settings_args.append(show('debug'))
else:
settings_args.append(hide('running'))
platform = config.get(host, 'platform')
if platform == 'windows':
settings_kwargs['shell'] = 'cmd /C'
else:
settings_kwargs['shell'] = '/bin/bash -l -c'
try:
with settings(**settings_kwargs):
# first, make sure that the connection is really working...
# on EC2 even if ssh connects there can be timeouts during authentication,
# so try to connect multiple times if there's a timeout
client = fab_connect(settings_kwargs['user'],
settings_kwargs['host_string'],
debug=debug)
if debug:
outf.write("<%s>: calling %s\n" %
(host, print_funct_call(funct, **kwargs)))
py = config.get(host, 'py')
if platform.startswith('win') and '.' in py:
# convert pythonX.Y form over to C:/PythonXY/python.exe
ver = py[6:]
py = 'C:/Python%s/python.exe' % ver.replace('.', '')
max_tries = 25
sleep = 10
for i in range(max_tries):
time.sleep(sleep)
if debug:
print "testing python call over fabric connection (try #%d)" % (
i + 1)
# even though the instance is running, it takes a while (on Windows) before
# python actually works, so try some simple python command until it works.
try:
retval = run('%s -c "dir()"' % py)
except:
pass
else:
if retval.failed or retval.return_code != 0:
continue
else:
break
else:
outf.write("\nrunning python via fabric on %s failed after %d"
" attempts. terminating...\n" % (host, max_tries))
terminate_instance(inst, host, outf, debug)
raise RuntimeError("couldn't run python on %s via fabric" %
host)
retval = funct(**kwargs)
except (SystemExit, Exception) as err:
outf.write(str(err))
if isinstance(err, SystemExit) and err.code is not None:
retval = err.code
else:
retval = -1
# try to retrieve console output if we can
try:
out = inst.get_console_output().output
with open('console.out', 'wb') as f:
f.write(out)
except:
outf.write("\ncouldn't retrieve console output\n")
keep = kwargs.get('keep', False)
if keep or not terminate:
outf.write("stopping %s instead of terminating it.\n" % host)
outf.write("%s will have to be terminated manually.\n" % host)
if not stop_instance(inst, host, orig_stdout, debug):
retval = -1
else:
if not terminate_instance(inst, host, orig_stdout, debug):
retval = -1
if retval != 0:
raise RuntimeError("return value = %s" % retval)
def run_on_ec2(host, config, conn, funct, outdir, **kwargs):
"""Runs the given function on an EC2 instance. The instance may be either
created from an image or may be an existing image that is stopped.
In either case, the instance will be started and the function will run.
If the instance was created from an image, it will be
terminated unless there is an error or keep==True, which will result in
the instance being stopped but not terminated. If the instance was
pre-existing, then it will just be stopped instead of terminated.
"""
# put fabric import here to prevent sphinx doc generation failure
# on windows when win32api isn't installed
from fabric.api import settings, hide, show
hostdir = os.path.join(outdir, host)
if not os.path.isdir(hostdir):
os.makedirs(hostdir)
os.chdir(hostdir)
orig_stdout = sys.stdout
orig_stderr = sys.stderr
sys.stdout = sys.stderr = open('run.out', 'wb')
outf = MultiFile(sys.stdout, orig_stdout)
settings_kwargs = {}
settings_args = []
settings_kwargs['key_filename'] = os.path.expanduser(
os.path.expandvars(config.get(host, 'identity').strip()))
settings_kwargs['user'] = config.get(host, 'user')
debug = config.getboolean(host, 'debug')
if config.has_option(host, 'instance_id'): # start a stopped instance
inst_id = config.get(host, 'instance_id')
outf.write("starting instance %s from stopped instance\n" % host)
inst = start_instance(conn, inst_id, debug=debug)
terminate = False
else: # stand up an instance of the specified image
outf.write("starting instance %s from image\n" % host)
inst = start_instance_from_image(conn, config, host, stream=outf)
terminate = True
settings_kwargs['host_string'] = inst.public_dns_name
settings_kwargs['disable_known_hosts'] = True
outf.write("instance %s was started successfully. dns=%s\n" %
(host, inst.public_dns_name))
if debug:
settings_args.append(show('debug'))
else:
settings_args.append(hide('running'))
platform = config.get(host, 'platform')
if platform == 'windows':
settings_kwargs['shell'] = 'cmd /C'
else:
settings_kwargs['shell'] = '/bin/bash -l -c'
try:
with settings(**settings_kwargs):
# first, make sure that the connection is really working...
# on EC2 even if ssh connects there can be timeouts during authentication,
# so try to connect multiple times if there's a timeout
client = fab_connect(settings_kwargs['user'],
settings_kwargs['host_string'], debug=debug)
if debug:
outf.write("<%s>: calling %s\n" %
(host, print_funct_call(funct, **kwargs)))
py = config.get(host, 'py')
if platform.startswith('win') and '.' in py:
# convert pythonX.Y form over to C:/PythonXY/python.exe
ver = py[6:]
py = 'C:/Python%s/python.exe' % ver.replace('.','')
max_tries = 25
sleep = 10
for i in range(max_tries):
time.sleep(sleep)
if debug:
print "testing python call over fabric connection (try #%d)" % (i+1)
# even though the instance is running, it takes a while (on Windows) before
# python actually works, so try some simple python command until it works.
try:
retval = run('%s -c "dir()"' % py)
except Exception as exc:
print ' caught exception:', exc
else:
if retval.failed:
print ' failed:', retval.failed
elif retval.return_code != 0:
print ' return code:', retval.return_code
else:
break
else:
outf.write("\nrunning python via fabric on %s failed after %d"
" attempts. terminating...\n" % (host, max_tries))
terminate_instance(inst, host, outf, debug)
raise RuntimeError("couldn't run python on %s via fabric" % host)
retval = funct(**kwargs)
except (SystemExit, Exception) as err:
outf.write(str(err))
if isinstance(err, SystemExit) and err.code is not None:
retval = err.code
else:
retval = -1
# try to retrieve console output if we can
try:
out = inst.get_console_output().output
with open('console.out', 'wb') as f:
f.write(out)
except:
outf.write("\ncouldn't retrieve console output\n")
keep = kwargs.get('keep', False)
if keep or not terminate:
outf.write("stopping %s instead of terminating it.\n" % host)
outf.write("%s will have to be terminated manually.\n" % host)
if not stop_instance(inst, host, orig_stdout, debug):
retval = -1
else:
if not terminate_instance(inst, host, orig_stdout, debug):
retval = -1
if retval != 0:
raise RuntimeError("return value = %s" % retval)
def run_on_ec2(host, config, conn, funct, outdir, **kwargs):
"""Runs the given function on an EC2 instance. The instance may be either
created from an image or may be an existing image that is stopped.
In either case, the instance will be started and the function will run.
If the instance was created from an image, it will be
terminated, unless there is an error or keep==True, which will result in
the instance being stopped but not terminated. If the instance was
pre-existing, then it will just be stopped instead of terminated.
"""
# put fabric import here to prevent sphinx doc generation failure
# on windows when win32api isn't installed
from fabric.api import settings, hide, show
hostdir = os.path.join(outdir, host)
if not os.path.isdir(hostdir):
os.makedirs(hostdir)
os.chdir(hostdir)
orig_stdout = sys.stdout
orig_stderr = sys.stderr
sys.stdout = sys.stderr = open('run.out', 'wb')
outf = MultiFile(sys.stdout, orig_stdout)
settings_kwargs = {}
settings_args = []
settings_kwargs['key_filename'] = os.path.expanduser(
os.path.expandvars(config.get(host, 'identity').strip()))
settings_kwargs['user'] = config.get(host, 'user')
debug = config.getboolean(host, 'debug')
if config.has_option(host, 'instance_id'): # start a stopped instance
inst_id = config.get(host, 'instance_id')
outf.write("starting instance %s from stopped instance\n" % host)
inst = start_instance(conn, inst_id, debug=debug)
terminate = False
else: # stand up an instance of the specified image
outf.write("starting instance %s from image\n" % host)
inst = start_instance_from_image(conn, config, host)
terminate = True
settings_kwargs['host_string'] = inst.public_dns_name
settings_kwargs['disable_known_hosts'] = True
outf.write("instance %s was started successfully. dns=%s\n" %
(host, inst.public_dns_name))
if debug:
settings_args.append(show('debug'))
else:
settings_args.append(hide('running'))
platform = config.get(host, 'platform')
if platform == 'windows':
settings_kwargs['shell'] = 'cmd /C'
else:
settings_kwargs['shell'] = '/bin/bash -l -c'
try:
with settings(**settings_kwargs):
# first, make sure that the connection is really working...
# on EC2 even if ssh connects there can be timeouts during authentication,
# so try to connect multiple times if there's a timeout
client = fab_connect(settings_kwargs['user'],
settings_kwargs['host_string'], debug=debug)
if debug:
outf.write("<%s>: calling %s\n" %
(host, print_funct_call(funct, **kwargs)))
retval = funct(**kwargs)
except (SystemExit, Exception) as err:
outf.write(str(err))
if isinstance(err, SystemExit) and err.code is not None:
retval = err.code
else:
retval = -1
# try to retrieve console output if we can
try:
out = inst.get_console_output().output
with open('console.out', 'wb') as f:
f.write(out)
except:
outf.write("\ncouldn't retrieve console output\n")
keep = kwargs.get('keep', False)
if retval == 0 or not keep:
if terminate is True:
if not terminate_instance(inst, host, orig_stdout, debug):
retval = -1
else:
if not stop_instance(inst, host, orig_stdout, debug):
retval = -1
else:
outf.write("run failed, so stopping %s instead of terminating it.\n" % host)
outf.write("%s will have to be terminated manually.\n" % host)
if not stop_instance(inst, host, orig_stdout, debug):
retval = -1
if retval != 0:
raise RuntimeError("return value = %s" % retval)
def run_on_ec2(host, config, conn, funct, outdir, **kwargs):
"""Runs the given function on an EC2 instance. The instance may be either
created from an image or may be an existing image that is stopped.
In either case, the instance will be started and the function will run.
If the instance was created from an image, it will be
terminated, unless there is an error or keep==True, which will result in
the instance being stopped but not terminated. If the instance was
pre-existing, then it will just be stopped instead of terminated.
"""
# put fabric import here to prevent sphinx doc generation failure
# on windows when win32api isn't installed
from fabric.api import settings, hide, show
hostdir = os.path.join(outdir, host)
if not os.path.isdir(hostdir):
os.makedirs(hostdir)
os.chdir(hostdir)
orig_stdout = sys.stdout
orig_stderr = sys.stderr
sys.stdout = sys.stderr = open('run.out', 'wb')
outf = MultiFile(sys.stdout, orig_stdout)
settings_kwargs = {}
settings_args = []
settings_kwargs['key_filename'] = os.path.expanduser(
os.path.expandvars(config.get(host, 'identity').strip()))
settings_kwargs['user'] = config.get(host, 'user')
debug = config.getboolean(host, 'debug')
if config.has_option(host, 'instance_id'): # start a stopped instance
inst_id = config.get(host, 'instance_id')
outf.write("starting instance %s from stopped instance\n" % host)
inst = start_instance(conn, inst_id, debug=debug)
terminate = False
else: # stand up an instance of the specified image
outf.write("starting instance %s from image\n" % host)
inst = start_instance_from_image(conn, config, host)
terminate = True
settings_kwargs['host_string'] = inst.public_dns_name
settings_kwargs['disable_known_hosts'] = True
outf.write("instance %s was started successfully. dns=%s\n" %
(host, inst.public_dns_name))
if debug:
settings_args.append(show('debug'))
else:
settings_args.append(hide('running'))
platform = config.get(host, 'platform')
if platform == 'windows':
settings_kwargs['shell'] = 'cmd /C'
else:
settings_kwargs['shell'] = '/bin/bash -l -c'
try:
with settings(**settings_kwargs):
# first, make sure that the connection is really working...
# on EC2 even if ssh connects there can be timeouts during authentication,
# so try to connect multiple times if there's a timeout
client = fab_connect(settings_kwargs['user'],
settings_kwargs['host_string'],
debug=debug)
if debug:
outf.write("<%s>: calling %s\n" %
(host, print_funct_call(funct, **kwargs)))
retval = funct(**kwargs)
except (SystemExit, Exception) as err:
outf.write(str(err))
if isinstance(err, SystemExit) and err.code is not None:
retval = err.code
else:
retval = -1
# try to retrieve console output if we can
try:
out = inst.get_console_output().output
with open('console.out', 'wb') as f:
f.write(out)
except:
outf.write("\ncouldn't retrieve console output\n")
keep = kwargs.get('keep', False)
if retval == 0 or not keep:
if terminate is True:
if not terminate_instance(inst, host, orig_stdout, debug):
retval = -1
else:
if not stop_instance(inst, host, orig_stdout, debug):
retval = -1
else:
outf.write("run failed, so stopping %s instead of terminating it.\n" %
host)
outf.write("%s will have to be terminated manually.\n" % host)
if not stop_instance(inst, host, orig_stdout, debug):
retval = -1
if retval != 0:
raise RuntimeError("return value = %s" % retval)
