def push_docs(commit_id, doc_host):
if DEVDOCS_DIR and doc_host is not None:
tarname = 'html.tar.gz'
tarpath = os.path.join(get_commit_dir(commit_id), 'host_results',
doc_host, tarname)
try:
if not os.path.isfile(tarpath):
raise OSError("docs tar file '%s' was not found" % tarfile)
cmds = [
'tar xzf %s' % tarname,
'rm -rf dev_docs',
'mv html dev_docs',
'rm -f %s' % tarname,
]
if socket.gethostname() == DOC_DEST_HOST: # local, so don't use fabric
print "docs are already local"
startdir = os.getcwd()
try:
print "changing dir to %s" % os.path.join(os.environ['HOME'], DEVDOCS_DIR)
os.chdir(os.path.join(os.environ['HOME'], DEVDOCS_DIR))
print "copying %s to %s" % (tarpath, tarname)
shutil.copy(tarpath, tarname)
for cmd in cmds:
print "running cmd: %s" % cmd
subprocess.check_call(cmd, cwd=os.getcwd(), shell=True)
finally:
os.chdir(startdir)
else:
print "docs are remote. push to doc dest host %s" % DOC_DEST_HOST
with settings(host_string='openmdao@%s' % DOC_DEST_HOST):
# tar up the docs so we can upload them to the server
# put the docs on the server and untar them
put(tarpath, '%s/%s' % (DEVDOCS_DIR, tarname))
with cd(DEVDOCS_DIR):
for cmd in cmds:
run(cmd)
except Exception as err:
log('ERROR: push_docs failed: %s' % str(err))
out = str(err)
ret = -1
else:
log('push_docs was successful')
out = 'Docs built successfully'
ret = 0
model.new_doc_info(commit_id, out)
return out, ret
else:
log('push_docs was skipped')
return '', 0 # allow update of production dev docs to be turned off during debugging
def push_docs(commit_id, doc_host):
if DEVDOCS_DIR and doc_host is not None:
tarname = 'html.tar.gz'
tarpath = os.path.join(get_commit_dir(commit_id), 'host_results',
doc_host, tarname)
try:
with settings(host_string='*****@*****.**'):
# tar up the docs so we can upload them to the server
# put the docs on the server and untar them
put(tarpath, '%s/%s' % (DEVDOCS_DIR, tarname))
with cd(DEVDOCS_DIR):
run('tar xzf %s' % tarname)
run('rm -rf dev_docs')
run('mv html dev_docs')
run('rm -f %s' % tarname)
except Exception as err:
log('ERROR: push_docs failed: %s' % str(err))
out = str(err)
ret = -1
else:
log('push_docs was successful')
out = 'Docs built successfully'
ret = 0
model.new_doc_info(commit_id, out)
return out, ret
else:
log('push_docs was skipped')
return '', 0 # allow update of production dev docs to be turned off during debugging
def run_on_ec2(host, config, conn, funct, outdir, **kwargs):
"""Runs the given function on an EC2 instance. The instance may be either
created from an image or may be an existing image that is stopped.
In either case, the instance will be started and the function will run.
If the instance was created from an image, it will be
terminated unless there is an error or keep==True, which will result in
the instance being stopped but not terminated. If the instance was
pre-existing, then it will just be stopped instead of terminated.
"""
# put fabric import here to prevent sphinx doc generation failure
# on windows when win32api isn't installed
from fabric.api import settings, hide, show
hostdir = os.path.join(outdir, host)
if not os.path.isdir(hostdir):
os.makedirs(hostdir)
os.chdir(hostdir)
orig_stdout = sys.stdout
orig_stderr = sys.stderr
sys.stdout = sys.stderr = open('run.out', 'wb')
outf = MultiFile(sys.stdout, orig_stdout)
settings_kwargs = {}
settings_args = []
settings_kwargs['key_filename'] = os.path.expanduser(
os.path.expandvars(config.get(host, 'identity').strip()))
settings_kwargs['user'] = config.get(host, 'user')
debug = config.getboolean(host, 'debug')
if config.has_option(host, 'instance_id'): # start a stopped instance
inst_id = config.get(host, 'instance_id')
outf.write("starting instance %s from stopped instance\n" % host)
inst = start_instance(conn, inst_id, debug=debug)
terminate = False
else: # stand up an instance of the specified image
outf.write("starting instance %s from image\n" % host)
inst = start_instance_from_image(conn, config, host, stream=outf)
terminate = True
settings_kwargs['host_string'] = inst.public_dns_name
settings_kwargs['disable_known_hosts'] = True
outf.write("instance %s was started successfully. dns=%s\n" %
(host, inst.public_dns_name))
if debug:
settings_args.append(show('debug'))
else:
settings_args.append(hide('running'))
platform = config.get(host, 'platform')
if platform == 'windows':
settings_kwargs['shell'] = 'cmd /C'
else:
settings_kwargs['shell'] = '/bin/bash -l -c'
try:
with settings(**settings_kwargs):
# first, make sure that the connection is really working...
# on EC2 even if ssh connects there can be timeouts during authentication,
# so try to connect multiple times if there's a timeout
client = fab_connect(settings_kwargs['user'],
settings_kwargs['host_string'],
debug=debug)
if debug:
outf.write("<%s>: calling %s\n" %
(host, print_funct_call(funct, **kwargs)))
py = config.get(host, 'py')
if platform.startswith('win') and '.' in py:
# convert pythonX.Y form over to C:/PythonXY/python.exe
ver = py[6:]
py = 'C:/Python%s/python.exe' % ver.replace('.', '')
max_tries = 25
sleep = 10
for i in range(max_tries):
time.sleep(sleep)
if debug:
print "testing python call over fabric connection (try #%d)" % (
i + 1)
# even though the instance is running, it takes a while (on Windows) before
# python actually works, so try some simple python command until it works.
try:
retval = run('%s -c "dir()"' % py)
except:
pass
else:
if retval.failed or retval.return_code != 0:
continue
else:
break
else:
outf.write("\nrunning python via fabric on %s failed after %d"
" attempts. terminating...\n" % (host, max_tries))
terminate_instance(inst, host, outf, debug)
raise RuntimeError("couldn't run python on %s via fabric" %
host)
retval = funct(**kwargs)
except (SystemExit, Exception) as err:
outf.write(str(err))
if isinstance(err, SystemExit) and err.code is not None:
retval = err.code
else:
retval = -1
# try to retrieve console output if we can
try:
out = inst.get_console_output().output
with open('console.out', 'wb') as f:
f.write(out)
except:
outf.write("\ncouldn't retrieve console output\n")
keep = kwargs.get('keep', False)
if keep or not terminate:
outf.write("stopping %s instead of terminating it.\n" % host)
outf.write("%s will have to be terminated manually.\n" % host)
if not stop_instance(inst, host, orig_stdout, debug):
retval = -1
else:
if not terminate_instance(inst, host, orig_stdout, debug):
retval = -1
if retval != 0:
raise RuntimeError("return value = %s" % retval)
def run_on_ec2(host, config, conn, funct, outdir, **kwargs):
"""Runs the given function on an EC2 instance. The instance may be either
created from an image or may be an existing image that is stopped.
In either case, the instance will be started and the function will run.
If the instance was created from an image, it will be
terminated unless there is an error or keep==True, which will result in
the instance being stopped but not terminated. If the instance was
pre-existing, then it will just be stopped instead of terminated.
"""
# put fabric import here to prevent sphinx doc generation failure
# on windows when win32api isn't installed
from fabric.api import settings, hide, show
hostdir = os.path.join(outdir, host)
if not os.path.isdir(hostdir):
os.makedirs(hostdir)
os.chdir(hostdir)
orig_stdout = sys.stdout
orig_stderr = sys.stderr
sys.stdout = sys.stderr = open('run.out', 'wb')
outf = MultiFile(sys.stdout, orig_stdout)
settings_kwargs = {}
settings_args = []
settings_kwargs['key_filename'] = os.path.expanduser(
os.path.expandvars(config.get(host, 'identity').strip()))
settings_kwargs['user'] = config.get(host, 'user')
debug = config.getboolean(host, 'debug')
if config.has_option(host, 'instance_id'): # start a stopped instance
inst_id = config.get(host, 'instance_id')
outf.write("starting instance %s from stopped instance\n" % host)
inst = start_instance(conn, inst_id, debug=debug)
terminate = False
else: # stand up an instance of the specified image
outf.write("starting instance %s from image\n" % host)
inst = start_instance_from_image(conn, config, host, stream=outf)
terminate = True
settings_kwargs['host_string'] = inst.public_dns_name
settings_kwargs['disable_known_hosts'] = True
outf.write("instance %s was started successfully. dns=%s\n" %
(host, inst.public_dns_name))
if debug:
settings_args.append(show('debug'))
else:
settings_args.append(hide('running'))
platform = config.get(host, 'platform')
if platform == 'windows':
settings_kwargs['shell'] = 'cmd /C'
else:
settings_kwargs['shell'] = '/bin/bash -l -c'
try:
with settings(**settings_kwargs):
# first, make sure that the connection is really working...
# on EC2 even if ssh connects there can be timeouts during authentication,
# so try to connect multiple times if there's a timeout
client = fab_connect(settings_kwargs['user'],
settings_kwargs['host_string'], debug=debug)
if debug:
outf.write("<%s>: calling %s\n" %
(host, print_funct_call(funct, **kwargs)))
py = config.get(host, 'py')
if platform.startswith('win') and '.' in py:
# convert pythonX.Y form over to C:/PythonXY/python.exe
ver = py[6:]
py = 'C:/Python%s/python.exe' % ver.replace('.','')
max_tries = 25
sleep = 10
for i in range(max_tries):
time.sleep(sleep)
if debug:
print "testing python call over fabric connection (try #%d)" % (i+1)
# even though the instance is running, it takes a while (on Windows) before
# python actually works, so try some simple python command until it works.
try:
retval = run('%s -c "dir()"' % py)
except Exception as exc:
print ' caught exception:', exc
else:
if retval.failed:
print ' failed:', retval.failed
elif retval.return_code != 0:
print ' return code:', retval.return_code
else:
break
else:
outf.write("\nrunning python via fabric on %s failed after %d"
" attempts. terminating...\n" % (host, max_tries))
terminate_instance(inst, host, outf, debug)
raise RuntimeError("couldn't run python on %s via fabric" % host)
retval = funct(**kwargs)
except (SystemExit, Exception) as err:
outf.write(str(err))
if isinstance(err, SystemExit) and err.code is not None:
retval = err.code
else:
retval = -1
# try to retrieve console output if we can
try:
out = inst.get_console_output().output
with open('console.out', 'wb') as f:
f.write(out)
except:
outf.write("\ncouldn't retrieve console output\n")
keep = kwargs.get('keep', False)
if keep or not terminate:
outf.write("stopping %s instead of terminating it.\n" % host)
outf.write("%s will have to be terminated manually.\n" % host)
if not stop_instance(inst, host, orig_stdout, debug):
retval = -1
else:
if not terminate_instance(inst, host, orig_stdout, debug):
retval = -1
if retval != 0:
raise RuntimeError("return value = %s" % retval)
