def run_on_host(host, config, conn, funct, outdir, **kwargs): """Runs the given funct on the specified host.""" from fabric.api import settings, show, hide hostdir = os.path.join(outdir, host) if not os.path.isdir(hostdir): os.makedirs(hostdir) os.chdir(hostdir) orig_stdout = sys.stdout orig_stderr = sys.stderr print '<%s>: writing stdout/stderr to %s' % (host, os.path.join(os.getcwd(), 'run.out')) sys.stdout = sys.stderr = open('run.out', 'wb') settings_kwargs = {} settings_args = [] debug = config.getboolean(host, 'debug') settings_kwargs['host_string'] = config.get(host, 'addr', None) if config.has_option(host, 'identity'): settings_kwargs['key_filename'] = os.path.expanduser( os.path.expandvars(config.get(host, 'identity'))) if config.has_option(host, 'user'): settings_kwargs['user'] = config.get(host, 'user') platform = config.get(host, 'platform') if platform == 'windows': settings_kwargs['shell'] = 'cmd /C' else: settings_kwargs['shell'] = '/bin/bash -l -c' if debug: settings_args.append(show('debug')) orig_stdout.write("<%s>: calling %s" % (host, print_funct_call(funct, **kwargs))) else: settings_args.append(hide('running')) with settings(*settings_args, **settings_kwargs): return funct(**kwargs)
def run_on_host(host, config, conn, funct, outdir, **kwargs): """Runs the given funct on the specified host.""" from fabric.api import settings, show, hide hostdir = os.path.join(outdir, host) if not os.path.isdir(hostdir): os.makedirs(hostdir) os.chdir(hostdir) orig_stdout = sys.stdout orig_stderr = sys.stderr print '<%s>: writing stdout/stderr to %s' % ( host, os.path.join(os.getcwd(), 'run.out')) sys.stdout = sys.stderr = open('run.out', 'wb') settings_kwargs = {} settings_args = [] debug = config.getboolean(host, 'debug') settings_kwargs['host_string'] = config.get(host, 'addr', None) if config.has_option(host, 'identity'): settings_kwargs['key_filename'] = os.path.expanduser( os.path.expandvars(config.get(host, 'identity'))) if config.has_option(host, 'user'): settings_kwargs['user'] = config.get(host, 'user') platform = config.get(host, 'platform') if platform == 'windows': settings_kwargs['shell'] = 'cmd /C' else: settings_kwargs['shell'] = '/bin/bash -l -c' if debug: settings_args.append(show('debug')) orig_stdout.write("<%s>: calling %s" % (host, print_funct_call(funct, **kwargs))) else: settings_args.append(hide('running')) with settings(*settings_args, **settings_kwargs): return funct(**kwargs)
def run_on_ec2(host, config, conn, funct, outdir, **kwargs): """Runs the given function on an EC2 instance. The instance may be either created from an image or may be an existing image that is stopped. In either case, the instance will be started and the function will run. If the instance was created from an image, it will be terminated unless there is an error or keep==True, which will result in the instance being stopped but not terminated. If the instance was pre-existing, then it will just be stopped instead of terminated. """ # put fabric import here to prevent sphinx doc generation failure # on windows when win32api isn't installed from fabric.api import settings, hide, show hostdir = os.path.join(outdir, host) if not os.path.isdir(hostdir): os.makedirs(hostdir) os.chdir(hostdir) orig_stdout = sys.stdout orig_stderr = sys.stderr sys.stdout = sys.stderr = open('run.out', 'wb') outf = MultiFile(sys.stdout, orig_stdout) settings_kwargs = {} settings_args = [] settings_kwargs['key_filename'] = os.path.expanduser( os.path.expandvars(config.get(host, 'identity').strip())) settings_kwargs['user'] = config.get(host, 'user') debug = config.getboolean(host, 'debug') if config.has_option(host, 'instance_id'): # start a stopped instance inst_id = config.get(host, 'instance_id') outf.write("starting instance %s from stopped instance\n" % host) inst = start_instance(conn, inst_id, debug=debug) terminate = False else: # stand up an instance of the specified image outf.write("starting instance %s from image\n" % host) inst = start_instance_from_image(conn, config, host, stream=outf) terminate = True settings_kwargs['host_string'] = inst.public_dns_name settings_kwargs['disable_known_hosts'] = True outf.write("instance %s was started successfully. dns=%s\n" % (host, inst.public_dns_name)) if debug: settings_args.append(show('debug')) else: settings_args.append(hide('running')) platform = config.get(host, 'platform') if platform == 'windows': settings_kwargs['shell'] = 'cmd /C' else: settings_kwargs['shell'] = '/bin/bash -l -c' try: with settings(**settings_kwargs): # first, make sure that the connection is really working... # on EC2 even if ssh connects there can be timeouts during authentication, # so try to connect multiple times if there's a timeout client = fab_connect(settings_kwargs['user'], settings_kwargs['host_string'], debug=debug) if debug: outf.write("<%s>: calling %s\n" % (host, print_funct_call(funct, **kwargs))) py = config.get(host, 'py') if platform.startswith('win') and '.' in py: # convert pythonX.Y form over to C:/PythonXY/python.exe ver = py[6:] py = 'C:/Python%s/python.exe' % ver.replace('.', '') max_tries = 25 sleep = 10 for i in range(max_tries): time.sleep(sleep) if debug: print "testing python call over fabric connection (try #%d)" % ( i + 1) # even though the instance is running, it takes a while (on Windows) before # python actually works, so try some simple python command until it works. try: retval = run('%s -c "dir()"' % py) except: pass else: if retval.failed or retval.return_code != 0: continue else: break else: outf.write("\nrunning python via fabric on %s failed after %d" " attempts. terminating...\n" % (host, max_tries)) terminate_instance(inst, host, outf, debug) raise RuntimeError("couldn't run python on %s via fabric" % host) retval = funct(**kwargs) except (SystemExit, Exception) as err: outf.write(str(err)) if isinstance(err, SystemExit) and err.code is not None: retval = err.code else: retval = -1 # try to retrieve console output if we can try: out = inst.get_console_output().output with open('console.out', 'wb') as f: f.write(out) except: outf.write("\ncouldn't retrieve console output\n") keep = kwargs.get('keep', False) if keep or not terminate: outf.write("stopping %s instead of terminating it.\n" % host) outf.write("%s will have to be terminated manually.\n" % host) if not stop_instance(inst, host, orig_stdout, debug): retval = -1 else: if not terminate_instance(inst, host, orig_stdout, debug): retval = -1 if retval != 0: raise RuntimeError("return value = %s" % retval)
def run_on_ec2(host, config, conn, funct, outdir, **kwargs): """Runs the given function on an EC2 instance. The instance may be either created from an image or may be an existing image that is stopped. In either case, the instance will be started and the function will run. If the instance was created from an image, it will be terminated unless there is an error or keep==True, which will result in the instance being stopped but not terminated. If the instance was pre-existing, then it will just be stopped instead of terminated. """ # put fabric import here to prevent sphinx doc generation failure # on windows when win32api isn't installed from fabric.api import settings, hide, show hostdir = os.path.join(outdir, host) if not os.path.isdir(hostdir): os.makedirs(hostdir) os.chdir(hostdir) orig_stdout = sys.stdout orig_stderr = sys.stderr sys.stdout = sys.stderr = open('run.out', 'wb') outf = MultiFile(sys.stdout, orig_stdout) settings_kwargs = {} settings_args = [] settings_kwargs['key_filename'] = os.path.expanduser( os.path.expandvars(config.get(host, 'identity').strip())) settings_kwargs['user'] = config.get(host, 'user') debug = config.getboolean(host, 'debug') if config.has_option(host, 'instance_id'): # start a stopped instance inst_id = config.get(host, 'instance_id') outf.write("starting instance %s from stopped instance\n" % host) inst = start_instance(conn, inst_id, debug=debug) terminate = False else: # stand up an instance of the specified image outf.write("starting instance %s from image\n" % host) inst = start_instance_from_image(conn, config, host, stream=outf) terminate = True settings_kwargs['host_string'] = inst.public_dns_name settings_kwargs['disable_known_hosts'] = True outf.write("instance %s was started successfully. dns=%s\n" % (host, inst.public_dns_name)) if debug: settings_args.append(show('debug')) else: settings_args.append(hide('running')) platform = config.get(host, 'platform') if platform == 'windows': settings_kwargs['shell'] = 'cmd /C' else: settings_kwargs['shell'] = '/bin/bash -l -c' try: with settings(**settings_kwargs): # first, make sure that the connection is really working... # on EC2 even if ssh connects there can be timeouts during authentication, # so try to connect multiple times if there's a timeout client = fab_connect(settings_kwargs['user'], settings_kwargs['host_string'], debug=debug) if debug: outf.write("<%s>: calling %s\n" % (host, print_funct_call(funct, **kwargs))) py = config.get(host, 'py') if platform.startswith('win') and '.' in py: # convert pythonX.Y form over to C:/PythonXY/python.exe ver = py[6:] py = 'C:/Python%s/python.exe' % ver.replace('.','') max_tries = 25 sleep = 10 for i in range(max_tries): time.sleep(sleep) if debug: print "testing python call over fabric connection (try #%d)" % (i+1) # even though the instance is running, it takes a while (on Windows) before # python actually works, so try some simple python command until it works. try: retval = run('%s -c "dir()"' % py) except Exception as exc: print ' caught exception:', exc else: if retval.failed: print ' failed:', retval.failed elif retval.return_code != 0: print ' return code:', retval.return_code else: break else: outf.write("\nrunning python via fabric on %s failed after %d" " attempts. terminating...\n" % (host, max_tries)) terminate_instance(inst, host, outf, debug) raise RuntimeError("couldn't run python on %s via fabric" % host) retval = funct(**kwargs) except (SystemExit, Exception) as err: outf.write(str(err)) if isinstance(err, SystemExit) and err.code is not None: retval = err.code else: retval = -1 # try to retrieve console output if we can try: out = inst.get_console_output().output with open('console.out', 'wb') as f: f.write(out) except: outf.write("\ncouldn't retrieve console output\n") keep = kwargs.get('keep', False) if keep or not terminate: outf.write("stopping %s instead of terminating it.\n" % host) outf.write("%s will have to be terminated manually.\n" % host) if not stop_instance(inst, host, orig_stdout, debug): retval = -1 else: if not terminate_instance(inst, host, orig_stdout, debug): retval = -1 if retval != 0: raise RuntimeError("return value = %s" % retval)
def run_on_ec2(host, config, conn, funct, outdir, **kwargs): """Runs the given function on an EC2 instance. The instance may be either created from an image or may be an existing image that is stopped. In either case, the instance will be started and the function will run. If the instance was created from an image, it will be terminated, unless there is an error or keep==True, which will result in the instance being stopped but not terminated. If the instance was pre-existing, then it will just be stopped instead of terminated. """ # put fabric import here to prevent sphinx doc generation failure # on windows when win32api isn't installed from fabric.api import settings, hide, show hostdir = os.path.join(outdir, host) if not os.path.isdir(hostdir): os.makedirs(hostdir) os.chdir(hostdir) orig_stdout = sys.stdout orig_stderr = sys.stderr sys.stdout = sys.stderr = open('run.out', 'wb') outf = MultiFile(sys.stdout, orig_stdout) settings_kwargs = {} settings_args = [] settings_kwargs['key_filename'] = os.path.expanduser( os.path.expandvars(config.get(host, 'identity').strip())) settings_kwargs['user'] = config.get(host, 'user') debug = config.getboolean(host, 'debug') if config.has_option(host, 'instance_id'): # start a stopped instance inst_id = config.get(host, 'instance_id') outf.write("starting instance %s from stopped instance\n" % host) inst = start_instance(conn, inst_id, debug=debug) terminate = False else: # stand up an instance of the specified image outf.write("starting instance %s from image\n" % host) inst = start_instance_from_image(conn, config, host) terminate = True settings_kwargs['host_string'] = inst.public_dns_name settings_kwargs['disable_known_hosts'] = True outf.write("instance %s was started successfully. dns=%s\n" % (host, inst.public_dns_name)) if debug: settings_args.append(show('debug')) else: settings_args.append(hide('running')) platform = config.get(host, 'platform') if platform == 'windows': settings_kwargs['shell'] = 'cmd /C' else: settings_kwargs['shell'] = '/bin/bash -l -c' try: with settings(**settings_kwargs): # first, make sure that the connection is really working... # on EC2 even if ssh connects there can be timeouts during authentication, # so try to connect multiple times if there's a timeout client = fab_connect(settings_kwargs['user'], settings_kwargs['host_string'], debug=debug) if debug: outf.write("<%s>: calling %s\n" % (host, print_funct_call(funct, **kwargs))) retval = funct(**kwargs) except (SystemExit, Exception) as err: outf.write(str(err)) if isinstance(err, SystemExit) and err.code is not None: retval = err.code else: retval = -1 # try to retrieve console output if we can try: out = inst.get_console_output().output with open('console.out', 'wb') as f: f.write(out) except: outf.write("\ncouldn't retrieve console output\n") keep = kwargs.get('keep', False) if retval == 0 or not keep: if terminate is True: if not terminate_instance(inst, host, orig_stdout, debug): retval = -1 else: if not stop_instance(inst, host, orig_stdout, debug): retval = -1 else: outf.write("run failed, so stopping %s instead of terminating it.\n" % host) outf.write("%s will have to be terminated manually.\n" % host) if not stop_instance(inst, host, orig_stdout, debug): retval = -1 if retval != 0: raise RuntimeError("return value = %s" % retval)