def thread():
def callback():
return not self.event_stop.is_set()
hub = Hub(taskconf.hub_apikey)
i = None
try:
for i, instance in enumerate(
hub.launch(new_workers,
VerboseLog(session_logs.manager),
callback, **taskconf.ec2_opts)):
launchq.put(instance)
except Exception, e:
unlaunched_workers = new_workers - (i + 1) \
if i is not None \
else new_workers
for i in range(unlaunched_workers):
launchq.put(None)
if not isinstance(e, hub.Stopped):
traceback.print_exc(file=session_logs.manager)
class CloudWorker:
SSH_PING_RETRIES = 3
Terminated = Parallelize.Worker.Terminated
class Error(Terminated):
pass
@classmethod
def _stop_handler(cls, event_stop):
def func():
if not event_stop:
return
if event_stop.is_set():
raise cls.Terminated
return func
def __init__(self,
session_logs,
taskconf,
sshkey,
ipaddress=None,
destroy=None,
event_stop=None,
launchq=None):
self.pid = os.getpid()
if event_stop:
signal.signal(signal.SIGINT, signal.SIG_IGN)
self.event_stop = event_stop
self.logs = session_logs
self.sshkey = sshkey
self.strikes = taskconf.strikes
self.strike = 0
self.timeout = taskconf.timeout
self.cleanup_command = taskconf.post
self.user = taskconf.user
self.ipaddress = ipaddress
self.instanceid = None
self.hub = None
self.ssh = None
if destroy is None:
if ipaddress:
destroy = False
else:
destroy = True
self.destroy = destroy
if not ipaddress:
if not taskconf.hub_apikey:
raise self.Error(
"can't auto launch a worker without a Hub API KEY")
self.hub = Hub(taskconf.hub_apikey)
if launchq:
with sighandle.sigignore(signal.SIGINT, signal.SIGTERM):
instance = launchq.get()
else:
class Bool:
value = False
stopped = Bool()
def handler(s, f):
stopped.value = True
with sighandle.sighandle(handler, signal.SIGINT,
signal.SIGTERM):
def callback():
return not stopped.value
instance = list(
self.hub.launch(1, VerboseLog(session_logs.manager),
callback, **taskconf.ec2_opts))[0]
if not instance or (event_stop and event_stop.is_set()):
raise self.Terminated
self.ipaddress, self.instanceid = instance
self.status("launched worker %s" % self.instanceid)
else:
self.status("using existing worker")
self.handle_stop = self._stop_handler(event_stop)
try:
self.ssh = SSH(self.ipaddress,
identity_file=self.sshkey.path,
login_name=taskconf.user,
callback=self.handle_stop)
except SSH.Error, e:
self.status("unreachable via ssh: " + str(e))
traceback.print_exc(file=self.logs.worker)
raise self.Error(e)
try:
self.ssh.copy_id(self.sshkey)
if taskconf.overlay:
self.ssh.apply_overlay(taskconf.overlay)
if taskconf.pre:
self.ssh.command(taskconf.pre).close()
except Exception, e:
self.status("setup failed")
traceback.print_exc(file=self.logs.worker)
raise self.Error(e)
