def _run_complete(self, session, mgr, future):
logger.debug('Run Plan completed')
try:
response = future.result()
logger.debug("Run response of: %s", response)
except:
logger.error("Run did an exception", exc_info=True)
async def shutdown(setlink):
try:
await self._stop_step(setlink)
except:
logger.error("Exception in shutdown.", exc_info=True)
setlink.step_record.completed_at = datetime.utcnow()
self._db_session.commit()
def _started(self, setlink, fut):
"""Runs after a setlink has started."""
setlink.collection.started = True
setlink.running.started_at = datetime.utcnow()
self._db_session.commit()
try:
fut.result()
except:
logger.error("Exception starting.", exc_info=True)
def _stopped(self, setlink, fut):
"""Runs after a setlink has stopped."""
setlink.running.completed_at = datetime.utcnow()
self._db_session.commit()
try:
fut.result()
except:
logger.error("Exception in shutdown.", exc_info=True)
async def _get_steps(self):
"""Request all the step instances needed from the pool
This is a separate method as both the recover run and new run
will need to run this identically.
"""
logger.debug('Getting steps & collections')
steps = self.run.plan.steps
collections = await gen.multi([
self._pool.request_instances(self.run.uuid,
s.uuid,
count=s.instance_count,
inst_type=s.instance_type,
region=s.instance_region,
plan=self.run.plan.name,
owner=self.run.owner,
run_max_time=s.run_delay +
s.run_max_time) for s in steps
])
try:
# First, setup some dicst, all keyed by step.uuid
steps_by_uuid = {x.uuid: x for x in steps}
step_records_by_uuid = {
x.step.uuid: x
for x in self.run.step_records
}
# Link the step/step_record/ec2_collection under a single
# StepRecordLink tuple
for coll in collections:
step = steps_by_uuid[coll.uuid]
step_record = step_records_by_uuid[coll.uuid]
setlink = StepRecordLink(step_record, step, coll)
self._set_links.append(setlink)
except Exception:
# Ensure we return collections if something bad happened
logger.error("Got an exception in runner, returning instances",
exc_info=True)
try:
await gen.multi(
[self._pool.release_instances(x) for x in collections])
except:
logger.error("Wat? Got an error returning instances.",
exc_info=True)
# Clear out the setlinks to make sure they aren't cleaned up
# again
self._set_links = []
def _get_container_sets(self):
"""Request all the container sets instances needed from the pool
This is a separate method as both the recover run and new run
will need to run this identically.
"""
csets = self.run.strategy.container_sets
collections = yield [
self._pool.request_instances(self.run.uuid, c.uuid,
count=c.instance_count,
inst_type=c.instance_type,
region=c.instance_region)
for c in csets]
try:
# Setup the collection lookup info
coll_by_uuid = {x.uuid: x for x in csets}
running_by_uuid = {x.container_set.uuid: x
for x in self.run.running_container_sets}
for coll in collections:
meta = coll_by_uuid[coll.uuid]
running = running_by_uuid[coll.uuid]
setlink = ContainerSetLink(running, meta, coll)
self._set_links.append(setlink)
except Exception:
# Ensure we return collections if something bad happened
logger.error("Got an exception in runner, returning instances",
exc_info=True)
try:
yield [self._pool.release_instances(x) for x in collections]
except:
logger.error("Wat? Got an error returning instances.",
exc_info=True)
# Clear out the setlinks to make sure they aren't cleaned up
# again
self._set_links = []
def _cleanup(self, exc=False):
if exc:
# Ensure we try and shut them down
logger.debug("Exception occurred, ensure containers terminated.",
exc_info=True)
try:
yield [self._stop_set(s) for s in self._set_links]
except Exception:
logger.error("Le sigh, error shutting down instances.",
exc_info=True)
# Ensure we always release the collections we used
logger.debug("Returning collections")
try:
yield [self._pool.release_instances(x.collection)
for x in self._set_links]
except Exception:
logger.error("Embarassing, error returning instances.",
exc_info=True)
self._set_links = []
async def _cleanup(self, exc=False):
if exc:
# Ensure we try and shut them down
logger.debug("Exception occurred, ensure containers terminated.",
exc_info=True)
try:
await gen.multi([self._stop_step(s) for s in self._set_links])
except Exception:
logger.error("Le sigh, error shutting down instances.",
exc_info=True)
# Ensure we always release the collections we used
logger.debug("Returning collections")
try:
await gen.multi([
self._pool.release_instances(x.ec2_collection)
for x in self._set_links
])
except Exception:
logger.error("Embarassing, error returning instances.",
exc_info=True)
self._set_links = []
def _test(self, session, mgr, future):
try:
response = yield future
logger.debug("Got response of: %s", response)
except:
logger.error("Got an exception", exc_info=True)
def _check_containers(self):
# If we have a dns map, we can use dns now
# This is done at the top to ensure prior containers had a
# chance to run first
if self._dns_map:
self._use_dns = True
# First, only consider collections not completed
running_collections = [x for x in self._set_links
if not x.running.completed_at]
# Bools of collections that were started
started = [x.collection.started for x in running_collections]
# Bools of collections that are done
dones = yield [self._is_done(x) for x in running_collections]
# Send shutdown to all finished collections, we're not going
# to wait on these futures, they will save when they complete
for done, setlink in zip(dones, self._set_links):
if not done or setlink.collection.finished:
continue
try:
_ = yield self._stop_set(setlink)
except:
logger.error("Exception in shutdown.", exc_info=True)
self._stopped(setlink)
# If all have started and are done, the run is complete.
if all(dones) and all(started):
return True
# Locate container sets that need to be started
starts = yield [self._should_start(x) for x in self._set_links]
# Startup containers that should be started
for start, setlink in zip(starts, self._set_links):
if not start or setlink.collection.started:
continue
# We tag the collection here since this may not actually run
# until another time through this loop due to async nature
setlink.collection.local_dns = self._use_dns
try:
_ = yield self._start_set(setlink)
except:
logger.error("Exception starting.", exc_info=True)
self._started(setlink)
# If this collection reg's a dns name, add this collections
# ip's to the name
if setlink.meta.dns_name:
ips = [x.instance.ip_address for x
in setlink.collection.instances]
self._dns_map[setlink.meta.dns_name] = ips
return False
async def _check_steps(self):
"""Checks steps for the plan to see if any existing steps
have finished, or new ones need to start.
When all the steps have run and completed, returns False
to indicate nothing remains for the plan.
"""
# Bools of collections that were started/finished
started = [x.ec2_collection.started for x in self._set_links]
finished = [x.ec2_collection.finished for x in self._set_links]
# If all steps were started and finished, the run is complete.
if all(started) and all(finished):
return True
# Locate all steps that have completed
dones = await gen.multi([self._is_done(x) for x in self._set_links])
dones = zip(dones, self._set_links)
# Send shutdown to steps that have completed, we can shut them all
# down in any order so we run in parallel
async def shutdown(setlink):
try:
await self._stop_step(setlink)
except:
logger.error("Exception in shutdown.", exc_info=True)
setlink.step_record.completed_at = datetime.utcnow()
self._db_session.commit()
await gen.multi([shutdown(s) for done, s in dones if done])
# Start steps that should be started, ordered by delay
starts = list(filter(self._should_start, self._set_links))
starts.sort(key=lambda x: x.step.run_delay)
# Start steps in order of lowest delay first, to ensure that steps
# started afterwards can use DNS names/etc from prior steps
for setlink in starts:
# We tag the collection here since this may not actually run
# until another time through this loop due to async nature
setlink.ec2_collection.local_dns = bool(self._dns_map)
try:
await self._start_step(setlink)
except:
logger.error("Exception starting.", exc_info=True)
setlink.step_record.failed = True
setlink.step_record.started_at = datetime.utcnow()
self._db_session.commit()
# If this collection reg's a dns name, add this collections
# ip's to the name
if setlink.step.dns_name:
ips = [
x.instance.ip_address
for x in setlink.ec2_collection.instances
]
self._dns_map[setlink.step.dns_name] = ips
return False
def _handle_request_exception(self, e):
logger.error(e)
self.write_error(status=500, message=str(e))