def terminate_pool_instances(self, pool, instances, config, terminateByPool=False):
""" Terminate an instance with the given configuration """
instance_ids_by_region = self.get_instance_ids_by_region(instances)
for region in instance_ids_by_region:
cluster = Laniakea(None)
try:
cluster.connect(region=region, aws_access_key_id=config.aws_access_key_id, aws_secret_access_key=config.aws_secret_access_key)
except Exception as msg:
logger.exception("[Pool %d] %s: laniakea failure: %s" % (pool.id, "terminate_pool_instances", msg))
return None
try:
if terminateByPool:
boto_instances = cluster.find(filters={"tag:SpotManager-PoolId" : str(pool.pk)})
# Data consistency checks
for boto_instance in boto_instances:
assert ((boto_instance.id in instance_ids_by_region[region])
or (boto_instance.state_code == INSTANCE_STATE['shutting-down']
or boto_instance.state_code == INSTANCE_STATE['terminated']))
cluster.terminate(boto_instances)
else:
logger.info("[Pool %d] Terminating %s instances in region %s" % (pool.id, len(instance_ids_by_region[region]),region))
cluster.terminate(cluster.find(instance_ids=instance_ids_by_region[region]))
except boto.exception.EC2ResponseError as msg:
logger.exception("[Pool %d] %s: boto failure: %s" % (pool.id, "terminate_pool_instances", msg))
return 1
def _terminate_pool_instances(pool, instances, config, terminateByPool=False):
""" Terminate an instance with the given configuration """
from .models import INSTANCE_STATE, PoolStatusEntry, POOL_STATUS_ENTRY_TYPE
instance_ids_by_region = _get_instance_ids_by_region(instances)
for region in instance_ids_by_region:
cluster = Laniakea(None)
try:
cluster.connect(region=region,
aws_access_key_id=config.aws_access_key_id,
aws_secret_access_key=config.aws_secret_access_key)
except Exception as msg:
# Log this error to the pool status messages
entry = PoolStatusEntry()
entry.type = POOL_STATUS_ENTRY_TYPE['unclassified']
entry.pool = pool
entry.msg = str(msg)
entry.isCritical = True
entry.save()
logger.exception(
"[Pool %d] terminate_pool_instances: laniakea failure: %s",
pool.id, msg)
return None
try:
if terminateByPool:
boto_instances = cluster.find(
filters={"tag:" + SPOTMGR_TAG + "-PoolId": str(pool.pk)})
# Data consistency checks
for boto_instance in boto_instances:
# state_code is a 16-bit value where the high byte is
# an opaque internal value and should be ignored.
state_code = boto_instance.state_code & 255
if not (
(boto_instance.id in instance_ids_by_region[region]) or
(state_code == INSTANCE_STATE['shutting-down']
or state_code == INSTANCE_STATE['terminated'])):
logger.error(
"[Pool %d] Instance with EC2 ID %s (status %d) "
"is not in region list for region %s", pool.id,
boto_instance.id, state_code, region)
cluster.terminate(boto_instances)
else:
logger.info("[Pool %d] Terminating %s instances in region %s",
pool.id, len(instance_ids_by_region[region]),
region)
cluster.terminate(
cluster.find(instance_ids=instance_ids_by_region[region]))
except (boto.exception.EC2ResponseError,
boto.exception.BotoServerError, ssl.SSLError,
socket.error) as msg:
logger.exception(
"[Pool %d] terminate_pool_instances: boto failure: %s",
pool.id, msg)
return 1
def terminate_pool_instances(self,
pool,
instances,
config,
terminateByPool=False):
""" Terminate an instance with the given configuration """
instance_ids_by_region = self.get_instance_ids_by_region(instances)
for region in instance_ids_by_region:
cluster = Laniakea(None)
try:
cluster.connect(
region=region,
aws_access_key_id=config.aws_access_key_id,
aws_secret_access_key=config.aws_secret_access_key)
except Exception as msg:
# Log this error to the pool status messages
entry = PoolStatusEntry()
entry.type = 0
entry.pool = pool
entry.msg = str(msg)
entry.isCritical = True
entry.save()
logger.exception("[Pool %d] %s: laniakea failure: %s" %
(pool.id, "terminate_pool_instances", msg))
return None
try:
if terminateByPool:
boto_instances = cluster.find(
filters={"tag:SpotManager-PoolId": str(pool.pk)})
# Data consistency checks
for boto_instance in boto_instances:
if not ((boto_instance.id
in instance_ids_by_region[region]) or
(boto_instance.state_code
== INSTANCE_STATE['shutting-down']
or boto_instance.state_code
== INSTANCE_STATE['terminated'])):
logger.error(
"[Pool %d] Instance with EC2 ID %s (status %d) is not in region list for region %s"
% (pool.id, boto_instance.id,
boto_instance.state_code, region))
cluster.terminate(boto_instances)
else:
logger.info(
"[Pool %d] Terminating %s instances in region %s" %
(pool.id, len(instance_ids_by_region[region]), region))
cluster.terminate(
cluster.find(
instance_ids=instance_ids_by_region[region]))
except boto.exception.EC2ResponseError as msg:
logger.exception("[Pool %d] %s: boto failure: %s" %
(pool.id, "terminate_pool_instances", msg))
return 1
def terminate_pool_instances(self,
pool,
instances,
config,
terminateByPool=False):
""" Terminate an instance with the given configuration """
instance_ids_by_region = self.get_instance_ids_by_region(instances)
for region in instance_ids_by_region:
cluster = Laniakea(None)
try:
cluster.connect(
region=region,
aws_access_key_id=config.aws_access_key_id,
aws_secret_access_key=config.aws_secret_access_key)
except Exception as msg:
logger.exception("[Pool %d] %s: laniakea failure: %s" %
(pool.id, "terminate_pool_instances", msg))
return None
try:
if terminateByPool:
boto_instances = cluster.find(
filters={"tag:SpotManager-PoolId": str(pool.pk)})
# Data consistency checks
for boto_instance in boto_instances:
assert ((boto_instance.id
in instance_ids_by_region[region])
or (boto_instance.state_code
== INSTANCE_STATE['shutting-down']
or boto_instance.state_code
== INSTANCE_STATE['terminated']))
cluster.terminate(boto_instances)
else:
logger.info(
"[Pool %d] Terminating %s instances in region %s" %
(pool.id, len(instance_ids_by_region[region]), region))
cluster.terminate(
cluster.find(
instance_ids=instance_ids_by_region[region]))
except boto.exception.EC2ResponseError as msg:
logger.exception("[Pool %d] %s: boto failure: %s" %
(pool.id, "terminate_pool_instances", msg))
return 1
def terminate_pool_instances(self, pool, instances, config, terminateByPool=False):
""" Terminate an instance with the given configuration """
instance_ids_by_region = self.get_instance_ids_by_region(instances)
for region in instance_ids_by_region:
cluster = Laniakea(None)
try:
cluster.connect(region=region, aws_access_key_id=config.aws_access_key_id, aws_secret_access_key=config.aws_secret_access_key)
except Exception as msg:
# Log this error to the pool status messages
entry = PoolStatusEntry()
entry.type = 0
entry.pool = pool
entry.msg = str(msg)
entry.isCritical = True
entry.save()
logger.exception("[Pool %d] %s: laniakea failure: %s" % (pool.id, "terminate_pool_instances", msg))
return None
try:
if terminateByPool:
boto_instances = cluster.find(filters={"tag:SpotManager-PoolId" : str(pool.pk)})
# Data consistency checks
for boto_instance in boto_instances:
# state_code is a 16-bit value where the high byte is
# an opaque internal value and should be ignored.
state_code = boto_instance.state_code & 255
if not ((boto_instance.id in instance_ids_by_region[region])
or (state_code == INSTANCE_STATE['shutting-down']
or state_code == INSTANCE_STATE['terminated'])):
logger.error("[Pool %d] Instance with EC2 ID %s (status %d) is not in region list for region %s" % (pool.id, boto_instance.id, state_code, region))
cluster.terminate(boto_instances)
else:
logger.info("[Pool %d] Terminating %s instances in region %s" % (pool.id, len(instance_ids_by_region[region]), region))
cluster.terminate(cluster.find(instance_ids=instance_ids_by_region[region]))
except (boto.exception.EC2ResponseError, boto.exception.BotoServerError, ssl.SSLError, socket.error) as msg:
logger.exception("[Pool %d] %s: boto failure: %s" % (pool.id, "terminate_pool_instances", msg))
return 1
