def sanity_check(regions):
spot_requests = []
for r in regions:
conn = get_aws_connection(r)
region_spot_requests = conn.get_all_spot_instance_requests()
if region_spot_requests:
spot_requests.extend(region_spot_requests)
all_spot_instances = aws_get_all_instances(regions)
instance_ids = [i.id for i in all_spot_instances]
for req in spot_requests:
if req.state in ["open", "failed"]:
if req.status.code in CANCEL_STATUS_CODES:
log.info("Cancelling request %s", req)
retry_aws_request(req.add_tag, "moz-cancel-reason", req.status.code)
req.cancel()
elif req.status.code not in IGNORABLE_STATUS_CODES:
log.error("Uknown status for request %s: %s", req,
req.status.code)
# Cancel all active request older than 30 mins without runing instances
elif req.state == "active" and \
parse_aws_time(req.create_time) + 30 * 60 < time.time() and \
req.instance_id not in instance_ids:
log.info("Cancelling request %s: %s is not running", req,
req.instance_id)
retry_aws_request(req.add_tag, "moz-cancel-reason", "no-running-instances")
req.cancel()
def sanity_check(regions):
spot_requests = []
for r in regions:
conn = get_aws_connection(r)
region_spot_requests = conn.get_all_spot_instance_requests()
if region_spot_requests:
spot_requests.extend(region_spot_requests)
all_spot_instances = aws_get_all_instances(regions)
instance_ids = [i.id for i in all_spot_instances]
for req in spot_requests:
if req.state in ["open", "failed"]:
if req.status.code in CANCEL_STATUS_CODES:
log.info("Cancelling request %s", req)
retry_aws_request(req.add_tag, "moz-cancel-reason",
req.status.code)
req.cancel()
elif req.status.code not in IGNORABLE_STATUS_CODES:
log.error("Uknown status for request %s: %s", req,
req.status.code)
# Cancel all active request older than 30 mins without runing instances
elif req.state == "active" and \
parse_aws_time(req.create_time) + 30 * 60 < time.time() and \
req.instance_id not in instance_ids:
log.info("Cancelling request %s: %s is not running", req,
req.instance_id)
retry_aws_request(req.add_tag, "moz-cancel-reason",
"no-running-instances")
req.cancel()
def _get_time_from_json(self, json_file):
"""reads a json log and returns the eventTime"""
try:
with open(json_file) as json_f:
data = json.loads(json_f.read())
event = parse_aws_time(data['eventTime'])
now = time.time()
tdelta = (now - event) / 3600
return tdelta
except TypeError:
# json_file is None; aws_sanity_checker has no events-dir set
pass
except IOError:
# file does not exist
pass
except ValueError:
# bad json filex
log.debug('JSON cannot load %s', json_file)
def _get_time_from_json(self, json_file):
"""reads a json log and returns the eventTime"""
try:
with open(json_file) as json_f:
data = json.loads(json_f.read())
event = parse_aws_time(data['eventTime'])
now = time.time()
tdelta = (now - event)/3600
return tdelta
except TypeError:
# json_file is None; aws_sanity_checker has no events-dir set
pass
except IOError:
# file does not exist
pass
except ValueError:
# bad json filex
log.debug('JSON cannot load %s', json_file)
def get_uptime(instance):
return (time.time() - parse_aws_time(instance.launch_time)) / 3600
