def _wait_until_completed(request_id):
for index in xrange(40):
response = call(
client,
'describe_spot_fleet_requests',
"Describe spot fleet request: {0} status.".format(request_id),
SpotFleetRequestIds=[request_id])
status = response.get('SpotFleetRequestConfigs',
[{}])[0].get('ActivityStatus', '')
if status == 'error':
error_handler(
"Can not start spot fleet {0} now. Please check your config.".
format(request_id), 'Failed to request a spot fleet.')
elif status == 'pending_fulfillment' or status == '':
logger.info('Please waiting for spot fleet request confirmed ...')
time.sleep(15)
elif status == 'fulfilled':
logger.info("Spot fleet {0} has fulfilled.".format(request_id))
return
else:
error_handler(
"Unsupported spot fleet request status: {0}.".format(status),
'Failed to request a spot fleet.')
def release_address(allocation_id):
if allocation_id:
call(client,
'release_address',
'Release EIP.',
AllocationId=allocation_id)
else:
error_handler('Release public IP failed.',
'Failed to delete EIP from spot instances.')
def _disassociate_address(association_id):
if association_id:
call(client,
'disassociate_address',
'Disassociate EIP from instance.',
AssociationId=association_id)
else:
error_handler('Disassociate public IP failed.',
'Failed to delete EIP from spot instances.')
def delete_eip(tag):
response = _describe_addresses(tag)
eips = response.get('Addresses', [])
if eips != []:
for eip in eips:
_disassociate_address(eip.get('AssociationId', None))
release_address(eip.get('AllocationId', None))
else:
error_handler('Describe public IP failed.',
'Failed to delete EIP from spot instances.')
def create_tag(instances_ids, tag):
if tag:
call(client,
'create_tags',
'Create tags for instances : ' + str(instances_ids),
Resources=instances_ids,
Tags=[{
'Key': 'EZSpot',
'Value': tag
}])
else:
error_handler('Unsupported tag type : None.',
'Failed to create instances tag.')
def cancel_persistent_instances(config, index):
tag = config.get_arr_attr('wld_fleet_tag', index)
request_ids = _describe_request_by_tag(tag)
logger.info('Get spot instance request ids successfully. Request IDs : ' + str(request_ids))
status = cancel_spot_request(request_ids, 'Can not cancel spot instance requests, please remember to delete it in your AWS console.')
if status:
logger.info('Cancel spot instance requests successfully.')
delete_eip(tag)
else:
error_handler('Can not cancel spot instance requests.', 'Failed to cancel spot instances')
logger.info('Spot instances all clear.')
def create_eip(instances_ids, tag):
elastic_ip_arr = []
for instance_id in instances_ids:
response = _allocate_address()
if response['PublicIp'] and response['AllocationId']:
elastic_ip_arr.append(response['PublicIp'])
_tag_address(response['AllocationId'], tag)
_associate_address(response['AllocationId'], instance_id)
else:
error_handler('Allocate public IP failed.',
'Failed to create EIP for spot instances.')
return elastic_ip_arr
def _get_request_ids_from_response(response):
requests = response.get('SpotInstanceRequests', [])
if requests == []:
error_handler('Can not request spot instances.', 'Failed to request spot instances.')
else:
request_ids = []
for request in requests:
request_id = request.get('SpotInstanceRequestId', None)
if request_id:
request_ids.append(request_id)
else:
error_handler('Can not get request id for spot instances.', 'Failed to request spot instances.')
return request_ids
def cancel_on_demand_instances(config, index):
tag = config.get_arr_attr('wld_fleet_tag', index)
instances_ids = _get_instances_ids(tag)
logger.info('Get on-demand instance ids successfully. Instances IDs : ' +
str(instances_ids))
status = terminate_instances(instances_ids)
if status:
logger.info('Cancel on-demand instances successfully.')
delete_eip(tag)
else:
error_handler('Can not cancel on-demand instances requests.',
'Failed to cancel on-demand instances.')
logger.info('On-demand instances all clear.')
def _get_instances_ids_from_response(response):
instances = response.get('Instances', [])
if instances == []:
error_handler('Can not request on-demand instances.',
'Failed to request on-demand instances.')
else:
instances_ids = []
for instance in instances:
instance_id = instance.get('InstanceId', None)
if instance_id:
instances_ids.append(instance_id)
else:
error_handler('Can not get instance id from response.',
'Failed to request on-demand instances.')
return instances_ids
def start_fleet(config, index):
request_id = _request(config, index)
if request_id:
logger.info('Request spot fleet successfully. Request ID : ' +
request_id)
else:
error_handler('Request spot fleet failed.',
'Failed to get a new spot fleet id.')
time.sleep(5)
_wait_until_completed(request_id)
instances_ids = _describe_fleet_instances(request_id)
eip_arr = create_eip(instances_ids,
config.get_arr_attr('wld_fleet_tag', index))
logger.info('Spot fleet ready.')
logger.info('Instances public IP: ' + str(eip_arr))
def cancel_fleet(config, index):
request_id = _get_fleet_request_id(config, index)
if request_id:
logger.info('Get fleet request id successfully. Request ID : ' +
request_id)
response = cancel_fleet_request(request_id)
if response.get('UnsuccessfulFleetRequests', []) == []:
logger.info('Cancel fleet request successfully.')
delete_eip(config.get_arr_attr('wld_fleet_tag', index))
else:
error_handler(
'Cancel fleet request failed, please check your config.',
'Get the following response: ' +
str(response.get('UnsuccessfulFleetRequests', [])))
else:
logger.error('Cancel fleet request failed, please check your config.')
logger.info('Spot fleet cancelled.')
def _describe_request(request_ids):
response = call(
client,
'describe_spot_instance_requests',
'Describe spot instance request by id : ' + str(request_ids),
SpotInstanceRequestIds=request_ids,
)
requests = response.get('SpotInstanceRequests', [])
if requests == []:
error_handler('Can not describe spot instances.', 'Failed to describe spot instances.')
else:
instances_ids = []
for request in requests:
instance_id = request.get('InstanceId', None)
if instance_id:
instances_ids.append(instance_id)
else:
error_handler('Can not get instance id for spot instances.', 'Failed to describe spot instances.')
return instances_ids
def _describe_fleet_instances(request_id):
response = call(
client,
'describe_spot_fleet_instances',
"Describe spot fleet instances, spot request: {0}".format(request_id),
SpotFleetRequestId=request_id)
instances = response.get('ActiveInstances', [])
if instances != []:
instances_ids = []
for instance in instances:
instance_id = instance.get('InstanceId', None)
if instance_id:
instances_ids.append(instance_id)
else:
error_handler('Get wrong intance response.',
'Failed to get spot instances information.')
return instances_ids
else:
error_handler('Describe spot fleet instances failed.',
'Failed to get spot instances information.')