def _tag_address(allocation_id, tag):
call(client,
'create_tags',
'Create tags to EIP.',
Resources=[allocation_id],
Tags=[{
'Key': 'EZSpot',
'Value': tag
}])
def _disassociate_address(association_id):
if association_id:
call(client,
'disassociate_address',
'Disassociate EIP from instance.',
AssociationId=association_id)
else:
error_handler('Disassociate public IP failed.',
'Failed to delete EIP from spot instances.')
def _associate_address(allocation_id, instance_id):
call(
client,
'associate_address',
"Associate EIP to instance: {0}".format(instance_id),
_runback_disassociate_address,
AllocationId=allocation_id,
InstanceId=instance_id,
)
def release_address(allocation_id):
if allocation_id:
call(client,
'release_address',
'Release EIP.',
AllocationId=allocation_id)
else:
error_handler('Release public IP failed.',
'Failed to delete EIP from spot instances.')
def _runback_disassociate_address(response):
association_id = response.get('AssociationId', None)
if association_id:
call(client,
'disassociate_address',
'Disassociate EIP from instance.',
AssociationId=association_id)
else:
logger.error(
'Can not disassociate EIP, please remember to disassociate it in your AWS console.'
)
def _runback_release_address(response):
allocation_id = response.get('AllocationId', None)
if allocation_id:
call(client,
'release_address',
"Release EIP : {0}.".format(response.get('PublicIp', '')),
AllocationId=allocation_id)
else:
logger.error(
'Can not release eip, please remember to release it in your AWS console.'
)
def create_tag(instances_ids, tag):
if tag:
call(client,
'create_tags',
'Create tags for instances : ' + str(instances_ids),
Resources=instances_ids,
Tags=[{
'Key': 'EZSpot',
'Value': tag
}])
else:
error_handler('Unsupported tag type : None.',
'Failed to create instances tag.')
def _get_instances_ids(tag):
response = call(
client,
'describe_tags',
"Describe instances which has tag EZSpot : {0}.".format(tag),
Filters=[{
'Name': 'key',
'Values': [
'EZSpot',
]
}, {
'Name': 'value',
'Values': [
tag,
]
}, {
'Name': 'resource-type',
'Values': [
'instance',
]
}],
)
tags = response.get('Tags', None)
instances_ids = []
if tags:
for item in tags:
instances_ids.append(item.get('ResourceId', None))
return instances_ids
def _get_role(name):
response = call(client,
'get_role',
'Get iam role successfully.',
RoleName=name)
return response.get('Role', {}).get('Arn', None)
def _wait_until_completed(request_id):
for index in xrange(40):
response = call(
client,
'describe_spot_fleet_requests',
"Describe spot fleet request: {0} status.".format(request_id),
SpotFleetRequestIds=[request_id])
status = response.get('SpotFleetRequestConfigs',
[{}])[0].get('ActivityStatus', '')
if status == 'error':
error_handler(
"Can not start spot fleet {0} now. Please check your config.".
format(request_id), 'Failed to request a spot fleet.')
elif status == 'pending_fulfillment' or status == '':
logger.info('Please waiting for spot fleet request confirmed ...')
time.sleep(15)
elif status == 'fulfilled':
logger.info("Spot fleet {0} has fulfilled.".format(request_id))
return
else:
error_handler(
"Unsupported spot fleet request status: {0}.".format(status),
'Failed to request a spot fleet.')
def _request(config, index):
kwargs = get_specification(config, index, request_type='on_demand')
kwargs['MaxCount'] = config.wld_instance_capacity[index]
kwargs['MinCount'] = config.wld_instance_capacity[index]
response = call(client, 'run_instances', 'Run on-demand instances.',
_runback_on_demand_instances, **kwargs)
return _get_instances_ids_from_response(response)
def _get_price_history(start_time, instance_type, product_descriptions):
response = call(client,
'describe_spot_price_history',
DryRun=False,
StartTime=start_time,
EndTime=datetime.datetime.now(),
InstanceTypes=[instance_type],
ProductDescriptions=[config.prc_product_description])
return response.get('SpotPriceHistory', [])
def get_ami_id():
response = call(client,
'describe_images',
'Get ami info successfully.',
Filters=[{
'Name': 'name',
'Values': [ami_name]
}],
Owners=['amazon'])
return response.get('Images', [{}])[0].get('ImageId', None)
def _get_fleet_request_price(config, index, request_id, end_time):
response = call(client,
'describe_spot_fleet_requests',
'Describe spot fleet request. Request ID: ' + request_id,
SpotFleetRequestIds=[request_id])
start_time = response.get('SpotFleetRequestConfigs',
[{}])[0].get('CreateTime', None)
return get_fleet_price(start_time, end_time,
config.get_arr_attr('wld_instance_type', index),
config.prc_product_description)
def _describe_addresses(tag):
return call(client,
'describe_addresses',
'Describe EIP by tag: ' + tag,
Filters=[{
'Name': 'tag-key',
'Values': ['EZSpot']
}, {
'Name': 'tag-value',
'Values': [tag]
}])
def terminate_instances(instances_ids):
response = call(client,
'terminate_instances',
'Terminate instances : ' + str(instances_ids),
InstanceIds=instances_ids)
if response.get('TerminatingInstances', []) == []:
logger.warning(
'Terminate no instances, please check if there is something wrong.'
)
return False
return True
def _request(config, index, host_type='one-time', block_duration=360):
response = call(
client,
'request_spot_instances',
'Request spot instances \'' + config.get_arr_attr('wld_fleet_tag', index) + '\'.',
_runback_spot_request,
InstanceCount=config.wld_instance_capacity[index],
LaunchSpecification=get_specification(config, index, 'spot_instance'),
Type=host_type,
BlockDurationMinutes=block_duration
)
return _get_request_ids_from_response(response)
def _allocate_address():
response = call(
client,
'allocate_address',
'Allocate EIP for spot instances',
_runback_release_address,
Domain='vpc',
)
newResponse = {}
newResponse['PublicIp'] = response.get('PublicIp', None)
newResponse['AllocationId'] = response.get('AllocationId', None)
return newResponse
def _request(config, index):
specification_arr = []
specification_arr.append(get_specification(config, index))
response = call(client,
'request_spot_fleet',
'Request spot fleet \'' +
config.get_arr_attr('wld_fleet_tag', index) + '\'.',
_runback_fleet_request,
SpotFleetRequestConfig={
'IamFleetRole': config.wld_iam_role,
'TargetCapacity': config.wld_instance_capacity[index],
'TerminateInstancesWithExpiration': False,
'LaunchSpecifications': specification_arr,
'ReplaceUnhealthyInstances': False
})
return response.get('SpotFleetRequestId', None)
def _describe_request_by_tag(tag):
response = call(
client,
'describe_spot_instance_requests',
'Describe spot instance requests by tag : ' + tag,
Filters=[ {
'Name': 'tag-key',
'Values': [
'EZSpot',
]
}, {
'Name': 'tag-value',
'Values': [
tag,
]
} ]
)
return _get_request_ids_from_response(response)
def cancel_spot_request(request_ids, error_message, wite_instances=True):
response = call(
client,
'cancel_spot_instance_requests',
'Cancel spot instances request : ' + str(request_ids),
SpotInstanceRequestIds=request_ids,
)
flag = True
if response.get('CancelledSpotInstanceRequests', []) == []:
logger.error(error_message)
flag = False
if wite_instances:
instances_ids = _describe_request(request_ids)
status = terminate_instances(instances_ids)
if flag and not status:
flag = status
return flag
def _describe_request(request_ids):
response = call(
client,
'describe_spot_instance_requests',
'Describe spot instance request by id : ' + str(request_ids),
SpotInstanceRequestIds=request_ids,
)
requests = response.get('SpotInstanceRequests', [])
if requests == []:
error_handler('Can not describe spot instances.', 'Failed to describe spot instances.')
else:
instances_ids = []
for request in requests:
instance_id = request.get('InstanceId', None)
if instance_id:
instances_ids.append(instance_id)
else:
error_handler('Can not get instance id for spot instances.', 'Failed to describe spot instances.')
return instances_ids
def _get_fleet_request_id(config, index):
response = call(
client,
'describe_spot_fleet_requests',
'Describe spot fleet requests.',
)
request_configs = response.get('SpotFleetRequestConfigs', [])
if request_configs != []:
for item in request_configs:
if item.get('ActivityStatus',
'') == 'pending_fulfillment' or item.get(
'ActivityStatus', '') == 'fulfilled':
specification = item.get('SpotFleetRequestConfig',
{}).get('LaunchSpecifications', [])
if len(specification) == 1:
tags = specification[0].get('TagSpecifications', [])
if len(tags) == 1 and tags[0]['Tags'][0][
'Key'] == 'EZSpot' and tags[0]['Tags'][0][
'Value'] == config.wld_fleet_tag[index]:
return item.get('SpotFleetRequestId', None)
return None
def _describe_fleet_instances(request_id):
response = call(
client,
'describe_spot_fleet_instances',
"Describe spot fleet instances, spot request: {0}".format(request_id),
SpotFleetRequestId=request_id)
instances = response.get('ActiveInstances', [])
if instances != []:
instances_ids = []
for instance in instances:
instance_id = instance.get('InstanceId', None)
if instance_id:
instances_ids.append(instance_id)
else:
error_handler('Get wrong intance response.',
'Failed to get spot instances information.')
return instances_ids
else:
error_handler('Describe spot fleet instances failed.',
'Failed to get spot instances information.')
def _describe_tags(resource_type):
response = call(client,
'describe_tags',
'Describe spot instances request ids.',
Filters=[{
'Name': 'resource-type',
'Values': [
resource_type,
]
}, {
'Name': 'key',
'Values': [
'EZSpot',
]
}])
request_arr = []
tags = response.get('Tags', [])
for tag in tags:
request = tag.get('ResourceId', None)
request_arr.append(request)
return request_arr
def get_fleet_price(start_time, current_time, instance_type,
product_descriptions):
response = call(
client,
'describe_spot_price_history',
"Describe spot price history from start time : {0}, instance type: {1}, product: {2}."
.format(start_time, instance_type, product_descriptions),
DryRun=False,
StartTime=start_time,
EndTime=current_time,
InstanceTypes=[instance_type],
ProductDescriptions=[product_descriptions])
price_history = response.get('SpotPriceHistory', [])
if len(price_history) == 0:
logger.debug('Get no price_history from aws spot price history.')
return None
else:
az_list = {}
for index in xrange(len(price_history) - 1, -1, -1):
az = price_history[index].get('AvailabilityZone', None)
price = price_history[index].get('SpotPrice', None)
if az and price:
price = float(price)
if az_list.has_key(az):
break
az_list[az] = price
else:
return None
logger.debug('AZ first price is : ' + str(az_list))
min = sys.maxsize
min_az = None
for az in az_list:
if az_list[az] < min:
min = az_list[az]
min_az = az
logger.debug("Choose AZ : {0} as calculate price az.".format(min_az))
total = 0.0
peer_time = start_time
peer_price = 0.0
logger.debug('Price claculate start at : ' + str(start_time))
if min_az:
for index in xrange(len(price_history) - 1, -1, -1):
if min_az == price_history[index].get('AvailabilityZone',
None):
price = float(price_history[index].get('SpotPrice', None))
timestamp = price_history[index].get('Timestamp', None)
if price and timestamp:
logger.debug("Get time : {0} at price : {1}.".format(
str(timestamp), price))
if time.mktime(timestamp.timetuple()) > time.mktime(
peer_time.timetuple()):
total += peer_price * (
time.mktime(timestamp.timetuple()) -
time.mktime(peer_time.timetuple())) / 3600
peer_time = timestamp
peer_price = price
total += peer_price * (time.mktime(current_time.timetuple()) -
time.mktime(peer_time.timetuple())) / 3600
return [
total,
(time.mktime(current_time.timetuple()) -
time.mktime(peer_time.timetuple())),
round((time.mktime(current_time.timetuple()) -
time.mktime(peer_time.timetuple())) / 3600)
]
else:
return None
def cancel_fleet_request(request_id):
return call(client,
'cancel_spot_fleet_requests',
'Cancel fleet request : ' + request_id,
SpotFleetRequestIds=[request_id],
TerminateInstances=True)
