def process( self, message ) :
""" This should never happen, so don't terminate the instance, leave it up so it can get SSH'ed into
to determine the cause of failure
:param message: SQS Message instance
"""
try:
spot_request_msg = SpotRequestMsg( raw_json=message.get_body() )
instance_termination_exception = spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_INSTANCE_TERMINATION_TIME_EXCEPTION ]
spot_request_item = get_spot_request_item( self.spot_request_table_name, spot_request_msg.spot_request_uuid, region_name=self.region_name, profile_name=self.profile_name )
logger.error( fmt_request_uuid_msg_hdr( spot_request_msg.spot_request_uuid ) + 'process_pending_termination_exception for spot_request_uuid, instance_termination_exception=' + instance_termination_exception )
ts_now = int( time.time() )
spot_request_row_partial_save( self.spot_request_table_name, spot_request_item,
{
TableSpotRequest.is_open:0,
TableSpotRequest.ts_end:ts_now,
TableSpotRequest.spot_request_state_code:SpotRequestStateCode.instance_force_termination_exception,
TableSpotRequest.instance_termination_exception:instance_termination_exception
},
region_name=self.region_name, profile_name=self.profile_name )
self.spot_request_sqs_message_durable.delete_message(message)
except StandardError as e:
logger.error( fmt_request_uuid_msg_hdr( spot_request_msg.spot_request_uuid ) + 'Exiting SpotRequestDispatcher due to exception' )
logger.error( fmt_request_uuid_msg_hdr( spot_request_msg.spot_request_uuid ) + str(e) )
logger.error( fmt_request_uuid_msg_hdr( spot_request_msg.spot_request_uuid ) + traceback.format_exc() )
def process( self, message ) :
""" Batch job (i.e. user script) threw an exception, try it again
TODO: need to check against some "max contiguous errors",
i.e. if this fails 3x in a row then terminate the request
:param message: SQS Message instance
"""
try:
spot_request_msg = SpotRequestMsg( raw_json=message.get_body() )
spot_request_uuid = spot_request_msg.spot_request_uuid
spot_master_uuid = spot_request_msg.spot_master_uuid
logger.info( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'process() for spot_master_uuid: ' + spot_master_uuid )
spot_request_item = get_spot_request_item( self.spot_request_table_name, spot_request_msg.spot_request_uuid, region_name=self.region_name, profile_name=self.profile_name )
ts_cmd_complete = spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_BATCH_PROCESS_COMPLETE_TIMESTAMP]
cmd_exception_message = spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_INSTANCE_BATCH_PROCESS_START_EXCEPTION_MESSAGE]
cmd_exception_traceback = spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_INSTANCE_BATCH_PROCESS_START_EXCEPTION_TRACEBACK]
key_value_pairs = {
TableSpotRequest.is_open:0,
TableSpotRequest.spot_request_state_code:SpotRequestStateCode.instance_complete_exception,
TableSpotRequest.ts_cmd_complete:ts_cmd_complete,
TableSpotRequest.cmd_exception_message:cmd_exception_message,
TableSpotRequest.cmd_exception_traceback:cmd_exception_traceback,
}
spot_request_row_partial_save( self.spot_request_table_name, spot_request_item, key_value_pairs, region_name=self.region_name, profile_name=self.profile_name )
self.spot_request_sqs_message_durable.delete_message(message)
except StandardError as e:
logger.error( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'Exiting SpotRequestDispatcher due to exception' )
logger.error( fmt_request_uuid_msg_hdr( spot_request_uuid ) + str(e) )
logger.error( fmt_request_uuid_msg_hdr( spot_request_uuid ) + traceback.format_exc() )
def process( self, message ) :
""" Process the message
:param message: SQS Message instance
"""
try:
spot_request_msg = SpotRequestMsg( raw_json=message.get_body() )
spot_request_uuid = spot_request_msg.spot_request_uuid
spot_master_uuid = spot_request_msg.spot_master_uuid
spot_request_id = spot_request_msg.spot_request_id
logger.info( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'process_check_status' )
# Get spot request row from DynamoDB and process based on state
spot_request_item = get_spot_request_item( self.spot_request_table_name, spot_request_uuid, region_name=self.region_name, profile_name=self.profile_name )
logger.info( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'spot request state=' + spot_request_item[TableSpotRequest.spot_request_state_code])
next_status_msg_delay_secs = 60
is_send_request_msg_check_status = True
spot_request_state_code = spot_request_item[TableSpotRequest.spot_request_state_code]
# Update the LastStateCheck timestamp
spot_request_row_partial_save( self.spot_request_table_name, spot_request_item, {
TableSpotRequest.ts_last_state_check:int( time.time() ),
},
region_name=self.region_name, profile_name=self.profile_name )
if SpotRequestStateCode.spot_request_in_progress == spot_request_state_code:
self.handle_state_request_spot_request_in_progress( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
elif SpotRequestStateCode.instance_starting == spot_request_state_code:
self.handle_state_request_instance_starting( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
elif SpotRequestStateCode.instance_running == spot_request_state_code:
self.handle_state_request_instance_running( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
elif SpotRequestStateCode.instance_complete == spot_request_state_code:
self.handle_state_request_instance_complete( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
is_send_request_msg_check_status = False
elif SpotRequestStateCode.instance_state_unknown == spot_request_state_code:
self.handle_state_request_instance_state_unknown( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
elif SpotRequestStateCode.constraint_encountered == spot_request_state_code:
self.handle_state_request_constraint_encountered( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
elif SpotRequestStateCode.instance_force_termination_pending == spot_request_state_code:
self.handle_state_instance_force_termination_pending( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
elif SpotRequestStateCode.instance_force_terminated == spot_request_state_code:
self.handle_state_request_instance_force_terminated( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
is_send_request_msg_check_status = False
if is_send_request_msg_check_status:
spot_request_msg_check_status = SpotRequestMsg( spot_request_uuid=spot_request_uuid,
spot_master_uuid=spot_master_uuid,
spot_request_msg_type=SpotRequestMsg.TYPE_CHECK_STATUS,
spot_request_id=spot_request_id )
message_attributes = create_microsvc_message_attributes( awsspotbatch.common.const.MICROSVC_REQUEST_CLASSNAME_SpotRequestMessageCheckStatus )
self.spot_request_sqs_message_durable.send_message( spot_request_msg_check_status.to_json(),
delay_seconds=next_status_msg_delay_secs,
message_attributes=message_attributes )
self.spot_request_sqs_message_durable.delete_message(message)
except StandardError as e:
logger.error( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'Exiting SpotRequestDispatcher due to exception' )
logger.error( fmt_request_uuid_msg_hdr( spot_request_uuid ) + str(e) )
logger.error( fmt_request_uuid_msg_hdr( spot_request_uuid ) + traceback.format_exc() )
def handle_state_request_instance_starting( self, spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid ):
""" Spot request fullfilled, the instance is starting.
Once the instance is running/ok, change status to instance_running
:param spot_request_msg:
:param spot_request_item:
:param spot_request_uuid:
:param spot_master_uuid:
"""
logger.info( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'handle_state_request_instance_starting' )
# Check if the instance is totally up (running and all status checks ok)
ec2_conn = awsext.ec2.connect_to_region( region_name=self.region_name, profile_name=self.profile_name )
instance_id = spot_request_item[ TableSpotRequest.instance_id ]
state_name, status = ec2_conn.get_instance_state_name_and_status( instance_id )
logger.info( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'spot instance state_name=' + state_name + ', status=' + status )
if state_name == 'running' and status == 'ok':
try:
instance = ec2_conn.get_only_instances(instance_ids=[instance_id])[0]
spot_request_row_partial_save( self.spot_request_table_name, spot_request_item, {
TableSpotRequest.spot_request_state_code:SpotRequestStateCode.instance_running,
TableSpotRequest.instance_public_ip_address:instance.ip_address,
},
region_name=self.region_name, profile_name=self.profile_name )
# Instance is running and ok - run the clientlaunch, which will start the wrapper, run some init cmds, launch user cmd on thread and
# start sending heartbeats
launch_remote_client( self.spot_batch_job_parm_table_name,
self.spot_rsa_key_table_name, spot_request_item,
region_name=self.region_name,
profile_name=self.profile_name)
except boto.exception.EC2ResponseError as e:
if e.code == 'InvalidInstanceID.NotFound':
logger.warning( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'Instance terminated after starting and before init complete, spot_master_uuid=' + spot_master_uuid)
# looks like spot instance was terminated between the time it was allocated and initialized
spot_request_row_partial_save( self.spot_request_table_name, spot_request_item, {
TableSpotRequest.spot_request_state_code:SpotRequestStateCode.instance_force_terminated,
},
region_name=self.region_name, profile_name=self.profile_name )
else:
logger.error( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'Instance unknown state, exception code=' + e.code + ', spot_master_uuid=' + spot_master_uuid)
logger.error( traceback.format_exc() )
spot_request_row_partial_save( self.spot_request_table_name, spot_request_item, {
TableSpotRequest.spot_request_state_code:SpotRequestStateCode.instance_state_unknown,
},
region_name=self.region_name, profile_name=self.profile_name )
elif state_name == 'terminated':
logger.warning( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'Instance terminated after starting and before init complete, spot_master_uuid=' + spot_master_uuid)
# looks like spot instance was terminated between the time it was allocated and initialized
spot_request_row_partial_save( self.spot_request_table_name, spot_request_item, {
TableSpotRequest.spot_request_state_code:SpotRequestStateCode.instance_force_terminated,
},
region_name=self.region_name, profile_name=self.profile_name )
def process( self, message ) :
""" Start SpotRequest process
1. Create item in SpotRequestItem table
2. queue up potRequestMessageCheckStatus, this will start the state-based process
:param message: SQS Message instance
"""
try:
spot_request_msg = SpotRequestMsg( raw_json=message.get_body() )
logger.info( fmt_request_uuid_msg_hdr( spot_request_msg.spot_request_uuid ) + 'process_spot_request_initiated for spot_master_uuid: ' + spot_request_msg.spot_master_uuid )
ts_now = int( time.time() )
dict_create_spot_request_item = {
TableSpotRequest.spot_request_uuid:spot_request_msg.spot_request_uuid,
TableSpotRequest.spot_master_uuid:spot_request_msg.spot_master_uuid,
TableSpotRequest.spot_request_id:spot_request_msg.spot_request_id,
TableSpotRequest.ts_last_state_check:ts_now,
TableSpotRequest.attempt_number:spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_ATTEMPT_NUMBER ],
TableSpotRequest.spot_price:spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_SPOT_PRICE ],
TableSpotRequest.instance_username:spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_INSTANCE_USERNAME ],
TableSpotRequest.is_open:1,
TableSpotRequest.spot_request_state_code:SpotRequestStateCode.spot_request_in_progress,
TableSpotRequest.ts_start:ts_now,
}
put_attempt_cnt = 0
put_attempt_max = 10
while True:
dynamodb_conn = boto.dynamodb2.connect_to_region( self.region_name, profile_name=self.profile_name )
spot_request_table = Table( self.spot_request_table_name, connection=dynamodb_conn )
result_spot_request_put = spot_request_table.put_item(data=dict_create_spot_request_item)
if result_spot_request_put: break
put_attempt_cnt += 1
if put_attempt_cnt == put_attempt_max:
raise awsspotbatch.common.exception.DynamoDbPutItemMaxAttemptsExceeded('Failed attempt to insert item in: ' + self.spot_request_table_name +
' for spot_request_uuid: ' + spot_request_msg.spot_request_uuid, self.spot_request_table_name )
time.sleep(6)
next_status_msg_delay_secs = 30
spot_request_msg_check_status = SpotRequestMsg(
spot_request_uuid=spot_request_msg.spot_request_uuid,
spot_master_uuid=spot_request_msg.spot_master_uuid,
spot_request_msg_type=SpotRequestMsg.TYPE_CHECK_STATUS,
spot_request_id=spot_request_msg.spot_request_id )
message_attributes = create_microsvc_message_attributes( awsspotbatch.common.const.MICROSVC_REQUEST_CLASSNAME_SpotRequestMessageCheckStatus )
self.spot_request_sqs_message_durable.send_message( spot_request_msg_check_status.to_json(), delay_seconds=next_status_msg_delay_secs, message_attributes=message_attributes )
self.spot_request_sqs_message_durable.delete_message(message)
except StandardError as e:
logger.error( fmt_request_uuid_msg_hdr( spot_request_msg.spot_request_uuid ) + 'Exiting SpotRequestDispatcher due to exception' )
logger.error( fmt_request_uuid_msg_hdr( spot_request_msg.spot_request_uuid ) + str(e) )
logger.error( fmt_request_uuid_msg_hdr( spot_request_msg.spot_request_uuid ) + traceback.format_exc() )
def handle_state_request_constraint_encountered( self, spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid ):
""" Constraint encountered after spot request initiated but before request fullfilled,
i.e. time limit expired
Submit another spot request
:param spot_request_msg:
:param spot_request_item:
:param spot_request_uuid:
:param spot_master_uuid:
"""
logger.info( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'handle_state_request_constraint_encountered' )
ts_now = int( time.time() )
spot_request_row_partial_save( self.spot_request_table_name, spot_request_item, {
TableSpotRequest.spot_request_state_code:SpotRequestStateCode.instance_complete,
TableSpotRequest.is_open:0,
TableSpotRequest.ts_end:ts_now
},
region_name=self.region_name, profile_name=self.profile_name )
# Create a new spot request based on the spot request that just failed
master_msg_resubmit_failed_request = SpotMasterMsg( spot_master_uuid=spot_request_msg.spot_master_uuid,
spot_master_msg_type=SpotMasterMsg.TYPE_RESUBMIT_FAILED_REQUEST,
spot_request_uuid=spot_request_msg.spot_request_uuid
)
message_attributes = create_microsvc_message_attributes( awsspotbatch.common.const.MICROSVC_MASTER_CLASSNAME_SpotMasterMessageResubmitFailedRequest )
spot_master_sqs_message_durable = SqsMessageDurable( self.spot_master_queue_name, self.region_name, profile_name=self.profile_name )
spot_master_sqs_message_durable.send_message( master_msg_resubmit_failed_request.to_json(),
message_attributes=message_attributes )
def handle_state_instance_force_termination_pending( self, spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid ):
""" AWS has started the termination process for this instance, i.e. the price has increased
This is the beginning of the two minute warning pending forced termination
Terminate the instance and start another spot request
:param spot_request_msg:
:param spot_request_item:
:param spot_request_uuid:
:param spot_master_uuid:
"""
logger.info( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'handle_state_instance_force_termination_pending' )
ts_now = int( time.time() )
spot_request_row_partial_save( self.spot_request_table_name, spot_request_item, {
TableSpotRequest.spot_request_state_code:SpotRequestStateCode.instance_complete,
TableSpotRequest.is_open:0,
TableSpotRequest.ts_end:ts_now
},
region_name=self.region_name, profile_name=self.profile_name )
# Create a new spot request based on the spot request that just failed
master_msg_resubmit_failed_request = SpotMasterMsg( spot_master_uuid=spot_request_msg.spot_master_uuid,
spot_master_msg_type=SpotMasterMsg.TYPE_RESUBMIT_FAILED_REQUEST,
spot_request_uuid=spot_request_msg.spot_request_uuid
)
message_attributes = create_microsvc_message_attributes( awsspotbatch.common.const.MICROSVC_MASTER_CLASSNAME_SpotMasterMessageResubmitFailedRequest )
spot_master_sqs_message_durable = SqsMessageDurable( self.spot_master_queue_name, self.region_name, profile_name=self.profile_name )
spot_master_sqs_message_durable.send_message( master_msg_resubmit_failed_request.to_json(),
message_attributes=message_attributes )
def handle_state_request_spot_request_in_progress( self, spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid ):
""" Check if spot request has been fullfilled, terminated or is still pending
:param spot_request_msg:
:param spot_request_item:
:param spot_request_uuid:
:param spot_master_uuid:
"""
logger.info( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'handle_state_request_spot_request_in_progress' )
ec2_conn = awsext.ec2.connect_to_region( region_name=self.region_name, profile_name=self.profile_name )
try:
spot_instance_requests = ec2_conn.get_all_spot_instance_requests(request_ids=[spot_request_item[TableSpotRequest.spot_request_id]] )
if spot_instance_requests != None and len(spot_instance_requests) == 1:
spot_instance_request = spot_instance_requests[0]
logger.info( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'spot_instance_request.status.code=' + spot_instance_request.status.code +
' for spot_request_id=' + spot_request_item[TableSpotRequest.spot_request_id ] +
', spot_request_uuid=' + spot_request_uuid )
if spot_instance_request.status.code == 'schedule-expired' or spot_instance_request.status.code in awsext.ec2.connection.AwsExtEC2Connection.SPOT_REQUEST_CONSTRAINTS:
# update status so that a new spot request can occur
spot_request_row_partial_save( self.spot_request_table_name, spot_request_item, {
TableSpotRequest.spot_request_state_code:SpotRequestStateCode.constraint_encountered,
TableSpotRequest.constraint_code:spot_instance_request.status.code,
},
region_name=self.region_name, profile_name=self.profile_name )
# Instance has been assigned, it's starting up now
if spot_instance_request.instance_id != None:
ec2_conn.create_tags( [spot_instance_request.instance_id],
{ 'Name':'spot_request_uuid_'+spot_request_uuid, 'spot_master_uuid':spot_master_uuid, 'spot_request_uuid':spot_request_uuid} )
spot_request_row_partial_save( self.spot_request_table_name, spot_request_item, {
TableSpotRequest.spot_request_state_code:SpotRequestStateCode.instance_starting,
TableSpotRequest.instance_id:spot_instance_request.instance_id,
},
region_name=self.region_name, profile_name=self.profile_name )
except boto.exception.EC2ResponseError as e:
logger.error( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'Instance unknown state, exception code=' + e.code + ', spot_request_uuid=' + spot_request_uuid + ', spot_master_uuid=' + spot_master_uuid)
logger.error( fmt_request_uuid_msg_hdr( spot_request_uuid ) + traceback.format_exc() )
spot_request_row_partial_save( self.spot_request_table_name, spot_request_item, {
TableSpotRequest.spot_request_state_code:SpotRequestStateCode.instance_state_unknown,
},
region_name=self.region_name, profile_name=self.profile_name )
def handle_state_request_instance_running( self, spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid ):
""" At this point, :class:awsspotbatch.client.clientlaunch is running on the spot instance,
it's sending back heartbeats, checking for spot termination and monitoring the users' batch job
:param spot_request_msg:
:param spot_request_item:
:param spot_request_uuid:
:param spot_master_uuid:
"""
logger.info( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'handle_state_request_instance_running' )
def handle_state_request_instance_state_unknown( self, spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid ):
""" Handle any future states -
:param spot_request_msg:
:param spot_request_item:
:param spot_request_uuid:
:param spot_master_uuid:
"""
logger.warning( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'handle_state_request_instance_state_unknown' )
ts_now = int( time.time() )
spot_request_row_partial_save( self.spot_request_table_name, spot_request_item, {
TableSpotRequest.spot_request_state_code:SpotRequestStateCode.instance_complete,
TableSpotRequest.is_open:0,
TableSpotRequest.ts_end:ts_now
},
region_name=self.region_name, profile_name=self.profile_name )
def handle_state_request_instance_complete( self, spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid ):
""" User's batch job has completed, start instance termination process
:param spot_request_msg:
:param spot_request_item:
:param spot_request_uuid:
:param spot_master_uuid:
"""
logger.info( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'handle_state_request_instance_complete' )
# terminate the instance, if it exists
if spot_request_item[TableSpotRequest.instance_id] != None:
ec2_conn = awsext.ec2.connect_to_region( self.region_name, profile_name=self.profile_name )
ec2_conn.terminate_instances( instance_ids=[ spot_request_item[TableSpotRequest.instance_id] ] )
ts_now = int( time.time() )
spot_request_row_partial_save( self.spot_request_table_name, spot_request_item,
{TableSpotRequest.is_open:0, TableSpotRequest.ts_end:ts_now },
region_name=self.region_name, profile_name=self.profile_name )