コード例 #1
0
    def process( self, message ) :
        """ Process the message

        :param message: SQS Message instance

        """
        try:
            spot_request_msg = SpotRequestMsg( raw_json=message.get_body() )
            spot_request_uuid = spot_request_msg.spot_request_uuid
            spot_master_uuid = spot_request_msg.spot_master_uuid
            spot_request_id = spot_request_msg.spot_request_id
            logger.info( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'process_check_status' )
            # Get spot request row from DynamoDB and process based on state
            spot_request_item = get_spot_request_item( self.spot_request_table_name, spot_request_uuid, region_name=self.region_name, profile_name=self.profile_name )
            logger.info( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'spot request state=' + spot_request_item[TableSpotRequest.spot_request_state_code])
             
            next_status_msg_delay_secs = 60
            is_send_request_msg_check_status = True
            spot_request_state_code = spot_request_item[TableSpotRequest.spot_request_state_code]
            # Update the LastStateCheck timestamp
            spot_request_row_partial_save( self.spot_request_table_name, spot_request_item, {
                                            TableSpotRequest.ts_last_state_check:int( time.time() ),
                                            },
                                            region_name=self.region_name, profile_name=self.profile_name )
             
            if SpotRequestStateCode.spot_request_in_progress == spot_request_state_code:
                self.handle_state_request_spot_request_in_progress( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
            elif SpotRequestStateCode.instance_starting == spot_request_state_code:
                self.handle_state_request_instance_starting( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
            elif SpotRequestStateCode.instance_running == spot_request_state_code:
                self.handle_state_request_instance_running( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
            elif SpotRequestStateCode.instance_complete == spot_request_state_code:
                self.handle_state_request_instance_complete( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
                is_send_request_msg_check_status = False
            elif SpotRequestStateCode.instance_state_unknown == spot_request_state_code:
                self.handle_state_request_instance_state_unknown( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
            elif SpotRequestStateCode.constraint_encountered == spot_request_state_code:
                self.handle_state_request_constraint_encountered( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )       
            elif SpotRequestStateCode.instance_force_termination_pending == spot_request_state_code:
                self.handle_state_instance_force_termination_pending( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
            elif SpotRequestStateCode.instance_force_terminated == spot_request_state_code:
                self.handle_state_request_instance_force_terminated( spot_request_msg, spot_request_item, spot_request_uuid, spot_master_uuid )
                is_send_request_msg_check_status = False
            
            if is_send_request_msg_check_status:
                spot_request_msg_check_status = SpotRequestMsg( spot_request_uuid=spot_request_uuid, 
                                                                spot_master_uuid=spot_master_uuid, 
                                                                spot_request_msg_type=SpotRequestMsg.TYPE_CHECK_STATUS, 
                                                                spot_request_id=spot_request_id )
                message_attributes = create_microsvc_message_attributes( awsspotbatch.common.const.MICROSVC_REQUEST_CLASSNAME_SpotRequestMessageCheckStatus )
                self.spot_request_sqs_message_durable.send_message( spot_request_msg_check_status.to_json(), 
                                                                    delay_seconds=next_status_msg_delay_secs, 
                                                                    message_attributes=message_attributes  )
        
            self.spot_request_sqs_message_durable.delete_message(message)

        except StandardError as e:
            logger.error( fmt_request_uuid_msg_hdr( spot_request_uuid ) + 'Exiting SpotRequestDispatcher due to exception'  )
            logger.error( fmt_request_uuid_msg_hdr( spot_request_uuid ) + str(e) )
            logger.error( fmt_request_uuid_msg_hdr( spot_request_uuid ) + traceback.format_exc() )    
    def process( self, message ) :
        """ Start SpotRequest process
            1. Create item in SpotRequestItem table
            2. queue up potRequestMessageCheckStatus, this will start the state-based process

        :param message: SQS Message instance

        """
        try:
            spot_request_msg = SpotRequestMsg( raw_json=message.get_body() )
            logger.info( fmt_request_uuid_msg_hdr( spot_request_msg.spot_request_uuid ) + 'process_spot_request_initiated for spot_master_uuid: ' + spot_request_msg.spot_master_uuid )
            ts_now = int( time.time() )
            dict_create_spot_request_item = {
                                        TableSpotRequest.spot_request_uuid:spot_request_msg.spot_request_uuid,
                                        TableSpotRequest.spot_master_uuid:spot_request_msg.spot_master_uuid,
                                        TableSpotRequest.spot_request_id:spot_request_msg.spot_request_id,
                                        TableSpotRequest.ts_last_state_check:ts_now,
                                        TableSpotRequest.attempt_number:spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_ATTEMPT_NUMBER ],
                                        TableSpotRequest.spot_price:spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_SPOT_PRICE ],
                                        TableSpotRequest.instance_username:spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_INSTANCE_USERNAME ],
                                        TableSpotRequest.is_open:1,
                                        TableSpotRequest.spot_request_state_code:SpotRequestStateCode.spot_request_in_progress,
                                        TableSpotRequest.ts_start:ts_now,
                                       }
            put_attempt_cnt = 0
            put_attempt_max = 10
            while True:
                dynamodb_conn = boto.dynamodb2.connect_to_region( self.region_name, profile_name=self.profile_name )
                spot_request_table = Table( self.spot_request_table_name, connection=dynamodb_conn ) 
                result_spot_request_put = spot_request_table.put_item(data=dict_create_spot_request_item)
                if result_spot_request_put: break
                put_attempt_cnt += 1
                if put_attempt_cnt == put_attempt_max: 
                    raise awsspotbatch.common.exception.DynamoDbPutItemMaxAttemptsExceeded('Failed attempt to insert item in: ' + self.spot_request_table_name + 
                                                             ' for spot_request_uuid: ' + spot_request_msg.spot_request_uuid, self.spot_request_table_name )
                time.sleep(6)
            
            next_status_msg_delay_secs = 30
            spot_request_msg_check_status = SpotRequestMsg( 
                                                           spot_request_uuid=spot_request_msg.spot_request_uuid, 
                                                           spot_master_uuid=spot_request_msg.spot_master_uuid,
                                                           spot_request_msg_type=SpotRequestMsg.TYPE_CHECK_STATUS, 
                                                           spot_request_id=spot_request_msg.spot_request_id )
            message_attributes = create_microsvc_message_attributes( awsspotbatch.common.const.MICROSVC_REQUEST_CLASSNAME_SpotRequestMessageCheckStatus )
            self.spot_request_sqs_message_durable.send_message( spot_request_msg_check_status.to_json(), delay_seconds=next_status_msg_delay_secs, message_attributes=message_attributes )
            self.spot_request_sqs_message_durable.delete_message(message) 
            
        except StandardError as e:
            logger.error( fmt_request_uuid_msg_hdr( spot_request_msg.spot_request_uuid ) + 'Exiting SpotRequestDispatcher due to exception'  )
            logger.error( fmt_request_uuid_msg_hdr( spot_request_msg.spot_request_uuid ) + str(e) )
            logger.error( fmt_request_uuid_msg_hdr( spot_request_msg.spot_request_uuid ) + traceback.format_exc() )    
    def process( self, message ) :
        """ Try to submit another Spot Request based on the one that just failed

        :param message: SQS Message instance

        """
        try: 
            spot_master_msg = SpotMasterMsg( raw_json=message.get_body() )
            spot_master_uuid = spot_master_msg.spot_master_uuid       
            logger.info( fmt_master_uuid_msg_hdr( spot_master_uuid ) + 'process_resubmit_failed_request')
            dynamodb_conn = boto.dynamodb2.connect_to_region( self.region_name, profile_name=self.profile_name )
            spot_master_table = Table( self.spot_master_table_name, connection=dynamodb_conn ) 
            spot_master_item = spot_master_table.get_item( spot_master_uuid=spot_master_uuid )
            spot_request_table = Table( self.spot_request_table_name, connection=dynamodb_conn ) 
            failed_spot_request_item = spot_request_table.get_item( spot_request_uuid=spot_master_msg.spot_request_uuid )
    
            # Request spot instance
            spot_instance_request = self.resubmit_failed_request_spot_instance( spot_master_item, failed_spot_request_item, dynamodb_conn )
    
            # Queue up a SpotRequestMsg     
            if spot_instance_request != None:
                spot_request_uuid = str(uuid.uuid1())
                spot_request_msg = SpotRequestMsg( spot_request_uuid=spot_request_uuid, 
                                                   spot_master_uuid=spot_master_item[ TableSpotMaster.spot_master_uuid ], 
                                                   spot_request_msg_type=SpotRequestMsg.TYPE_SPOT_REQUEST_INITIATED, 
                                                   spot_request_id=spot_instance_request.id )
                spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_SPOT_PRICE ] = str( spot_instance_request.price )
                spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_INSTANCE_USERNAME ] = spot_master_item[ TableSpotMaster.instance_username ]
                spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_ATTEMPT_NUMBER ] = int( failed_spot_request_item[ TableSpotRequest.attempt_number ] + 1 )
                
                spot_request_sqs_message_durable = SqsMessageDurable( self.spot_request_queue_name, self.region_name, profile_name=self.profile_name )
                spot_request_sqs_message_durable.send_message( spot_request_msg.to_json(), message_attributes=create_microsvc_message_attributes( awsspotbatch.common.const.MICROSVC_REQUEST_CLASSNAME_SpotRequestMessageSpotRequestInitiated ) )
                self.spot_master_sqs_message_durable.delete_message(message) 
            # No instances available - resubmit this message with a delay timer so it will get reprocessed in future
            else:
                logger.warning( fmt_master_uuid_msg_hdr( spot_master_uuid ) + 'No spot instances available, will try again in ' + str(awsspotbatch.common.const.NO_SPOT_INSTANCES_AVAILABLE_RECHECK_MINUTES) + ' minutes')
                delay_seconds = awsspotbatch.common.const.NO_SPOT_INSTANCES_AVAILABLE_RECHECK_MINUTES * 60
                self.spot_master_sqs_message_durable.send_message( message.get_body(), 
                                                                   message_attributes=create_microsvc_message_attributes( awsspotbatch.common.const.MICROSVC_MASTER_CLASSNAME_SpotMasterMessageResubmitFailedRequest ), 
                                                                   delay_seconds=delay_seconds )
                self.spot_master_sqs_message_durable.delete_message(message)

        except StandardError as e:
            logger.error( fmt_master_item_msg_hdr( spot_master_item ) + str(e) )
            logger.error( fmt_master_item_msg_hdr( spot_master_item ) + traceback.format_exc() )
コード例 #4
0
    def handle_state_master_role_policy_in_progress(self, spot_master_item, dynamodb_conn ):
        """ Verify the Policy is added to the Role 

        :param spot_master_item: 
        :param dynamodb_conn: 

        """
        logger.info( fmt_master_item_msg_hdr( spot_master_item ) + 'handle_state_master_role_policy_in_progress')
        iam_conn = awsext.iam.connect_to_region( self.region_name, profile_name=self.profile_name )
        is_role_policy_added = iam_conn.is_role_policy_added( role_name=spot_master_item[ TableSpotMaster.role_name ], 
                                       policy_name=spot_master_item[ TableSpotMaster.policy_name ])
        if not is_role_policy_added: return
        # For some bizarre timing reason, is_role_policy_added can return True but the spot request fails on IAM role not attached to instance profile
        #  - give it a few seconds to clear
        time.sleep(5)
    
        spot_master_state_code = SpotMasterStateCode.waiting_for_instances_complete
        
        # Request spot instances
        spot_instance_requests = submit_request_spot_instances( spot_master_item, self.region_name, self.profile_name )
        
        # Queue up a SpotRequestMsg for each spot request - this will manage all states for SpotRequest        
        if spot_instance_requests != None:
            spot_request_sqs_message_durable = SqsMessageDurable( self.spot_request_queue_name, self.region_name, profile_name=self.profile_name )
            for spot_instance_request in spot_instance_requests:
                spot_request_uuid = str(uuid.uuid1())
                spot_request_msg = SpotRequestMsg( spot_request_uuid=spot_request_uuid, 
                                                   spot_master_uuid=spot_master_item[ TableSpotMaster.spot_master_uuid ], 
                                                   spot_request_msg_type=SpotRequestMsg.TYPE_SPOT_REQUEST_INITIATED, 
                                                   spot_request_id=spot_instance_request.id )
                spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_SPOT_PRICE ] = str( spot_master_item[TableSpotMaster.cheapest_price])
                spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_INSTANCE_USERNAME ] = spot_master_item[ TableSpotMaster.instance_username ]
                spot_request_msg.name_value_pairs[ SpotRequestMsg.PAIR_NAME_ATTEMPT_NUMBER ] = 1
                message_attributes = create_microsvc_message_attributes( awsspotbatch.common.const.MICROSVC_REQUEST_CLASSNAME_SpotRequestMessageSpotRequestInitiated )
                spot_request_sqs_message_durable.send_message( spot_request_msg.to_json(), message_attributes=message_attributes  )
        else: spot_master_state_code = SpotMasterStateCode.no_instances_available
        
        spot_master_row_partial_save( self.spot_master_table_name, spot_master_item, {
                                                              TableSpotMaster.spot_master_state_code:spot_master_state_code
                                                              },
                                      region_name=self.region_name, profile_name=self.profile_name  )