def checkSubs(seq_hash, msg_list): incoming_msg = publishsubscribe.receive_message(sub_sockets) # This assumes that the return is in the format [seq_num, message] if incoming_msg[ 1]: # If there was an incoming message from the subscribe ports algorithm.add_seq_num( seq_hash, incoming_msg[0][0]) # Adds the seq_num to the hash msg_list.append(incoming_msg[0]) # Stores the message in the list
def running_loop(): try: conn = boto.sqs.connect_to_region("us-west-2") if conn == None: sys.stderr.write( "Could not connect to AWS region '{0}'\n".format("us-west-2")) sys.exit(1) # Assuming that the queues will have already been created elsewhere q_in = conn.get_queue(args.in_queue) q_out = conn.get_queue(args.out_queue) except Exception as e: sys.stderr.write("Exception connecting to SQS\n") sys.stderr.write(str(e)) sys.exit(1) global seq_num seq_hash = [] last_performed_num = 0 stored_messages = [] print "Starting up instance: {0}".format(args.my_name) # Actual work gets done here while (1 < 2): # lol # grab a message off SQS_IN rs = q_in.get_messages( message_attributes=["action", "id", "name", "activities"]) if (len(rs) < 1): checkSubs(seq_hash, stored_messages) #Checks sub ports calculated_num, next_in_seq = algorithm.compare_seq_num( seq_hash, last_performed_num) if next_in_seq: # If the next number is indeed last_performed_num + 1 last_performed_num = catchup(calculated_num, stored_messages, last_performed_num) time.sleep( POLL_INTERVAL ) # wait before checking for messages again (in seconds) continue m = rs[0] q_in.delete_message( m) # remove message from queue so it's not read multiple times print "Received message: " + m.get_body() seq_num += 1 #increment loc_seq_num = seq_num.last_set #store value to local variable op_list = parse_sqs_msg(m) publishsubscribe.send_message(pub_socket, [loc_seq_num, op_list]) algorithm.add_seq_num(seq_hash, loc_seq_num) # Publish to pub_socket the seq_num and the message # Runs the algorithm to check if the next number in hash is one above the last performed seq num calculated_num, next_in_seq = algorithm.compare_seq_num( seq_hash, last_performed_num) # This is the catch-up loop, running until the number after last performed seq num is loc_seq_num while (calculated_num < loc_seq_num): checkSubs(seq_hash, stored_messages) #Checks sub ports if next_in_seq: # If the next number is indeed last_performed_num + 1 last_performed_num = catchup(calculated_num, stored_messages, last_performed_num) calculated_num, next_in_seq = algorithm.compare_seq_num( seq_hash, last_performed_num) # NOTE: after it finally exits this loop, the seq_num should be equal to calculated_num # This means that calculated_num == loc_seq_num, so it is the current operation # If not, then something has gone wrong with the algorithm (Needs to be debugged) # Actually perform the operation on the db here, grab the response, # and put a message on the output queue message_out = perform_operation_msg(m) last_performed_num += 1 # Send out the message to the output queue q_out.write(message_out)
def running_loop(): try: conn = boto.sqs.connect_to_region("us-west-2") if conn == None: sys.stderr.write("Could not connect to AWS region '{0}'\n".format("us-west-2")) sys.exit(1) # Assuming that the queues will have already been created elsewhere q_in = conn.get_queue(args.in_queue) q_out = conn.get_queue(args.out_queue) except Exception as e: sys.stderr.write("Exception connecting to SQS\n") sys.stderr.write(str(e)) sys.exit(1) global seq_num seq_hash = [] last_performed_num = 0 stored_messages = [] print "Starting up instance: {0}".format(args.my_name) # Actual work gets done here while (1 < 2): # lol # grab a message off SQS_IN rs = q_in.get_messages(message_attributes=["action", "id", "name", "activities"]) if (len(rs) < 1): checkSubs(seq_hash, stored_messages) #Checks sub ports calculated_num, next_in_seq = algorithm.compare_seq_num(seq_hash, last_performed_num) if next_in_seq: # If the next number is indeed last_performed_num + 1 last_performed_num = catchup(calculated_num, stored_messages, last_performed_num) time.sleep(POLL_INTERVAL) # wait before checking for messages again (in seconds) continue m = rs[0] q_in.delete_message(m) # remove message from queue so it's not read multiple times print "Received message: " + m.get_body() seq_num += 1 #increment loc_seq_num = seq_num.last_set #store value to local variable op_list = parse_sqs_msg(m) publishsubscribe.send_message(pub_socket, [loc_seq_num, op_list]) algorithm.add_seq_num(seq_hash, loc_seq_num) # Publish to pub_socket the seq_num and the message # Runs the algorithm to check if the next number in hash is one above the last performed seq num calculated_num, next_in_seq = algorithm.compare_seq_num(seq_hash, last_performed_num) # This is the catch-up loop, running until the number after last performed seq num is loc_seq_num while (calculated_num < loc_seq_num): checkSubs(seq_hash, stored_messages) #Checks sub ports if next_in_seq: # If the next number is indeed last_performed_num + 1 last_performed_num = catchup(calculated_num, stored_messages, last_performed_num) calculated_num, next_in_seq = algorithm.compare_seq_num(seq_hash, last_performed_num) # NOTE: after it finally exits this loop, the seq_num should be equal to calculated_num # This means that calculated_num == loc_seq_num, so it is the current operation # If not, then something has gone wrong with the algorithm (Needs to be debugged) # Actually perform the operation on the db here, grab the response, # and put a message on the output queue message_out = perform_operation_msg(m) last_performed_num += 1 # Send out the message to the output queue q_out.write(message_out)
def checkSubs(seq_hash, msg_list): incoming_msg = publishsubscribe.receive_message(sub_sockets) # This assumes that the return is in the format [seq_num, message] if incoming_msg[1]: # If there was an incoming message from the subscribe ports algorithm.add_seq_num(seq_hash, incoming_msg[0][0]) # Adds the seq_num to the hash msg_list.append(incoming_msg[0]) # Stores the message in the list