def main(): if len(sys.argv) < 3: print "Too few arguments." print HELP elif len(sys.argv) > 3: print "Too many arguments." print HELP else: # Parse command line args, open input file data_file = open(sys.argv[1], "r") local_port = int(sys.argv[2]) # Open the server socket to accept connections srv_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) localhost = netutils.get_ip_addr() srv_socket.bind((localhost, local_port)) srv_socket.listen(0) (sink, sink_address) = srv_socket.accept() # Now that we have someone listening, start talking. for line in data_file: sink.send(line) # End the data transmission. sink.send("\0") # Clean up. sink.shutdown(socket.SHUT_RDWR) sink.close()
def level_one_main(): """Main function for a level 1 reducer. A level 1 reducer receives data from either a mapper or another level 1 reducer, and combines values which have the same key. When the reducer receives null ('\\0') in the input, it ends the reduction. """ if len(sys.argv) < 5: print "Too few arguments." print HELP elif len(sys.argv) > 5: print "Too many arguments." print HELP else: # Parse command line arguments. source_ip = sys.argv[2] source_port = int(sys.argv[3]) local_port = int(sys.argv[4]) localhost = netutils.get_ip_addr() logging.info("Preparing for level 1 reduce on host", localhost) logging.info("Source:", source_ip, source_port) # Set up sockets for remote communications. source = socket.socket(socket.AF_INET, socket.SOCK_STREAM) srv_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # Enable logging to file. logging.basicConfig(filename="reducer-%s.log" % localhost, level=logging.DEBUG) # Bind to local address and listen for new connections (from srv_socket) logging.debug("Binding to local address %s.", localhost) srv_socket.bind((localhost, local_port)) srv_socket.listen(0) # Accept the sink connection. Now we're ready to start reducing. (sink, sink_address) = srv_socket.accept() logging.info("Received sink connection from", sink_address) srv_socket.close() # Reduce! source.connect((source_ip, source_port)) logging.info("Beginning reduce.") reducer = Reducer() receive_values(reducer, source) send_values(reducer, sink) # Clean up after. logging.info("Closing connections.") sink.close() source.close()