def main():
if len(sys.argv) < 3:
print "Too few arguments."
print HELP
elif len(sys.argv) > 3:
print "Too many arguments."
print HELP
else:
# Parse command line args, open input file
data_file = open(sys.argv[1], "r")
local_port = int(sys.argv[2])
# Open the server socket to accept connections
srv_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
localhost = netutils.get_ip_addr()
srv_socket.bind((localhost, local_port))
srv_socket.listen(0)
(sink, sink_address) = srv_socket.accept()
# Now that we have someone listening, start talking.
for line in data_file:
sink.send(line)
# End the data transmission.
sink.send("\0")
# Clean up.
sink.shutdown(socket.SHUT_RDWR)
sink.close()
def level_one_main():
"""Main function for a level 1 reducer.
A level 1 reducer receives data from either a mapper or another level 1
reducer, and combines values which have the same key. When the reducer
receives null ('\\0') in the input, it ends the reduction.
"""
if len(sys.argv) < 5:
print "Too few arguments."
print HELP
elif len(sys.argv) > 5:
print "Too many arguments."
print HELP
else:
# Parse command line arguments.
source_ip = sys.argv[2]
source_port = int(sys.argv[3])
local_port = int(sys.argv[4])
localhost = netutils.get_ip_addr()
logging.info("Preparing for level 1 reduce on host", localhost)
logging.info("Source:", source_ip, source_port)
# Set up sockets for remote communications.
source = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
srv_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Enable logging to file.
logging.basicConfig(filename="reducer-%s.log" % localhost,
level=logging.DEBUG)
# Bind to local address and listen for new connections (from srv_socket)
logging.debug("Binding to local address %s.", localhost)
srv_socket.bind((localhost, local_port))
srv_socket.listen(0)
# Accept the sink connection. Now we're ready to start reducing.
(sink, sink_address) = srv_socket.accept()
logging.info("Received sink connection from", sink_address)
srv_socket.close()
# Reduce!
source.connect((source_ip, source_port))
logging.info("Beginning reduce.")
reducer = Reducer()
receive_values(reducer, source)
send_values(reducer, sink)
# Clean up after.
logging.info("Closing connections.")
sink.close()
source.close()