def reduceData(s):
'''
The message received could have been
from either a mapper sender or a
master relayer, so the name of the message
is a little misleading...
The master relayers are trying to take
the reduced data while the mappers are
trying to pass of their mapped data to
the reducer
'''
mapperMessage = comms_pb2.AMessage()
mapperMessage.ParseFromString(s)
global endResult, roster, isReady, path
#Check who the message is from
if mapperMessage.data[0:6] == "SERVER":
###A reducer will be ready if it has received from all mappers
if isReady:
toMasterMessage = comms_pb2.AMessage()
toMasterMessage.data = str(endResult)
isReduced.pop(0)
print("[Reducer " + redID +
"] handing off finished data to master relayer")
return toMasterMessage.SerializeToString()
else:
print("NOT READY!")
return bytes("NOT READY", encoding='utf8')
else:
###If it's from a mapper we just reduce and add it to the current dictionary
aReduction = subprocess.Popen([
'python.exe',
os.path.join(path, "Reducer", mapperMessage.functionFileName)
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
aReduction.stdin.write(bytes(mapperMessage.data, encoding='utf8'))
outs, errs = aReduction.communicate(timeout=10)
theData = repr(outs)[2:len(repr(outs)) - 1].replace('\\n', '')
theData = theData.replace('\\r', '')
theMapper = mapperMessage.theSender.name
print("[Reducer " + redID + "] received data from " + theMapper)
roster.remove(theMapper)
if len(roster) == 0:
isReady = True
ini_dict = [endResult, eval(theData)]
counter = collections.Counter()
for d in ini_dict:
counter.update(d)
endResult = dict(counter)
#print(endResult)
return bytes("Thanks", encoding='utf8')
print("Where ya goin?")
return bytes("Thank you, i'll handle this, master", encoding='utf8')
def mapData(s):
global path
masterMessage = comms_pb2.AMessage()
masterMessage.ParseFromString(s)
functions = masterMessage.functionFileName.split(" ")
mapperFn = functions[0]
aMapping = subprocess.Popen(
['python.exe', os.path.join(path, "Mapper", mapperFn)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
aMapping.stdin.write(bytes(masterMessage.data, encoding='utf8'))
outs, errs = aMapping.communicate(timeout=10)
theData = repr(outs)[2:len(repr(outs)) - 1].replace('\\n', '')
theData = theData.replace('\\r', '')
theReducers = masterMessage.others
protoHelper = []
for reducer in theReducers:
protoHelper.append([[reducer.host, reducer.port, functions[1]],
reducer.range])
toBeSent = theData + "vxyxv" + str(protoHelper)
print(toBeSent)
isMapped.pop(0)
return bytes("Thank you, i'll handle this, master", encoding='utf8')
def service_connection(key, mask):
global isServer, finalMessage
sock = key.fileobj
data = key.data
if mask & selectors.EVENT_READ:
recv_data = sock.recv(1024) # Should be ready to read
if recv_data:
#print("[dataRelayer] received", repr(recv_data), "from the server")
mess = repr(recv_data)
if isServer:
#print(mess[2:len(mess)-1])
if (mess[3:len(mess) - 1] == "NOT READY"):
messages = [finalMessage]
start_connections(host, int(port))
else:
endData = comms_pb2.AMessage()
endData.ParseFromString(recv_data)
print(endData.data)
data.recv_total += len(recv_data)
if not recv_data or data.recv_total == data.msg_total:
#print("[Mapper] closing connection to server")
sel.unregister(sock)
sock.close()
if mask & selectors.EVENT_WRITE:
if not data.outb and data.messages:
#print("[Mapper] these are the messages still to be sent : ", data.messages)
data.outb = data.messages.pop(0)
if data.outb:
#print("[Mapper] sending", repr(data.outb), "to server")
sent = sock.send(data.outb) # Should be ready to write
time.sleep(
1
) # This sleep allows the server to receive each message individually
data.outb = data.outb[sent:]
def takeAttendance(s):
theMapOrRedMessage = comms_pb2.AMessage()
theMapOrRedMessage.ParseFromString(s)
senderName = theMapOrRedMessage.theSender.name
rcvrHost = theMapOrRedMessage.theFriend.host
rcvrPort = theMapOrRedMessage.theFriend.port
print(senderName + " " + rcvrHost + " " + rcvrPort +
" ") ### prints to stdout where it is piped to parent
if senderName is None:
print("The senders name was none!")
sys.exit(1)
#return bytes("OW!", encoding='utf8')
roster.remove(senderName)
return bytes("Thanks", encoding='utf8')
Pretty simple here.
Just get the args and store them in the message
'''
if len(sys.argv) != 7:
print(
"usage:", sys.argv[0],
"<host> <port> <id> <rec host/start range> <rec port/ end range> <reducer fn>"
)
sys.exit(1)
host = sys.argv[1]
port = sys.argv[2]
mapID = sys.argv[3]
thisMessage = comms_pb2.AMessage()
thisMessage.data = sys.stdin.readline()
thisMessage.functionFileName = sys.argv[6]
thisMessage.theSender.name = "Mapper" + mapID
thisMessage.theSender.host = port
thisMessage.theSender.port = host
thisMessage.theFriend.name = "MDRcvr" + mapID
thisMessage.theFriend.host = sys.argv[4]
thisMessage.theFriend.port = sys.argv[5]
finalMessage = thisMessage.SerializeToString()
messages = [finalMessage]
#A work in progress... not necessary
hasDropped = False