def handleUDP(data):
global ACTIVE
global BLOCK_FLAG
global log
global state
global quiet_lock
global num_client
global num_blocked
sock = data
tdata = threading.local()
while True:
try:
tdata.data, tdata.addr = sock.recvfrom(buffSize)
except socket.timeout:
exit(0)
tdata.t, tdata.s, tdata.c = messager.decodeMsg(
tdata.data.decode('utf-8'))
if tdata.t == messager.MSG_CHECKPOINT:
with __lock:
if tdata.s == 1:
tdata.data = messager.encodeMsg(messager.MSG_CHECKPOINT, 2,
state)
sock.sendto(bytes(tdata.data, 'utf-8'),
(tdata.c[0], tdata.c[1]))
elif tdata.s == 2:
print("after checkpoint state is : ", tdata.c)
else:
tdata.cid, tdata.sid, tdata.seq, tdata.ts, tdata.state = messager.decodeUpdateState(
tdata.c)
print("checkpoint state is : ", tdata.state)
state = tdata.state
for i in range(len(log)):
if log[i][0] == tdata.cid and log[i][2] == tdata.seq:
break
log = log[i:]
print("obtain useful log:")
for i in log:
print(i)
for i in log:
state = i[4]
logging.info(
"client: %i ,server: %i, seq: %i, time: %s,state: %s",
i[0], i[1], i[2], i[3], i[4])
log = []
quiet_lock = False
if tdata.t == messager.MSG_LFD_SERVER_INIT:
print('recv lfd init')
sudp.sendto(tdata.data, tdata.addr)
# Heartbeating
if tdata.t == messager.MSG_PING:
if sid == messager.decodeHeartbeat(tdata.c):
tdata.data = messager.encodeMsg(messager.MSG_BEATING, tdata.s,
tdata.c)
sock.sendto(bytes(tdata.data, 'utf-8'), tdata.addr)
# Should Never happen, but just in case
else:
tdata.data = messager.encodeMsg(messager.MSG_ERROR, tdata.s,
tdata.c)
sock.sendto(bytes(tdata.data, 'utf-8'), tdata.addr)
elif tdata.t == messager.MSG_ACTIVE:
with __lock:
ACTIVE = True
logging.info("Become active")
elif tdata.t == messager.MSG_QUIET:
with __lock:
quiet_lock = True
logging.info("Become quiet")
elif tdata.t == messager.MSG_CHECK_REMIND:
with __lock:
tosid = int(tdata.c)
print("sid:", tosid)
if tosid == 1:
sendToCheck = 10087
if tosid == 2:
sendToCheck = 10088
if tosid == 3:
sendToCheck = 10089
BLOCK_FLAG = True
check_state = state
print("checkpointing")
print("send checkpoint. state:", check_state)
#logging.info("%d %d", num_blocked, num_client)
while num_blocked < num_client:
continue
#logging.info("%d %d", num_blocked, num_client)
with __lock:
tdata.c = messager.encodeUpdateState(log[0][0], log[0][1],
log[0][2], log[0][3],
check_state)
msg = messager.encodeMsg(messager.MSG_CHECKPOINT, 0, tdata.c)
sock.sendto(msg.encode('utf-8'), ("localhost", sendToCheck))
BLOCK_FLAG = False
log = []
num_client = 0
def handleTCP(data):
global state
global log
global num_client
global num_blocked
sock = data[0]
tdata = threading.local()
tdata.cid = 0
recovered = False
with __lock:
num_client += 1
while True:
tdata.data = sock.recv(buffSize)
if not tdata.data:
# logging.info("CID: %i exit (0 is UNKNOWN)", tdata.cid)
with __lock:
num_client -= 1
sock.close()
return
tdata.t, tdata.s, tdata.c = messager.decodeMsg(
tdata.data.decode('utf-8'))
# Update state, may be better to have a dedicate function
if tdata.t == messager.MSG_UPDATESTATE:
# cid, seq, state
tdata.cid, tdata.sid, tdata.seq, tdata.ts, tdata.state = messager.decodeUpdateState(
tdata.c)
item = [tdata.cid, tdata.sid, tdata.seq, tdata.ts, tdata.state]
with __lock:
if ACTIVE == False:
logging.error("NOT ACTIVE")
log.append([
tdata.cid, tdata.sid, tdata.seq, tdata.ts, tdata.state
])
print("log:")
for i in log:
print(i)
tdata.c = "SERVER NOT ACTIVE"
tdata.data = messager.encodeMsg(messager.MSG_ERROR,
len(tdata.c), tdata.c)
sock.send(bytes(tdata.data, 'utf-8'))
#with __lock:
#num_client -= 1
#sock.close()
continue
with __lock:
# Send
if quiet_lock:
logging.info("NOT ACTIVE")
log.append([
tdata.cid, tdata.sid, tdata.seq, tdata.ts, tdata.state
])
print("log:")
for i in log:
print(i)
tdata.data = messager.encodeMsg(messager.MSG_UPDATESTATE,
0, tdata.c)
sock.send(bytes(tdata.data, 'utf-8'))
continue
__lock.acquire() # Racing Condition
state = tdata.state
logging.info("client: %i ,server: %i, seq: %i, time: %s,state: %s",
tdata.cid, tdata.sid, tdata.seq, tdata.ts, state)
if BLOCK_FLAG:
num_blocked += 1
log.append(
[tdata.cid, tdata.sid, tdata.seq, tdata.ts, tdata.state])
recovered = True
__lock.release()
while BLOCK_FLAG: # Busy wait for very short period
continue
with __lock:
tdata.data = messager.encodeMsg(messager.MSG_UPDATESTATE, 0,
tdata.c)
sock.send(tdata.data.encode('utf-8'))
if recovered:
time.sleep(0.2)
recovered = False
def stateUpdate(stcp):
global seq
global recovering
global resend
# Currently for easy debug -- Later will be feeded by script
if recovering:
state = resend
print("send recover", state)
else:
state = messager.inputState()
# send msg
for i in stcp:
# check connection
if i[2]:
timestamp = str(datetime.now())
update_str = messager.encodeUpdateState(cid, i[0], seq, timestamp,
state)
if recovering:
msg = messager.encodeMsg(messager.MSG_RECOVERY,
len(update_str) + 1, update_str)
else:
msg = messager.encodeMsg(messager.MSG_UPDATESTATE,
len(update_str) + 1, update_str)
i[1].send(bytes(msg, 'utf-8'))
if not recovering:
print("msg send to server id:", i[0], state)
else:
# reconnect
try:
i[1] = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
i[1].connect(server_address[i[0] - 1])
i[2] = True
timestamp = str(datetime.now())
update_str = messager.encodeUpdateState(
cid, i[0], seq, timestamp, state)
if recovering:
msg = messager.encodeMsg(messager.MSG_RECOVERY,
len(update_str) + 1, update_str)
else:
msg = messager.encodeMsg(messager.MSG_UPDATESTATE,
len(update_str) + 1, update_str)
i[1].send(bytes(msg, 'utf-8'))
if not recovering:
print("msg send to server id:", i[0], state)
except socket.error:
continue
# duplicate detector
recovering = True
for i in range(len(stcp)):
if stcp[i][2]:
try:
data = stcp[i][1].recv(buffSize)
# error
if not data:
stcp[i][1].close()
stcp[i][2] = False
if stcp[i][3] == 'p':
recovering = True
stcp[i][3] = 'b'
continue
i_type, i_len, i_c = messager.decodeMsg(data.decode('utf-8'))
if i_type == messager.MSG_UPDATESTATE:
i_cid, i_sid, i_seq, i_timestamp, i_state = messager.decodeUpdateState(
i_c)
# only implementaiton on reply
if i_state == "backup is sleeping":
continue
if i_seq >= seq:
print(messager.decodeUpdateState(i_c))
stcp[i][3] = 'p'
recovering = False
seq += 1
else:
print("duplicate reply detected:")
print(messager.decodeUpdateState(i_c))
# error
except socket.error:
stcp[i][1].close()
stcp[i][2] = False
if stcp[i][3] == 'p':
recovering = True
stcp[i][3] = 'b'
continue
if recovering:
resend = state
time.sleep(1)
return
def handleTCP(data, udp):
global state
global Checkpoint_turn
global Log
sock = data[0]
tdata = threading.local()
tdata.cid = 0
checkpoint_count = 0
while True:
tdata.data = sock.recv(buffSize)
if not tdata.data:
logging.info("CID: %i exit (0 is UNKNOWN)", tdata.cid)
sock.close()
return
tdata.t, tdata.s, tdata.c = messager.decodeMsg(
tdata.data.decode('utf-8'))
# Update state, may be better to have a dedicate function
if tdata.t == messager.MSG_UPDATESTATE:
# Client side do not have a way to handle this at this point
if ACTIVE == False:
logging.error("NOT ACTIVE")
tdata.c = "SERVER NOT ACTIVE"
tdata.data = messager.encodeMsg(messager.MSG_ERROR,
len(tdata.c), tdata.c)
# sock.send(bytes(tdata.data, 'utf-8'))
sock.close()
exit(0)
# cid, seq, state
tdata.cid, tdata.sid, tdata.seq, tdata.ts, tdata.state = messager.decodeUpdateState(
tdata.c)
__lock.acquire() # Racing Condition
# state = tdata.state
# logging.info("client: %i ,server: %i, seq: %i, state: %s", tdata.cid, tdata.sid,tdata.seq, state)
# sock.send(tdata.c.encode('utf-8'))
# Checkpoint_turn = True
##checkpoint_count += 1
if primaryFlag == True:
state = tdata.state
logging.info(
"client: %i ,server: %i, seq: %i, time: %s, state: %s",
tdata.cid, tdata.sid, tdata.seq, tdata.ts, state)
tdata.data = messager.encodeMsg(messager.MSG_UPDATESTATE, 0,
tdata.c)
sock.send(tdata.data.encode('utf-8'))
Checkpoint_turn += 1
else:
logging.info("backup is sleeping")
tdata.c = messager.encodeUpdateState(tdata.cid, tdata.sid,
tdata.seq, tdata.ts,
"backup is sleeping")
tdata.data = messager.encodeMsg(messager.MSG_UPDATESTATE, 0,
tdata.c)
sock.send(tdata.data.encode('utf-8'))
Log.append([
tdata.cid, tdata.sid, tdata.seq, tdata.ts, tdata.state, 0
])
print("log:")
for i in Log:
print(i[:5])
__lock.release()
# recovery request
elif tdata.t == messager.MSG_RECOVERY:
#print("recovering")
# cid, seq, state
tdata.cid, tdata.sid, tdata.seq, tdata.ts, tdata.state = messager.decodeUpdateState(
tdata.c)
if primaryFlag == True:
while True:
#process log
flag = False
for i in range(len(Log)):
if Log[i][0] == tdata.cid and Log[i][2] == tdata.seq:
#current first
if i == 0 or Log[i - 1][5] == 0 or Log[i -
1][5] == 2:
flag = True
break
else:
break
if flag:
break
time.sleep(0.2)
with __lock:
#process log
for i in range(len(Log)):
if Log[i][0] == tdata.cid and Log[i][2] == tdata.seq:
state = Log[i][4]
Log[i][5] = 2
logging.info(
"client: %i ,server: %i, seq: %i, time: %s, state: %s",
Log[i][0], Log[i][1], Log[i][2], Log[i][3],
state)
tdata.c = messager.encodeUpdateState(
Log[i][0], Log[i][1], Log[i][2], Log[i][3],
state)
tdata.data = messager.encodeMsg(
messager.MSG_UPDATESTATE, 0, tdata.c)
sock.send(tdata.data.encode('utf-8'))
Checkpoint_turn += 1
#prune log
if i == (len(Log) - 1):
Log = []
print("Recovery done. Current state:", state)
break
elif Log[i][5] == 0:
state = Log[i][4]
Log[i][5] = 2
else:
with __lock:
tdata.c = messager.encodeUpdateState(
tdata.cid, tdata.sid, tdata.seq, tdata.ts,
"backup is sleeping")
tdata.data = messager.encodeMsg(messager.MSG_UPDATESTATE,
0, tdata.c)
sock.send(tdata.data.encode('utf-8'))
#mark as unreplied
for i in range(len(Log)):
if Log[i][0] == tdata.cid and Log[i][2] == tdata.seq:
Log[i][5] = 1
# send checkpoint out if it is the primary and meets the frequency
if primaryFlag and (Checkpoint_turn >= checkpoint_frequency):
with __lock:
checkpoint = messager.encodeMsg(messager.MSG_CHECKPOINT,
len(state) + 1, state)
sendCheck(udp, checkpoint)
Checkpoint_turn = 0
def stateUpdate(stcp):
global seq
# Currently for easy debug -- Later will be feeded by script
state = messager.inputState()
# send msg
for i in stcp:
# check connection
if i[2]:
timestamp = str(datetime.now())
update_str = messager.encodeUpdateState(cid, i[0], seq, timestamp,
state)
msg = messager.encodeMsg(messager.MSG_UPDATESTATE,
len(update_str) + 1, update_str)
i[1].send(bytes(msg, 'utf-8'))
print("msg send to server id:", i[0], "msg:", state)
else:
# reconnect
try:
i[1] = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
i[1].connect(server_address[i[0] - 1])
i[2] = True
timestamp = str(datetime.now())
update_str = messager.encodeUpdateState(
cid, i[0], seq, timestamp, state)
msg = messager.encodeMsg(messager.MSG_UPDATESTATE,
len(update_str) + 1, update_str)
i[1].send(bytes(msg, 'utf-8'))
print("msg send to server id:", i[0], "msg:", state)
except socket.error:
print("not add ", server_address[i[0] - 1])
continue
# duplicate detector
for i in range(len(stcp)):
if stcp[i][2]:
try:
data = stcp[i][1].recv(buffSize)
# error
if not data:
stcp[i][1].close()
stcp[i][2] = False
continue
i_type, i_len, i_c = messager.decodeMsg(data.decode('utf-8'))
if i_type == messager.MSG_UPDATESTATE:
i_cid, i_sid, i_seq, i_timestamp, i_state = messager.decodeUpdateState(
i_c)
# only implementaiton on reply
if i_state == "backup is sleeping":
continue
if i_seq >= seq:
print("Reply:", messager.decodeUpdateState(i_c))
seq += 1
else:
print("Duplicate reply detected:")
print(messager.decodeUpdateState(i_c))
elif i_type == messager.MSG_ERROR:
if i_c == "SERVER NOT ACTIVE":
continue
elif i_c == "SERVER QUIESCENCE":
print("quiet")
continue
# error
except socket.error:
stcp[i][1].close()
stcp[i][2] = False
continue
return