def _send_thread(self):
# Get Messages from DB
self.msgs.extend(self._get_all())
logger.info('Total %d messages to send' % len(self.msgs))
while True:
if self.stop_flag:
self.send_done = True
break
while len(self.msgs) > 0:
msg = self.msgs.pop(0)
if not self._send_msg(msg):
# ERROR, exit thread
self.stop_flag = True
break
time.sleep(TIMEOUT)
if (not self.send_done) and len(self.msgs) == 0:
# Send SEND_DONE when all messages are sent
# Thread does not die until recv thread receives SEND_DONE
logger.debug('SEND_DONE')
self._send(self.conn_send, SEND_DONE)
self.send_done = True
logger.debug('send_thread exist')
def _send_thread(self):
# Get Messages from DB
self.msgs.extend(self._get_all())
logger.info('Total %d messages to send' % len(self.msgs))
while True:
if self.stop_flag:
self.send_done = True
break
while len(self.msgs) > 0:
msg = self.msgs.pop(0)
if not self._send_msg(msg):
# ERROR, exit thread
self.stop_flag = True
break
time.sleep(TIMEOUT)
if (not self.send_done) and len(self.msgs) == 0:
# Send SEND_DONE when all messages are sent
# Thread does not die until recv thread receives SEND_DONE
logger.debug('SEND_DONE')
self._send(self.conn_send, SEND_DONE)
self.send_done = True
logger.debug('send_thread exist')
def __init__(self, conn_send, conn_recv, my_sh, sh, target, sm, cb=None, server_conn=False):
threading.Thread.__init__(self)
#self.mode = mode
self.conn_send = conn_send
self.conn_recv = conn_recv
self.conn_recv.settimeout(4)
self.sm = sm
self.my_sh = my_sh
self.sh = sh
logger.debug('my_sh %s, sh %s' %(my_sh, sh))
# FIXME need? Callback function
self.cb = cb
# Target the other Site Manager needs
# * or list of ids
self.target = target
self.server_conn = server_conn
self.msgs = list()
# Flags
self.stop_flag = False
self.keep_alive = False
self.send_done = False
self.recv_done = False
self.buf = ''
def handle_bcast_listen(self, s):
"""docstring for handle_bcast_listen"""
logger.debug('Recv bcast')
msg, addr = s.recvfrom(65535)
# send back DTN Port
s.sendto('%s %d' % (self.my_ip, self.dtn_port), addr)
def handle_bcast_listen(self, s):
"""docstring for handle_bcast_listen"""
logger.debug('Recv bcast')
msg, addr = s.recvfrom(65535)
# send back DTN Port
s.sendto('%s %d' % (self.my_ip, self.dtn_port), addr)
def notify_monitors(self, msg):
""" Notify Monitors when receiving a message """
for m in self.monitor_sockets:
logger.debug('sending to monitor %s' % msg)
try:
m.send(msg + '\n')
except socket.error:
print "Error sending to ", m
self.monitor_sockets.remove(m)
def notify_monitors(self, msg):
""" Notify Monitors when receiving a message """
for m in self.monitor_sockets:
logger.debug('sending to monitor %s'% msg)
try:
m.send(msg + '\n')
except socket.error:
print "Error sending to ", m
self.monitor_sockets.remove(m)
def _recv_thread(self):
while True:
if self.stop_flag:
logger.debug('recv_thread is killed')
self.recv_done = True
break
try:
self.buf += self.conn_recv.recv(1024)
while self.buf.find('\n') >= 0:
msg, self.buf = self.buf.split('\n', 1)
if msg == SEND_DONE:
logger.debug('recv SEND_DONE')
self.recv_done = True
continue
self._recv_handle(msg)
if self.cb is not None:
self.cb(msg)
except socket.timeout:
logger.debug("no incoming data")
except socket.error:
logger.warn("recv_thread connection lost")
self.stop_flag = True
self.recv_done = True
break
logger.debug('recv_thread exist')
def _recv_thread(self):
while True:
if self.stop_flag:
logger.debug('recv_thread is killed')
self.recv_done = True
break
try:
self.buf += self.conn_recv.recv(1024)
while self.buf.find('\n') >= 0:
msg, self.buf = self.buf.split('\n', 1)
if msg == SEND_DONE:
logger.debug('recv SEND_DONE')
self.recv_done = True
continue
self._recv_handle(msg)
if self.cb is not None:
self.cb(msg)
except socket.timeout:
logger.debug("no incoming data")
except socket.error:
logger.warn("recv_thread connection lost")
self.stop_flag = True
self.recv_done = True
break
logger.debug('recv_thread exist')
def _conn_thread(self):
if self.server_port == 0:
return
while True:
import time
time.sleep(5)
if self.stop_flag:
break
if self.server_connected:
continue
logger.debug('Client try to connect to server')
if self.connect_to_sm(self.server_ip, self.server_port, True):
self.server_connected = True
time.sleep(10)
def _conn_thread(self):
if self.server_port == 0:
return
while True:
import time
time.sleep(5)
if self.stop_flag:
break
if self.server_connected:
continue
logger.debug('Client try to connect to server')
if self.connect_to_sm(self.server_ip, self.server_port, True):
self.server_connected = True
time.sleep(10)
def _daemon_thread(self):
while True:
#if not self.send_thread.isAlive() and self.recv_done:#not self.recv_thread.isAlive():
if (self.send_done and self.recv_done) or ((not self.send_thread.isAlive()) and (not
self.recv_thread.isAlive())):#not self.recv_thread.isAlive():
if self.stop_flag:
logger.error('Error happended or stop by user')
break
time.sleep(5)
# Work is done, this instance of DTNConnection could be removed
self.stop_flag = True
self.send_thread.join()
self.recv_thread.join()
self.clean()
logger.debug('clean done')
logger.info('job done')
def handle_vclient_listen(self, s):
chunk = s.recv(1024)
if chunk == '':
print "UDP error", s
sys.exit(1)
else:
logger.debug('recv from vclient ' + chunk)
msg = DTNMessage()
msg.handle(chunk, self.server_sh)
msg.src = self.sh
if msg.type == 'PING':
# Notify Monitors
self.notify_monitors(chunk)
# Save into database
try:
self.db.insert_msg(msg)
except:
logger.debug('error in inserting PING')
def handle_vclient_listen(self, s):
chunk = s.recv(1024)
if chunk == '':
print "UDP error", s
sys.exit(1)
else:
logger.debug('recv from vclient ' + chunk)
msg = DTNMessage()
msg.handle(chunk, self.server_sh)
msg.src = self.sh
if msg.type == 'PING':
# Notify Monitors
self.notify_monitors(chunk)
# Save into database
try:
self.db.insert_msg(msg)
except:
logger.debug('error in inserting PING')
def _daemon_thread(self):
while True:
#if not self.send_thread.isAlive() and self.recv_done:#not self.recv_thread.isAlive():
if (self.send_done and self.recv_done) or (
(not self.send_thread.isAlive()) and
(not self.recv_thread.isAlive())
): #not self.recv_thread.isAlive():
if self.stop_flag:
logger.error('Error happended or stop by user')
break
time.sleep(5)
# Work is done, this instance of DTNConnection could be removed
self.stop_flag = True
self.send_thread.join()
self.recv_thread.join()
self.clean()
logger.debug('clean done')
logger.info('job done')
def __init__(self,
conn_send,
conn_recv,
my_sh,
sh,
target,
sm,
cb=None,
server_conn=False):
threading.Thread.__init__(self)
#self.mode = mode
self.conn_send = conn_send
self.conn_recv = conn_recv
self.conn_recv.settimeout(4)
self.sm = sm
self.my_sh = my_sh
self.sh = sh
logger.debug('my_sh %s, sh %s' % (my_sh, sh))
# FIXME need? Callback function
self.cb = cb
# Target the other Site Manager needs
# * or list of ids
self.target = target
self.server_conn = server_conn
self.msgs = list()
# Flags
self.stop_flag = False
self.keep_alive = False
self.send_done = False
self.recv_done = False
self.buf = ''
def connect_to_sm(self, ip, port, server_conn=False):
""" connect to specific IP:PORT
"""
logger.debug('Try to connect to Site Manager and establish DTN connection')
conn_send = DTN._tcp_connect(ip, port)
if conn_send is not None:
# send information
conn_send.send('%s %s\n' % (self.sh, self.target))
data = conn_send.recv(1024)
port_recv = 0
if data == EXIST_ERR:
logger.debug('DTN connection already exists between these two Site Managers')
return False
sh = data.split()[0]
port_recv = int(data.split()[1])
tar = ' '.join(data.split()[2:])
logger.debug('SH: %s, Port: %d, Target: %s' % (sh, port_recv, tar))
# check if site manager is connected
if self.dtn.has_key(sh) and self.dtn[sh] is not None:
conn_send.send(EXIST_ERR)
logger.debug('DTN connection already exists between these two Site Managers')
return False
else:
conn_send.send(SUCCESS_INFO)
# connect to ip using another port
conn_recv = DTN._tcp_connect(ip, port_recv)
if conn_recv is not None:
# Ready
dtn_conn = DTNConnection(conn_send, conn_recv, self.sh, sh, tar, self, server_conn=server_conn)
self.dtn[sh] = dtn_conn
if not self.last_hash.has_key(sh):
self.last_hash[sh] = ''
logger.info('New DTN connection established')
dtn_conn.start()
return True
return False
def _send_msg(self, msg, n=0):
""" Send DTNMessage
If error occurs, message will be resent 3 times max
"""
if n > 2:
return False
try:
data = msg.to_msg()
self._send(self.conn_send, data)
logger.debug("send %s : %s" % (msg.type, data))
# Waiting ACK
res = self.conn_send.recv(1024)
#logger.debug('waiting ack and recv ' + res)
if res == 'ACK %s\n' % msg.hash:
logger.debug('recv ACK')
# update last hash
self.sm.last_hash[self.sh] = msg.hash
#reach destination, set ack=1
#if msg.dst == self.sh:
#self.sm.db.update('ack=1', "hash='%s'"%msg.hash)
return True
elif res == UNKNOWN_MSG:
return True
else:
logger.debug('not ACK, try to send again')
# Error, try to send again
time.sleep(TIMEOUT)
return self._send_msg(msg, n+1)
except socket.timeout:
logger.debug('timeout, no ACK, try to send again')
return self._send_msg(msg, n+1)
except socket.error:
self.stop_flag = True
self.send_done = True
logger.warn("send_thread connection lost")
return False
return True
def _send_msg(self, msg, n=0):
""" Send DTNMessage
If error occurs, message will be resent 3 times max
"""
if n > 2:
return False
try:
data = msg.to_msg()
self._send(self.conn_send, data)
logger.debug("send %s : %s" % (msg.type, data))
# Waiting ACK
res = self.conn_send.recv(1024)
#logger.debug('waiting ack and recv ' + res)
if res == 'ACK %s\n' % msg.hash:
logger.debug('recv ACK')
# update last hash
self.sm.last_hash[self.sh] = msg.hash
#reach destination, set ack=1
#if msg.dst == self.sh:
#self.sm.db.update('ack=1', "hash='%s'"%msg.hash)
return True
elif res == UNKNOWN_MSG:
return True
else:
logger.debug('not ACK, try to send again')
# Error, try to send again
time.sleep(TIMEOUT)
return self._send_msg(msg, n + 1)
except socket.timeout:
logger.debug('timeout, no ACK, try to send again')
return self._send_msg(msg, n + 1)
except socket.error:
self.stop_flag = True
self.send_done = True
logger.warn("send_thread connection lost")
return False
return True
def bcast(self, bcast_port):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
s.sendto(BCAST_MSG, ('<broadcast>', bcast_port))
logger.debug('send broadcast')
s.settimeout(2)
try:
buf, addr = s.recvfrom(2048)
logger.info('Received from %s: %s' % (buf.split()[0], buf.split()[1]))
s.close()
return buf.split()[0], int(buf.split()[1])
except socket.timeout:
logger.debug('no feedback')
except :
logger.debug('bcast socket error')
s.close()
return None
def bcast(self, bcast_port):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
s.sendto(BCAST_MSG, ('<broadcast>', bcast_port))
logger.debug('send broadcast')
s.settimeout(2)
try:
buf, addr = s.recvfrom(2048)
logger.info('Received from %s: %s' %
(buf.split()[0], buf.split()[1]))
s.close()
return buf.split()[0], int(buf.split()[1])
except socket.timeout:
logger.debug('no feedback')
except:
logger.debug('bcast socket error')
s.close()
return None
def _get_last_key(self, hash):
msgs = self.sm.db.select_msg("hash='%s'" % hash)
if len(msgs) == 1:
logger.debug('Last ID : %d' % msgs[0].id)
return msgs[0].id
return -1
def handle_dtn_listen(self, s):
logger.info('DTN connection request')
conn_recv, remote = s.accept()
data = conn_recv.recv(1024)
sh = data.split()[0]
tar = ' '.join(data.split()[1:])
logger.debug('SH: %s, Target: %s' % (sh, tar))
if self.dtn.has_key(sh):
logger.debug(self.dtn[sh])
if self.dtn.has_key(sh) and self.dtn[sh] is not None:
conn_recv.send(EXIST_ERR)
logger.debug(
'DTN connection already exists between these two Site Managers'
)
return False
# Generate a random port for listening
random_n = 1
listener = None
while True:
try:
listener = DTN._tcp_listen(self.my_ip,
self.dtn_port + random_n)
except:
random_n += 1
continue
break
conn_recv.send('%s %d %s' %
(self.sh, self.dtn_port + random_n, self.target))
data = conn_recv.recv(1024)
if data == EXIST_ERR:
logger.debug(
'DTN connection already exists between these two Site Managers'
)
return False
# wait for connection
listener.settimeout(5)
try:
conn_send, remote = listener.accept()
except socket.timeout:
logger.debug('timeout')
return False
# remote[0] is IP
# remote[1] is PORT
#conn_send = DTN._tcp_connect(remote[0], port)
if conn_send is not None:
# send my SH info
#conn_send.send('%s %s' % (self.sh, self.target))
#data = conn_send.recv(1024)
#if data == EXIST_ERR:
#logger.debug('DTN connection already exists between these two Site Managers')
#return
#else:
#logger.debug('Good')
# Ready
dtn_conn = DTNConnection(conn_send, conn_recv, self.sh, sh, tar,
self)
self.dtn[sh] = dtn_conn
if not self.last_hash.has_key(sh):
self.last_hash[sh] = ''
#if sh == SERVER_SH_INFO:
#self.server_connected = True
logger.info('New DTN connection established')
dtn_conn.start()
return True
return False
def connect_to_sm(self, ip, port, server_conn=False):
""" connect to specific IP:PORT
"""
logger.debug(
'Try to connect to Site Manager and establish DTN connection')
conn_send = DTN._tcp_connect(ip, port)
if conn_send is not None:
# send information
conn_send.send('%s %s\n' % (self.sh, self.target))
data = conn_send.recv(1024)
port_recv = 0
if data == EXIST_ERR:
logger.debug(
'DTN connection already exists between these two Site Managers'
)
return False
sh = data.split()[0]
port_recv = int(data.split()[1])
tar = ' '.join(data.split()[2:])
logger.debug('SH: %s, Port: %d, Target: %s' % (sh, port_recv, tar))
# check if site manager is connected
if self.dtn.has_key(sh) and self.dtn[sh] is not None:
conn_send.send(EXIST_ERR)
logger.debug(
'DTN connection already exists between these two Site Managers'
)
return False
else:
conn_send.send(SUCCESS_INFO)
# connect to ip using another port
conn_recv = DTN._tcp_connect(ip, port_recv)
if conn_recv is not None:
# Ready
dtn_conn = DTNConnection(conn_send,
conn_recv,
self.sh,
sh,
tar,
self,
server_conn=server_conn)
self.dtn[sh] = dtn_conn
if not self.last_hash.has_key(sh):
self.last_hash[sh] = ''
logger.info('New DTN connection established')
dtn_conn.start()
return True
return False
def _recv_handle(self, msg):
dtn_msg = DTNMessage()
if not dtn_msg.handle(msg):
logger.debug('cannot recognize message ' + msg)
self._send(self.conn_recv, UNKNOWN_MSG)
return
if dtn_msg.type == 'DST_ACK':
self.sm.db.insert_dst_ack(dtn_msg)
# set the match data ack=1
self.sm.db.update('ack=1', "hash='%s'" % dtn_msg.data)
else:
self.sm.db.insert_msg(dtn_msg)
logger.debug('recv %s : %s' % (dtn_msg.type, msg))
# Any messages but ACK should send ACK
if dtn_msg.type != 'ACK':
self._send(self.conn_recv, 'ACK ' + dtn_msg.hash)
logger.debug('send ACK ' + dtn_msg.hash)
else: # ACK
return
# Message reaches destination
if dtn_msg.dst == self.my_sh:
self.sm.db.update('ack=1', "hash='%s'" % dtn_msg.hash)
# New DST_ACK message
if dtn_msg.type != 'DST_ACK':
dst_msg = DTNMessage()
dst_msg.time = int(time.time() * 1000)
dst_msg.ack = 1
dst_msg.ip = self.sm.my_ip
dst_msg.port = self.sm.dtn_port
dst_msg.dst = dtn_msg.src
dst_msg.src = self.my_sh
dst_msg.type = 'DST_ACK'
dst_msg.data = dtn_msg.hash
dst_msg.hash = dst_msg.get_hash()
self.sm.db.insert_dst_ack(dst_msg)
self.msgs.append(dst_msg)
# CMD
if dtn_msg.type == 'CMD':
# Run
cmd = dtn_msg.data
logger.debug('recv CMD : %s', cmd)
import commands
res = commands.getstatusoutput(cmd)
logger.debug('CMD result :' + str(res))
# Send CMD_RES
res_msg = DTNMessage()
res_msg.time = int(time.time() * 1000)
res_msg.ack = 0
res_msg.ip = self.sm.my_ip
res_msg.port = self.sm.dtn_port
res_msg.dst = dtn_msg.src
res_msg.src = self.my_sh
res_msg.type = 'CMD_RES'
res_msg.data = str(res)
res_msg.hash = res_msg.get_hash()
self.sm.db.insert_msg(res_msg)
self.msgs.append(res_msg)
return True
def handle_dtn_listen(self, s):
logger.info('DTN connection request')
conn_recv, remote = s.accept()
data = conn_recv.recv(1024)
sh = data.split()[0]
tar = ' '.join(data.split()[1:])
logger.debug('SH: %s, Target: %s' % (sh, tar))
if self.dtn.has_key(sh):
logger.debug(self.dtn[sh])
if self.dtn.has_key(sh) and self.dtn[sh] is not None:
conn_recv.send(EXIST_ERR)
logger.debug('DTN connection already exists between these two Site Managers')
return False
# Generate a random port for listening
random_n = 1
listener = None
while True:
try:
listener = DTN._tcp_listen(self.my_ip, self.dtn_port+random_n)
except:
random_n += 1
continue
break
conn_recv.send('%s %d %s' % (self.sh, self.dtn_port+random_n, self.target))
data = conn_recv.recv(1024)
if data == EXIST_ERR:
logger.debug('DTN connection already exists between these two Site Managers')
return False
# wait for connection
listener.settimeout(5)
try:
conn_send, remote = listener.accept()
except socket.timeout:
logger.debug('timeout')
return False
# remote[0] is IP
# remote[1] is PORT
#conn_send = DTN._tcp_connect(remote[0], port)
if conn_send is not None:
# send my SH info
#conn_send.send('%s %s' % (self.sh, self.target))
#data = conn_send.recv(1024)
#if data == EXIST_ERR:
#logger.debug('DTN connection already exists between these two Site Managers')
#return
#else:
#logger.debug('Good')
# Ready
dtn_conn = DTNConnection(conn_send, conn_recv, self.sh, sh, tar, self)
self.dtn[sh] = dtn_conn
if not self.last_hash.has_key(sh):
self.last_hash[sh] = ''
#if sh == SERVER_SH_INFO:
#self.server_connected = True
logger.info('New DTN connection established')
dtn_conn.start()
return True
return False
def _get_last_key(self, hash):
msgs = self.sm.db.select_msg("hash='%s'" % hash)
if len(msgs) == 1:
logger.debug('Last ID : %d' % msgs[0].id)
return msgs[0].id
return -1
def _recv_handle(self, msg):
dtn_msg = DTNMessage()
if not dtn_msg.handle(msg):
logger.debug('cannot recognize message ' + msg)
self._send(self.conn_recv, UNKNOWN_MSG)
return
if dtn_msg.type == 'DST_ACK':
self.sm.db.insert_dst_ack(dtn_msg)
# set the match data ack=1
self.sm.db.update('ack=1', "hash='%s'"%dtn_msg.data)
else:
self.sm.db.insert_msg(dtn_msg)
logger.debug('recv %s : %s' % (dtn_msg.type, msg))
# Any messages but ACK should send ACK
if dtn_msg.type != 'ACK':
self._send(self.conn_recv, 'ACK ' + dtn_msg.hash)
logger.debug('send ACK ' + dtn_msg.hash)
else: # ACK
return
# Message reaches destination
if dtn_msg.dst == self.my_sh:
self.sm.db.update('ack=1', "hash='%s'"%dtn_msg.hash)
# New DST_ACK message
if dtn_msg.type != 'DST_ACK':
dst_msg = DTNMessage()
dst_msg.time = int(time.time()*1000)
dst_msg.ack = 1
dst_msg.ip = self.sm.my_ip
dst_msg.port = self.sm.dtn_port
dst_msg.dst = dtn_msg.src
dst_msg.src = self.my_sh
dst_msg.type = 'DST_ACK'
dst_msg.data = dtn_msg.hash
dst_msg.hash = dst_msg.get_hash()
self.sm.db.insert_dst_ack(dst_msg)
self.msgs.append(dst_msg)
# CMD
if dtn_msg.type == 'CMD':
# Run
cmd = dtn_msg.data
logger.debug('recv CMD : %s', cmd)
import commands
res = commands.getstatusoutput(cmd)
logger.debug('CMD result :' + str(res))
# Send CMD_RES
res_msg = DTNMessage()
res_msg.time = int(time.time()*1000)
res_msg.ack = 0
res_msg.ip = self.sm.my_ip
res_msg.port = self.sm.dtn_port
res_msg.dst = dtn_msg.src
res_msg.src = self.my_sh
res_msg.type = 'CMD_RES'
res_msg.data = str(res)
res_msg.hash = res_msg.get_hash()
self.sm.db.insert_msg(res_msg)
self.msgs.append(res_msg)
return True
