def sched_udp_proc(self, coro=None):
coro.set_daemon()
sched_sock = asyncoro.AsyncSocket(
socket.socket(socket.AF_INET, socket.SOCK_DGRAM))
sched_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sched_sock.bind(('', self.scheduler_port))
while 1:
msg, addr = yield sched_sock.recvfrom(1024)
if (not msg.startswith('PING:'.encode())
or not self.scheduler_ip_addrs or not self.scheduler_port):
logger.debug('Ignoring ping message from %s (%s)', addr[0],
addr[1])
continue
try:
info = asyncoro.unserialize(msg[len('PING:'.encode()):])
logger.debug('sched_sock: %s' % info)
assert info['version'] == __version__
# assert isinstance(info['cpus'], int)
except:
logger.debug(traceback.format_exc())
msg = {
'ip_addrs': self.scheduler_ip_addrs,
'port': self.scheduler_port,
'version': __version__
}
if info.get('relay', None):
logger.debug('Ignoring ping back from %s: %s', addr[0], info)
continue
msg['relay'] = 'y'
relay_sock = asyncoro.AsyncSocket(
socket.socket(socket.AF_INET, socket.SOCK_DGRAM))
yield relay_sock.sendto('PING:'.encode() + asyncoro.serialize(msg),
(info['ip_addr'], info['port']))
relay_sock.close()
def listen_udp_proc(self, coro=None):
coro.set_daemon()
bc_sock = asyncoro.AsyncSocket(
socket.socket(socket.AF_INET, socket.SOCK_DGRAM))
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
if self.scheduler_ip_addrs and self.scheduler_port:
relay_request = {
'ip_addrs': self.scheduler_ip_addrs,
'port': self.scheduler_port,
'version': __version__,
'sign': None
}
bc_sock.sendto(
'PING:'.encode() + asyncoro.serialize(relay_request),
('<broadcast>', self.node_port))
bc_sock.close()
listen_sock = asyncoro.AsyncSocket(
socket.socket(socket.AF_INET, socket.SOCK_DGRAM))
listen_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listen_sock.bind(('', self.listen_port))
while 1:
msg, addr = yield listen_sock.recvfrom(1024)
if not msg.startswith('PING:'.encode()):
logger.debug('Ignoring message "%s" from %s',
msg[:min(len(msg), 5)], addr[0])
continue
logger.debug('Ping message from %s (%s)', addr[0], addr[1])
try:
info = asyncoro.unserialize(msg[len('PING:'.encode()):])
if info['version'] != __version__:
logger.warning(
'Ignoring %s due to version mismatch: %s / %s',
info['ip_addrs'], info['version'], __version__)
continue
self.scheduler_ip_addrs = info['ip_addrs'] + [addr[0]]
self.scheduler_port = info['port']
except:
logger.debug('Ignoring ping message from %s (%s)', addr[0],
addr[1])
logger.debug(traceback.format_exc())
continue
if info.get('relay', None):
logger.debug('Ignoring ping back (from %s)', addr[0])
continue
logger.debug('relaying ping from %s / %s' %
(info['ip_addrs'], addr[0]))
if self.node_port == self.listen_port:
info[
'relay'] = 'y' # 'check if this message loops back to self
bc_sock = asyncoro.AsyncSocket(
socket.socket(socket.AF_INET, socket.SOCK_DGRAM))
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
yield bc_sock.sendto('PING:'.encode() + asyncoro.serialize(info),
('<broadcast>', self.node_port))
bc_sock.close()
def sched_udp_proc(self, coro=None):
coro.set_daemon()
sched_sock = asyncoro.AsyncSocket(
socket.socket(self.addrinfo[0], socket.SOCK_DGRAM))
if self.addrinfo[0] == socket.AF_INET:
sched_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
addr = ('', self.scheduler_port)
else: # self.addrinfo[0] == socket.AF_INET6
sched_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP,
self.mreq)
addr = list(self.addrinfo[4])
addr[0] = ''
addr[1] = self.scheduler_port
addr = tuple(addr)
sched_sock.bind(addr)
while 1:
msg, addr = yield sched_sock.recvfrom(1024)
if (not msg.startswith('PING:'.encode())
or not self.scheduler_ip_addrs or not self.scheduler_port):
logger.debug('Ignoring ping message from %s (%s)', addr[0],
addr[1])
continue
try:
info = asyncoro.deserialize(msg[len('PING:'.encode()):])
assert info['version'] == __version__
# assert isinstance(info['cpus'], int)
except:
logger.debug(traceback.format_exc())
msg = {
'ip_addrs': self.scheduler_ip_addrs,
'port': self.scheduler_port,
'version': __version__
}
if info.get('relay', None):
logger.debug('Ignoring ping back from %s: %s', addr[0], info)
continue
msg['relay'] = 'y'
relay_sock = asyncoro.AsyncSocket(
socket.socket(self.addrinfo[0], socket.SOCK_DGRAM))
if self.addrinfo[0] == socket.AF_INET:
addr = (info['ip_addr'], info['port'])
else: # self.sock_family == socket.AF_INET6
addr = list(self.addrinfo[4])
addr[1] = 0
bc_sock.bind(tuple(addr))
addr[0] = info['ip_addr']
addr[1] = info['port']
addr = tuple(addr)
yield relay_sock.sendto('PING:'.encode() + asyncoro.serialize(msg),
addr)
relay_sock.close()
def server_proc(host, port, coro=None):
global count1
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock = asyncoro.AsyncSocket(sock)
try:
yield sock.bind((host, port))
except socket.error as msg:
print 'Bind failed. Error Code : ' + str(msg[0]) + ' Message ' + msg[1]
sys.exit()
print 'Socket bind complete'
sock.listen(1280)
print 'Socket now listening'
count1 = 0
while True:
conn, addr = yield sock.accept()
print 'Connected with ' + addr[0] + ':' + str(addr[1])
if count1 == 0:
#rt1 = currenttime()
rt1 = time.time()
print rt1
count1 += 1
asyncoro.Coro(process, conn, addr, sock)
sock.close()
def client_recv(conn, coro=None):
conn = asyncoro.AsyncSocket(conn)
while True:
line = yield conn.recv_msg()
if not line:
break
print(line.decode())
def client_proc(host, port, coro=None):
sock = asyncoro.AsyncSocket(socket.socket(socket.AF_INET, socket.SOCK_DGRAM))
msg = 'client socket: %s' % (sock.fileno())
if sys.version_info.major >= 3:
msg = bytes(msg, 'ascii')
yield sock.sendto(msg, (host, port))
sock.close()
def client(host, port, sock, coro=None):
global filename
global numchunks
global numconn
x = sock
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
sock = asyncoro.AsyncSocket(sock)
try:
yield sock.bind((host, port))
except socket.error as err:
print 'Bind Failed. Error: ' + str(err[0]) + 'Message' + err[1]
sys.exit()
host1 = '194.47.150.244'
port3 = 8012
yield sock.connect((host1, port3))
for n in xrange(div3[x], div3[x+1]):
msg = '%d:data' % (n) + str(numchunks) + ':data' + filename + ' ' + datalist[n]
yield sock.send_msg(msg)
#print "DAta sent through ", x
yield sock.send_msg('terminate')
sock.close()
def AppSocket(host, uid, coro=None):
coro.set_daemon()
astore.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
astore.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
astore.sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 10)
astore.sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 60)
astore.sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 120)
astore.sock = asyncoro.AsyncSocket(astore.sock)
astore.fd = astore.sock.fileno()
astore.mynum = 0
astore.srcsock = 0xFFFFFFFF
astore.dstsock = 0xFFFFFFFF
astore.jluid = uid
astore.user = uid
astore.pwd = uid
astore.host = host
astore.recv = ''
astore.sbuf = []
astore.errqueue = errqueue
astore.statqueue = statqueue
yield astore.sock.connect(host)
astore.sbuf = stun_register_request()
yield appsocket_write()
while True:
data = yield astore.sock.recv(SOCK_BUFSIZE)
if not data:
print "no data"
return
astore.recv += hexlify(data)
handle_packet_first()
def client(host, port, n, coro=None):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock = asyncoro.AsyncSocket(sock)
yield sock.connect((host, port))
msg = '%d: ' % n + '-' * random.randint(100, 300) + '/'
msg = msg.encode()
yield sock.sendall(msg)
sock.close()
def listen_tcp_proc(self, coro=None):
coro.set_daemon()
tcp_sock = asyncoro.AsyncSocket(
socket.socket(socket.AF_INET, socket.SOCK_STREAM))
tcp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcp_sock.bind(('', self.listen_port))
tcp_sock.listen(8)
auth_len = len(dispy.auth_code('', ''))
def tcp_task(conn, addr, coro=None):
conn.settimeout(5)
try:
msg = yield conn.recvall(auth_len)
msg = yield conn.recv_msg()
except:
logger.debug(traceback.format_exc())
logger.debug('Ignoring invalid TCP message from %s:%s' %
(addr[0], addr[1]))
raise StopIteration
finally:
conn.close()
logger.debug('Ping message from %s (%s)', addr[0], addr[1])
try:
info = asyncoro.unserialize(msg[len('PING:'.encode()):])
if info['version'] != __version__:
logger.warning(
'Ignoring %s due to version mismatch: %s / %s',
info['ip_addrs'], info['version'], __version__)
raise StopIteration
# TODO: since dispynetrelay is not aware of computations
# closing, if more than one client sends ping, nodes will
# respond to different clients
self.scheduler_ip_addrs = info['ip_addrs'] + [addr[0]]
self.scheduler_port = info['port']
except:
logger.debug('Ignoring ping message from %s (%s)', addr[0],
addr[1])
logger.debug(traceback.format_exc())
raise StopIteration
if info.get('relay', None):
logger.debug('Ignoring ping back (from %s)', addr[0])
raise StopIteration
logger.debug('relaying ping from %s / %s' %
(info['ip_addrs'], addr[0]))
if self.node_port == self.listen_port:
info[
'relay'] = 'y' # 'check if this message loops back to self
bc_sock = asyncoro.AsyncSocket(
socket.socket(socket.AF_INET, socket.SOCK_DGRAM))
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
yield bc_sock.sendto('PING:'.encode() + asyncoro.serialize(info),
('<broadcast>', self.node_port))
bc_sock.close()
while 1:
conn, addr = yield tcp_sock.accept()
asyncoro.Coro(tcp_task, conn, addr)
def server(host, port, coro=None):
coro.set_daemon()
sock = asyncoro.AsyncSocket(socket.socket(socket.AF_INET, socket.SOCK_STREAM))
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((host, port))
sock.listen(5000)
while True:
conn, addr = yield sock.accept()
asyncoro.Coro(process, conn)
def client_worker(conn, addr, server, coro=None):
# given conn is synchronous, convert it to asynchronous
conn = asyncoro.AsyncSocket(conn)
server.send(('joined'.encode(), (conn, addr)))
while True:
line = yield conn.recv_msg()
if not line:
server.send(('left'.encode(), (conn, addr)))
break
server.send(('broadcast'.encode(), (conn, line)))
def EpollServer(errqueue, statqueue, coro=None):
coro.set_daemon()
mstore.srvsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
mstore.srvsocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
mstore.srvsocket.setsockopt(socket.SOL_SOCKET, socket.TCP_NODELAY, 1)
mstore.srvsocket.setsockopt(socket.SOL_SOCKET, socket.TCP_QUICKACK, 1)
mstore.srvsocket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
mstore.srvsocket.setsockopt(socket.SOL_SOCKET, socket.SO_DEBUG, 0)
mstore.srvsocket.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 6)
mstore.srvsocket.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 60)
mstore.srvsocket.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, 120)
mstore.srvsocket = asyncoro.AsyncSocket(mstore.srvsocket)
mstore.srvsocket.bind(('0.0.0.0', 3478))
mstore.srvsocket.listen(1024)
mstore.errqueue = errqueue
mstore.statqueue = statqueue
mstore.db_engine = QueryDB().get_engine()
mstore.execute = mstore.db_engine.connect().execute
[setattr(mstore, x, {}) for x in store]
mstore.prefunc = {
STUN_METHOD_ALLOCATE: handle_allocate_request, # 小机登录方法
STUN_METHOD_CHECK_USER: handle_chkuser_request,
STUN_METHOD_REGISTER: handle_register_request,
STUN_METHOD_BINDING: handle_app_login_request # app端登录方法
}
#认证后的处理
mstore.postfunc = {
STUN_METHOD_CHANNEL_BIND: handle_app_bind_device, # APP 绑定小机的命令
#STUN_METHOD_REFRESH:mstore.handle_refresh_request,
STUN_METHOD_MODIFY: handle_modify_bind_item, #修改绑定的信息
STUN_METHOD_DELETE: handle_delete_bind_item, #删除现有的绑定
STUN_METHOD_PULL: handle_app_pull
}
while True:
try:
nsock, addr = yield mstore.srvsocket.accept()
nf = nsock.fileno()
mstore.clients[nf] = nsock
mstore.hosts[nf] = addr
mstore.requests[nf] = ''
asyncoro.Coro(handle_new_accept, nsock)
#mstore.gpool.spawn_n(mstore.handle_new_accept,nsock)
except (SystemExit, KeyboardInterrupt):
print "Server exit"
break
def client_proc(host, port, input, coro=None):
# client reads input file and sends data in chunks
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock = asyncoro.AsyncSocket(sock)
yield sock.connect((host, port))
# data can be written to this asynchronous socket; however, for
# illustration, convert its file descriptor to asynchronous file
# and write to that instead
afd = asyncoro.asyncfile.AsyncFile(sock)
input = open(input)
csum = hashlib.sha1()
while True:
data = os.read(input.fileno(), 16 * 1024)
if not data:
break
csum.update(data)
n = yield afd.write(data, full=True)
afd.close()
print('client sha1 csum: %s' % csum.hexdigest())
def listen_udp_proc(self, coro=None):
coro.set_daemon()
bc_sock = asyncoro.AsyncSocket(
socket.socket(self.addrinfo[0], socket.SOCK_DGRAM))
if self.addrinfo[0] == socket.AF_INET:
bc_sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
addr = (self._broadcast, self.node_port)
else: # self.sock_family == socket.AF_INET6
bc_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS,
struct.pack('@i', 1))
addr = list(self.addrinfo[4])
addr[1] = 0
bc_sock.bind(tuple(addr))
addr[0] = self._broadcast
addr[1] = self.node_port
addr = tuple(addr)
if self.scheduler_ip_addrs and self.scheduler_port:
relay_request = {
'ip_addrs': self.scheduler_ip_addrs,
'port': self.scheduler_port,
'version': __version__,
'sign': None
}
bc_sock.sendto(
'PING:'.encode() + asyncoro.serialize(relay_request), addr)
listen_sock = asyncoro.AsyncSocket(
socket.socket(self.addrinfo[0], socket.SOCK_DGRAM))
if self.addrinfo[0] == socket.AF_INET:
listen_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
addr = ('', self.listen_port)
else: # self.addrinfo[0] == socket.AF_INET6
listen_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP,
self.mreq)
addr = list(self.addrinfo[4])
addr[0] = ''
addr[1] = self.listen_port
addr = tuple(addr)
listen_sock.bind(addr)
while 1:
msg, addr = yield listen_sock.recvfrom(1024)
if not msg.startswith('PING:'.encode()):
logger.debug('Ignoring message "%s" from %s',
msg[:min(len(msg), 5)], addr[0])
continue
logger.debug('Ping message from %s (%s)', addr[0], addr[1])
try:
info = asyncoro.deserialize(msg[len('PING:'.encode()):])
if info['version'] != __version__:
logger.warning(
'Ignoring %s due to version mismatch: %s / %s',
info['ip_addrs'], info['version'], __version__)
continue
self.scheduler_ip_addrs = info['ip_addrs'] + [addr[0]]
self.scheduler_port = info['port']
except:
logger.debug('Ignoring ping message from %s (%s)', addr[0],
addr[1])
logger.debug(traceback.format_exc())
continue
if info.get('relay', None):
logger.debug('Ignoring ping back (from %s)', addr[0])
continue
logger.debug('relaying ping from %s / %s', info['ip_addrs'],
addr[0])
if self.node_port == self.listen_port:
info[
'relay'] = 'y' # 'check if this message loops back to self
yield bc_sock.sendto('PING:'.encode() + asyncoro.serialize(info),
addr)
host = sys.argv[1]
else:
host = ''
# port used by server is arg2
if len(sys.argv) > 2:
port = int(sys.argv[2])
else:
port = 1234
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, port))
asyncoro.Coro(client_recv, sock)
# wrap it with asyncoro's synchronous socket so 'send_msg' can be
# used
conn = asyncoro.AsyncSocket(sock, blocking=True)
if sys.version_info.major > 2:
read_input = input
else:
read_input = raw_input
while True:
try:
line = read_input().strip()
if line.lower() in ('quit', 'exit'):
break
if not line:
continue
except:
break
conn.send_msg(line.encode())
# receive all requests (and program may need to be terminated
# explicitly).
import sys, socket
import asyncoro
def server_proc(n, sock, coro=None):
for i in range(n):
msg, addr = yield sock.recvfrom(1024)
print('Received "%s" from %s:%s' % (msg, addr[0], addr[1]))
sock.close()
def client_proc(host, port, coro=None):
sock = asyncoro.AsyncSocket(socket.socket(socket.AF_INET, socket.SOCK_DGRAM))
msg = 'client socket: %s' % (sock.fileno())
if sys.version_info.major >= 3:
msg = bytes(msg, 'ascii')
yield sock.sendto(msg, (host, port))
sock.close()
if __name__ == '__main__':
sock = asyncoro.AsyncSocket(socket.socket(socket.AF_INET, socket.SOCK_DGRAM))
sock.bind(('127.0.0.1', 0))
host, port = sock.getsockname()
n = 50
server_coro = asyncoro.Coro(server_proc, n, sock)
for i in range(n):
asyncoro.Coro(client_proc, host, port)
server_coro.value()
def DevSocket(host, uid, coro=None):
coro.set_daemon()
mstore.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
mstore.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
mstore.sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 10)
mstore.sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 60)
mstore.sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 120)
mstore.sock = asyncoro.AsyncSocket(mstore.sock)
mstore.fileno = mstore.sock.fileno()
mstore.mynum = 0
mstore.srcsock = 0xFFFFFFFF
mstore.dstsock = 0xFFFFFFFF
mstore.uid = uid
mstore.errqueue = errqueue
mstore.statqueue = statqueue
mstore.sbuf = ''
mstore.recv = ''
yield mstore.sock.connect(host)
# except socket.timeout:
# mstore.errqueue.put('sock %d timeout %f' % (mstore.fileno,time.time()-n))
# return None
# except socket.error:
# mstore.errqueue.put('sock %d socket.error %f' % (mstore.fileno,time.time()-n))
# return None
mstore.sbuf = device_struct_allocate()
yield devsocket_write()
while True:
#try:
data = yield mstore.sock.recv(SOCK_BUFSIZE)
# except socket.timeout:
# mstore.errqueue.put('sock %d recv timeout %f' % (mstore.fileno,time.time()-n))
# break
# except IOError:
# mstore.errqueue.put('sock %d recv IOError ' % (mstore.fileno))
# break
# if not data:
# mstore.errqueue.put('sock %d recv recv no data,maybe disconnect' % (mstore.fileno))
# break
mstore.recv += hexlify(data)
l = mstore.recv.count(HEAD_MAGIC) #没有找到JL关键字
if not l:
mstore.errqueue.put('sock %d, recv no HEAD_MAGIC packet %s' %
(mstore.fileno, mstore.recv))
return
plen = len(mstore.recv)
if l > 1:
#mstore.errqueue.put('sock %d,recv unkown msg %s' % (fileno,mstore.requests[:l])
mstore.statqueue.put("sock %d,recv multi buf,len %d, buf: %s" %
(mstore.fileno, plen, mstore.recv))
#hbuf = hbuf[l:] # 从找到标识头开始处理
pos = sum([len(v) for v in split_requests_buf(mstore.recv)])
mstore.recv = mstore.recv[pos:]
[process_dev_loop(n) for n in split_requests_buf(mstore.recv)]
else: # 找到一个标识，还不知在什么位置
pos = mstore.recv.index(HEAD_MAGIC)
mstore.recv = mstore.recv[pos:]
nlen = int(mstore.recv[8:12], 16) * 2
if len(mstore.recv) < nlen:
mstore.errqueue.put("sock %d, recv packet not complete, %s" %
(mstore.fileno, mstore.recv))
return
onepack = mstore.recv[:nlen]
mstore.recv = mstore.recv[nlen:]
yield process_dev_loop(onepack)
