def main_program(content, ob=None, paras=paras):
'''
statu:1
content:iterated_times
'''
max_iter = paras['niter']
if content < max_iter:
msg = message(2, content + 1001)
ob.send_list.put(msg.msg_encode())
ob.show_debug("message is ready to send to loop.")
else:
msg = message(499, 'Completed! Waiting for the final result...')
ob.send_list.put(msg.msg_encode())
def __init__(self,recv_list,send_list,name='master',host='localhost',port=50001):
Process.__init__(self)
self.s=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
self.host=host
self.port=port
self.recv_list=recv_list
self.send_list=send_list
if type(port) is int:
print("init the server on port %d."%port)
try:
self.s.bind((self.host,self.port))
except:
print("bind error.")
exit()
else:
counter=0
while True:
try:
self.port=np.random.randint(50010,59999)
self.s.bind((self.host,self.port))
break
except:
counter+=1
if counter>=10:
print("Tried 10 times but failed, check your code plz.")
break
continue
#if this isnot the master server,then registe its port.
if port!=50001:
print("Now registing the port.")
msg=message(50001,[101,[self.name,self.port]]).msg_encode()
self.send_list.put(msg)
def get_result(content, ob=None, paras=paras):
'''
statu:10
content:result array
'''
print("The final result is:\n", content)
msg = message(499, "result gotton.")
ob.send_list.put(msg.msg_encode())
def init(content, ob=None, paras=paras):
'''
statu:0
content:one_iter_num,generally,it's a int.
'''
msg = message(1, 0)
ob.send_list.put(msg.msg_encode())
ob.show_debug('init over,wait for respons.')
def run(self):
self.s.listen(50)
while True:
conn,_=self.s.accept()
#print('Connect with:',addr)
data=conn.recv(512000)
print("server recvd data:",data)
msg=data
statu,content=json.loads(data.decode())
if statu==101:
#port register
print("The registry msg 101 is:",content)
self.recv_list.put(msg)
elif statu==102:
_,file_name,file_path=content
filename=file_path+file_name #+'_new
f=open(filename,'wb')
recvd_size=0
m=hashlib.md5()
conn.send('wait for size...'.encode())
respon=conn.recv(1024)
file_size=int(respon.decode())
conn.send('Ready for recv...'.encode())
while recvd_size<file_size:
size=0
if file_size-recvd_size>1024:
size=1024
else:
size=file_size-recvd_size
data=conn.recv(size)
data_len=len(data)
recvd_size+=data_len
print("\r recvd:",int(recvd_size/file_size*100),"%")
m.update(data)
f.write(data)
f.close()
print("real recvd size:",recvd_size)
md5_s=conn.recv(1024).decode()
md5_c=m.hexdigest()
print("original md5:",md5_s)
if md5_c==md5_s:
print("recv success.")
conn.send('s'.encode())
else:
print("recv failure.")
conn.send('f'.encode())
self.recv_list.put(msg)
continue
elif statu<400 or statu>=500:
self.recv_list.put(msg)
conn.send(message(466,'recved').msg_encode())
else:
print("Testing msg recvd:",statu,content)
def run(self):
self.s.listen(10)
while True:
conn,_=self.s.accept()
#print('Connect with:',addr)
data=conn.recv(512000)
print("server recvd data:",data)
statu,content=json.loads(data.decode())
if statu==905:
file_name,file_path=content
if os.path.isfile(file_path+file_name):
size=os.stat(file_path+file_name).st_size
conn.send(str(size).encode())
conn.recv(1024)
m=hashlib.md5()
f=open(file_path+file_name,'rb')
for line in f:
conn.send(line)
m.update(line)
f.close()
md5=m.hexdigest()
conn.send(md5.encode())
print("md5:",md5)
else:
conn.send("file you needed is not found.")
print("file not found...")
elif statu==907:
new_msg=message(904,content)
self.send_list.put(new_msg.msg_encode())
elif statu<400 or statu>=500:
msg=message(statu,content)
self.recv_list.put(msg.msg_encode())
conn.send(message(466,'recved').msg_encode())
else:
print("Testing msg recvd:",statu,content)
def run(self):
while True:
if self.recv_list.empty():
time.sleep(2)
#print("No msg in recvlist")
continue
else:
msg=self.recv_list.get()
print("There is a msg in recv_list",msg.decode())
new_msg=message.b2m(msg)
statu,content=new_msg.statu,new_msg.content
if statu==101:
self.route[content[0]]=content[1]
print("new service is registed:",content)
else:
if statu in self.algo_route:
next_name=self.algo_route[statu]
if next_name in self.route:
next_port=self.route[next_name]
send_msg=message(next_port,[msg.statu,msg.content]).msg_encode()
self.send_list.put(send_msg)
else:
self.recv_list.put(msg)
print("the port has not been registed ,please wait.")
time.sleep(2)
else:
#self.recv_list.put(msg)
print("the algo route is not exist.")
# class micro_service:
# def __init__(self,recv_list,send_list,route,*args,\
# task_config='task_config.json',host='localhost',port=50001,\
# name='master',debug=True,**kargs):
# #super().__init__()
# self.recv_list=recv_list
# self.send_list=send_list
# self.args=args
# self.name=name
# self.kargs=kargs
# self.route=route
# self.route['master']=50001
# self.debug=debug
# #statu to name.
# if task_config!='':
# with open(task_config,'r') as ff:
# tmp=json.loads(ff.read())
# self.help=tmp['comment']
# self.algo_route=tmp['algo_statu']
# self.commom_statu=tmp['commom_statu']
# else:
# self.help=dict()
# self.algo_route=dict()
# self.commom_statu=dict()
# self.server=server(self.recv_list,self.send_list,name=self.name,host=host,port=port)
# self.messager=messager(self.recv_list,self.send_list,host=host)
# self.worker=worker(self.recv_list,self.send_list,algo_route=self.algo_route)
# def put_to_send_list(self,port,content):
# new_msg=message(port,content).msg_encode()
# self.send_list.put(new_msg)
# def show_debug(self,*args):
# if self.debug==True:
# for i in args:
# print(i)
# else:
# return
# def run(self):
# self.server.start()
# self.messager.start()
# self.worker.start()
# if __name__=="__main__":
# recv_list=multiprocessing.Queue()
# send_list=multiprocessing.Queue()
#ms=micro_service(recv_list,send_list,dict())
#ms.run()
#k=wait_file('./','init.py')
#print(k)
def run(self):
while True:
if self.send_list.empty():
time.sleep(2)
continue
else:
# self.route=self.ob.route
msg=self.send_list.get()
statu,content=json.loads(msg.decode())
if statu<100:
next_port=50001
new_msg=msg
elif statu==102:
s=socket.socket()
_name,file_name,file_path=content
s.connect(('localhost',50001))
s.send(msg)
s_resp=s.recv(1024)
print(s_resp.decode())
file_size=os.stat(file_path+file_name).st_size
s.send(str(file_size).encode())
s_resp=s.recv(1024)
print(s_resp.decode())
f=open(file_path+file_name,'rb')
m=hashlib.md5()
for line in f:
s.send(line)
m.update(line)
f.close()
md5=m.hexdigest()
s.send(md5.encode())
print("md5:",md5)
print("sending over.")
flag=s.recv(1024).decode()
if flag=='s':
print("sending success.")
else:
self.send_list.put(msg)
print("sending fall,try again.")
s.close()
continue
elif 400<=statu<500:
print("Test or Info msg:",statu,content)
continue
elif statu>10000:
next_port=statu
new_msg=message(content[0],content[1]).msg_encode()
else:
print("error :wrong statu ",statu)
continue
s=socket.socket()
addr=(self.host,next_port)
s.connect(addr)
# new_msg=message(content[0],content[1]).msg_encode()
s.send(new_msg)
recv=s.recv(1024)
print(recv)
s.close()
def func4(x, ob=None):
print("this is the final function.")
msg = message(50001, [401, 'finished.'])
ob.send_list.put(msg.msg_encode())
import numpy as np
import json
from functools import partial
import time
import sys, os
from func_lib import message
func1 = lambda x, ob=None: ob.recv_list.put(message(1, x * 2).msg_encode())
func2 = lambda x, ob=None: ob.recv_list.put(message(2, x * 3).msg_encode())
func3 = lambda x, ob=None: ob.recv_list.put(message(3, x * 5).msg_encode())
def func4(x, ob=None):
print("this is the final function.")
msg = message(50001, [401, 'finished.'])
ob.send_list.put(msg.msg_encode())
def get_dct():
funcs = dict()
funcs[0] = func1
funcs[1] = func2
funcs[2] = func3
funcs[3] = func4
return funcs
from base import *
from test_custom_func import get_dct
from func_lib import message
if __name__ == "__main__":
recv_list = multiprocessing.Queue()
send_list = multiprocessing.Queue()
#m=Manager()
route = dict()
d = get_dct()
ms = micro_service(recv_list,
send_list,
route,
func_dct=d,
name='init',
port='s')
ms.run()
#ms.recv_list.put(message(0,11).msg_encode())
ms.send_list.put(
message(906, ["master", "test_file.dmg", "./test_fold/"]).msg_encode())
print("msg put into the recv list.")
from base import *
from test_custom_func import get_dct
from func_lib import message
if __name__ == "__main__":
recv_list = multiprocessing.Queue()
send_list = multiprocessing.Queue()
route = dict()
ms = micro_service(recv_list, send_list, route, name='init', port='s')
ms.run()
ms.send_list.put(
message(102, [1, "test_file.dmg", "./test_fold/"]).msg_encode())
print("msg put into the recv list.")
def put_to_send_list(self,port,content):
new_msg=message(port,content).msg_encode()
self.send_list.put(new_msg)
def run(self):
while True:
if self.send_list.empty():
time.sleep(2)
continue
else:
# self.route=self.ob.route
msg=self.send_list.get()
statu,content=json.loads(msg.decode())
if statu<100:
if statu in self.algo_route:
next_name=self.algo_route[statu]
if next_name in self.ob.route:
next_port=self.ob.route[next_name]
else:
print('Try to get the needed port.',statu,content)
if self.name=='master':
self.send_list.put(msg)
time.sleep(4)
continue
new_msg=message(102,[self.port,next_name])
self.send_list.put(new_msg.msg_encode)
print("waiting the port respons")
self.send_list.put(msg)
continue
else:
print("This statu is not defined in the algorithm, plz check.")
print("Error info:",statu,content)
continue
elif 100<=statu<200:
#message with statu in this range will be send to the master server.
next_port=50001
elif 400<=statu<500:
print("Test or Info msg:",statu,content)
elif statu==904:
'''
message(904,[server_name,file_name])
request for a file from server with file name.
'''
next_name,file_name,file_path=content
if next_name in self.ob.route:
next_port=self.ob.route[next_name]
else:
print("Now the route is :",self.ob.route)
print('Try to get the needed port.',statu,content)
if self.name=='master':
self.send_list.put(msg)
time.sleep(4)
continue
new_msg=message(102,[self.port,next_name])
self.send_list.put(new_msg.msg_encode)
print("waiting the port respons")
self.send_list.put(msg)
continue
self.s=socket.socket()
self.s.connect((self.host,next_port))
self.s.send(message(905,[file_name,file_path]).msg_encode())
server_respons=self.s.recv(1024)
try:
file_size=int(server_respons.decode())
except:
print(server_respons.decode())
continue
print("recvd size:",file_size)
self.s.send("ready for recv file....".encode())
filename=file_path+"new_"+file_name
f=open(filename,'wb')
recvd_size=0
m=hashlib.md5()
while recvd_size<file_size:
size=0
if file_size-recvd_size>1024:
size=1024
else:
size=file_size-recvd_size
data=self.s.recv(size)
data_len=len(data)
recvd_size+=data_len
print("\r\n recvd:",int(recvd_size/file_size*100),"%")
m.update(data)
f.write(data)
print("real recvd size:",recvd_size)
md5_server=self.s.recv(1024).decode()
md5_client=m.hexdigest()
print("md5 on service:",md5_server)
print("md5 on client:",md5_client)
if md5_client==md5_server:
print("sending success.")
self.s.close()
continue
else:
print("sending fail.preparad for next try....")
self.send_list.put(msg)
self.s.close()
continue
elif statu==906:
next_name,file_name,file_path=content
if next_name in self.ob.route:
next_port=self.ob.route[next_name]
else:
print('Try to get the needed port.',statu,content)
if self.name=='master':
self.send_list.put(msg)
time.sleep(4)
continue
new_msg=message(102,[self.port,next_name])
self.send_list.put(new_msg.msg_encode)
print("waiting the port respons")
self.send_list.put(msg)
continue
new_msg=message(907,[self.name,file_name,file_path]).msg_encode()
self.s=socket.socket()
addr=(self.host,next_port)
self.s.connect(addr)
self.s.send(new_msg)
recv=self.s.recv(52000)
print(recv)
self.s.close()
elif statu>10000:
next_port=statu
else:
print("error :wrong statu ",statu)
continue
self.s=socket.socket()
addr=(self.host,next_port)
self.s.connect(addr)
new_msg=message(content[0],content[1]).msg_encode()
self.s.send(new_msg)
recv=self.s.recv(52000)
print(recv)
self.s.close()
