def send_file(conn, filename, flog):
#将文件打包压缩
mylog(1, flog, "进入文件发送函数...")
filename = filename.replace(".dat",".tar.gz")
t = tarfile.open("/tmp/"+filename, "w:gz")
filename = filename.replace(".tar.gz",".dat")
t.add("/tmp/"+filename)
t.close()
mylog(1, flog, "文件打包完成...")
filename = filename.replace(".dat",".tar.gz")
s_filename = filename.encode("UTF-8")
#向client端发送文件头，包括压缩包名称和大小
#msg = 'OK'
#conn.sendall(msg.encode('utf-8'))
fhead = struct.pack('<128s11I', s_filename, 0, 0, 0, 0, 0, 0, 0, 0, os.stat("/tmp/"+filename).st_size, 0, 0)
conn.send(fhead)
mylog(1, flog, "发送压缩包名称和大小...")
#向client端发送压缩包
t = open("/tmp/" + filename, 'rb')
while 1:
filedata = t.read(1024)
if not filedata: break
conn.sendall(filedata)
t.close()
mylog(1, flog, "向client端发送压缩包...")
os.remove("/tmp/"+filename)
filename = filename.replace(".tar.gz",".dat")
os.remove("/tmp/"+filename)
mylog(1, flog, "清除本地缓存...")
def startthread(f, myxmldoc):
# Called by handleconnection when a new thread is needed.
# Note: lockpool is already acquired when this function is called.
mylog(3, f, "DEBUG:Starting new client processor thread.")
t = Thread(target=threadworker, args=(f, myxmldoc))
t.setDaemon(1)
t.start()
def __init__(self, pidfile,logfile,stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
self.stdin = stdin
self.stdout = stdout
self.stderr = stderr
self.pidfile = pidfile
#self.Fparamfilename = paramfilename
self.flog = logfile
mylog(1,self.flog,"DEBUG:CREATE BaseDaemon.....")
mylog(1,self.flog,"DEBUG:PIDFile="+pidfile)
def socketmain(paramfilename, flog):
strusername = getpass.getuser()
#f = open("/tmp/huangzb-pyserver-"+strusername+".log", "w")
#probe something about os,mpi,etc.
mylog(1, flog, '\n\nSTART LISTENER......................\n\n')
myxmldoc = xml.dom.minidom.parse(paramfilename)
listener(flog, myxmldoc)
while 1:
mylog(3, flog, 'Idle.... %s\n' % time.ctime(time.time()))
time.sleep(100)
def handleconnection(clientsock, f, myxmldoc):
"""Handle an incoming client connection."""
lockpool.acquire()
mylog(3, f, "DEBUG:Received new client connection.")
try:
if len(waitinglist) == 0 and (activeCount() - 1) >= MAXTHREADS:
# Too many connections. Just close it and exit.
clientsock.close()
return
if len(waitinglist) == 0:
startthread(f, myxmldoc)
queue.append(clientsock)
sem.release()
finally:
lockpool.release()
def my_socket(flog):
cli, db, fs = pyMongo.get_cli_db_fs()
mylog(1, flog, '\n\n数据库链接完毕......................\n\n')
print(" 数据库链接完毕")
s = socket.socket()
host = socket.gethostname()
port = 9999
s.bind((host, port))
s.listen(5)
mylog(1, flog, '\n\n服务正在启动......................\n\n')
print("服务正在启动...")
while True:
sock, addr = s.accept() # 接收一个新连接
t1 = threading.Thread(target=conn_thread, kwargs={"conn": sock, "addr": addr, "cli": cli, "db": db, "fs": fs, "flog":flog})
t1.start()
def conn_thread(conn, addr, cli, db, fs, flog): # TCP服务器端处理逻辑
print('Accept new connection from %s:%s.' % addr) # 接受新的连接请求
mylog(1, flog, 'Accept new connection from %s:%s.' % addr)
msg = conn.recv(1024)
req = eval(msg.decode('utf-8')) # 将客户端的请求信息转换为字典类型
mylog(1, flog, req)
print(req)
handle_req(conn, req, cli, db, fs, flog)
mylog(1, flog, "本次请求处理完毕，正在断开连接...")
print("本次请求处理完毕，正在断开连接...")
conn.close()
mylog(1, flog, "连接已关闭...\n")
print("连接已关闭...\n")
def threadworker(f, myxmldoc):
global waitinglist, lockpool, busylist
mylog(3, f, "DEBUG:starting....threadworker.....")
time.sleep(1) # Simulate expensive startup
name = currentThread().getName()
try:
lockpool.acquire()
try:
waitinglist[name] = 1
finally:
lockpool.release()
processclients(f, myxmldoc)
finally:
# Clean up if the thread is dying for some reason.
# Can't lock here -- we may already hold the lock, but it's OK
print("** WARNING** Thread %s died" % name)
if name in waitinglist:
del waitinglist[name]
if name in busylist:
del busylist[name]
# Start a replacement thread.
startthread()
def start(self):
"""
Start the daemon
"""
# Check for a pidfile to see if the daemon already runs
print("DEBUG:Running the daemon......")
mylog(1,self.flog,"DEBUG:CHECK for a pidfile to see if the daemon already runs")
try:
pf = open(self.pidfile,'r')
pid = int(pf.read().strip())
pf.close()
except IOError:
pid = None
if pid:
message = "DEBUG:pidfile %s already exist. Daemon already running?\n"
sys.stderr.write(message % self.pidfile)
mylog(1,self.flog,message % self.pidfile)
sys.exit(1)
# Start the daemon
mylog(1,self.flog,"DEBUG:Start the daemon......self.daemonize()")
self.daemonize()
self.run()
def daemonize(self):
"""
do the UNIX double-fork magic, see Stevens' "Advanced
Programming in the UNIX Environment" for details (ISBN 0201563177)
"""
try:
pid = os.fork()
mylog(1,self.flog,"DEBUG:CREATE the child process by os.fork()...PID=%d" % pid)
if pid > 0:
# exit first parent
sys.exit(0)
mylog(1,self.flog,"DEBUG:EXIT first parent process.")
except OSError as e:
sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror))
mylog(1,self.flog,"DEBUG:fork #1 failed: %d (%s)\n" % (e.errno, e.strerror))
sys.exit(1)
mylog(1,self.flog,"DEBUG:Child Process......PID=%d" % pid)
mylog(1,self.flog,'DEBUG:GETCWD()=%s\n'%os.getcwd())#返回当前工作目录
mylog(1,self.flog,'DEBUG:PWD=%s\n'%os.environ['PWD'])#显示整个路径名？
mylog(1,self.flog,'DEBUG:PATH=%s\n'%sys.argv[0])
mylog(1,self.flog,'DEBUG:%s\n'%os.path.split(sys.argv[0])[0])
mylog(1,self.flog,'DEBUG:%d\n'%os.getcwd().find(os.environ['PWD']))
#if (os.getcwd().find(os.path.split(sys.argv[0])[0])<0):
# sys.stderr.write('Running Directory is not in the script Directory!!\n!!PLEASE CHECK!!\n\n')
#self.Fparamfilename=os.getcwd()+"/"+self.Fparamfilename
# decouple from parent environment
os.chdir("/")
os.setsid()
os.umask(0)
# do second fork
try:
pid = os.fork()
mylog(1,self.flog,"DEBUG:CREATE the child process by os.fork()...PID=%d" % pid)
if pid > 0:
# exit from second parent
sys.exit(0)
mylog(1,self.flog,"DEBUG:EXIT second parent process.")
except OSError as e:
sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror))
sys.exit(1)
# redirect standard file descriptors
sys.stdout.flush()
sys.stderr.flush()
si = open(self.stdin, 'r')
so = open(self.stdout, 'a+')
se = open(self.stderr, 'a+')
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
# write pidfile
atexit.register(self.delpid)
pid = str(os.getpid())
open(self.pidfile,'w+').write("%s\n" % pid)
import sys, time
from daemon1 import Daemon
from Huangzblog1 import mylog
from Agent2 import *
class MyDaemon(Daemon):
def run(self):
f = open('/tmp/test2.log', "w")
my_socket(f)
if __name__ == "__main__":
f = open('/tmp/test.log', "w")
mylog(1, f, "DEBUG: log is build")
daemon = MyDaemon('/tmp/daemon-example.pid', f)
if len(sys.argv) == 2:
if 'start' == sys.argv[1]:
daemon.start()
elif 'stop' == sys.argv[1]:
daemon.stop()
elif 'restart' == sys.argv[1]:
daemon.restart()
else:
print("Unknown command")
sys.exit(2)
sys.exit(0)
else:
print("usage: %s start|stop|restart" % sys.argv[0])
sys.exit(2)
def handle_req(conn, req, cli, db, fs, flog): # 处理前端请求
mylog(1, flog, '\n处理客户端请求。。。')
if req['type'] == 'list_all_shapes': # 前端向后台发送请求，获取最外层目录
mylog(1, flog, '\nlist_all_shapes')
response = {
"type": "list_all_shapes",
"all_shapenames": list_all_shapes(cli, db, fs, flog)
}
mylog(1, flog, response)
filename = req['clientfilename']
send_j_response(conn,response,filename, flog)
mylog(1, flog, '\nlist_all_shapes 执行完毕')
elif req['type'] == 'list_doc_tree': # 前端向后台发送请求，获取指定内层目录
doc_tree = list_doc_tree(req['doc_tree_name'], cli, db, fs, flog)
doc_tree = eval(doc_tree)
filename = req['clientfilename']
send_j_response(conn, doc_tree, filename, flog)
elif req['type'] == 'log_in': # 前端向后台发送请求，请求登录
response = {
"type": "log_in",
"error_message": check_user(req['user_name'], req['user_password'], cli, db, fs, flog),
"user_name": req['user_name'],
"user_authority": get_user_authority(req['user_name'], cli, db, fs, flog)
}
send_j_response(conn, response, flog)
elif req['type'] == 'modify': # 前端向后端发送修改用户信息请求
response = {
"type": "modify",
"error_message": modify_user(req['user_name'], req['user_password'], req['user_authority'], cli, db, fs, flog)
}
send_j_response(conn, response, flog)
elif req['type'] == 'list_all_users': # 前端要求列出所有用户
response = {
"type": "list_all_users",
"users_num": count_users(cli, db, fs, flog),
"all_users": get_user_list(cli, db, fs, flog)
}
send_j_response(conn, response, flog)
elif req['type'] == 'add_user': # 前端要求添加用户
response = {
"type": "add_user",
"error_message": add_user(req['user_name'], req['user_password'], req['user_authority'], cli, db, fs, flog)
}
send_j_response(conn, response, flog)
elif req['type'] == 'delete_user': # 前端要求删除用户
response = {
"type": "delete_user",
"error_message": del_user(req['user_name'], cli, db, fs, flog)
}
send_j_response(conn, response, flog)
elif req['type'] == 'download_file': # 前端向后台发送请求，获得指定文件
new_filename = name_switch(req['file_name']['which_file'])
send_file(conn, find_file(new_filename, req['file_name'],req['clientfilename'], cli, db, fs, flog), flog)
# if req['ack'] == 1:
# file = find_file(new_filename, req['file_name'])
# print(conn.send(file))
# else:
# file_length, file_authority = get_file_information(new_filename, req['file_name'])
# response = {
# "type": "download_file",
# "file_length": file_length,
# "file_authority": file_authority,
# "file_name": req['file_name']['which_file']
# }
# send_j_response(conn, response)
else:
print('Wrong request!')
def send_j_response(conn, res, filename, flog): # 向前台发送json压缩包
#将json写入文件
#filename = 'req_ALEX_20170708142032_list_all_shapes.json'
mylog(1, flog, "进入json发送函数")
mylog(1, flog, filename)
fp = open("/tmp/"+filename, 'w')
fp.write(json.dumps(res))
fp.close()
mylog(1, flog, "将json写入文件...")
filename = filename.replace(".json",".tar.gz")
t = tarfile.open("/tmp/"+filename, "w:gz")
filename = filename.replace(".tar.gz",".json")
t.add("/tmp/"+filename)
t.close()
filename = filename.replace(".json",".tar.gz")
s_filename = filename.encode("UTF-8")
#向client端发送文件头，包括压缩包名称和大小
# msg = 'OK'
#conn.sendall(msg.encode('utf-8'))
fhead = struct.pack('<128s11I', s_filename, 0, 0, 0, 0, 0, 0, 0, 0, os.stat("/tmp/"+filename).st_size, 0, 0)
conn.send(fhead)
mylog(1, flog, "向client端发送文件头，包括压缩包名称和大小...")
#向client端发送压缩包
t = open("/tmp/"+filename, 'rb')
while 1:
filedata = t.read(1024)
if not filedata: break
conn.sendall(filedata)
t.close()
mylog(1, flog, "向client端发送压缩包...")
os.remove("/tmp/"+filename)
filename = filename.replace(".tar.gz",".json")
os.remove("/tmp/"+filename)
mylog(1, flog, "清除本地缓存...")
def processclients(f, myxmldoc):
global sem, queue, waitinglist, busylist, lockpool
name = currentThread().getName()
while 1:
sem.acquire()
lockpool.acquire()
try:
clientsock = queue.pop(0)
del waitinglist[name]
busylist[name] = 1
finally:
lockpool.release()
try:
mylog(
1, f,
"\n\nSTART TO DEAL WITH THE Request from Clients........\n\n")
mylog(
3, f, "processclients;[%s] Got connection from %s \n" %
(name, clientsock.getpeername()))
data = clientsock.recv(2048)
mylog(
3, f, "\nprocessclients;data=" + data +
" data.find(ASKING)=%d\n" % data.find("ASKING"))
if (data.find("ASKING") >= 0):
clientsock.sendall("GREETI")
data = clientsock.recv(1024)
mylog(3, f, "processclients;data=%s" % data)
while ((len(data) > 0) and (data.find("OVER") == -1)):
handle_req(clientsock, data)
else:
mylog(
3, f, "\nprocessclients;data=" + data +
" len(data)=%d" % len(data))
except (KeyboardInterrupt, SystemExit):
raise
except:
traceback.print_exc()
info = sys.exc_info()
mylog(1, f,
"...ERROR...info[0]=%s.....info[1]=%s." % (info[0], info[1]))
# Close the connection
try:
clientsock.close()
except KeyboardInterrupt:
raise
except:
traceback.print_exc()
lockpool.acquire()
try:
del busylist[name]
waitinglist[name] = 1
finally:
lockpool.release()
def ReceiveFileOfUploadedFileFromClient_Fu1fle(clientsock, f):
mylog(3, f, "\n\nReceiveFileOfUploadedFileFromClient:START....")
FBUFSIZE = 1024
SBUFSIZE = 1024
FILEINFO_SIZE = struct.calcsize('<128s32sI8s')
fhead = clientsock.recv(FILEINFO_SIZE)
mylog(3, f, "\n\nReceiveFileOfUploadedFileFromClient:recv fhead....")
srvfilepath, temp1, filesize, temp2 = struct.unpack('<128s32sI8s', fhead)
mylog(
3, f,
"ReceiveFileOfUploadedFileFromClient:unpack filesize..srvfilepath=%s...."
% srvfilepath)
srvfilepath = srvfilepath.strip("\00")
if (os.path.isdir(srvfilepath)):
clientsock.sendall('OK')
FILEINFO_SIZE = struct.calcsize('<128s32sI8s')
fhead = clientsock.recv(FILEINFO_SIZE)
# mylog(3,f,"FU1FLE,receiving fhead.....")
mylog(3, f, "ReceiveFileOfUploadedFileFromClient: recv fhead")
filename, temp1, filesize, temp2 = struct.unpack('<128s32sI8s', fhead)
mylog(
3, f, "ReceiveFileOfUploadedFileFromClient:unpack...filename=%s" %
filename)
# filename = srvfilepath+string.join(random.sample(['a','b','c','d','e','f','g','h','i','j','k','L','m','n','o','p','q','r','s','t'], 8)).replace(" ","")+"_"+filename.strip('\00')
filename = srvfilepath + filename.strip('\00')
mylog(
3, f,
"ReceiveFileOfUploadedFileFromClient:full file path and name=%s" %
filename)
fp = open(filename, 'wb')
restsize = filesize
# mylog(3,f,"ReceiveFileOfUploadedFileFromClient:Upload File To Cluster...filesize=%d"%filesize)
while 1:
if restsize > SBUFSIZE:
filedata = clientsock.recv(SBUFSIZE)
else:
filedata = clientsock.recv(restsize)
if not filedata: break
fp.write(filedata)
restsize = restsize - len(filedata)
if restsize == 0: break
fp.flush()
fp.close()
mylog(3, f,
'ReceiveFileOfUploadedFileFromClient:all data over...OK\n\n')
time.sleep(0.1)
clientsock.sendall("OK")
mylog(3, f, "ReceiveFileOfUploadedFileFromClient:send OK to client")
mylog(3, f, 'ReceiveFileOfUploadedFileFromClient:END.\n\n')
return filename
else:
clientsock.sendall('ERROR')
mylog(3, f,
"ReceiveFileOfUploadedFileFromClient:send ERROR to client\n\n")
pass
def SendServerFileToClient_Fd1fle(clientsock, f):
fullfilename = ""
filename = ""
mylog(3, f, "\n\nSendServerFileToClient:START....")
FILEINFO_SIZE = struct.calcsize('<128s32sI8s')
fhead = clientsock.recv(FILEINFO_SIZE) ####谁告诉的文件头？
mylog(3, f, "SendServerFileToClient,receiving fhead.....")
filename, temp1, filesize, temp2 = struct.unpack('<128s32sI8s', fhead)
mylog(
3, f,
"SendServerFileToClient:Unpack filename=%s,filesize=%d..........." %
(filename, filesize))
fullfilename = filename.decode("utf-16")
filename = filename.strip("\00")
# print "FD1FLE,",filename
if (os.path.isfile(filename)): ####这判断又是干嘛的？？？
try:
fileStats = os.stat(filename)
except:
info = sys.exc_info()
mylog(
3, f,
"SendServerFileToClient:...ERROR...info[0]=%s.....info[1]=%s..."
% (info[0], info[1]))
clientsock.sendall('OK')
mylog(3, f, "SendServerFileToClient:send OK to client")
fullfilename = filename
filename = filename.encode("UTF-8")
FILEINFO_SIZE = struct.calcsize('<128s32sI8s')
fhead = struct.pack('<128s11I', filename, 0, 0, 0, 0, 0, 0, 0, 0,
os.stat(fullfilename).st_size, 0, 0)
clientsock.send(fhead)
mylog(3, f, "SendServerFileToClient,sending fhead.....")
fp = open(fullfilename, 'rb')
while 1:
filedata = fp.read(BUFSIZE)
if not filedata: break
clientsock.sendall(filedata)
fp.close()
mylog(3, f, 'SendServerFileToClient:Send all data over......OK')
clientsock.sendall("OK")
mylog(3, f, "SendServerFileToClient:send OK to client")
mylog(3, f, "SendServerFileToClient:END.....\n\n")
return fullfilename
else:
clientsock.sendall('ERROR')
mylog(3, f, "SendServerFileToClient:send ERROR to client\n\n")
pass