def download(filename, lk1, btnDld):
lk1.acquire() #acquire the lock before status can
try:
if mainq.empty(): return 0
url = mainq.get(1, 1) #block=True, timeout=None
# filename = url.split('/')[-1]
utils.printlog("Start download: {0}".format(url))
with get(url, stream=True, timeout=3) as r:
filesize = r.headers['Content-length'].encode("utf8")
utils.printlog("Download size: {0} kilobytes".format(filesize))
with open(filename + ".tmp", 'wb') as fh:
fh.write(filesize)
with open(filename, 'wb') as f:
lk1.release()
copyfileobj(r.raw, f)
return filename, filesize
except exceptions.ConnectionError as e:
utils.printlog("Error during connect to URL: {0}".format(e))
# messagebox.showerror("Error", "Connection to {0} timeout. Please check your internet connection.".format(url))
return 0, 0
except exceptions.MissingSchema as e:
utils.printlog("Invalid URL: {0}".format(e))
except Exception as e:
raise (e)
finally:
utils.printlog("Finalizing with qsize: {0}".format(mainq.qsize()))
if lk1.locked(): lk1.release()
cmdq.put((tonggle_button, (btnDld, ), {})) #toggle the button
def start_download(btnDld, ent, progress, lblSave):
try:
# cmdq.put((messagebox.showinfo, ('title', 'message'), {}))
cmdq.put((tonggle_button, (btnDld, ), {}))
# get some inputs
url = ent.get()
filename = lblSave['text']
if url and url != "": mainq.put_nowait(url)
utils.printlog("Queue size: {0}".format(mainq.qsize()))
lk1 = Lock()
# start DOWNLOAD thread
dld = Thread(name="DOWNLOAD",
target=download,
daemon=True,
args=(filename, lk1, btnDld))
dld.start()
# start STATUS thread
stt = Thread(name="STATUS",
target=status,
daemon=True,
args=(filename, lk1, progress))
stt.start()
# Additional
# JOINING - Blocking
# dld.join()
# stt.join()
# mainq.join() ## block until all tasks are done -> q.task_done()
except Exception as e:
raise (e)
finally:
pass
def controller(l, q, ev, br):
printlog("controller started")
l.acquire()
[q.put(i) for i in range(10) if not q.full()]
l.release()
# time.sleep(10)
# ev.set() #set the flag using event
i = br.wait() #set the flag using barrier
printlog("controller stopped")
def getdiskspace(self, cmd="df -h\n"):
if self.chan is not None:
self.getfeedback()
printlog("Begin get disk space ...")
self.chan.send(cmd)
r1 = self.getfeedback()
self.chan.send("exit\n")
# printlog(r1)
return r1
else:
printlog("Unable to accomply because channel is NULL")
return "Unable to accomply because channel is NULL"
def worker(i, l, q):
printlog("worker {0} started".format(i))
data = None
while not EXITFLAG and not GRACEFULEXITFLAG:
l.acquire()
if not q.empty():
data = q.get()
l.release()
printwait("{0}".format(data), 10)
else:
l.release()
time.sleep(10)
def connect(self, ip, uname, pw, debug=False):
"""connect to server with ip, username, password"""
try:
self.client.connect(ip, username=uname, password=pw)
self.chan = self.client.invoke_shell()
if debug == True: printlog("Successfull connected to %s" % ip)
self.chan.settimeout(5.0)
self.error = 'No error founds while connecting'
return True
except Exception as e:
self.error = e
self.chan = None
printlog(e)
return False
def runcmd(self, cmd, regcode=']#', debug=False):
try:
if self.chan is not None:
self.getfeedback()
printlog("Begin runcmd: {0}".format(cmd))
self.chan.send(cmd)
r1 = self.getfeedback()
self.chan.send("exit\n")
# printlog(r1)
return r1
except Exception as e:
printlog("[runcmd] {0}".format(e))
# if any other
return False
def connectWithKey(self, ip, uname, privatekeypath, debug=False):
try:
# create RSAKey from private key file mykey.pem
mykey = paramiko.RSAKey.from_private_key_file(privatekeypath)
# Connect with pkey
self.client.connect(hostname=ip, username=uname, pkey=mykey)
self.chan = self.client.invoke_shell()
if debug == True: printlog("Successfull connected to %s" % ip)
self.chan.settimeout(5.0)
self.error = 'No error founds while connecting'
return True
except Exception as e:
self.error = e
self.chan = None
printlog(e)
return False
def download(url, filename):
try:
lk.acquire()
filename = url.split('/')[-1]
with requests.get(url, stream=True) as r:
filesize = r.headers['Content-length'].encode("utf8")
printlog("Start download {0} kilobytes".format(filesize))
with open(filename + ".tmp", 'wb') as fh:
fh.write(filesize)
with open(filename, 'wb') as f:
lk.release()
shutil.copyfileobj(r.raw, f)
return filename, filesize
except Exception as e:
print(e)
return False, 0
finally:
if lk.locked():
lk.release()
def status(filename, lk1, progress):
lk1.acquire()
try:
with open(filename + ".tmp", "rb") as fh:
filesize = fh.read()
filesize = int(filesize)
utils.printlog("Got filesize = {0}".format(filesize))
os.remove(filename + ".tmp")
while 1: #Return true if the lock is acquired
thesize = os.stat(filename).st_size
if thesize == filesize:
utils.printlog("Download finished")
if lk1.locked(): lk1.release()
cmdq.put((setProgress, (progress, 100), {})) #set progress
break
else:
percent = (thesize * 100 / filesize)
utils.printwait(
"{0}/{1} - Percentage: {2:.2f}% complete".format(
thesize, filesize, percent), 1)
cmdq.put((setProgress, (progress, percent), {})) #set progress
except FileNotFoundError as e:
utils.printlog("File not found")
return 0
except Exception as e:
raise (e)
finally:
if lk1.locked(): lk1.release()
def percentage(filename):
try:
lk.acquire()
with open(filename + ".tmp", "rb") as fh:
filesize = fh.read()
filesize = int(filesize)
printlog("Got filesize = {0}".format(filesize))
os.remove(filename + ".tmp")
while 1:
thesize = os.stat(filename).st_size
if thesize == filesize:
printlog("Download finished")
lk.release()
break
else:
printwait(
"{0}/{1} - Percentage: {2:.2f}%".format(
thesize, filesize, (thesize * 100 / filesize)), 1)
except Exception as e:
print(e)
if lk.locked(): lk.release()
return False
def checkProcess(self,
cmd="systemctl status sshd | grep Active\n",
evid="active (running)",
debug=False):
"""Check process running."""
if self.chan is not None:
self.getfeedback(debug=debug)
if debug:
printlog("Begin check process with command: {0}".format(cmd))
self.chan.send(cmd)
r1 = self.getfeedback(debug=debug)
self.chan.send("exit\n")
if debug: printlog(r1)
if (evid in r1):
printlog("[checkProcess] The process is ALIVE")
return True
else:
printlog("[checkProcess] The process is DIED")
return False
else:
printlog(
"[checkProcess] Unable to accomply because channel is NULL")
return False
def getDiskSpaceHtml(self, cmd="df -h /\n", isDebug=False):
#Remember not add r"" as we need to translate to Enter button pressed"
if self.chan is not None:
self.getfeedbackASCII(debug=isDebug)
printlog("Begin get disk space with command: {0}...".format(cmd))
self.chan.send(cmd)
r1 = self.getfeedbackASCII(debug=isDebug)
self.chan.send("exit\n")
if isDebug: printlog("r1: {0}".format(r1))
ss = r1.split(r"\r\n")[1].split()
if isDebug: printlog("ss: {0}".format(ss))
xs = r1.split(r"\r\n")[2].split()
# printlog(xs)
if len(xs) == 1:
ys = r1.split(r"\r\n")[3].split()
percentt = float(ys[3].split("%")[0])
percentt = '<strong style="color: green;">{0}%</strong>'.format(
percentt
) if percentt < 50 else '<strong style="color: red;">{0}%</strong>'.format(
percentt)
ret = """<table border="1" style="width:100%" align="center"><tbody>
<tr><td>{0}</td><td>{1}</td><td>{2}</td><td>{3}</td><td>{4}</td><td>{5}</td></tr>
<tr><td>{6}</td><td>{7}</td><td>{8}</td><td>{9}</td><td>{10}</td><td>{11}</td></tr>
</tbody></table>""".format(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], xs[0],
ys[0], ys[1], ys[2], percentt, ys[4])
else:
percentt = float(xs[4].split("%")[0])
percentt = '<strong style="color: green;">{0}</strong>'.format(
xs[4]
) if percentt < 50 else '<strong style="color: red;">{0}</strong>'.format(
xs[4])
ret = """<table border="1" style="width:100%" align="center"><tbody>
<tr><td>{0}</td><td>{1}</td><td>{2}</td><td>{3}</td><td>{4}</td><td>{5}</td></tr>
<tr><td>{6}</td><td>{7}</td><td>{8}</td><td>{9}</td><td>{10}</td><td>{11}</td></tr>
</tbody></table>""".format(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], xs[0],
xs[1], xs[2], xs[3], percentt, xs[5])
return ret
else:
printlog("Unable to accomply because channel is NULL")
return "Unable to accomply because channel is NULL"
def getfeedbackASCII(self, regcode=']#', debug=False):
"""getfeedback from server command line."""
try:
if debug:
printlog("[getfeedback] Waiting for reply until timeout...")
buff = ''
while buff.find(regcode) == -1:
resp = self.chan.recv(9999999)
buff = "{0}{1}".format(buff, resp)
# printlog(buff)
return buff #return UNICODE String
except socket.timeout:
printlog(
"[getfeedback] Time out while waiting for response from server. Program will exits."
)
sys.exit(1)
except Exception as e:
printlog("[getfeedbackASCII] {0}".format(e))
return False
printlog("controller started")
l.acquire()
[q.put(i) for i in range(10) if not q.full()]
l.release()
# time.sleep(10)
# ev.set() #set the flag using event
i = br.wait() #set the flag using barrier
printlog("controller stopped")
ts = [
threading.Thread(target=worker, args=(i, l, q), daemon=True)
for i in range(2)
]
printlog("Fill the queue")
ctl = threading.Thread(target=controller, args=(l, q, ev, br), daemon=True)
ctl.start()
# block = ev.wait(10) #wait in 10 sec using event
# if block:
try:
i = br.wait(
5
) #main thread will wait for controller because main finished its job sooner than controller. Wait 5s or raise threading.BrokenBarrierError exception
for t in ts:
t.start()
printlog("Wait for queue to be executed")
while not q.empty():
percent = 100 - q.qsize() * 100 / 10
def func(d, l):
printlog(d)
printlog(l)
'''
Created Date: Saturday August 22nd 2020
Author: Leo Tran (https://github.com/loitd)
-----
Last Modified: Saturday August 22nd 2020 11:16:21 pm
Modified By: Leo Tran (https://github.com/loitd)
-----
HISTORY:
Date By Comments
---------- ------ ---------------------------------------------------------
22-08-2020 loitd Initialize the file
'''
from multiprocessing import Pool, Process, Queue, Pipe, Lock, Value, Array, Manager
import os
from lutils.utils import printlog
def func(d, l):
printlog(d)
printlog(l)
if __name__ == "__main__":
# Functionality within this package requires that the __main__ module be importable by the children
with Manager() as man:
d = man.dict()
l = man.list([1, 2, 3, 4, 5, 6])
with Pool(processes=3) as pool:
res = [pool.apply_async(os.getpid, ()) for i in l]
printlog([i.get(timeout=100) for i in res])
print("No more pool")
def tonggle_button(btn):
utils.printlog("Current BTN state is: {0}".format(btn['state']))
if btn['state'] != 'disabled':
btn['state'] = 'disabled'
else:
btn['state'] = 'normal'
