def status_icon_run(cluster): if gui_get()==True: global statusicon if cluster==False: statusicon.setIcon(icon_get("ball_red")) else: statusicon.setIcon(icon_get("ball_red4"))
def status_icon_stop(cluster): if gui_get()==True: global statusicon if cluster==False: statusicon.setIcon(icon_get("ball_green")) else: statusicon.setIcon(icon_get("ball_green4"))
# with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # ## @package gui_util # GUI utilities. # from cal_path import get_image_file_path import os #qt from gui_enable import gui_get if gui_get() == True: from PyQt5.QtWidgets import QTextEdit, QAction, QTableWidgetItem, QComboBox, QMessageBox, QDialog, QDialogButtonBox from PyQt5.QtWidgets import QListWidgetItem, QListView, QLineEdit, QWidget, QHBoxLayout, QPushButton from PyQt5.uic import loadUi from PyQt5.QtGui import QPixmap from PyQt5.QtCore import QSize, Qt, QTimer from QComboBoxLang import QComboBoxLang from PyQt5.QtGui import QIcon from gtkswitch import gtkswitch from gpvdm_select_material import gpvdm_select_material from gpvdm_select_emission import gpvdm_select_emission from leftright import leftright from gpvdm_select import gpvdm_select from QComboBoxLang import QComboBoxLang from QColorPicker import QColorPicker from QComboBoxNewtonSelect import QComboBoxNewtonSelect
def status_icon_init(): if gui_get()==True: global statusicon statusicon=tray_icon() statusicon.show()
def status_icon_get(): if gui_get()==True: global statusicon return statusicon
def get_exe_args(): if gui_get() == True: return "--gui --html" #--english else: return ""
class cluster: if gui_get() == True: load_update = pyqtSignal() jobs_update = pyqtSignal() new_message = pyqtSignal(str) def load_server_ip(self): self.server_ip = inp_get_token_value(os.path.join( get_sim_path(), "cluster"), "#cluster_ip", search_active_file=True) def cluster_init(self): self.socket = False self.cluster = False self.interactive_cluster = False self.nodes = [] def connect(self): if self.cluster == False: encrypt_load() self.load_server_ip() self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) port = int( inp_get_token_value(os.path.join(get_sim_path(), "cluster"), "#port", search_active_file=True)) try: self.socket.connect((self.server_ip, port)) except: print("Failed to connect to ", self.server_ip) return False self.cluster = True data = tx_struct() data.id = "gpvdmregistermaster" self.tx_packet(data) if self.running == False: self.mylock = False self.thread = threading.Thread(target=self.listen) self.thread.daemon = True self.thread.start() print("conected to cluster") return True def cluster_disconnect(self): if self.cluster == True: #try: self.stop_listening = True self.thread.join() #except: # print("error stopping thread",sys.exc_info()[0]) self.socket.close() self.cluster = False print("not conected to cluster") return True def get_nodes_load(self): cpus = 0.0 load = 0.0 for i in range(0, len(self.nodes)): cpus = cpus + self.nodes[i].cpus load = load + self.nodes[i].load if cpus == 0: return 0 return 100.0 * (load / cpus) def wake_nodes(self): if self.cluster == True: s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) port = 8888 s.sendto("wake_nodes", (self.server_ip, port)) s.close() def clear_cache(self): #if self.cluster==True: # if self.running==True: # print "Clear cache on server" # self.mylock=True # s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # port = 8888; # s.sendto("clear_cache#", (self.server_ip, port)) # s.close() # print "I should be waiting for mylock" # self.wait_lock() # print "I have finished waiting" print("cache clear") def recvall(self, n): data = bytearray() while len(data) < n: packet = self.socket.recv(n - len(data)) if not packet: print("not packet") return None data += packet #print(100.0*len(data)/n) data = decrypt(data) return data def cluster_make(self): data = tx_struct() data.id = "gpvdmheadexe" data.dir_name = "src" data.command = inp_get_token_value(os.path.join( get_sim_path(), "cluster"), "#make_command", search_active_file=True) self.tx_packet(data) def sync_files(self, files, target): count = 0 banned = [] sums = "" for fname in files: f = open(fname, 'rb') bytes = f.read() size = len(bytes) f.close() m = hashlib.md5() m.update(bytes) bin = m.digest() key_hash = "" for i in range(0, m.digest_size): key_hash = key_hash + format(ord(bin[i]), '02x') sums = sums + fname + "\n" + key_hash + "\n" sums = sums[0:len(sums) - 1] #print("sending",sums0 size = len(sums) tx_size = ((int(size) / int(512)) + 1) * 512 data = bytearray(tx_size) for i in range(0, size): data[i] = sums[i] head = "gpvdm_sync_packet_one\n#size\n" + str( size) + "\n#target\n" + target + "\n#src\n" + path + "\n#end" start_len = len(head) head_buf = bytearray(512) for i in range(0, len(head)): head_buf[i] = head[i] buf = head_buf + data buf = encrypt(buf) self.socket.sendall(buf) def sync_dir(self, path, target): count = 0 banned = [] sums = "" files = self.gen_dir_list(path) self.sync_files(files, target) def gen_dir_list(self, path, banned_types=None, banned_dirs=None): if banned_types == None: banned_types = [ ".pdf", ".png", ".dll", ".o", ".so", ".so", ".a", ".dat", ".aprox", ".ods", ".xls", ".xlsx", ".log", ".pptx", ".dig", ".old", ".bak", ".opj", ".csv" ] if banned_dirs == None: banned_dirs = [ "equilibrium", "man_src", "images", "snapshots", "plot", "pub", "gui", "debian", "desktop", "device_lib" ] file_list = [] for root, dirs, files in os.walk(path): for name in files: fname = os.path.join(root, name) tx = True for i in range(0, len(banned_types)): if name.endswith(banned_types[i]) == True: tx = False if tx == True: fname = fname[len(path):] fname = strip_slash(fname) file_list.append(fname) f = [] for i in range(0, len(file_list)): if file_list[i].endswith("gpvdm_gui_config.inp"): banned_dirs.append(os.path.dirname(file_list[i])) for i in range(0, len(file_list)): add = True for ii in range(0, len(banned_dirs)): if file_list[i].startswith(banned_dirs[ii] + os.path.sep) == True: add = False break if add == True: f.append(file_list[i]) return f def send_dir(self, src, target): print("Sending dir", src, target) files = self.gen_dir_list(src) self.send_files(target, src, files) def tx_packet2(self, data): #If the user wants to tx a string conver it into byte sfirst data_size = len(data.data) if type(data.data) == str: dat = str.encode(data.data) else: dat = data.data if data.zip == True: data.uzipsize = len(dat) dat = zlib.compress(dat) header = "gpvdm " header = header + "#id\n" + data.id + "\n" if data.file_name != "": header = header + "#file_name\n" + data.file_name + "\n" header = header + "#size\n" + str(len(dat)) + "\n" if data.target != "": header = header + "#target\n" + data.target + "\n" header = header + "#stat\n" + str(data.stat) + "\n" if data.dir_name != "": header = header + "#dir_name\n" + data.dir_name + "\n" if data.exe_name != "": header = header + "#exe_name\n" + data.exe_name + "\n" if data.command != "": header = header + "#command\n" + data.command + "\n" hash = random.getrandbits(128) header = header + "#token\n" + data.token + "\n" if data.cpus != -1: header = header + "#cpus\n" + str(data.cpus) + "\n" if data.zip == True: header = header + "#zip\n" + "1" + "\n#uzipsize\n" + str( data.uzipsize) + "\n" header = header + "#end" tx_size = len(header) + len(dat) packet_size = ((int)(tx_size / 16) + 1) * 16 packet = bytearray(packet_size) for i in range(0, len(header)): packet[i] = ord(header[i]) text_size = str(packet_size - 32) pos = 5 for i in range(0, len(text_size)): packet[pos] = ord(text_size[i]) pos = pos + 1 pos = pos + 1 text_size = str(len(header) - 32) for i in range(0, len(text_size)): packet[pos] = ord(text_size[i]) pos = pos + 1 packet[len(header):len(header) + len(dat)] = dat #buf=buf+bytes #print("I am encrypting2",len(buf),data.id,len(buf),len(dat)) buf = encrypt2(packet) #print("I am sending",len(buf),data.id) self.socket.sendall(packet) def tx_packet(self, data): self.tx_packet2(data) return #If the user wants to tx a string conver it into byte sfirst if type(data.data) == str: dat = str.encode(data.data) else: dat = data.data if data.zip == True: dat = zlib.compress(dat) tx_size = len(dat) if tx_size != 0: expand = ((int)(tx_size / 512) + 1) * 512 - tx_size expand = int(expand) zeros = bytearray(expand) dat += zeros header = data.id + "\n" if data.file_name != "": header = header + "#file_name\n" + data.file_name + "\n" header = header + "#size\n" + str(tx_size) + "\n" if data.target != "": header = header + "#target\n" + data.target + "\n" header = header + "#stat\n" + str(data.stat) + "\n" if data.dir_name != "": header = header + "#dir_name\n" + data.dir_name + "\n" if data.exe_name != "": header = header + "#exe_name\n" + data.exe_name + "\n" if data.command != "": header = header + "#command\n" + data.command + "\n" if data.cpus != -1: header = header + "#cpus\n" + str(data.cpus) + "\n" if data.zip == True: header = header + "#zip\n" + "1" + "\n#uzipsize\n" + str( data.uzipsize) + "\n" header = header + "#end" buf = bytearray(512) for i in range(0, len(header)): buf[i] = ord(header[i]) buf += dat #buf=buf+bytes print("I am encrypting", len(buf), data.id, len(buf), len(dat)) buf = encrypt(buf) print("I am sending", len(buf), data.id) self.socket.sendall(buf) def send_files(self, target, src, files): progress_window = progress_class() progress_window.show() progress_window.start() process_events() count = 0 banned = [] for i in range(0, len(files)): data = tx_struct() progress_window.set_fraction(float(i) / float(len(files))) progress_window.set_text("Sending " + files[i]) process_events() full_path = os.path.normpath(os.path.join(src, files[i])) data.stat = os.stat(full_path)[ST_MODE] f = open(full_path, 'rb') bytes = f.read() f.close() orig_size = len(bytes) print("tx file:", full_path) if target == "": data.target = src else: data.target = target data.id = "gpvdmfile" data.uzipsize = len(bytes) data.data = bytes data.zip = True data.file_name = strip_slash(files[i]) count = count + 1 self.tx_packet(data) #if count>2: # break print("status>>>>>>>>", i, len(files)) progress_window.stop() print("total=", count) def process_node_list(self, data): self.nodes = [] data = data.data data = data.decode("utf-8") data = data.split("\n") for i in range(0, len(data) - 1): n = node() d = data[i].split(":") n.name = d[0] n.ip = d[1] n.cpus = float(d[2]) n.load = float(d[4]) n.max_cpus = float(d[5]) n.last_seen = d[6] self.nodes.append(n) def process_job_list(self, data): ret = data.data lines = ret.decode("utf-8").split("\n") self.jobs = [] for i in range(1, len(lines)): act = lines[i].split() if len(act) > 1: j = job() j.name = act[1] j.path = act[4] j.ip = act[5] j.start = act[7] j.stop = act[9] j.cpus = int(act[10]) j.status = int(act[3]) self.jobs.append(j) #print(act[0], act[1], act[2], act[3],act[4], act[5], act[6], act[7]) #self.cluster_jobs.append([act[0], act[1], act[2], act[3],act[4], act[5], act[6], act[7]]) self.jobs_update.emit() #print(ret.data,"jim") def process_sync_packet_two(self, data): lines = data.split("\n") target = inp_search_token_value(lines, "#target") src = inp_search_token_value(lines, "#src") size = int(inp_search_token_value(lines, "#size")) packet_len = int(int(size) / int(512) + 1) * 512 data = self.recvall(packet_len) data = data[0:size] lines = data.split("\n") pos = 0 copy_list = [] #print(lines) for i in range(0, len(lines)): fname = lines[pos] pos = pos + 1 if fname != "": copy_list.append(fname) self.send_files(target, src, copy_list) def process_sync_packet_one(self, data): print(data) def rx_packet(self): head = self.recvall(32) if head == None: return None head = decrypt(head) head = head[5:] lengths = [int(s) for s in head.split() if s.isdigit()] #print(lengths) packet_len = lengths[0] head_len = lengths[1] packet = self.recvall(packet_len) if len(packet) != packet_len: print("packet len does not match size", len(packet), packet_len) packet = decrypt(packet) ret = tx_struct() lines = packet[0:head_len].decode("utf-8").split("\n") ret.id = inp_search_token_value(lines, "#id") ret.file_name = inp_search_token_value(lines, "#file_name") ret.size = int(inp_search_token_value(lines, "#size")) ret.target = inp_search_token_value(lines, "#target") ret.zip = int(inp_search_token_value(lines, "#zip")) ret.uzipsize = int(inp_search_token_value(lines, "#uzipsize")) ret.message = inp_search_token_value(lines, "#message") ret.token = inp_search_token_value(lines, "#token") #print(lines) #print(ret.file_name,ret.size,ret.uzipsize,len(data)) if ret.size != 0: ret.data = packet[head_len:head_len + ret.size] if ret.zip == 1: ret.data = zlib.decompress(ret.data) ret.size = len(ret.data) #print(lines,ret.data) return ret def set_cluster_loads(self, ip, loads): packet = "" for i in range(0, len(ip)): packet = packet + ip[i] + "\n" + str(loads[i]) + "\n" packet = packet[:-1] data = tx_struct() data.id = "gpvdm_set_max_loads" data.data = packet data.size = len(packet) self.tx_packet(data) def copy_src_to_cluster_fast(self): if self.cluster == True: path = get_src_path() if path == None: return self.sync_dir(path, "src") path = inp_get_token_value(os.path.join(get_sim_path(), "cluster"), "#path_to_libs", search_active_file=True) self.sync_dir(path, "src") def copy_src_to_cluster(self): if self.cluster == True: path = get_src_path() if path == None: return banned_types = [ ".pdf", ".png", ".dll", ".o", ".so", ".so", ".a", ".dat", ".aprox", ".ods", ".rpm", ".deb" ] banned_types.extend([ ".xls", ".xlsx", ".log", ".pptx", ".dig", ".old", ".bak", ".opj", ".csv", ".jpg", ".so.3", ".so.5", "gpvdm_core" ]) banned_types.extend(["so.0", ".zip"]) banned_dirs = [ "equilibrium", "man_src", "images", "snapshots", "plot", "pub", "gui", "debian", "desktop", "device_lib", "sim", "cluster", "cluster_", "cluster_libs" ] files = self.gen_dir_list(path, banned_types=banned_types, banned_dirs=banned_dirs) self.send_files("src", path, files) path = inp_get_token_value(os.path.join(get_sim_path(), "cluster"), "#path_to_libs", search_active_file=True) if path == "autosearch": path = get_cluster_libs_path() self.send_dir(path, "src") def cluster_get_data(self): if self.cluster == True: data = tx_struct() data.id = "gpvdmgetdata" self.tx_packet(data) def cluster_get_info(self): if self.cluster == True: data = tx_struct() data.id = "gpvdmsendnodelist" self.tx_packet(data) def cluster_quit(self): if self.cluster == True: data = tx_struct() data.id = "gpvdmquit" self.tx_packet(data) def cluster_killall(self): data = tx_struct() data.id = "gpvdm_stop_all_jobs" self.tx_packet(data) data.id = "gpvdmkillall" self.tx_packet(data) sleep(1) data.id = "gpvdm_delete_all_jobs" self.tx_packet(data) self.stop() def cluster_run_jobs(self): exe_name = inp_get_token_value(os.path.join(get_sim_path(), "cluster"), "#exe_name", search_active_file=True) data = tx_struct() data.id = "gpvdmrunjobs" data.exe_name = exe_name self.tx_packet(data) def sleep(self): if self.cluster == True: data = tx_struct() data.id = "gpvdmsleep" self.tx_packet(data) def poweroff(self): if self.cluster == True: data = tx_struct() data.id = "gpvdmpoweroff" self.tx_packet(data) def add_remote_job(self, job_orig_path): data = tx_struct() data.id = "gpvdmaddjob" data.target = job_orig_path data.cpus = int( inp_get_token_value(os.path.join(get_sim_path(), "cluster"), "#cluster_cpus", search_active_file=True)) if data.cpus == 0: data.cpus = 1 wait_add(data) self.tx_packet(data) wait_wait(data) def cluster_clean(self): data = tx_struct() data.id = "gpvdm_master_clean" self.tx_packet(data) def cluster_list_jobs(self): data = tx_struct() data.id = "gpvdm_send_job_list" self.tx_packet(data) def rx_file(self, data): pwd = get_sim_path() print(data, data.target, data.file_name, pwd) target = data.target + data.file_name if target.startswith(pwd): my_dir = os.path.dirname(target) if os.path.isdir(my_dir) == False: os.makedirs(my_dir) if data.size > 0: print("write:", target) f = open(target, "wb") f.write(data.data[0:data.size]) f.close() else: f = open(target, "wb") f.close() else: print("not writing target", pwd, target) def listen(self): self.running = True self.stop_listening = False while (1): understood = False if self.stop_listening == True: break data = self.rx_packet() if data == None: break #print(data.id) if data.id == "gpvdmnodelist": self.process_node_list(data) self.load_update.emit() understood = True if data.id == "gpvdm_job_list": self.process_job_list(data) understood = True if data.id == "gpvdmfinished": self.stop() understood = True if data.id == "gpvdmheadquit": self.stop() print("Server quit!") understood = True if data.id == "gpvdm_message": self.new_message.emit(data.message) understood = True if data.id == "gpvdmfile": print("from here", data.id) self.rx_file(data) understood = True if data.id == "gpvdmpercent": lines = data.split("\n") percent = float(inp_search_token_value(lines, "#percent")) self.progress_window.set_fraction(percent / 100.0) understood = True if data.id == "gpvdmjobfinished": lines = data.split("\n") name = inp_search_token_value(lines, "#job_name") self.label.set_text(gui_print_path("Finished: ", name, 60)) understood = True if data.id == "gpvdmthingdone": wait_update(data.token) understood = True if data.id == "gpvdm_sync_packet_two": self.process_sync_packet_two(data) understood = True if data.id == "gpvdm_sync_packet_one": self.process_sync_packet_one(data) understood = True if understood == False: print("Command >", data, "<not understood")
def process_events(): if gui_get()==True: QApplication.processEvents()