def receive(self, preprocess=None): e_size = struct.calcsize("!Q") r_size = 0 stub = ppc.b_("") data = stub while r_size < e_size: msg = self.socket.recv(e_size - r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('_sr', l, self.socket, msg))+'\n') if msg == stub: raise RuntimeError("Socket connection is broken") r_size += len(msg) data += msg e_size = struct.unpack("!Q", ppc.b_(data))[0] # get size of msg r_size = 0 data = stub while r_size < e_size: msg = self.socket.recv(e_size - r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('sr_', l, self.socket, msg))+'\n') if msg == stub: raise RuntimeError("Socket connection is broken") r_size += len(msg) data += msg data = ppc.b_(data) return data
def receive(self, preprocess=None): e_size = struct.calcsize("!Q") # 8 c_size = struct.calcsize("!c") # 1 r_size = 0 stub = ppc.b_("") if (self.has_rb or self.r.mode == 'rb') else "" data = stub while r_size < e_size: msg = self.rb.read(e_size - r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('_r', l, self.r, msg))+'\n') if msg == stub: raise RuntimeError("Communication pipe read error") if stub == "" and msg.startswith('['): #HACK to get str_ length while not msg.endswith('{B}'): msg += self.rb.read(c_size) r_size += len(msg) data += msg e_size = struct.unpack("!Q", ppc.b_(data))[0] # get size of msg r_size = 0 data = stub while r_size < e_size: msg = self.rb.read(e_size - r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('r_', l, self.r, msg))+'\n') if msg == stub: raise RuntimeError("Communication pipe read error") r_size += len(msg) data += msg data = ppc.b_(data) if preprocess is None: preprocess = lambda x: x return tuple(map(preprocess, (data, )))[0]
def receive(self, preprocess=None): e_size = struct.calcsize("!Q") # 8 c_size = struct.calcsize("!c") # 1 r_size = 0 stub = ppc.b_("") if (self.has_rb or self.r.mode == 'rb') else "" data = stub while r_size < e_size: msg = self.rb.read(e_size-r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('_r', l, self.r, msg))+'\n') if msg == stub: raise RuntimeError("Communication pipe read error") if stub == "" and msg.startswith('['): #HACK to get str_ length while not msg.endswith('{B}'): msg += self.rb.read(c_size) r_size += len(msg) data += msg e_size = struct.unpack("!Q", ppc.b_(data))[0] # get size of msg r_size = 0 data = stub while r_size < e_size: msg = self.rb.read(e_size-r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('r_', l, self.r, msg))+'\n') if msg == stub: raise RuntimeError("Communication pipe read error") r_size += len(msg) data += msg data = ppc.b_(data) if preprocess is None: preprocess = lambda x:x return tuple(map(preprocess, (data, )))[0]
def receive(self, preprocess=None): e_size = struct.calcsize("!Q") r_size = 0 stub = ppc.b_("") data = stub while r_size < e_size: msg = self.socket.recv(e_size-r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('_sr', l, self.socket, msg))+'\n') if msg == stub: raise RuntimeError("Socket connection is broken") r_size += len(msg) data += msg e_size = struct.unpack("!Q", ppc.b_(data))[0] # get size of msg r_size = 0 data = stub while r_size < e_size: msg = self.socket.recv(e_size-r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('sr_', l, self.socket, msg))+'\n') if msg == stub: raise RuntimeError("Socket connection is broken") r_size += len(msg) data += msg data = ppc.b_(data) return data
def csend(self, msg): #if hasattr(self, 'w'): # open('/tmp/pp.debug', 'a+').write(repr(('cs', self.w, msg))+'\n') #else: # open('/tmp/pp.debug', 'a+').write(repr(('cs', self.socket, msg))+'\n') msg = ppc.b_(msg) hash1 = self.hash(msg) if hash1 in self.scache: self.send(ppc.b_("H" + hash1)) else: self.send(ppc.b_("N") + msg) self.scache[hash1] = True
def broadcast(self): """Sends a broadcast""" if self.isclient: self.base.logger.debug("Client sends initial broadcast to (%s, %i)" % self.broadcast_addr) self.bsocket.sendto(ppc.b_("C"), self.broadcast_addr) else: while True: if self.base._exiting: return self.base.logger.debug("Server sends broadcast to (%s, %i)" % self.broadcast_addr) self.bsocket.sendto(ppc.b_("S"), self.broadcast_addr) time.sleep(BROADCAST_INTERVAL)
def creceive(self, preprocess=None): msg = self.receive() #if hasattr(self, 'r'): # open('/tmp/pp.debug', 'a+').write(repr(('cr', self.r, msg))+'\n') #else: # open('/tmp/pp.debug', 'a+').write(repr(('cr', self.socket, msg))+'\n') msg = ppc.b_(msg) if msg[:1] == ppc.b_('H'): hash1 = ppc.str_(msg[1:]) else: msg = msg[1:] hash1 = self.hash(msg) if preprocess is None: preprocess = lambda x: x self.rcache[hash1] = tuple(map(preprocess, (msg, )))[0] return self.rcache[hash1]
def creceive(self, preprocess=None): msg = self.receive() #if hasattr(self, 'r'): # open('/tmp/pp.debug', 'a+').write(repr(('cr', self.r, msg))+'\n') #else: # open('/tmp/pp.debug', 'a+').write(repr(('cr', self.socket, msg))+'\n') msg = ppc.b_(msg) if msg[:1] == ppc.b_('H'): hash1 = ppc.str_(msg[1:]) else: msg = msg[1:] hash1 = self.hash(msg) if preprocess is None: preprocess = lambda x:x self.rcache[hash1] = tuple(map(preprocess, (msg, )))[0] return self.rcache[hash1]
def listen(self): """Listens for broadcasts from other clients/servers""" self.base.logger.debug("Listening (%s, %i)" % self.interface_addr) self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) self.socket.settimeout(5) self.socket.bind(self.interface_addr) ppc.start_thread("broadcast", self.broadcast) while True: try: if self.base._exiting: return message, (host, port) = self.socket.recvfrom(1024) message = ppc.str_(message) remote_address = (host, self.broadcast_addr[1]) hostid = host + ":" + str(self.broadcast_addr[1]) self.base.logger.debug("Discovered host (%s, %i) message=%c" % (remote_address + (message[0], ))) if not self.base.autopp_list.get(hostid, 0) and self.isclient \ and message[0] == 'S': self.base.logger.debug("Connecting to host %s" % (hostid, )) ppc.start_thread("ppauto_connect1", self.base.connect1, remote_address+(False, )) if not self.isclient and message[0] == 'C': self.base.logger.debug("Replying to host %s" % (hostid, )) self.bsocket.sendto(ppc.b_("S"), self.broadcast_addr) except socket.timeout: pass except: self.base.logger.error("An error has occured during execution of " "Discover.listen") sys.excepthook(*sys.exc_info())
def authenticate(self, secret): remote_version = ppc.str_(self.receive()) if version != remote_version: logging.error("PP version mismatch (local: pp-%s, remote: pp-%s)" % (version, remote_version)) logging.error("Please install the same version of PP on all nodes") return False srandom = ppc.b_(self.receive()) secret = ppc.b_(secret) answer = sha_new(srandom+secret).hexdigest() self.send(answer) response = ppc.b_(self.receive()) if response == ppc.b_("OK"): return True else: return False
def authenticate(self, secret): remote_version = ppc.str_(self.receive()) if version != remote_version: logging.error("PP version mismatch (local: pp-%s, remote: pp-%s)" % (version, remote_version)) logging.error("Please install the same version of PP on all nodes") return False srandom = ppc.b_(self.receive()) secret = ppc.b_(secret) answer = sha_new(srandom + secret).hexdigest() self.send(answer) response = ppc.b_(self.receive()) if response == ppc.b_("OK"): return True else: return False
def md5test(hash, start, end): """Calculates md5 of the integers between 'start' and 'end' and compares it with 'hash'""" from ppcommon import b_ for x in range(start, end): if hashlib.md5(b_(str(x))).hexdigest() == hash: return x
def crun(self, csocket): """Authenticates client and handles its jobs""" mysocket = pptransport.CSocketTransport(csocket, self.socket_timeout) #send PP version mysocket.send(version) #generate a random string srandom = "".join( [random.choice(string.ascii_letters) for i in range(16)]) mysocket.send(srandom) answer = sha_new(ppc.b_(srandom + self.secret)).hexdigest() clientanswer = ppc.str_(mysocket.receive()) if answer != clientanswer: self.logger.warning( "Authentication failed, client host=%s, port=%i" % csocket.getpeername()) mysocket.send("FAILED") csocket.close() return else: mysocket.send("OK") ctype = ppc.str_(mysocket.receive()) self.logger.debug("Control message received: " + ctype) self.ncon_add(1) try: if ctype == "STAT": #reset time at each new connection self.get_stats()["local"].time = 0.0 #open('/tmp/pp.debug', 'a+').write('STAT: \n') mysocket.send(str(self.get_ncpus())) #open('/tmp/pp.debug', 'a+').write('STAT: get_ncpus\n') while 1: mysocket.receive() #open('/tmp/pp.debug', 'a+').write('STAT: recvd\n') mysocket.send(str(self.get_stats()["local"].time)) #open('/tmp/pp.debug', 'a+').write('STAT: _\n') elif ctype == "EXEC": while 1: #open('/tmp/pp.debug', 'a+').write('EXEC: \n') sfunc = mysocket.creceive() #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sfunc,))+'\n') sargs = mysocket.receive() #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sargs,))+'\n') fun = self.insert(sfunc, sargs) sresult = fun(True) #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sresult,))+'\n') mysocket.send(sresult) #open('/tmp/pp.debug', 'a+').write('EXEC: _\n') except: if self._exiting: return if pp.SHOW_EXPECTED_EXCEPTIONS: self.logger.debug( "Exception in crun method (possibly expected)", exc_info=True) self.logger.debug("Closing client socket") csocket.close() self.ncon_add(-1)
def __unpickle(self): """Unpickles the result of the task""" self.result, sout = pickle.loads(ppc.b_(self.sresult)) self.unpickled = True if len(sout) > 0: six.print_(sout, end=' ') if self.callback: args = self.callbackargs + (self.result, ) self.callback(*args)
def crun(self, csocket): """Authenticates client and handles its jobs""" mysocket = pptransport.CSocketTransport(csocket, self.socket_timeout) #send PP version mysocket.send(version) #generate a random string srandom = "".join([random.choice(string.ascii_letters) for i in range(16)]) mysocket.send(srandom) answer = sha_new(ppc.b_(srandom+self.secret)).hexdigest() clientanswer = ppc.str_(mysocket.receive()) if answer != clientanswer: self.logger.warning("Authentication failed, client host=%s, port=%i" % csocket.getpeername()) mysocket.send("FAILED") csocket.close() return else: mysocket.send("OK") ctype = ppc.str_(mysocket.receive()) self.logger.debug("Control message received: " + ctype) self.ncon_add(1) try: if ctype == "STAT": #reset time at each new connection self.get_stats()["local"].time = 0.0 #open('/tmp/pp.debug', 'a+').write('STAT: \n') mysocket.send(str(self.get_ncpus())) #open('/tmp/pp.debug', 'a+').write('STAT: get_ncpus\n') while 1: mysocket.receive() #open('/tmp/pp.debug', 'a+').write('STAT: recvd\n') mysocket.send(str(self.get_stats()["local"].time)) #open('/tmp/pp.debug', 'a+').write('STAT: _\n') elif ctype=="EXEC": while 1: #open('/tmp/pp.debug', 'a+').write('EXEC: \n') sfunc = mysocket.creceive() #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sfunc,))+'\n') sargs = mysocket.receive() #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sargs,))+'\n') fun = self.insert(sfunc, sargs) sresult = fun(True) #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sresult,))+'\n') mysocket.send(sresult) #open('/tmp/pp.debug', 'a+').write('EXEC: _\n') except: if self._exiting: return if pp.SHOW_EXPECTED_EXCEPTIONS: self.logger.debug("Exception in crun method (possibly expected)", exc_info=True) self.logger.debug("Closing client socket") csocket.close() self.ncon_add(-1)
def preprocess(msg): fname, fsources, imports = pickle.loads(ppc.b_(msg)) fobjs = [compile(fsource, '<string>', 'exec') for fsource in fsources] for module in imports: try: if not module.startswith("from ") and not module.startswith("import "): module = "import " + module six.exec_(module) globals().update(locals()) except: print("An error has occured during the module import") sys.excepthook(*sys.exc_info()) return fname, fobjs
def send(self, msg): #l = len(ppc.b_(msg)) if (self.has_wb or self.w.mode == 'wb') else len(ppc.str_(msg)) #open('/tmp/pp.debug', 'a+').write(repr(('s', l, self.w, msg))+'\n') if self.has_wb or self.w.mode == 'wb': msg = ppc.b_(msg) self.wb.write(struct.pack("!Q", len(msg))) self.w.flush() else: #HACK: following may be > 8 bytes, needed for len(msg) >= 256 msg = ppc.str_(msg) self.wb.write(ppc.str_(struct.pack("!Q", len(msg)))) self.w.flush() self.wb.write(msg) self.w.flush()
def send(self, data): #l = len(ppc.b_(data)) #open('/tmp/pp.debug', 'a+').write(repr(('ss', l, self.socket, data))+'\n') data = ppc.b_(data) size = struct.pack("!Q", len(data)) t_size = struct.calcsize("!Q") s_size = ppc.long(0) while s_size < t_size: p_size = self.socket.send(size[s_size:]) if p_size == 0: raise RuntimeError("Socket connection is broken") s_size += p_size t_size = len(data) s_size = ppc.long(0) while s_size < t_size: p_size = self.socket.send(data[s_size:]) if p_size == 0: raise RuntimeError("Socket connection is broken") s_size += p_size
def run(self): try: #execution cycle while 1: __fname, __fobjs = self.t.creceive(preprocess) __sargs = self.t.receive() for __fobj in __fobjs: try: six.exec_(__fobj) globals().update(locals()) except: print("An error has occured during the " + \ "function import") sys.excepthook(*sys.exc_info()) __args = pickle.loads(ppc.b_(__sargs)) __f = locals()[ppc.str_(__fname)] try: __result = __f(*__args) except: print("An error has occured during the function execution") sys.excepthook(*sys.exc_info()) __result = None __sresult = pickle.dumps((__result, self.sout.getvalue()), self.pickle_proto) self.t.send(__sresult) self.sout.truncate(0) except: print("A fatal error has occured during the function execution") sys.excepthook(*sys.exc_info()) __result = None __sresult = pickle.dumps((__result, self.sout.getvalue()), self.pickle_proto) self.t.send(__sresult)
def hash(self, msg): return md5_new(ppc.b_(msg)).hexdigest()
#ppservers = ("10.0.0.1","10.0.0.2") # list of static IPs ppservers = () if len(sys.argv) > 1: ncpus = int(sys.argv[1]) # Creates jobserver with ncpus workers job_server = pp.Server(ncpus, ppservers=ppservers) else: # Creates jobserver with automatically detected number of workers job_server = pp.Server(ppservers=ppservers) print("Starting pp with %s workers" % job_server.get_ncpus()) #Calculates md5 hash from the given number from ppcommon import b_ hash = hashlib.md5(b_("1829182")).hexdigest() print("hash = %s" % hash) #Now we will try to find the number with this hash value start = 1 end = 2000000 # Since jobs are not equal in the execution time, division of the problem # into a 128 of small subproblems leads to a better load balancing parts = 128 step = int((end - start) / parts + 1) jobs = [] for index in range(parts): starti = start + index * step