def receive(self, preprocess=None):
e_size = struct.calcsize("!Q")
r_size = 0
stub = ppc.b_("")
data = stub
while r_size < e_size:
msg = self.socket.recv(e_size - r_size)
#l = len(msg)
#open('/tmp/pp.debug', 'a+').write(repr(('_sr', l, self.socket, msg))+'\n')
if msg == stub:
raise RuntimeError("Socket connection is broken")
r_size += len(msg)
data += msg
e_size = struct.unpack("!Q", ppc.b_(data))[0] # get size of msg
r_size = 0
data = stub
while r_size < e_size:
msg = self.socket.recv(e_size - r_size)
#l = len(msg)
#open('/tmp/pp.debug', 'a+').write(repr(('sr_', l, self.socket, msg))+'\n')
if msg == stub:
raise RuntimeError("Socket connection is broken")
r_size += len(msg)
data += msg
data = ppc.b_(data)
return data
def receive(self, preprocess=None):
e_size = struct.calcsize("!Q") # 8
c_size = struct.calcsize("!c") # 1
r_size = 0
stub = ppc.b_("") if (self.has_rb or self.r.mode == 'rb') else ""
data = stub
while r_size < e_size:
msg = self.rb.read(e_size - r_size)
#l = len(msg)
#open('/tmp/pp.debug', 'a+').write(repr(('_r', l, self.r, msg))+'\n')
if msg == stub:
raise RuntimeError("Communication pipe read error")
if stub == "" and msg.startswith('['): #HACK to get str_ length
while not msg.endswith('{B}'):
msg += self.rb.read(c_size)
r_size += len(msg)
data += msg
e_size = struct.unpack("!Q", ppc.b_(data))[0] # get size of msg
r_size = 0
data = stub
while r_size < e_size:
msg = self.rb.read(e_size - r_size)
#l = len(msg)
#open('/tmp/pp.debug', 'a+').write(repr(('r_', l, self.r, msg))+'\n')
if msg == stub:
raise RuntimeError("Communication pipe read error")
r_size += len(msg)
data += msg
data = ppc.b_(data)
if preprocess is None: preprocess = lambda x: x
return tuple(map(preprocess, (data, )))[0]
def receive(self, preprocess=None):
e_size = struct.calcsize("!Q") # 8
c_size = struct.calcsize("!c") # 1
r_size = 0
stub = ppc.b_("") if (self.has_rb or self.r.mode == 'rb') else ""
data = stub
while r_size < e_size:
msg = self.rb.read(e_size-r_size)
#l = len(msg)
#open('/tmp/pp.debug', 'a+').write(repr(('_r', l, self.r, msg))+'\n')
if msg == stub:
raise RuntimeError("Communication pipe read error")
if stub == "" and msg.startswith('['): #HACK to get str_ length
while not msg.endswith('{B}'):
msg += self.rb.read(c_size)
r_size += len(msg)
data += msg
e_size = struct.unpack("!Q", ppc.b_(data))[0] # get size of msg
r_size = 0
data = stub
while r_size < e_size:
msg = self.rb.read(e_size-r_size)
#l = len(msg)
#open('/tmp/pp.debug', 'a+').write(repr(('r_', l, self.r, msg))+'\n')
if msg == stub:
raise RuntimeError("Communication pipe read error")
r_size += len(msg)
data += msg
data = ppc.b_(data)
if preprocess is None: preprocess = lambda x:x
return tuple(map(preprocess, (data, )))[0]
def receive(self, preprocess=None):
e_size = struct.calcsize("!Q")
r_size = 0
stub = ppc.b_("")
data = stub
while r_size < e_size:
msg = self.socket.recv(e_size-r_size)
#l = len(msg)
#open('/tmp/pp.debug', 'a+').write(repr(('_sr', l, self.socket, msg))+'\n')
if msg == stub:
raise RuntimeError("Socket connection is broken")
r_size += len(msg)
data += msg
e_size = struct.unpack("!Q", ppc.b_(data))[0] # get size of msg
r_size = 0
data = stub
while r_size < e_size:
msg = self.socket.recv(e_size-r_size)
#l = len(msg)
#open('/tmp/pp.debug', 'a+').write(repr(('sr_', l, self.socket, msg))+'\n')
if msg == stub:
raise RuntimeError("Socket connection is broken")
r_size += len(msg)
data += msg
data = ppc.b_(data)
return data
def csend(self, msg):
#if hasattr(self, 'w'):
# open('/tmp/pp.debug', 'a+').write(repr(('cs', self.w, msg))+'\n')
#else:
# open('/tmp/pp.debug', 'a+').write(repr(('cs', self.socket, msg))+'\n')
msg = ppc.b_(msg)
hash1 = self.hash(msg)
if hash1 in self.scache:
self.send(ppc.b_("H" + hash1))
else:
self.send(ppc.b_("N") + msg)
self.scache[hash1] = True
def csend(self, msg):
#if hasattr(self, 'w'):
# open('/tmp/pp.debug', 'a+').write(repr(('cs', self.w, msg))+'\n')
#else:
# open('/tmp/pp.debug', 'a+').write(repr(('cs', self.socket, msg))+'\n')
msg = ppc.b_(msg)
hash1 = self.hash(msg)
if hash1 in self.scache:
self.send(ppc.b_("H" + hash1))
else:
self.send(ppc.b_("N") + msg)
self.scache[hash1] = True
def broadcast(self):
"""Sends a broadcast"""
if self.isclient:
self.base.logger.debug("Client sends initial broadcast to (%s, %i)"
% self.broadcast_addr)
self.bsocket.sendto(ppc.b_("C"), self.broadcast_addr)
else:
while True:
if self.base._exiting:
return
self.base.logger.debug("Server sends broadcast to (%s, %i)"
% self.broadcast_addr)
self.bsocket.sendto(ppc.b_("S"), self.broadcast_addr)
time.sleep(BROADCAST_INTERVAL)
def broadcast(self):
"""Sends a broadcast"""
if self.isclient:
self.base.logger.debug("Client sends initial broadcast to (%s, %i)"
% self.broadcast_addr)
self.bsocket.sendto(ppc.b_("C"), self.broadcast_addr)
else:
while True:
if self.base._exiting:
return
self.base.logger.debug("Server sends broadcast to (%s, %i)"
% self.broadcast_addr)
self.bsocket.sendto(ppc.b_("S"), self.broadcast_addr)
time.sleep(BROADCAST_INTERVAL)
def creceive(self, preprocess=None):
msg = self.receive()
#if hasattr(self, 'r'):
# open('/tmp/pp.debug', 'a+').write(repr(('cr', self.r, msg))+'\n')
#else:
# open('/tmp/pp.debug', 'a+').write(repr(('cr', self.socket, msg))+'\n')
msg = ppc.b_(msg)
if msg[:1] == ppc.b_('H'):
hash1 = ppc.str_(msg[1:])
else:
msg = msg[1:]
hash1 = self.hash(msg)
if preprocess is None: preprocess = lambda x: x
self.rcache[hash1] = tuple(map(preprocess, (msg, )))[0]
return self.rcache[hash1]
def creceive(self, preprocess=None):
msg = self.receive()
#if hasattr(self, 'r'):
# open('/tmp/pp.debug', 'a+').write(repr(('cr', self.r, msg))+'\n')
#else:
# open('/tmp/pp.debug', 'a+').write(repr(('cr', self.socket, msg))+'\n')
msg = ppc.b_(msg)
if msg[:1] == ppc.b_('H'):
hash1 = ppc.str_(msg[1:])
else:
msg = msg[1:]
hash1 = self.hash(msg)
if preprocess is None: preprocess = lambda x:x
self.rcache[hash1] = tuple(map(preprocess, (msg, )))[0]
return self.rcache[hash1]
def listen(self):
"""Listens for broadcasts from other clients/servers"""
self.base.logger.debug("Listening (%s, %i)" % self.interface_addr)
self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
self.socket.settimeout(5)
self.socket.bind(self.interface_addr)
ppc.start_thread("broadcast", self.broadcast)
while True:
try:
if self.base._exiting:
return
message, (host, port) = self.socket.recvfrom(1024)
message = ppc.str_(message)
remote_address = (host, self.broadcast_addr[1])
hostid = host + ":" + str(self.broadcast_addr[1])
self.base.logger.debug("Discovered host (%s, %i) message=%c"
% (remote_address + (message[0], )))
if not self.base.autopp_list.get(hostid, 0) and self.isclient \
and message[0] == 'S':
self.base.logger.debug("Connecting to host %s" % (hostid, ))
ppc.start_thread("ppauto_connect1", self.base.connect1,
remote_address+(False, ))
if not self.isclient and message[0] == 'C':
self.base.logger.debug("Replying to host %s" % (hostid, ))
self.bsocket.sendto(ppc.b_("S"), self.broadcast_addr)
except socket.timeout:
pass
except:
self.base.logger.error("An error has occured during execution of "
"Discover.listen")
sys.excepthook(*sys.exc_info())
def listen(self):
"""Listens for broadcasts from other clients/servers"""
self.base.logger.debug("Listening (%s, %i)" % self.interface_addr)
self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
self.socket.settimeout(5)
self.socket.bind(self.interface_addr)
ppc.start_thread("broadcast", self.broadcast)
while True:
try:
if self.base._exiting:
return
message, (host, port) = self.socket.recvfrom(1024)
message = ppc.str_(message)
remote_address = (host, self.broadcast_addr[1])
hostid = host + ":" + str(self.broadcast_addr[1])
self.base.logger.debug("Discovered host (%s, %i) message=%c"
% (remote_address + (message[0], )))
if not self.base.autopp_list.get(hostid, 0) and self.isclient \
and message[0] == 'S':
self.base.logger.debug("Connecting to host %s" % (hostid, ))
ppc.start_thread("ppauto_connect1", self.base.connect1,
remote_address+(False, ))
if not self.isclient and message[0] == 'C':
self.base.logger.debug("Replying to host %s" % (hostid, ))
self.bsocket.sendto(ppc.b_("S"), self.broadcast_addr)
except socket.timeout:
pass
except:
self.base.logger.error("An error has occured during execution of "
"Discover.listen")
sys.excepthook(*sys.exc_info())
def authenticate(self, secret):
remote_version = ppc.str_(self.receive())
if version != remote_version:
logging.error("PP version mismatch (local: pp-%s, remote: pp-%s)"
% (version, remote_version))
logging.error("Please install the same version of PP on all nodes")
return False
srandom = ppc.b_(self.receive())
secret = ppc.b_(secret)
answer = sha_new(srandom+secret).hexdigest()
self.send(answer)
response = ppc.b_(self.receive())
if response == ppc.b_("OK"):
return True
else:
return False
def authenticate(self, secret):
remote_version = ppc.str_(self.receive())
if version != remote_version:
logging.error("PP version mismatch (local: pp-%s, remote: pp-%s)" %
(version, remote_version))
logging.error("Please install the same version of PP on all nodes")
return False
srandom = ppc.b_(self.receive())
secret = ppc.b_(secret)
answer = sha_new(srandom + secret).hexdigest()
self.send(answer)
response = ppc.b_(self.receive())
if response == ppc.b_("OK"):
return True
else:
return False
def md5test(hash, start, end):
"""Calculates md5 of the integers between 'start' and 'end' and
compares it with 'hash'"""
from ppcommon import b_
for x in range(start, end):
if hashlib.md5(b_(str(x))).hexdigest() == hash:
return x
def crun(self, csocket):
"""Authenticates client and handles its jobs"""
mysocket = pptransport.CSocketTransport(csocket, self.socket_timeout)
#send PP version
mysocket.send(version)
#generate a random string
srandom = "".join(
[random.choice(string.ascii_letters) for i in range(16)])
mysocket.send(srandom)
answer = sha_new(ppc.b_(srandom + self.secret)).hexdigest()
clientanswer = ppc.str_(mysocket.receive())
if answer != clientanswer:
self.logger.warning(
"Authentication failed, client host=%s, port=%i" %
csocket.getpeername())
mysocket.send("FAILED")
csocket.close()
return
else:
mysocket.send("OK")
ctype = ppc.str_(mysocket.receive())
self.logger.debug("Control message received: " + ctype)
self.ncon_add(1)
try:
if ctype == "STAT":
#reset time at each new connection
self.get_stats()["local"].time = 0.0
#open('/tmp/pp.debug', 'a+').write('STAT: \n')
mysocket.send(str(self.get_ncpus()))
#open('/tmp/pp.debug', 'a+').write('STAT: get_ncpus\n')
while 1:
mysocket.receive()
#open('/tmp/pp.debug', 'a+').write('STAT: recvd\n')
mysocket.send(str(self.get_stats()["local"].time))
#open('/tmp/pp.debug', 'a+').write('STAT: _\n')
elif ctype == "EXEC":
while 1:
#open('/tmp/pp.debug', 'a+').write('EXEC: \n')
sfunc = mysocket.creceive()
#open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sfunc,))+'\n')
sargs = mysocket.receive()
#open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sargs,))+'\n')
fun = self.insert(sfunc, sargs)
sresult = fun(True)
#open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sresult,))+'\n')
mysocket.send(sresult)
#open('/tmp/pp.debug', 'a+').write('EXEC: _\n')
except:
if self._exiting:
return
if pp.SHOW_EXPECTED_EXCEPTIONS:
self.logger.debug(
"Exception in crun method (possibly expected)",
exc_info=True)
self.logger.debug("Closing client socket")
csocket.close()
self.ncon_add(-1)
def __unpickle(self):
"""Unpickles the result of the task"""
self.result, sout = pickle.loads(ppc.b_(self.sresult))
self.unpickled = True
if len(sout) > 0:
six.print_(sout, end=' ')
if self.callback:
args = self.callbackargs + (self.result, )
self.callback(*args)
def __unpickle(self):
"""Unpickles the result of the task"""
self.result, sout = pickle.loads(ppc.b_(self.sresult))
self.unpickled = True
if len(sout) > 0:
six.print_(sout, end=' ')
if self.callback:
args = self.callbackargs + (self.result, )
self.callback(*args)
def crun(self, csocket):
"""Authenticates client and handles its jobs"""
mysocket = pptransport.CSocketTransport(csocket, self.socket_timeout)
#send PP version
mysocket.send(version)
#generate a random string
srandom = "".join([random.choice(string.ascii_letters)
for i in range(16)])
mysocket.send(srandom)
answer = sha_new(ppc.b_(srandom+self.secret)).hexdigest()
clientanswer = ppc.str_(mysocket.receive())
if answer != clientanswer:
self.logger.warning("Authentication failed, client host=%s, port=%i"
% csocket.getpeername())
mysocket.send("FAILED")
csocket.close()
return
else:
mysocket.send("OK")
ctype = ppc.str_(mysocket.receive())
self.logger.debug("Control message received: " + ctype)
self.ncon_add(1)
try:
if ctype == "STAT":
#reset time at each new connection
self.get_stats()["local"].time = 0.0
#open('/tmp/pp.debug', 'a+').write('STAT: \n')
mysocket.send(str(self.get_ncpus()))
#open('/tmp/pp.debug', 'a+').write('STAT: get_ncpus\n')
while 1:
mysocket.receive()
#open('/tmp/pp.debug', 'a+').write('STAT: recvd\n')
mysocket.send(str(self.get_stats()["local"].time))
#open('/tmp/pp.debug', 'a+').write('STAT: _\n')
elif ctype=="EXEC":
while 1:
#open('/tmp/pp.debug', 'a+').write('EXEC: \n')
sfunc = mysocket.creceive()
#open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sfunc,))+'\n')
sargs = mysocket.receive()
#open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sargs,))+'\n')
fun = self.insert(sfunc, sargs)
sresult = fun(True)
#open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sresult,))+'\n')
mysocket.send(sresult)
#open('/tmp/pp.debug', 'a+').write('EXEC: _\n')
except:
if self._exiting:
return
if pp.SHOW_EXPECTED_EXCEPTIONS:
self.logger.debug("Exception in crun method (possibly expected)", exc_info=True)
self.logger.debug("Closing client socket")
csocket.close()
self.ncon_add(-1)
def preprocess(msg):
fname, fsources, imports = pickle.loads(ppc.b_(msg))
fobjs = [compile(fsource, '<string>', 'exec') for fsource in fsources]
for module in imports:
try:
if not module.startswith("from ") and not module.startswith("import "):
module = "import " + module
six.exec_(module)
globals().update(locals())
except:
print("An error has occured during the module import")
sys.excepthook(*sys.exc_info())
return fname, fobjs
def send(self, msg):
#l = len(ppc.b_(msg)) if (self.has_wb or self.w.mode == 'wb') else len(ppc.str_(msg))
#open('/tmp/pp.debug', 'a+').write(repr(('s', l, self.w, msg))+'\n')
if self.has_wb or self.w.mode == 'wb':
msg = ppc.b_(msg)
self.wb.write(struct.pack("!Q", len(msg)))
self.w.flush()
else: #HACK: following may be > 8 bytes, needed for len(msg) >= 256
msg = ppc.str_(msg)
self.wb.write(ppc.str_(struct.pack("!Q", len(msg))))
self.w.flush()
self.wb.write(msg)
self.w.flush()
def preprocess(msg):
fname, fsources, imports = pickle.loads(ppc.b_(msg))
fobjs = [compile(fsource, '<string>', 'exec') for fsource in fsources]
for module in imports:
try:
if not module.startswith("from ") and not module.startswith("import "):
module = "import " + module
six.exec_(module)
globals().update(locals())
except:
print("An error has occured during the module import")
sys.excepthook(*sys.exc_info())
return fname, fobjs
def send(self, msg):
#l = len(ppc.b_(msg)) if (self.has_wb or self.w.mode == 'wb') else len(ppc.str_(msg))
#open('/tmp/pp.debug', 'a+').write(repr(('s', l, self.w, msg))+'\n')
if self.has_wb or self.w.mode == 'wb':
msg = ppc.b_(msg)
self.wb.write(struct.pack("!Q", len(msg)))
self.w.flush()
else: #HACK: following may be > 8 bytes, needed for len(msg) >= 256
msg = ppc.str_(msg)
self.wb.write(ppc.str_(struct.pack("!Q", len(msg))))
self.w.flush()
self.wb.write(msg)
self.w.flush()
def send(self, data):
#l = len(ppc.b_(data))
#open('/tmp/pp.debug', 'a+').write(repr(('ss', l, self.socket, data))+'\n')
data = ppc.b_(data)
size = struct.pack("!Q", len(data))
t_size = struct.calcsize("!Q")
s_size = ppc.long(0)
while s_size < t_size:
p_size = self.socket.send(size[s_size:])
if p_size == 0:
raise RuntimeError("Socket connection is broken")
s_size += p_size
t_size = len(data)
s_size = ppc.long(0)
while s_size < t_size:
p_size = self.socket.send(data[s_size:])
if p_size == 0:
raise RuntimeError("Socket connection is broken")
s_size += p_size
def send(self, data):
#l = len(ppc.b_(data))
#open('/tmp/pp.debug', 'a+').write(repr(('ss', l, self.socket, data))+'\n')
data = ppc.b_(data)
size = struct.pack("!Q", len(data))
t_size = struct.calcsize("!Q")
s_size = ppc.long(0)
while s_size < t_size:
p_size = self.socket.send(size[s_size:])
if p_size == 0:
raise RuntimeError("Socket connection is broken")
s_size += p_size
t_size = len(data)
s_size = ppc.long(0)
while s_size < t_size:
p_size = self.socket.send(data[s_size:])
if p_size == 0:
raise RuntimeError("Socket connection is broken")
s_size += p_size
def run(self):
try:
#execution cycle
while 1:
__fname, __fobjs = self.t.creceive(preprocess)
__sargs = self.t.receive()
for __fobj in __fobjs:
try:
six.exec_(__fobj)
globals().update(locals())
except:
print("An error has occured during the " + \
"function import")
sys.excepthook(*sys.exc_info())
__args = pickle.loads(ppc.b_(__sargs))
__f = locals()[ppc.str_(__fname)]
try:
__result = __f(*__args)
except:
print("An error has occured during the function execution")
sys.excepthook(*sys.exc_info())
__result = None
__sresult = pickle.dumps((__result, self.sout.getvalue()),
self.pickle_proto)
self.t.send(__sresult)
self.sout.truncate(0)
except:
print("A fatal error has occured during the function execution")
sys.excepthook(*sys.exc_info())
__result = None
__sresult = pickle.dumps((__result, self.sout.getvalue()),
self.pickle_proto)
self.t.send(__sresult)
def run(self):
try:
#execution cycle
while 1:
__fname, __fobjs = self.t.creceive(preprocess)
__sargs = self.t.receive()
for __fobj in __fobjs:
try:
six.exec_(__fobj)
globals().update(locals())
except:
print("An error has occured during the " + \
"function import")
sys.excepthook(*sys.exc_info())
__args = pickle.loads(ppc.b_(__sargs))
__f = locals()[ppc.str_(__fname)]
try:
__result = __f(*__args)
except:
print("An error has occured during the function execution")
sys.excepthook(*sys.exc_info())
__result = None
__sresult = pickle.dumps((__result, self.sout.getvalue()),
self.pickle_proto)
self.t.send(__sresult)
self.sout.truncate(0)
except:
print("A fatal error has occured during the function execution")
sys.excepthook(*sys.exc_info())
__result = None
__sresult = pickle.dumps((__result, self.sout.getvalue()),
self.pickle_proto)
self.t.send(__sresult)
def hash(self, msg):
return md5_new(ppc.b_(msg)).hexdigest()
#ppservers = ("10.0.0.1","10.0.0.2") # list of static IPs
ppservers = ()
if len(sys.argv) > 1:
ncpus = int(sys.argv[1])
# Creates jobserver with ncpus workers
job_server = pp.Server(ncpus, ppservers=ppservers)
else:
# Creates jobserver with automatically detected number of workers
job_server = pp.Server(ppservers=ppservers)
print("Starting pp with %s workers" % job_server.get_ncpus())
#Calculates md5 hash from the given number
from ppcommon import b_
hash = hashlib.md5(b_("1829182")).hexdigest()
print("hash = %s" % hash)
#Now we will try to find the number with this hash value
start = 1
end = 2000000
# Since jobs are not equal in the execution time, division of the problem
# into a 128 of small subproblems leads to a better load balancing
parts = 128
step = int((end - start) / parts + 1)
jobs = []
for index in range(parts):
starti = start + index * step
def hash(self, msg):
return md5_new(ppc.b_(msg)).hexdigest()
