def _receiver(self):
from accelerator.compat import QueueFull, pickle, itervalues
while True:
try:
hdr = recvall(self.sock, 5)
if not hdr:
break
op, length = struct.unpack('<cI', hdr)
data = recvall(self.sock, length)
cookie, data = pickle.loads(data)
q = self._waiters.pop(cookie)
q.put(data)
except Exception:
print_exc()
break
# All is lost, unblock anyone waiting
for q in itervalues(self._waiters):
try:
q.put(None, block=False)
except QueueFull:
pass
break
except ValueError:
pass
r1, r2 = resource.getrlimit(resource.RLIMIT_NOFILE)
if r1 < r2:
print("WARNING: Failed to raise RLIMIT_NOFILE to %d. Set to %d." % (
r2,
r1,
))
if r1 < 5000:
print("WARNING: RLIMIT_NOFILE is %d, that's not much." % (r1, ))
while True:
op, length = struct.unpack('<cI', recvall(sock, 5, True))
data = recvall(sock, length, True)
cookie, data = pickle.loads(data)
if op == b'm':
res = load_methods(*data)
respond(cookie, res)
elif op == b's':
res = launch_start(data)
respond(cookie, res)
elif op == b'f':
# waits until job is done, so must run on a separate thread
Thread(
target=launch_finish,
args=(
cookie,
data,
),
name=data[3], # jobid