def taskFinished(self, tid, tried, result, update): i = self.tidToIndex[tid] self.finished[i] = True self.tasksFinished += 1 task = self.tasks[i] task.used += time.time() - task.start self.total_used += task.used if sys.stderr.isatty(): title = "Job %d: task %s finished in %.1fs (%d/%d) " % (self.id, tid, task.used, self.tasksFinished, self.numTasks) logger.info("Task %s finished in %.1fs (%d/%d) \x1b]2;%s\x07\x1b[1A", tid, task.used, self.tasksFinished, self.numTasks, title) from dpark.schedule import Success self.sched.taskEnded(task, Success(), result, update) for t in range(task.tried): if t + 1 != tried: self.sched.killTask(self.id, task.id, t + 1) if self.tasksFinished == self.numTasks: ts = [t.used for t in self.tasks] tried = [t.tried for t in self.tasks] logger.info("Job %d finished in %.1fs: min=%.1fs, avg=%.1fs, max=%.1fs, maxtry=%d", self.id, time.time()-self.start, min(ts), sum(ts)/len(ts), max(ts), max(tried)) from dpark.accumulator import LocalReadBytes, RemoteReadBytes lb, rb = LocalReadBytes.reset(), RemoteReadBytes.reset() if rb > 0: logger.info("read %s (%d%% localized)", readable(lb+rb), lb*100/(rb+lb)) self.sched.jobFinished(self)
def taskFinished(self, tid, tried, result, update): i = self.tidToIndex[tid] self.finished[i] = True self.tasksFinished += 1 task = self.tasks[i] task.used += time.time() - task.start self.total_used += task.used if getattr(sys.stderr, 'isatty', lambda: False)(): title = 'Job %d: task %s finished in %.1fs (%d/%d) ' % ( self.id, tid, task.used, self.tasksFinished, self.numTasks) logger.info( 'Task %s finished in %.1fs (%d/%d)' ' \x1b]2;%s\x07\x1b[1A', tid, task.used, self.tasksFinished, self.numTasks, title) from dpark.schedule import Success self.sched.taskEnded(task, Success(), result, update) for t in range(task.tried): if t + 1 != tried: self.sched.killTask(self.id, task.id, t + 1) if self.tasksFinished == self.numTasks: ts = [t.used for t in self.tasks] tried = [t.tried for t in self.tasks] logger.info( 'Job %d finished in %.1fs: min=%.1fs, ' 'avg=%.1fs, max=%.1fs, maxtry=%d', self.id, time.time() - self.start, min(ts), sum(ts) / len(ts), max(ts), max(tried)) from dpark.accumulator import LocalReadBytes, RemoteReadBytes lb, rb = LocalReadBytes.reset(), RemoteReadBytes.reset() if rb > 0: logger.info('read %s (%d%% localized)', readable(lb + rb), lb * 100 / (rb + lb)) self.sched.jobFinished(self)
def read_chunk(host, port, chunkid, version, size, offset=0): if offset + size > CHUNKSIZE: raise ValueError("size too large %s > %s" % (size, CHUNKSIZE-offset)) from dpark.accumulator import RemoteReadBytes conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) conn.settimeout(10) conn.connect((host, port)) msg = pack(CUTOCS_READ, uint64(chunkid), version, offset, size) n = conn.send(msg) while n < len(msg): if not n: raise IOError("write failed") msg = msg[n:] n = conn.send(msg) def recv(n): d = conn.recv(n) while len(d) < n: nd = conn.recv(n-len(d)) if not nd: raise IOError("not enough data") d += nd return d while size > 0: cmd, l = unpack("II", recv(8)) if cmd == CSTOCU_READ_STATUS: if l != 9: raise Exception("readblock: READ_STATUS incorrect message size") cid, code = unpack("QB", recv(l)) if cid != chunkid: raise Exception("readblock; READ_STATUS incorrect chunkid") conn.close() return elif cmd == CSTOCU_READ_DATA: if l < 20 : raise Exception("readblock; READ_DATA incorrect message size") cid, bid, boff, bsize, crc = unpack("QHHII", recv(20)) if cid != chunkid: raise Exception("readblock; READ_STATUS incorrect chunkid") if l != 20 + bsize: raise Exception("readblock; READ_DATA incorrect message size ") if bsize == 0 : # FIXME raise Exception("readblock; empty block") #yield "" #continue if bid != offset >> 16: raise Exception("readblock; READ_DATA incorrect block number") if boff != offset & 0xFFFF: raise Exception("readblock; READ_DATA incorrect block offset") breq = 65536 - boff if size < breq: breq = size if bsize != breq: raise Exception("readblock; READ_DATA incorrect block size") while breq > 0: data = conn.recv(breq) if not data: #print chunkid, version, offset, size, bsize, breq raise IOError("unexpected ending: need %d" % breq) RemoteReadBytes.add(len(data)) yield data breq -= len(data) offset += bsize size -= bsize else: raise Exception("readblock; unknown message: %s" % cmd) conn.close()
def read_chunk(host, port, chunkid, version, size, offset=0): if offset + size > CHUNKSIZE: raise ValueError("size too large %s > %s" % (size, CHUNKSIZE - offset)) from dpark.accumulator import RemoteReadBytes conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) conn.settimeout(10) conn.connect((host, port)) msg = pack(CUTOCS_READ, uint64(chunkid), version, offset, size) n = conn.send(msg) while n < len(msg): if not n: raise IOError("write failed") msg = msg[n:] n = conn.send(msg) def recv(n): d = conn.recv(n) while len(d) < n: nd = conn.recv(n - len(d)) if not nd: raise IOError("not enough data") d += nd return d while size > 0: cmd, l = unpack("II", recv(8)) if cmd == CSTOCU_READ_STATUS: if l != 9: raise Exception( "readblock: READ_STATUS incorrect message size") cid, code = unpack("QB", recv(l)) if cid != chunkid: raise Exception("readblock; READ_STATUS incorrect chunkid") conn.close() return elif cmd == CSTOCU_READ_DATA: if l < 20: raise Exception("readblock; READ_DATA incorrect message size") cid, bid, boff, bsize, crc = unpack("QHHII", recv(20)) if cid != chunkid: raise Exception("readblock; READ_STATUS incorrect chunkid") if l != 20 + bsize: raise Exception("readblock; READ_DATA incorrect message size ") if bsize == 0: # FIXME raise Exception("readblock; empty block") #yield "" #continue if bid != offset >> 16: raise Exception("readblock; READ_DATA incorrect block number") if boff != offset & 0xFFFF: raise Exception("readblock; READ_DATA incorrect block offset") breq = 65536 - boff if size < breq: breq = size if bsize != breq: raise Exception("readblock; READ_DATA incorrect block size") while breq > 0: data = conn.recv(breq) if not data: #print chunkid, version, offset, size, bsize, breq raise IOError("unexpected ending: need %d" % breq) RemoteReadBytes.add(len(data)) yield data breq -= len(data) offset += bsize size -= bsize else: raise Exception("readblock; unknown message: %s" % cmd) conn.close()