def report(self): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) while True: time.sleep(1) ##sleep for next report. if int(time.time()) % 60 == 0: Log.errinfo(time.strftime('%Y-%m-%d %H:%M:%S')) prog = "mj:%d/%s,aj:%d/(%d,%d,%d)" % (self._mjob_count, self._mjob_all, self._job_count, self.job_queue.qsize(), self.job_queue2.qsize(), self.job_queue3.qsize()) if isinstance(self.curjobid, dict) and self.curjobid.has_key('url'): cjstr = util.utf8str(self.curjobid['url']) else: cjstr = util.utf8str(self.curjobid.__str__()) cjstr = re.sub(r'\r|\n', '', cjstr) if len(cjstr) > 100: cjstr = cjstr[0:100] message = "[pid=%d]job:%s prog:%s\n" % (os.getpid(), cjstr, prog) try: sock.sendto(message, ("127.0.0.1", self._logport)) except Exception as e: pass if self._end_mark: message = "[pid=%d] DONE\n" % (os.getpid()) try: sock.sendto(message, ("127.0.0.1", self._logport)) except: pass return
def msg_str(self, mqsz, qsz): progargs = (self.mjob_count, mqsz, self.job_count, qsz) prog = "mj:%d/%d,aj:%d/%d" % progargs if self.curjob.has_key('url'): cjstr = util.utf8str(self.curjob['url']) else: cjstr = util.utf8str(self.curjob) cjstr = re.sub(r'\r|\n', '', cjstr) if len(cjstr) > 100: cjstr = cjstr[0:100] return "[pid=%d]job:%s prog:%s\n" % (os.getpid(), cjstr, prog)
def add_main_job_range(self, j, begin, end, step=1): assert isinstance(j, dict) r = self._jobq.add_main_job_range(util.utf8str(j), begin, end, step) self.condtion.acquire() self.condtion.notifyAll() self.condtion.release() return r
def add_main_job_file(self, j, filename, begline=0, endline=0): assert isinstance(j, dict) r = self._jobq.add_main_job_file(util.utf8str(j), filename, begline, endline) self.condtion.acquire() self.condtion.notifyAll() self.condtion.release() return r
def need_split(self, url, level, isLast): tol = util.utf8str(url).strip() if tol in self.oldjobs: return False raise RuntimeError('virtual function called.')
def log_url(self, url): tol = util.utf8str(url).strip() if tol in self.oldjobs: return self.fs.append(tol)
def re_add_job(self, j): assert isinstance(j, dict) return self._jobq.readd_job(util.utf8str(j))
def add_main_job_file(self, j, filename, begline=0, endline=0): assert isinstance(j, dict) return self._jobq.add_main_job_file(util.utf8str(j), filename, begline, endline)
def add_main_job_range(self, j, begin, end, step=1): assert isinstance(j, dict) return self._jobq.add_main_job_range(util.utf8str(j), begin, end, step)
def add_main_job(self, j): assert isinstance(j, dict) return self._jobq.add_main_job(util.utf8str(j))
def log_param(self, param): tol = util.utf8str(param).strip() self.fs.append(tol)
def dump_job(self, job): header = "[tid:%d t:%s]job:" % ( self.get_tid(), int(time.time() - self._stat.start_time)) jobstr = util.utf8str(job) print header, jobstr