def report(self):
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
while True:
time.sleep(1) ##sleep for next report.
if int(time.time()) % 60 == 0:
Log.errinfo(time.strftime('%Y-%m-%d %H:%M:%S'))
prog = "mj:%d/%s,aj:%d/(%d,%d,%d)" % (self._mjob_count, self._mjob_all, self._job_count,
self.job_queue.qsize(), self.job_queue2.qsize(), self.job_queue3.qsize())
if isinstance(self.curjobid, dict) and self.curjobid.has_key('url'):
cjstr = util.utf8str(self.curjobid['url'])
else:
cjstr = util.utf8str(self.curjobid.__str__())
cjstr = re.sub(r'\r|\n', '', cjstr)
if len(cjstr) > 100:
cjstr = cjstr[0:100]
message = "[pid=%d]job:%s prog:%s\n" % (os.getpid(), cjstr, prog)
try:
sock.sendto(message, ("127.0.0.1", self._logport))
except Exception as e:
pass
if self._end_mark:
message = "[pid=%d] DONE\n" % (os.getpid())
try:
sock.sendto(message, ("127.0.0.1", self._logport))
except:
pass
return
def msg_str(self, mqsz, qsz):
progargs = (self.mjob_count, mqsz, self.job_count, qsz)
prog = "mj:%d/%d,aj:%d/%d" % progargs
if self.curjob.has_key('url'):
cjstr = util.utf8str(self.curjob['url'])
else:
cjstr = util.utf8str(self.curjob)
cjstr = re.sub(r'\r|\n', '', cjstr)
if len(cjstr) > 100:
cjstr = cjstr[0:100]
return "[pid=%d]job:%s prog:%s\n" % (os.getpid(), cjstr, prog)
def add_main_job_range(self, j, begin, end, step=1):
assert isinstance(j, dict)
r = self._jobq.add_main_job_range(util.utf8str(j), begin, end, step)
self.condtion.acquire()
self.condtion.notifyAll()
self.condtion.release()
return r
def add_main_job_file(self, j, filename, begline=0, endline=0):
assert isinstance(j, dict)
r = self._jobq.add_main_job_file(util.utf8str(j), filename, begline,
endline)
self.condtion.acquire()
self.condtion.notifyAll()
self.condtion.release()
return r
def need_split(self, url, level, isLast):
tol = util.utf8str(url).strip()
if tol in self.oldjobs:
return False
raise RuntimeError('virtual function called.')
def log_url(self, url):
tol = util.utf8str(url).strip()
if tol in self.oldjobs:
return
self.fs.append(tol)
def re_add_job(self, j):
assert isinstance(j, dict)
return self._jobq.readd_job(util.utf8str(j))
def add_main_job_file(self, j, filename, begline=0, endline=0):
assert isinstance(j, dict)
return self._jobq.add_main_job_file(util.utf8str(j), filename, begline,
endline)
def add_main_job_range(self, j, begin, end, step=1):
assert isinstance(j, dict)
return self._jobq.add_main_job_range(util.utf8str(j), begin, end, step)
def add_main_job(self, j):
assert isinstance(j, dict)
return self._jobq.add_main_job(util.utf8str(j))
def log_param(self, param):
tol = util.utf8str(param).strip()
self.fs.append(tol)
def dump_job(self, job):
header = "[tid:%d t:%s]job:" % (
self.get_tid(), int(time.time() - self._stat.start_time))
jobstr = util.utf8str(job)
print header, jobstr
