def test(): runque = RedisQueueConnection('scan').conn size = runque.qsize() print size sleep(1) cnt = 0 if size: while cnt < size: i = runque.get() print i runque.put(i) cnt += 1 runque.flushdb() exit(0) f = open('seeds995k.txt') urls = f.read().strip().split('\n') if size == 0: i = 0 st = time() for url in urls: runque.put(url)
def main(): if len(sys.argv) != 4: print "wrong param" exit(0) port = int(sys.argv[1]) cnt = int(sys.argv[2]) cur = int(sys.argv[3]) ol = 'ret_%s.txt' % (port) f = open(il) i = 0 slist = list() lines = f.read().strip().split('\n') slist = lines[ cur : len(lines) : cnt ] scancnt = len(slist) tmp = "scanips_%s_%s_%s" % (port, cnt, cur) if os.path.isfile(tmp): os.unlink(tmp) f = open(tmp,'w') for ip in slist: f.write(ip + '\n') f.close() cmd = scancmd % (port, tmp, ol) print cmd run(cmd) # when done # make bitmap to store the scanret and then insert into redis print "run done, we collect the ips" tmp = open(ol).read().split('\n')[1:-2] ips = list() for ip in tmp: ips.append(ip.split()[3]) alivecnt = len(ips) print "SCAN: %d ALIVE: %d " % (scancnt, alivecnt) scanque = RedisQueueConnection('scan').conn ipd = dict() for ip in ips: h = calc(ip) ht = ip.split('.')[-1] if h in ipd: ipd[h].append(ht) else: ipd[h] = [ht] for h in ipd: i = [port, h, ipd[h]] item = pickle.dumps(i) scanque.put(item) print "Insert into redis done " print "Total: %d" % (scanque.qsize())
def __init__(self): self.showpercounts = 10 self.timeout = 20 self.poolsize = 100 self.down_pool = Pool(size=self.poolsize) self.run_que = RedisQueueConnection('running').conn self.doneque = RedisQueueConnection('robots').conn self.tempque = Queue() self.done = 1 self.sent = 0 self.quit = False self.err = Error() self.https_enable = 0 self.httpget = self.httpget_requests # down method self.httpget_requests | httpget_curl self.totalnettime = 0 self.totaldownsize = 0 self.starttime = time() self.ip = getip() self.headers = { 'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.6', 'Accept-Encoding': 'gzip,deflate', 'Connection': 'close', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36' }
def insert(): runque = RedisQueueConnection('extracturls').conn urls = flist('urlstogetrobots1.txt')[100000:300000] print len(urls) for url in urls: runque.put(url)
def __init__(self, done_que): self.showpercounts = 100 self.timeout = 5 self.starttime = time() self.quit = False self.run_que = RedisQueueConnection('running').conn self.done_que = done_que self.tasks = [] self.done = 1 self.errdone = set() self.err = Error() self.https_enable = 0 self.httpget = self.httpget_requests # down method self.httpget_requests | httpget_curl self.poolsize = 100 self.down_pool = Pool(size=self.poolsize) self.totalnettime = 0 self.totaldownsize = 0 self.ip = getip()
def extract(): icmd = "insert into orignal (dbname, dbid, url, cms, headers, head) values (%s, %s, %s, %s, %s, %s)" runque = RedisQueueConnection('cms').conn size = runque.qsize() print "total:" , size i = 0 while i < size: item = runque.get() #runque.put(item) data = pickle.loads(item) ndata = [] for item in data: if isinstance(item, unicode): item = item.encode('utf8') ndata.append(item) #print ndata if ndata[3]: print ndata[3], ndata[2] cur.execute(icmd , ndata) i += 1 conn.commit() print "done" print runque.qsize()
def run(): runque = RedisQueueConnection('running').conn #########runque.flushdb() size = runque.qsize() print size raw_input('flush runing') runque.flushdb()
def main(): if len(sys.argv) != 4: print "wrong param" exit(0) port = int(sys.argv[1]) cnt = int(sys.argv[2]) cur = int(sys.argv[3]) ol = 'ret_%s.txt' % (port) f = open(il) i = 0 slist = list() lines = f.read().strip().split('\n') slist = lines[cur:len(lines):cnt] scancnt = len(slist) tmp = "scanips_%s_%s_%s" % (port, cnt, cur) if os.path.isfile(tmp): os.unlink(tmp) f = open(tmp, 'w') for ip in slist: f.write(ip + '\n') f.close() cmd = scancmd % (port, tmp, ol) print cmd run(cmd) # when done # make bitmap to store the scanret and then insert into redis print "run done, we collect the ips" tmp = open(ol).read().split('\n')[1:-2] ips = list() for ip in tmp: ips.append(ip.split()[3]) alivecnt = len(ips) print "SCAN: %d ALIVE: %d " % (scancnt, alivecnt) scanque = RedisQueueConnection('scan').conn ipd = dict() for ip in ips: h = calc(ip) ht = ip.split('.')[-1] if h in ipd: ipd[h].append(ht) else: ipd[h] = [ht] for h in ipd: i = [port, h, ipd[h]] item = pickle.dumps(i) scanque.put(item) print "Insert into redis done " print "Total: %d" % (scanque.qsize())
def daemon(tempque): st = random.random() + 0.5 doneque = RedisQueueConnection('robots').conn while True: if not tempque.empty(): dat = tempque.get() doneque.put(dat) else: sleep(st)
def inserturls(): runque = RedisQueueConnection('extracturls').conn print runque.qsize() raw_input('flushdb?') runque.flushdb() urls = flist('urlstogetip.txt') for url in urls: runque.put(url) print runque.qsize()
def test(): runque = RedisQueueConnection('running').conn size = runque.qsize() print size exit(0) if size == 0: f = open('seeds995k.txt') for c in f: url = c.strip() runque.put(url)
def test(): runque = RedisQueueConnection('scan').conn #########runque.flushdb() size = runque.qsize() t = 0 cnt = 0 port = 0 data = [] if size == 0: return tmp = runque.get() runque.put(tmp) port, iph, ipl = tmp.split(' ') print port, size raw_input('confirm:') while runque.qsize() > 0: tmp = runque.get() try: t += len(tmp.split(' ')[-1].split(',')) data.append(tmp) except: pass cnt += 1 f = "china%s_%s_%s.txt" % (port, size, t) fp = open(f, 'w') for item in data: fp.write(item + '\n') fp.close()
def test(): runque = RedisQueueConnection('running').conn size = runque.qsize() print size exit(0) if size == 0: f = open('seeds995k.txt') for c in f: url = c.strip() runque.put(url)
def test(): runque = RedisQueueConnection('scan').conn #########runque.flushdb() size = runque.qsize() t = 0 cnt = 0 port = 0 data = [] if size == 0: return tmp = runque.get() runque.put(tmp) port, iph, ipl = tmp.split(' ') print port, size raw_input('confirm:') while runque.qsize() > 0: tmp = runque.get() try: t += len(tmp.split(' ')[-1].split(',')) data.append(tmp) except: pass cnt += 1 f = "china%s_%s_%s.txt" % (port, size, t) fp = open(f, 'w') for item in data: fp.write(item + '\n') fp.close()
def test(): runque = RedisQueueConnection('robots').conn #########runque.flushdb() size = runque.qsize() item = runque.get() runque.put(item) print pickle.loads(item) print size return raw_input('cofrim') s = flist('urlstogetrobots.txt') for url in s: runque.put(url) print runque.qsize()
def __init__(self, done_que): self.cnt = 0 self.showpercounts = 100 self.dbsize = 0 self.dbsizelimit = 536870912 # 512M 536870912 self.spend = 0 #queue for daemon recieve downloaded websites info self.done_que = done_que #urls queue to put filtered urls extracted from the webpage self.urls_que = RedisQueueConnection('extracturls').conn self.urlfilter = Filter() self.ip = getip() self.fname = self.getdbname() self.conn = sqlite3.connect(self.fname) self.conn.execute( "create table if not exists mainpages (id integer primary key autoincrement, url TEXT,headers TEXT,content BLOB)" ) #when recv the ctrl+c signal, run out the extractation jobs and then quit self.quit = False
def __init__(self, done_que): self.cnt = 0 self.showpercounts = 100 self.dbsize = 0 self.dbsizelimit = 536870912 # 512M 536870912 self.spend = 0 #queue for daemon recieve downloaded websites info self.done_que = done_que #urls queue to put filtered urls extracted from the webpage self.urls_que = RedisQueueConnection('extracturls').conn self.urlfilter = Filter() self.ip = getip() self.fname = self.getdbname() self.conn = sqlite3.connect(self.fname) self.conn.execute("create table if not exists mainpages (id integer primary key autoincrement, url TEXT,headers TEXT,content BLOB)") #when recv the ctrl+c signal, run out the extractation jobs and then quit self.quit = False
def show(name): runque = RedisQueueConnection(name).conn cnt = 0 while cnt < runque.qsize(): data = runque.get() runque.put(data) data = pickle.loads(data) seed = data['seed'] data = data['content'].replace('\r', '\n').replace('\n\n','\n').strip() if not data: continue if data.find('<') >= 0: #html page print seed continue robots = data.split('\n') print seed print print "\n".join(robots) print cnt += 1
class Daemon: def __init__(self, done_que): self.cnt = 0 self.showpercounts = 100 self.dbsize = 0 self.dbsizelimit = 536870912 # 512M 536870912 self.spend = 0 #queue for daemon recieve downloaded websites info self.done_que = done_que #urls queue to put filtered urls extracted from the webpage self.urls_que = RedisQueueConnection('extracturls').conn self.urlfilter = Filter() self.ip = getip() self.fname = self.getdbname() self.conn = sqlite3.connect(self.fname) self.conn.execute("create table if not exists mainpages (id integer primary key autoincrement, url TEXT,headers TEXT,content BLOB)") #when recv the ctrl+c signal, run out the extractation jobs and then quit self.quit = False def getdbname(self, create=False): path = "/work/db" tf = "%Y%m%d-%H%M%S" pre = "sitedata" suf = ".db" dbsize = 0 ip = getip() findname = "%s%s" % (ip, suf) if create == True: date = time.strftime(tf, time.localtime()) lastname = "_".join([pre, date, ip]) + suf self.dbsize = 0 print "Create db: ", lastname return os.path.join(path, lastname) fnames = os.listdir(path) last = 0 lastname = "" for fname in fnames: if fname.endswith(findname): fnow = fname.split('_')[1] fnown = int(time.mktime(time.strptime(fnow, tf))) if fnown > last: last = fnown lastname = fname #can not found the newest db file, so create it if not last: date = time.strftime(tf, time.localtime()) lastname = "_".join([pre, date, ip]) + suf print "Create db: ", lastname self.dbsize = 0 else: print "Reuse the last db: ", lastname self.dbsize = os.stat(os.path.join(path, lastname)).st_size return os.path.join(path, lastname) def geturls(self, seed, content): urls = [] returls = [] if not content or len(content) == 0: return [] try: urls = re.findall(self.urlfilter.urlpatern, content) returls = self.urlfilter.filter_urls(seed,urls) except: pass return returls def run(self): #backend job, sleep(2) while True: try: if self.done_que.empty(): if self.quit == True: #the speed to extract urls is more higher than crawler sleep(1) if not self.done_que.empty(): continue print "Daemon run done and quit successfuly" exit(0) #print "Downloaded queue empty, wait crawler ..." sleep(10) continue data = cPickle.loads(self.done_que.get()) seed = data['seed'] content = data['content'] headers = str(data['headers']) urls = self.geturls(seed, content) #put the extracted urls to urls_que for url in urls: self.urls_que.put(url) #use level 1 to compress data , we get enough compress ratio and speed gziphtml = sqlite3.Binary(gzip.zlib.compress(content, 1)) self.dbsize += ( len(gziphtml) + len(seed) + len(headers) ) self.conn.execute("insert into mainpages (url,headers,content) values (?,?,?)", (seed, headers, gziphtml)) self.cnt += 1 if self.cnt % self.showpercounts == 0: self.conn.commit() print "\n%s\n\tExtract done:%d todo:%d size:%dM" % \ (self.ip, self.cnt, self.done_que.qsize(), self.dbsize/1024/1024) if self.dbsize > self.dbsizelimit: self.fname = self.getdbname(True) self.conn.close() self.conn = sqlite3.connect(self.fname) self.conn.execute("create table if not exists mainpages (id integer primary key autoincrement, url TEXT,headers TEXT,content BLOB)") except Exception as e: print e except KeyboardInterrupt: print "Daemon recv quit singal, waiting for queue empty" self.quit = True
#coding:utf8 import re, os import requests from db_driver import db_driver import db_file from redis_inc import RedisQueueConnection import cPickle as pickle import multiprocessing as mp cmsque = RedisQueueConnection('cms').conn pscript = re.compile(u"<script.*?>(.*?)</script>", re.S) pstyle = re.compile(u"<style.*?>(.*?)</style>", re.S) ptitle = re.compile(u"<title>(.*?)</title>", re.S) pby = re.compile(u'powered by (.*?)["\'</>]', re.S) headtag = "</head>" bodytag = "<body>" htmltag = re.compile(r'<[^>]+>', re.S) # check version fllowed by the cms name # dz x2.5 # phpcms v9 # we believe the version follows by cms must contains numbers def body_version(poweredby): l = poweredby.strip().split() if len(l) == 1: return poweredby
from gevent import monkey import sys from gevent.pool import Pool import requests from redis_inc import RedisQueueConnection monkey.patch_all(thread=False) size = 100 pool = Pool(size) runque = RedisQueueConnection('running').conn robotsque = RedisQueueConnection('robots').conn def httpget(url): url = url + "/robots.txt" con = "" try: with gevent.Timeout(2) as timeout: req = requests.get(url,timeout=(2,2)) con = req.content #print url, len(con) req.close() except:
# this script to test the urls.txt in some tests #! /bin/python from pybloomfilter import BloomFilter import sqlite3 import multiprocessing as mp import sqlite3 import zlib import sys, os, getopt import hashlib import time import re import Queue from redis_inc import RedisQueueConnection #!!! db=1 use db1 to store the seeds r = RedisQueueConnection('test').conn cmd = "select id from mainpages" #worker is a singal process for each cpu on each computer def worker(queue, lock, cpuid, outque): cpuurls = set() innerque = Queue.Queue() while queue.qsize() > 0: db = queue.get() lock.acquire() print "CPU-%s:runing : %s" % (cpuid, db) lock.release()
class Daemon: def __init__(self, done_que): self.cnt = 0 self.showpercounts = 100 self.dbsize = 0 self.dbsizelimit = 536870912 # 512M 536870912 self.spend = 0 #queue for daemon recieve downloaded websites info self.done_que = done_que #urls queue to put filtered urls extracted from the webpage self.urls_que = RedisQueueConnection('extracturls').conn self.urlfilter = Filter() self.ip = getip() self.fname = self.getdbname() self.conn = sqlite3.connect(self.fname) self.conn.execute( "create table if not exists mainpages (id integer primary key autoincrement, url TEXT,headers TEXT,content BLOB)" ) #when recv the ctrl+c signal, run out the extractation jobs and then quit self.quit = False def getdbname(self, create=False): path = "/work/db" tf = "%Y%m%d-%H%M%S" pre = "sitedata" suf = ".db" dbsize = 0 ip = getip() findname = "%s%s" % (ip, suf) if create == True: date = time.strftime(tf, time.localtime()) lastname = "_".join([pre, date, ip]) + suf self.dbsize = 0 print "Create db: ", lastname return os.path.join(path, lastname) fnames = os.listdir(path) last = 0 lastname = "" for fname in fnames: if fname.endswith(findname): fnow = fname.split('_')[1] fnown = int(time.mktime(time.strptime(fnow, tf))) if fnown > last: last = fnown lastname = fname #can not found the newest db file, so create it if not last: date = time.strftime(tf, time.localtime()) lastname = "_".join([pre, date, ip]) + suf print "Create db: ", lastname self.dbsize = 0 else: print "Reuse the last db: ", lastname self.dbsize = os.stat(os.path.join(path, lastname)).st_size return os.path.join(path, lastname) def geturls(self, seed, content): urls = [] returls = [] if not content or len(content) == 0: return [] try: urls = re.findall(self.urlfilter.urlpatern, content) returls = self.urlfilter.filter_urls(seed, urls) except: pass return returls def run(self): #backend job, sleep(2) while True: try: if self.done_que.empty(): if self.quit == True: #the speed to extract urls is more higher than crawler sleep(1) if not self.done_que.empty(): continue print "Daemon run done and quit successfuly" exit(0) #print "Downloaded queue empty, wait crawler ..." sleep(10) continue data = cPickle.loads(self.done_que.get()) seed = data['seed'] content = data['content'] headers = str(data['headers']) urls = self.geturls(seed, content) #put the extracted urls to urls_que for url in urls: self.urls_que.put(url) #use level 1 to compress data , we get enough compress ratio and speed gziphtml = sqlite3.Binary(gzip.zlib.compress(content, 1)) self.dbsize += (len(gziphtml) + len(seed) + len(headers)) self.conn.execute( "insert into mainpages (url,headers,content) values (?,?,?)", (seed, headers, gziphtml)) self.cnt += 1 if self.cnt % self.showpercounts == 0: self.conn.commit() print "\n%s\n\tExtract done:%d todo:%d size:%dM" % \ (self.ip, self.cnt, self.done_que.qsize(), self.dbsize/1024/1024) if self.dbsize > self.dbsizelimit: self.fname = self.getdbname(True) self.conn.close() self.conn = sqlite3.connect(self.fname) self.conn.execute( "create table if not exists mainpages (id integer primary key autoincrement, url TEXT,headers TEXT,content BLOB)" ) except Exception as e: print e except KeyboardInterrupt: print "Daemon recv quit singal, waiting for queue empty" self.quit = True
import gevent from gevent import monkey import sys from gevent.pool import Pool import requests from redis_inc import RedisQueueConnection monkey.patch_all(thread=False) size = 100 pool = Pool(size) runque = RedisQueueConnection('running').conn robotsque = RedisQueueConnection('robots').conn def httpget(url): url = url + "/robots.txt" con = "" try: with gevent.Timeout(2) as timeout: req = requests.get(url, timeout=(2, 2)) con = req.content #print url, len(con) req.close() except: pass data = (url, con) cb(data)
class Crawler: def __init__(self, done_que): self.showpercounts = 100 self.timeout = 5 self.starttime = time() self.quit = False self.run_que = RedisQueueConnection('running').conn self.done_que = done_que self.tasks = [] self.done = 1 self.errdone = set() self.err = Error() self.https_enable = 0 self.httpget = self.httpget_requests # down method self.httpget_requests | httpget_curl self.poolsize = 100 self.down_pool = Pool(size=self.poolsize) self.totalnettime = 0 self.totaldownsize = 0 self.ip = getip() #callback function when greenlet of httpget run done def cb_httpget(self, data = None): if not data: return seed, err, headers, content = data if err: self.handle_error(err,seed) return data={'seed':seed,'headers':headers,'content':content} dat = cPickle.dumps(data) #self.done_que.put_nowait(dat) #print "done", seed if self.done % self.showpercounts == 0: self.out(seed) def out(self, seed): spendtime = time() - self.starttime spendtime = 1 if spendtime == 0 else spendtime nowh = str(int(spendtime)/3600)+":" if spendtime>3600 else "" now = "%s%02d:%02d" % (nowh, spendtime%3600/60, spendtime%60 ) print "\n%s\t%s D:%-4d R:%-7d [QPS: %.2f %.2f] %s" % (self.ip, now, (self.done), self.run_que.qsize(), \ self.done/spendtime, self.done/self.totalnettime , str(self.err) ) def run(self): while self.quit == False: try: if self.run_que.qsize() == 0: print "run que empty" sleep(10) continue url = self.run_que.get() self.run_que.put(url) #self.down_pool.apply_cb(self.httpget, (url,), callback=self.cb_httpget) # spawn is more fast? #url = 'http://www.sdust.edu.cn' self.down_pool.spawn(self.httpget, url) self.done += 1 except KeyboardInterrupt: print "Crawler recv quit singal" self.quit = True self.down_pool.join() print "Crawler over, quit" def handle_error(self,e,url): self.err.lasterrurl = url # do not record the err url, but record the least err url to show if e.find('DNSError') > 0 : self.err.dns += 1 #self.err.rdns.append(url) elif e.find('reset') > 0 :#Connection reset self.err.reset += 1 #self.err.rreset.append(url) elif e.find('Max retries') > 0 or e.find('Connection aborted'): # self.err.conntimeout += 1 #self.err.rconntimeout.append(url) elif e.find('refused') > 0: #Connection refused self.err.refuse += 1 #self.err.rrefuse.append(url) else: self.err.others +=1 #self.err.rothers.append(url) print "Error", url, e # requests is better than curl in tests def httpget_requests(self, url): #return data data = None st = time() con = "" e = "" res_headers = "" headers = { 'Accept-Language':'zh-CN,zh;q=0.8,zh-TW;q=0.6', 'Accept-Encoding':'gzip,deflate', 'Connection':'close', 'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36' } res = None done = False try: with gevent.Timeout(3, False) as timeout: #req.max_redirects = 2 res = requests.get(url, headers = headers ) con = res.content res.close() done = True except KeyboardInterrupt: raise except Exception as e: e = str(e) if res: res.close() #as for spawn, no callback , we should call by ourself data = (url, e, None, None) #return url,e,None,None et = time() self.totalnettime += (et-st) #spawn if done: data = (url, e, res.headers, con) self.cb_httpget(data)
ThreadRunning = True serverips = ['127.0.0.1'] MAX_RUNNING_COUNT = 2**19 # 2**19 = 524 288 MAX_QPS = 1000 # 700+ showpercounts = 1000 #print out every when every 1000 querys sent s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind(('', 5310)) cur = 0 s.settimeout(0.1) done_sites_fname = 'done_sites.bin' bfdone = BloomFilter.open(done_sites_fname) urlsque = RedisQueueConnection('extracturls').conn runque = RedisQueueConnection('running').conn #query count send out, and success query url cont querysent = 0 querysuc = 0 #dnslocalserver = "dns_server_ip" dns_server_died = False def insert_redis(data): #perform extract and reform url to runque #domain format:
#coding:utf8 import re, os import redis_inc from redis_inc import RedisQueueConnection import cPickle as pickle import multiprocessing as mp import requests import mysql_inc cmsque = RedisQueueConnection('robots').conn tempset = redis_inc.RedisConnection('test').conn conn, cur = mysql_inc.gethandler() cms = dict() cms['disallow'] = dict() cd = cms['disallow'] cd['dedecms'] = ['ad_js.php', 'mytag_js.php', 'feedback_js.php'] cd['phpcms'] = ['phpcms', 'phpsso_server'] cd['wordpress'] = ['wp-admin', 'wp-content', 'wp-includes'] cd['xiaocms'] = ['Print.aspx'] cd['discuz'] = ['forum.php?mod='] cd['yiqicms'] = ['captcha'] cd['ecshop'] = ['goods_script.php'] cd['empirecms'] = ['e/enews'] # orinal: /e/enew/ forks = list() for key in cms.keys(): disk = cms[key] for cmsk in disk.keys():
def getsize(name): runque = RedisQueueConnection(name).conn print runque.qsize() i = runque.get() runque.put(i) print i
class Crawler: def __init__(self): self.showpercounts = 10 self.timeout = 20 self.poolsize = 100 self.down_pool = Pool(size=self.poolsize) self.run_que = RedisQueueConnection('running').conn self.doneque = RedisQueueConnection('robots').conn self.tempque = Queue() self.done = 1 self.sent = 0 self.quit = False self.err = Error() self.https_enable = 0 self.httpget = self.httpget_requests # down method self.httpget_requests | httpget_curl self.totalnettime = 0 self.totaldownsize = 0 self.starttime = time() self.ip = getip() self.headers = { 'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.6', 'Accept-Encoding': 'gzip,deflate', 'Connection': 'close', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36' } #callback function when greenlet of httpget run done def cb_httpget(self, data=None): if not data: return seed, err, headers, content = data if err: self.handle_error(err, seed) return if len(content) <= 0: return data = {'seed': seed, 'headers': headers, 'content': content} #content is robots.txt, normally it's pure text dat = cPickle.dumps(data) self.tempque.put(dat) self.done += 1 if self.done % self.showpercounts == 0: self.out(seed) def out(self, seed): spendtime = time() - self.starttime spendtime = 1 if spendtime == 0 else spendtime print "\n%s D:%-4d DT: %4d R:%-7d [QPS: %.2f %.2f] %s" % (self.ip, self.done,self.doneque.qsize(), self.run_que.qsize(), \ self.done/spendtime, self.done/self.totalnettime , str(self.err) ) def run(self): while self.quit == False: try: if self.run_que.qsize() == 0: print "run que empty" sleep(60) url = self.run_que.get() self.down_pool.spawn(self.httpget, url) self.sent += 1 except KeyboardInterrupt: print "Crawler recv quit singal" self.quit = True self.down_pool.join() print "Crawler over, quit" def handle_error(self, e, url): self.err.lasterrurl = url # do not record the err url, but record the least err url to show if e.find('DNSError') > 0: self.err.dns += 1 #self.err.rdns.append(url) elif e.find('reset') > 0: #Connection reset self.err.reset += 1 #self.err.rreset.append(url) elif e.find('Max retries') > 0 or e.find('Connection aborted'): # self.err.conntimeout += 1 #self.err.rconntimeout.append(url) elif e.find('refused') > 0: #Connection refused self.err.refuse += 1 #self.err.rrefuse.append(url) else: self.err.others += 1 #self.err.rothers.append(url) print "Error", url, e # requests is better than curl in tests def httpget_requests(self, url): #return data data = None st = time() con = "" e = "" res_headers = "" res = None done = False try: with gevent.Timeout(self.timeout, False) as timeout: url = url + '/robots.txt' res = requests.get(url, headers=self.headers) if res.status_code == 200: con = res.content done = True res.close() except KeyboardInterrupt: raise except Exception as e: e = str(e) if res: res.close() data = (url, e, None, None) et = time() self.totalnettime += (et - st) #spawn if done: data = (url, e, res.headers, con) #self.cb_httpget(data) if not data: return seed, err, headers, content = data if err: self.handle_error(err, seed) return if len(content) <= 0: return data = {'seed': seed, 'headers': headers, 'content': content} #content is robots.txt, normally it's pure text dat = cPickle.dumps(data) self.tempque.put(dat) self.done += 1 if self.done % self.showpercounts == 0: #self.out(seed) spendtime = time() - self.starttime spendtime = 1 if spendtime == 0 else spendtime print "\n%s D:%-4dDT:%4d R:%-7d [QPS: %.2f %.2f] %s" % (self.ip, self.done,self.doneque.qsize(), self.run_que.qsize(), \ self.done/spendtime, self.sent/spendtime , str(self.err) )
def rmdb(test): runque = RedisQueueConnection(test).conn print runque.qsize() raw_input('yes?') runque.flushdb()