示例#1
0
class RequestThread(BaseThread):
    def __init__(self, iqueue, oqueue, delay, **kw):
        BaseThread.__init__(self, **kw)
        self.log = logging.getLogger('robber.request.thread')
        self._init_fetch()
        self.iqueue = iqueue
        self.oqueue = oqueue
        self.delay = delay

    def _init_fetch(self):
        self.fetch = Fetch()

    def process(self):
        self.log.debug('requestthread process ..')
        while True:
            urldata = self.iqueue.get()
            self.log.debug('iqueue get url: %s' % urldata.url)
            while(urldata.trycount > 0):
                try:
                    self.fetch.request(urldata)
                    self.log.debug('request url: %s' % urldata.url)
                    break
                except Exception:
                    urldata.trycount -= 1
                    traceback.print_exc()
                    self.log.error('request error: %s' % urldata.url)
            self.iqueue.task_done()
            while (self.oqueue.full()):
                self.log.warning('oquque is full, wait...')
                time.sleep(QUEUE_FULL_WAIT)
            self.log.debug('oqueue put url: %s' % urldata.url)
            self.oqueue.put(urldata)
            time.sleep(self.delay)
 def test_request(self):
     req_url = 'https://unknowndomain99.com'
     urld = UrlData(req_url)
     fet = Fetch()
     fet.request(urld)
     self.assertEqual(urld.code, -1)
     self.assertEqual(urld.trueurl, '')
     self.assertEqual(urld.msg, '')
 def test_request(self):
     req_url = 'http://www.sina.com.cn/404.html'
     urld = UrlData(req_url)
     fet = Fetch()
     fet.request(urld)
     self.assertEqual(urld.code, 404)
     self.assertEqual(req_url, urld.trueurl)
     self.assertEqual(urld.msg, 'Not Found')
 def test_request(self):
     req_url = 'http://www.sina.com.cn'
     urld = UrlData(req_url)
     fet = Fetch()
     fet.request(urld)
     self.assertEqual(urld.code, 200)
     self.assertEqual(req_url, urld.trueurl)
     self.assertEqual(urld.msg, 'OK')
示例#5
0
 def _init_fetch(self):
     self.fetch = Fetch()