예제 #1
0
 def _get_response(self, url, data=None):
     
     response = BaseCrawler._get_response(self, url, data)
            
     fixer = HTMLFixer(self._url_regex, url, response.raw_html)        
     html = fixer.get_fixed_html()
     
     return html
예제 #2
0
    def _get_response(self, url, data=None):

        response = BaseCrawler._get_response(self, url, data)

        fixer = HTMLFixer(self._url_regex, url, response.raw_html)
        html = fixer.get_fixed_html()

        return html
예제 #3
0
파일: fast.py 프로젝트: wgfi110/crawley
    def __init__(self, *args, **kwargs):

        BaseCrawler.__init__(self, *args, **kwargs)
        self.request_manager = FastRequestManager()
예제 #4
0
 def __init__(self):
     BaseCrawler.__init__(self)
     self.stop_request = False
     self.pause_request = False
     self.status = 'init'
     self._set_status('ready')
예제 #5
0
 def delay(self, multiplier=1):
     self._set_status('paused')
     BaseCrawler.delay(self, multiplier)
     while (self.pause_request) and not self.stop_request:
         pass
     self._set_status('running')
예제 #6
0
 def __init__(self, *args, **kwargs):
     
     BaseCrawler.__init__(self, *args, **kwargs)
예제 #7
0
파일: fast.py 프로젝트: 4iji/crawley
    def __init__(self, *args, **kwargs):

        BaseCrawler.__init__(self, *args, **kwargs)
        self.request_manager = FastRequestManager()
예제 #8
0
    def __init__(self, *args, **kwargs):

        BaseCrawler.__init__(self, *args, **kwargs)