Exemplo n.º 1
0
class ImpMakerParser(Parser):
    def __init__(self, opener=None, url=None, **kw):
        super(ImpMakerParser, self).__init__(opener=opener, url=url, **kw)
        self.logger = kw.get('logger')

    def parse(self, url=None):
        url = url or self.url
        if 'click' not in url:
            times = random.randrange(2, 5)
        else:
            times = 1
        i = 0

        self.opener = MechanizeOpener(user_agent=random_user_agent())
        odds = random.randint(0, 100)
        if 'click' not in url or odds <= 5:
            # add proxy
            p_ = get_ip_proxy()
            if p_:

                self.opener.remove_proxy()
                self.opener.add_proxy(p_)
            while i < times:
                html = self.opener.open(url)
                #print(html)
                i = i + 1
                time.sleep(.1)

        return url