Example #1
0
    def convertToProxy(self, row): 
        proxy = Proxy()
        proxy.addr = row[0].decode('utf-8')
        proxy.port = row[1].decode('utf-8')
        proxy.protocol = row[2].decode('utf-8').lower()
        
        proxy.info = row[3].decode('utf-8') if row[3] else None
        proxy.source = row[4].decode('utf-8') if row[4] else None

        proxy.testCount = row[5]
        proxy.successCount = row[6]
        proxy.averageTime = row[7]
        return proxy
Example #2
0
 def parse(self, html, source):
     """Parse a html page, and return all contained proxy."""
     document = BeautifulSoup(html)
     content = document.find('p')
     proxies = []
     if content:
         for string in content.stripped_strings:
             proxy = Proxy.parse(string, source)
             if proxy:
                 proxies.append(proxy)
     else:
         log.warning('Can not find content element.')
     return proxies
Example #3
0
def testProxyPool():
    proxy1 = Proxy()
    proxy1.addr = u'1.1.1.1'
    proxy1.port = u'1'
    proxy1.protocol = u'http'
    proxy1.info = u'1234'
    proxy1.source = u'3321' 
    proxy1.testCount = 5
    proxy1.successCount = 5
    proxy1.averageTime = 20
    
    proxy2 = Proxy()
    proxy2.addr = u'2.2.2.2'
    proxy2.port = u'2'
    proxy2.protocol = u'http'
    proxy2.info = u'信息'
    proxy2.source = u'来源' 
    proxy2.testCount = 5
    proxy2.successCount = 5
    proxy2.averageTime = 200
    
    proxy3 = Proxy()
    proxy3.addr = u'3.3.3.3'
    proxy3.port = u'3'
    proxy3.protocol = None
    proxy3.info = None
    proxy3.source = None 
    proxy3.testCount = 5
    proxy3.successCount = 3
    proxy3.averageTime = 12000
    
    proxy4 = Proxy()
    proxy4.addr = u'4.4.4.4'
    proxy4.port = u'4'
    proxy4.protocol = u'http'
    proxy4.info = None
    proxy4.source = None 

    pool = createTestProxyPool()

    assert len(pool.getAllProxies()) == 0
    pool.insertProxy(proxy1)
    pool.insertProxy(proxy2)
    pool.insertProxy(proxy3)
    pool.insertProxy(proxy4)
    assert len(pool.getAllProxies()) == 4

    proxies = pool.getProxies(2)
    assert len(proxies) == 2
    assertProxyEqual(proxies[0], proxy1)
    assertProxyEqual(proxies[1], proxy2)

    proxies = pool.getProxies(4)
    assert len(proxies) == 3
    assertProxyEqual(proxies[0], proxy1)
    assertProxyEqual(proxies[1], proxy2)
    assertProxyEqual(proxies[2], proxy3)

    proxy3.averageTime = 100
    pool.updateProxy(proxy3)
    proxies = pool.getProxies(4)
    assert len(proxies) == 3
    assertProxyEqual(proxies[0], proxy1)
    assertProxyEqual(proxies[1], proxy3)
    assertProxyEqual(proxies[2], proxy2)

    pool.deleteProxy(proxy1)
    assert len(pool.getAllProxies()) == 3
    pool.deleteAllProxies()
    assert len(pool.getAllProxies()) == 0

    pool.close()