def crawlFromItmop(self):
prefix = 'http://www.itmop.com/proxy/post/'
parser = PageParser()
threads = []
for i in range(1000, 1331 + 1):
url = prefix + str(i) + '.html'
thread = CrawlingThread(url, parser, self)
threads.append(thread)
pool = ThreadPool(40)
pool.start(threads)
def testAll(self):
startTime = time.time()
threads = []
proxies = self.proxiesToTest
for proxy in proxies:
tester = ProxyTester(proxy)
threads.append(tester)
pool = ThreadPool(200)
pool.start(threads)
log.info( 'Test finish. Total test time: ' +\
str(time.time() - startTime) + 's')
def importFromFile(fname):
log.config(GC.LOG_FILE_DIR + 'import_accounts', 'info', 'info')
fileName = fname
accounts = []
pool = createProdRenrenAccountPool()
with open(fileName) as importedFile:
lines = importedFile.readlines()
for line in lines:
strs = line.split()
if len(strs) < 2:
continue # May be not a valid account
username = strs[0] # User name first.
password = strs[1] # And then password.
log.info("Find username: "******" " +\
"password: "******"Finish importing..........\n" +\
"Success on verify accounts number: " +\
str(verifySuccessCount) + "\n" +\
"Fail on verify accounts number: " +\
str(verifyFailCount))
log.info('Success imported number: %s' % importSuccessCount)
log.info('Fail imported number: %s' % importFailCount)
