def crawlFromItmop(self): prefix = 'http://www.itmop.com/proxy/post/' parser = PageParser() threads = [] for i in range(1000, 1331 + 1): url = prefix + str(i) + '.html' thread = CrawlingThread(url, parser, self) threads.append(thread) pool = ThreadPool(40) pool.start(threads)
def testAll(self): startTime = time.time() threads = [] proxies = self.proxiesToTest for proxy in proxies: tester = ProxyTester(proxy) threads.append(tester) pool = ThreadPool(200) pool.start(threads) log.info( 'Test finish. Total test time: ' +\ str(time.time() - startTime) + 's')
def importFromFile(fname): log.config(GC.LOG_FILE_DIR + 'import_accounts', 'info', 'info') fileName = fname accounts = [] pool = createProdRenrenAccountPool() with open(fileName) as importedFile: lines = importedFile.readlines() for line in lines: strs = line.split() if len(strs) < 2: continue # May be not a valid account username = strs[0] # User name first. password = strs[1] # And then password. log.info("Find username: "******" " +\ "password: "******"Finish importing..........\n" +\ "Success on verify accounts number: " +\ str(verifySuccessCount) + "\n" +\ "Fail on verify accounts number: " +\ str(verifyFailCount)) log.info('Success imported number: %s' % importSuccessCount) log.info('Fail imported number: %s' % importFailCount)