def test_parallel_get_campaigns(self):
# Because the client established in setup does not have PyCurlMixin,
# make another client to handle parallel get campaigns.
# All the options are the same, except mixins is PyCurlMixin
opts = helper.get_options()
opts['mixins'] = [PyCurlMixin]
self.parallel_client = NurturingClient(**opts)
self.parallel_client.get_campaigns()
self.parallel_client.get_campaigns()
results = self.parallel_client.process_queue()
for result in results:
self.assertTrue('data' in result)
self.assertTrue('code' in result)
self._check_result(result['data'])
def test_parallel_get_campaigns(self):
# Because the client established in setup does not have PyCurlMixin,
# make another client to handle parallel get campaigns.
# All the options are the same, except mixins is PyCurlMixin
opts = helper.get_options()
opts['mixins'] = [PyCurlMixin]
self.parallel_client = NurturingClient(**opts)
self.parallel_client.get_campaigns()
self.parallel_client.get_campaigns()
results = self.parallel_client.process_queue()
for result in results:
self.assertTrue('data' in result)
self.assertTrue('code' in result)
self._check_result(result['data'])
def setUp(self):
self.client = ProspectsClient(**helper.get_options())
def setUp(self):
self.client = NurturingClient(**helper.get_options())
def setUp(self):
self.client = ContactsClient(**helper.get_options())
def setUp(self):
self.client = LeadsClient(**helper.get_options())
def setUp(self): self.client = ContactListsClient(**helper.get_options()) self.contacts_client = ContactsClient(**helper.get_options()) self.lists = [] self.contacts =[]
def setUp(self):
self.client = SettingsClient(**helper.get_options())
def setUp(self):
self.client = KeywordsClient(**helper.get_options())
self.keyword_guids = None
def setUp(self):
self.client = NurturingClient(**helper.get_options())
def setUp(self):
self.client = BroadcastClient(**helper.get_options())
self.broadcast_guids = None
def setUp(self):
self.client = SettingsClient(**helper.get_options())
def setUp(self):
self.client = KeywordsClient(**helper.get_options())
self.keyword_guids = None
def setUp(self):
self.client = ContactsClient(**helper.get_options())
self.contacts = []
def setUp(self):
self.client = ProspectsClient(**helper.get_options())
def setUp(self):
self.client = ListsClient(**helper.get_options())
def setUp(self):
self.client = BlogClient(**helper.get_options())
# print "current_thread: ",thread_current
try:
urllib.urlretrieve(image_url,realpath)
except:
print "[Error]:[retrieve_net_image]",sys.exc_info()[:2]
def __completed(self):
"""检查是否满足条件停止程序"""
if self.stop == False and self.limits != 0 and self.tasks >= self.limits:
self.stop = True
return True
else:
print "Limits:",self.limits,"Threads:",len(threading.enumerate())," Completed:",self.tasks
self.tasks += 1
return False
if __name__ == "__main__":
options = helper.get_options()
mm_crawler = MMCrawler(options.threads,options.output,options.limits)
#22MM美女网
# mm_crawler.recursion_category("http://www.22mm.cc/mm/qingliang/index.html")
# mm_crawler.recursion_category("http://www.22mm.cc/mm/jingyan/index.html")
# mm_crawler.recursion_category("http://www.22mm.cc/mm/bagua/index.html")
# mm_crawler.recursion_category("http://www.22mm.cc/mm/suren/index.html")
#美女私房照
mm_crawler.recursion_category("http://www.mnsfz.com/h/qingchun/index.html")
# mm_crawler.recursion_category("http://www.mnsfz.com/h/meihuo/index.html")
# mm_crawler.recursion_category("http://www.mnsfz.com/h/yangguang/index.html")
# mm_crawler.recursion_category("http://www.mnsfz.com/h/qiaopi/index.html")
def setUp(self):
self.client = BlogClient(**helper.get_options())
def setUp(self):
self.client = BroadcastClient(**helper.get_options())
self.broadcast_guids = None