class TwitterSpiderTest(SocialSpider, unittest.TestCase): def setUp(self): self.spider = TwitterSpider() self.spider.name = 'twitter-test' self.spider.profile = { 'name': 'twtestname', 'profile_image_url': {'data': {'url': 'http://test.tw.photo.uri.com'}}, 'description': 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit', 'followers_count': '4321', 'id': 456456, 'search_name': 'twsearchname', } def test_parse_offline(self): """ Test with regards to implementation time api spec """ results = self.spider.parse({}) self._test_item_results(results) def test_parse_online(self): """ Test with regards to current api spec """ self.spider.profile = None try: response = self.spider.api_call('barackobama') except TwythonAuthError: return else: results = self.spider.parse(response) self._test_item_results(results)
def setUp(self): self.spider = TwitterSpider() self.spider.name = 'twitter-test' self.spider.profile = { 'name': 'twtestname', 'profile_image_url': { 'data': { 'url': 'http://test.tw.photo.uri.com' } }, 'description': 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit', 'followers_count': '4321', 'id': 456456, 'search_name': 'twsearchname', }
def setUp(self): self.spider = TwitterSpider() self.spider.name = 'twitter-test' self.spider.profile = { 'name': 'twtestname', 'profile_image_url': {'data': {'url': 'http://test.tw.photo.uri.com'}}, 'description': 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit', 'followers_count': '4321', 'id': 456456, 'search_name': 'twsearchname', }
class TwitterSpiderTest(SocialSpider, unittest.TestCase): def setUp(self): self.spider = TwitterSpider() self.spider.name = 'twitter-test' self.spider.profile = { 'name': 'twtestname', 'profile_image_url': { 'data': { 'url': 'http://test.tw.photo.uri.com' } }, 'description': 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit', 'followers_count': '4321', 'id': 456456, 'search_name': 'twsearchname', } def test_parse_offline(self): """ Test with regards to implementation time api spec """ results = self.spider.parse({}) self._test_item_results(results) def test_parse_online(self): """ Test with regards to current api spec """ self.spider.profile = None try: response = self.spider.api_call('barackobama') except TwythonAuthError: return else: results = self.spider.parse(response) self._test_item_results(results)
def crawl(username, media): """ Start crawling celery task, """ assert media in ('twitter', 'facebook'), \ 'media :`{}` not supported yet!'.format(media) spider = TwitterSpider(username) if media == 'twitter' else FacebookSpider( username) settings = get_project_settings() crawler = Crawler(settings) crawler.signals.connect(reactor.stop, signal=signals.spider_closed) crawler.configure() crawler.crawl(spider) crawler.start() log.start() reactor.run(installSignalHandlers=0)