class FacebookSpiderTest(SocialSpider, unittest.TestCase): def setUp(self): self.spider = FacebookSpider() self.spider.name = 'facebook-test' self.spider.profile = { 'name': 'fbtestname', 'picture': {'data': {'url': 'http://test.fb.photo.uri.com'}}, 'description': 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit', 'friend_count': '1234', 'id': 98765, 'search_name': 'fbtestsearch', } def test_parse_offline(self): """ Test with regards to implementation time api spec """ results = self.spider.parse({}) self._test_item_results(results) def test_parse_online(self): """ Test with regards to current api spec """ self.spider.profile = None response = self.spider.api_call('baracobama') results = self.spider.parse(response) self._test_item_results(results)
class FacebookSpiderTest(SocialSpider, unittest.TestCase): def setUp(self): self.spider = FacebookSpider() self.spider.name = 'facebook-test' self.spider.profile = { 'name': 'fbtestname', 'picture': { 'data': { 'url': 'http://test.fb.photo.uri.com' } }, 'description': 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit', 'friend_count': '1234', 'id': 98765, 'search_name': 'fbtestsearch', } def test_parse_offline(self): """ Test with regards to implementation time api spec """ results = self.spider.parse({}) self._test_item_results(results) def test_parse_online(self): """ Test with regards to current api spec """ self.spider.profile = None response = self.spider.api_call('baracobama') results = self.spider.parse(response) self._test_item_results(results)
def setUp(self): self.spider = FacebookSpider() self.spider.name = 'facebook-test' self.spider.profile = { 'name': 'fbtestname', 'picture': { 'data': { 'url': 'http://test.fb.photo.uri.com' } }, 'description': 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit', 'friend_count': '1234', 'id': 98765, 'search_name': 'fbtestsearch', }
def setUp(self): self.spider = FacebookSpider() self.spider.name = 'facebook-test' self.spider.profile = { 'name': 'fbtestname', 'picture': {'data': {'url': 'http://test.fb.photo.uri.com'}}, 'description': 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit', 'friend_count': '1234', 'id': 98765, 'search_name': 'fbtestsearch', }
def crawl(username, media): """ Start crawling celery task, """ assert media in ('twitter', 'facebook'), \ 'media :`{}` not supported yet!'.format(media) spider = TwitterSpider(username) if media == 'twitter' else FacebookSpider( username) settings = get_project_settings() crawler = Crawler(settings) crawler.signals.connect(reactor.stop, signal=signals.spider_closed) crawler.configure() crawler.crawl(spider) crawler.start() log.start() reactor.run(installSignalHandlers=0)