class TwitterSpiderTest(SocialSpider, unittest.TestCase):
def setUp(self):
self.spider = TwitterSpider()
self.spider.name = 'twitter-test'
self.spider.profile = {
'name': 'twtestname',
'profile_image_url': {'data': {'url': 'http://test.tw.photo.uri.com'}},
'description': 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit',
'followers_count': '4321',
'id': 456456,
'search_name': 'twsearchname',
}
def test_parse_offline(self):
"""
Test with regards to implementation time api spec
"""
results = self.spider.parse({})
self._test_item_results(results)
def test_parse_online(self):
"""
Test with regards to current api spec
"""
self.spider.profile = None
try:
response = self.spider.api_call('barackobama')
except TwythonAuthError:
return
else:
results = self.spider.parse(response)
self._test_item_results(results)
def setUp(self):
self.spider = TwitterSpider()
self.spider.name = 'twitter-test'
self.spider.profile = {
'name': 'twtestname',
'profile_image_url': {
'data': {
'url': 'http://test.tw.photo.uri.com'
}
},
'description':
'Lorem ipsum dolor sit amet, consectetuer adipiscing elit',
'followers_count': '4321',
'id': 456456,
'search_name': 'twsearchname',
}
def setUp(self):
self.spider = TwitterSpider()
self.spider.name = 'twitter-test'
self.spider.profile = {
'name': 'twtestname',
'profile_image_url': {'data': {'url': 'http://test.tw.photo.uri.com'}},
'description': 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit',
'followers_count': '4321',
'id': 456456,
'search_name': 'twsearchname',
}
class TwitterSpiderTest(SocialSpider, unittest.TestCase):
def setUp(self):
self.spider = TwitterSpider()
self.spider.name = 'twitter-test'
self.spider.profile = {
'name': 'twtestname',
'profile_image_url': {
'data': {
'url': 'http://test.tw.photo.uri.com'
}
},
'description':
'Lorem ipsum dolor sit amet, consectetuer adipiscing elit',
'followers_count': '4321',
'id': 456456,
'search_name': 'twsearchname',
}
def test_parse_offline(self):
"""
Test with regards to implementation time api spec
"""
results = self.spider.parse({})
self._test_item_results(results)
def test_parse_online(self):
"""
Test with regards to current api spec
"""
self.spider.profile = None
try:
response = self.spider.api_call('barackobama')
except TwythonAuthError:
return
else:
results = self.spider.parse(response)
self._test_item_results(results)
def crawl(username, media):
"""
Start crawling celery task,
"""
assert media in ('twitter', 'facebook'), \
'media :`{}` not supported yet!'.format(media)
spider = TwitterSpider(username) if media == 'twitter' else FacebookSpider(
username)
settings = get_project_settings()
crawler = Crawler(settings)
crawler.signals.connect(reactor.stop, signal=signals.spider_closed)
crawler.configure()
crawler.crawl(spider)
crawler.start()
log.start()
reactor.run(installSignalHandlers=0)
