def crawl(username, media):
"""
Start crawling celery task,
"""
assert media in ('twitter', 'facebook'), \
'media :`{}` not supported yet!'.format(media)
spider = TwitterSpider(username) if media == 'twitter' else FacebookSpider(
username)
settings = get_project_settings()
crawler = Crawler(settings)
crawler.signals.connect(reactor.stop, signal=signals.spider_closed)
crawler.configure()
crawler.crawl(spider)
crawler.start()
log.start()
reactor.run(installSignalHandlers=0)
def setUp(self):
self.spider = TwitterSpider()
self.spider.name = 'twitter-test'
self.spider.profile = {
'name': 'twtestname',
'profile_image_url': {
'data': {
'url': 'http://test.tw.photo.uri.com'
}
},
'description':
'Lorem ipsum dolor sit amet, consectetuer adipiscing elit',
'followers_count': '4321',
'id': 456456,
'search_name': 'twsearchname',
}
