def crawl(username, media):
"""
Start crawling celery task,
"""
assert media in ('twitter', 'facebook'), \
'media :`{}` not supported yet!'.format(media)
spider = TwitterSpider(username) if media == 'twitter' else FacebookSpider(
username)
settings = get_project_settings()
crawler = Crawler(settings)
crawler.signals.connect(reactor.stop, signal=signals.spider_closed)
crawler.configure()
crawler.crawl(spider)
crawler.start()
log.start()
reactor.run(installSignalHandlers=0)
def setUp(self):
self.spider = FacebookSpider()
self.spider.name = 'facebook-test'
self.spider.profile = {
'name': 'fbtestname',
'picture': {
'data': {
'url': 'http://test.fb.photo.uri.com'
}
},
'description':
'Lorem ipsum dolor sit amet, consectetuer adipiscing elit',
'friend_count': '1234',
'id': 98765,
'search_name': 'fbtestsearch',
}
