def is_update_twitter(self, twaccount): if not twaccount: return False try: # max_id_status = TwStatus.objects.raw('select max(id) from ChasingSomeoneApp_twstatus') status = TwStatus.objects.all().aggregate(Max('id')) max_id_status = status['id__max'] except TwStatus.DoesNotExist: print 'Can not get status from TwStatus' max_id_status = None crawler = Crawler_twitter() status_list = None if max_id_status: status_list = crawler.get_status(id=twaccount.act_id, screen_name=twaccount.screen_name, since_id=int(max_id_status)) else: status_list = crawler.get_status(id=twaccount.act_id, screen_name=twaccount.screen_name) if not status_list: print 'Can not get posts from user\'s homepage' return False try: self.update_twitter_status(twaccount, status_list) except KeyError: print 'Error occured when updates twitter status' return False return True
class CrawlerTestCase(unittest.TestCase): def setUp(self): self.crawler = Crawler_twitter() def tearDown(self): self.crawler = None def test_add_follower(self): self.assertFalse(self.crawler.add_follower(), 'add follower with empty information') self.assertFalse(self.crawler.add_follower(id = '51248642'), 'add follower with wrong information') self.assertIsNotNone(self.crawler.add_follower(screen_name = 'matthewperryfan'), 'add follower with correct screen name') self.assertEqual(self.crawler.add_follower(id = '30653573').get('id'), '30653573', 'check return id if is equal to given correct id') def test_get_status(self): self.assertEqual(self.crawler.get_status(), [], 'get status with empty information') self.assertEqual(self.crawler.get_status(id = '51248642'), [], 'get status with incorret information') self.assertNotEqual(len(self.crawler.get_status(screen_name = 'matthewperryfan')), 0, 'get status with correct screen name') self.assertNotEqual(len(self.crawler.get_status(id = '30653573')), 0, 'get status with correct id')
def is_update_twitter(self, twaccount): if not twaccount: return False try: # max_id_status = TwStatus.objects.raw('select max(id) from ChasingSomeoneApp_twstatus') status = TwStatus.objects.all().aggregate(Max("id")) max_id_status = status["id__max"] except TwStatus.DoesNotExist: print "Can not get status from TwStatus" max_id_status = None crawler = Crawler_twitter() status_list = None if max_id_status: status_list = crawler.get_status( id=twaccount.act_id, screen_name=twaccount.screen_name, since_id=int(max_id_status) ) else: status_list = crawler.get_status(id=twaccount.act_id, screen_name=twaccount.screen_name) if not status_list: print "Can not get posts from user's homepage" return False try: self.update_twitter_status(twaccount, status_list) except KeyError: print "Error occured when updates twitter status" return False return True
class CrawlerTestCase(unittest.TestCase): def setUp(self): self.crawler = Crawler_twitter() def tearDown(self): self.crawler = None def test_add_follower(self): self.assertFalse(self.crawler.add_follower(), 'add follower with empty information') self.assertFalse(self.crawler.add_follower(id='51248642'), 'add follower with wrong information') self.assertIsNotNone( self.crawler.add_follower(screen_name='matthewperryfan'), 'add follower with correct screen name') self.assertEqual( self.crawler.add_follower(id='30653573').get('id'), '30653573', 'check return id if is equal to given correct id') def test_get_status(self): self.assertEqual(self.crawler.get_status(), [], 'get status with empty information') self.assertEqual(self.crawler.get_status(id='51248642'), [], 'get status with incorret information') self.assertNotEqual( len(self.crawler.get_status(screen_name='matthewperryfan')), 0, 'get status with correct screen name') self.assertNotEqual(len(self.crawler.get_status(id='30653573')), 0, 'get status with correct id')
def refresh_status(act_list, since_id=None): tw_crawler = Crawler_twitter() status_list = [] for act in act_list: temp_list = tw_crawler.get_status(id=act.act_id, screen_name=act.screen_name, since_id=since_id) if temp_list: temp_list = transfer_status_list(temp_list) save_status(act, temp_list) status_list.extend(temp_list) if len(status_list) == 0: return [] sorted_status_list = sorted(status_list, key=lambda tw_status: tw_status['id'], cmp=status_cmp) if len(sorted_status_list) > 10: sorted_status_list = sorted_status_list[:10] for status in sorted_status_list: status['act_type'] = 'twitter' status['time_stamp'] = status['created_at'] return sorted_status_list