def is_update_twitter(self, twaccount): if not twaccount: return False try: # max_id_status = TwStatus.objects.raw('select max(id) from ChasingSomeoneApp_twstatus') status = TwStatus.objects.all().aggregate(Max('id')) max_id_status = status['id__max'] except TwStatus.DoesNotExist: print 'Can not get status from TwStatus' max_id_status = None crawler = Crawler_twitter() status_list = None if max_id_status: status_list = crawler.get_status(id=twaccount.act_id, screen_name=twaccount.screen_name, since_id=int(max_id_status)) else: status_list = crawler.get_status(id=twaccount.act_id, screen_name=twaccount.screen_name) if not status_list: print 'Can not get posts from user\'s homepage' return False try: self.update_twitter_status(twaccount, status_list) except KeyError: print 'Error occured when updates twitter status' return False return True
def is_update_twitter(self, twaccount): if not twaccount: return False try: # max_id_status = TwStatus.objects.raw('select max(id) from ChasingSomeoneApp_twstatus') status = TwStatus.objects.all().aggregate(Max("id")) max_id_status = status["id__max"] except TwStatus.DoesNotExist: print "Can not get status from TwStatus" max_id_status = None crawler = Crawler_twitter() status_list = None if max_id_status: status_list = crawler.get_status( id=twaccount.act_id, screen_name=twaccount.screen_name, since_id=int(max_id_status) ) else: status_list = crawler.get_status(id=twaccount.act_id, screen_name=twaccount.screen_name) if not status_list: print "Can not get posts from user's homepage" return False try: self.update_twitter_status(twaccount, status_list) except KeyError: print "Error occured when updates twitter status" return False return True
class CrawlerTestCase(unittest.TestCase): def setUp(self): self.crawler = Crawler_twitter() def tearDown(self): self.crawler = None def test_add_follower(self): self.assertFalse(self.crawler.add_follower(), 'add follower with empty information') self.assertFalse(self.crawler.add_follower(id = '51248642'), 'add follower with wrong information') self.assertIsNotNone(self.crawler.add_follower(screen_name = 'matthewperryfan'), 'add follower with correct screen name') self.assertEqual(self.crawler.add_follower(id = '30653573').get('id'), '30653573', 'check return id if is equal to given correct id') def test_get_status(self): self.assertEqual(self.crawler.get_status(), [], 'get status with empty information') self.assertEqual(self.crawler.get_status(id = '51248642'), [], 'get status with incorret information') self.assertNotEqual(len(self.crawler.get_status(screen_name = 'matthewperryfan')), 0, 'get status with correct screen name') self.assertNotEqual(len(self.crawler.get_status(id = '30653573')), 0, 'get status with correct id')
class CrawlerTestCase(unittest.TestCase): def setUp(self): self.crawler = Crawler_twitter() def tearDown(self): self.crawler = None def test_add_follower(self): self.assertFalse(self.crawler.add_follower(), 'add follower with empty information') self.assertFalse(self.crawler.add_follower(id='51248642'), 'add follower with wrong information') self.assertIsNotNone( self.crawler.add_follower(screen_name='matthewperryfan'), 'add follower with correct screen name') self.assertEqual( self.crawler.add_follower(id='30653573').get('id'), '30653573', 'check return id if is equal to given correct id') def test_get_status(self): self.assertEqual(self.crawler.get_status(), [], 'get status with empty information') self.assertEqual(self.crawler.get_status(id='51248642'), [], 'get status with incorret information') self.assertNotEqual( len(self.crawler.get_status(screen_name='matthewperryfan')), 0, 'get status with correct screen name') self.assertNotEqual(len(self.crawler.get_status(id='30653573')), 0, 'get status with correct id')
def verify_account(**verify_kwargs): if verify_kwargs['flr_name'] is not None: if TwAccount.objects.filter(follower__name=verify_kwargs['flr_name']).exists(): return False tw_crawler = Crawler_twitter() if verify_kwargs['act_id'] == u'None': verify_kwargs['act_id'] = None if verify_kwargs['screen_name'] == u'None': verify_kwargs['screen_name'] = None crawler_kwargs = {'id': verify_kwargs['act_id'], 'screen_name': verify_kwargs['screen_name']} is_exist_follower = tw_crawler.add_follower(**crawler_kwargs) if not is_exist_follower: return '404' # rename for consistency is_exist_follower['act_id'] = is_exist_follower.pop('id') return is_exist_follower
def verify_account(**verify_kwargs): if verify_kwargs['flr_name'] is not None: if TwAccount.objects.filter( follower__name=verify_kwargs['flr_name']).exists(): return False tw_crawler = Crawler_twitter() if verify_kwargs['act_id'] == u'None': verify_kwargs['act_id'] = None if verify_kwargs['screen_name'] == u'None': verify_kwargs['screen_name'] = None crawler_kwargs = { 'id': verify_kwargs['act_id'], 'screen_name': verify_kwargs['screen_name'] } is_exist_follower = tw_crawler.add_follower(**crawler_kwargs) if not is_exist_follower: return '404' # rename for consistency is_exist_follower['act_id'] = is_exist_follower.pop('id') return is_exist_follower
def refresh_status(act_list, since_id=None): tw_crawler = Crawler_twitter() status_list = [] for act in act_list: temp_list = tw_crawler.get_status(id=act.act_id, screen_name=act.screen_name, since_id=since_id) if temp_list: temp_list = transfer_status_list(temp_list) save_status(act, temp_list) status_list.extend(temp_list) if len(status_list) == 0: return [] sorted_status_list = sorted(status_list, key=lambda tw_status: tw_status['id'], cmp=status_cmp) if len(sorted_status_list) > 10: sorted_status_list = sorted_status_list[:10] for status in sorted_status_list: status['act_type'] = 'twitter' status['time_stamp'] = status['created_at'] return sorted_status_list
def add_follower(request): if request.method == 'POST': tw_crawler = Crawler_twitter() follower_id = request.POST['follower_id'] screen_name = request.POST['screen_name'] kwargs = {'screen_name': screen_name, 'id': follower_id} is_exist_follower = tw_crawler.add_follower(**kwargs) if not is_exist_follower: return HttpResponse(False) else: if TwFollower.objects.filter(user_id=request.user.id).filter( follower_id=is_exist_follower['id']).exists(): return HttpResponse(False) new_follower = TwFollower() new_follower.follower_id = is_exist_follower['id'] new_follower.screen_name = is_exist_follower['screen_name'] new_follower.user = request.user new_follower.save() return HttpResponse(True) else: raise Http404
def add_follower(request): if request.method == 'POST': tw_crawler = Crawler_twitter() follower_id = request.POST['follower_id'] screen_name = request.POST['screen_name'] kwargs = {'screen_name': screen_name, 'id': follower_id} is_exist_follower = tw_crawler.add_follower(**kwargs) if not is_exist_follower: return HttpResponse(False) else: if TwFollower.objects.filter(user_id=request.user.id).filter(follower_id=is_exist_follower['id']).exists(): return HttpResponse(False) new_follower = TwFollower() new_follower.follower_id = is_exist_follower['id'] new_follower.screen_name = is_exist_follower['screen_name'] new_follower.user = request.user new_follower.save() return HttpResponse(True) else: raise Http404
def setUp(self): self.crawler = Crawler_twitter()