def test_get_pm_for_page(self, fake_driver): puller = FacebookHistoryScrapper(self.channel) pm = puller.get_page_private_messages("fake_page_id") # TODO: not passing # assert fake_driver.obtain_new_page_token.call_count == 1 # assert fake_driver.request.call_count == 1 self.assertIsInstance(pm, mock.MagicMock)
def test_handle_post(self): user = self.default_user self.post_to_user(user) scrapper = FacebookHistoryScrapper(self.channel) posts = scrapper.get_posts(user['id'], self.since, self.until, self.limit) post = scrapper.handle_data_item(posts['data'][0], FBDataHandlerFactory.get_instance(FBDataHandlerFactory.POST), user['id']) self.assertIsNotNone(post) self.assertIsNotNone(post['content'])
def test_scrap_posts(self): user = self.default_user for i in range(0, self.limit): self.post_to_user(user) scrapper = FacebookHistoryScrapper(self.channel) posts = scrapper.get_posts(user['id'], self.since, self.until, self.limit) self.assertEqual(len(posts['data']), self.limit)
def test_handle_comments(self): user = self.default_user post = self.post_to_user(user) self.comment_post(user, post) scrapper = FacebookHistoryScrapper(self.channel) comment = scrapper.get_comments(post['id'], self.since, self.until, self.limit) comment = scrapper.handle_data_item(comment['data'][0], FBDataHandlerFactory.get_instance(FBDataHandlerFactory.COMMENT), user['id']) self.assertIsNotNone(comment) self.assertIsNotNone(comment['content'])
def test_scrap_comments(self): user = self.default_user post = self.post_to_user(user) self.comment_post(user, post) self.comment_post(user, post) scrapper = FacebookHistoryScrapper(self.channel) comments = scrapper.get_comments(post['id'], self.since, self.until, self.limit) self.assertEqual(len(comments['data']), 2)
def fb_get_comments_for(channel, target, user, since): from solariat_bottle.db.post.utils import factory_by_user from solariat_bottle.daemons.facebook.facebook_data_handlers import FBDataHandlerFactory from solariat_bottle.daemons.facebook.facebook_history_scrapper import FacebookHistoryScrapper puller = FacebookHistoryScrapper(channel, user=user) comments = puller.get_comments(target, since=since)['data'] for comment in comments: comment = puller.handle_data_item( data=comment, handler=FBDataHandlerFactory.get_instance( FBDataHandlerFactory.COMMENT), target_id=target) factory_by_user(user, sync=True, **comment)
def fb_get_post_comments(channel, user, target, since, until): from solariat_bottle.db.post.utils import factory_by_user from solariat_bottle.daemons.facebook.facebook_data_handlers import FBDataHandlerFactory from solariat_bottle.daemons.facebook.facebook_history_scrapper import FacebookHistoryScrapper puller = FacebookHistoryScrapper(channel, user) new_posts = puller.get_posts(since=since, until=until, target=target) for post in new_posts['data']: #avoid handling posts with no data useful to parsing if 'message' not in post and 'link' not in post: continue try: post = puller.handle_data_item( data=post, handler=FBDataHandlerFactory.get_instance( FBDataHandlerFactory.POST), target_id=target) except Exception, ex: logger.error(ex) continue comments = post.pop('comments', []) factory_by_user(user, sync=True, **post) for comment in comments: post_id = post['id'] try: comment = puller.handle_data_item( data=comment, handler=FBDataHandlerFactory.get_instance( FBDataHandlerFactory.COMMENT), target_id=post_id) except Exception, ex: logger.error(ex) continue factory_by_user(user, sync=True, **comment)
def fb_get_private_messages(channel, page_id, user, since, until): from solariat_bottle.db.post.utils import factory_by_user from solariat_bottle.daemons.facebook.facebook_data_handlers import FBDataHandlerFactory from solariat_bottle.daemons.facebook.facebook_history_scrapper import FacebookHistoryScrapper puller = FacebookHistoryScrapper(channel, user=user) message_threads = puller.get_page_private_messages(page_id, since, until)['data'] for thread in message_threads: if 'messages' in thread and thread['messages']['data']: data = thread['messages'][ 'data'][::-1] #get array of messages in reversed order root_message_id = data[0]['id'] if thread['id'] in channel.tracked_fb_message_threads_ids: check = lambda msg, since: utc( parser.parse(msg['created_time'])) > utc(since) messages_to_handle = [msg for msg in data if check(msg, since)] else: channel.tracked_fb_message_threads_ids.append(thread['id']) channel.save() messages_to_handle = data conversation_id = thread['id'] for msg in messages_to_handle: if msg['id'] != root_message_id: msg['root_post'] = root_message_id msg['conversation_id'] = conversation_id msg['page_id'] = page_id msg = puller.handle_data_item( msg, FBDataHandlerFactory.get_instance(FBDataHandlerFactory.PM), thread['id']) factory_by_user(user, sync=True, **msg)
def test_scrapper_creation(self): history_scrapper = FacebookHistoryScrapper(self.channel) self.assertIsNotNone(history_scrapper)