def test_get_pm_for_page(self, fake_driver):
puller = FacebookHistoryScrapper(self.channel)
pm = puller.get_page_private_messages("fake_page_id")
# TODO: not passing
# assert fake_driver.obtain_new_page_token.call_count == 1
# assert fake_driver.request.call_count == 1
self.assertIsInstance(pm, mock.MagicMock)
def test_handle_post(self):
user = self.default_user
self.post_to_user(user)
scrapper = FacebookHistoryScrapper(self.channel)
posts = scrapper.get_posts(user['id'], self.since, self.until, self.limit)
post = scrapper.handle_data_item(posts['data'][0], FBDataHandlerFactory.get_instance(FBDataHandlerFactory.POST), user['id'])
self.assertIsNotNone(post)
self.assertIsNotNone(post['content'])
def test_scrap_posts(self):
user = self.default_user
for i in range(0, self.limit):
self.post_to_user(user)
scrapper = FacebookHistoryScrapper(self.channel)
posts = scrapper.get_posts(user['id'], self.since, self.until, self.limit)
self.assertEqual(len(posts['data']), self.limit)
def test_handle_comments(self):
user = self.default_user
post = self.post_to_user(user)
self.comment_post(user, post)
scrapper = FacebookHistoryScrapper(self.channel)
comment = scrapper.get_comments(post['id'], self.since, self.until, self.limit)
comment = scrapper.handle_data_item(comment['data'][0], FBDataHandlerFactory.get_instance(FBDataHandlerFactory.COMMENT), user['id'])
self.assertIsNotNone(comment)
self.assertIsNotNone(comment['content'])
def test_scrap_comments(self):
user = self.default_user
post = self.post_to_user(user)
self.comment_post(user, post)
self.comment_post(user, post)
scrapper = FacebookHistoryScrapper(self.channel)
comments = scrapper.get_comments(post['id'], self.since, self.until, self.limit)
self.assertEqual(len(comments['data']), 2)
def fb_get_comments_for(channel, target, user, since):
from solariat_bottle.db.post.utils import factory_by_user
from solariat_bottle.daemons.facebook.facebook_data_handlers import FBDataHandlerFactory
from solariat_bottle.daemons.facebook.facebook_history_scrapper import FacebookHistoryScrapper
puller = FacebookHistoryScrapper(channel, user=user)
comments = puller.get_comments(target, since=since)['data']
for comment in comments:
comment = puller.handle_data_item(
data=comment,
handler=FBDataHandlerFactory.get_instance(
FBDataHandlerFactory.COMMENT),
target_id=target)
factory_by_user(user, sync=True, **comment)
def fb_get_post_comments(channel, user, target, since, until):
from solariat_bottle.db.post.utils import factory_by_user
from solariat_bottle.daemons.facebook.facebook_data_handlers import FBDataHandlerFactory
from solariat_bottle.daemons.facebook.facebook_history_scrapper import FacebookHistoryScrapper
puller = FacebookHistoryScrapper(channel, user)
new_posts = puller.get_posts(since=since, until=until, target=target)
for post in new_posts['data']:
#avoid handling posts with no data useful to parsing
if 'message' not in post and 'link' not in post:
continue
try:
post = puller.handle_data_item(
data=post,
handler=FBDataHandlerFactory.get_instance(
FBDataHandlerFactory.POST),
target_id=target)
except Exception, ex:
logger.error(ex)
continue
comments = post.pop('comments', [])
factory_by_user(user, sync=True, **post)
for comment in comments:
post_id = post['id']
try:
comment = puller.handle_data_item(
data=comment,
handler=FBDataHandlerFactory.get_instance(
FBDataHandlerFactory.COMMENT),
target_id=post_id)
except Exception, ex:
logger.error(ex)
continue
factory_by_user(user, sync=True, **comment)
def fb_get_private_messages(channel, page_id, user, since, until):
from solariat_bottle.db.post.utils import factory_by_user
from solariat_bottle.daemons.facebook.facebook_data_handlers import FBDataHandlerFactory
from solariat_bottle.daemons.facebook.facebook_history_scrapper import FacebookHistoryScrapper
puller = FacebookHistoryScrapper(channel, user=user)
message_threads = puller.get_page_private_messages(page_id, since,
until)['data']
for thread in message_threads:
if 'messages' in thread and thread['messages']['data']:
data = thread['messages'][
'data'][::-1] #get array of messages in reversed order
root_message_id = data[0]['id']
if thread['id'] in channel.tracked_fb_message_threads_ids:
check = lambda msg, since: utc(
parser.parse(msg['created_time'])) > utc(since)
messages_to_handle = [msg for msg in data if check(msg, since)]
else:
channel.tracked_fb_message_threads_ids.append(thread['id'])
channel.save()
messages_to_handle = data
conversation_id = thread['id']
for msg in messages_to_handle:
if msg['id'] != root_message_id:
msg['root_post'] = root_message_id
msg['conversation_id'] = conversation_id
msg['page_id'] = page_id
msg = puller.handle_data_item(
msg,
FBDataHandlerFactory.get_instance(FBDataHandlerFactory.PM),
thread['id'])
factory_by_user(user, sync=True, **msg)
def test_scrapper_creation(self):
history_scrapper = FacebookHistoryScrapper(self.channel)
self.assertIsNotNone(history_scrapper)