def test_batch(): graph = GraphAPI(TEST_USER_ACCESS_TOKEN) response.content = json.dumps([ { 'code': 200, 'headers': [ { 'name': 'Content-Type', 'value': 'text/javascript; charset=UTF-8' } ], 'body': '{"foo": "bar"}' } ]) requests = [ { 'method': 'GET', 'relative_url': 'me/friends' }, { 'method': 'GET', 'relative_url': 'me/photos' } ] batch = graph.batch( requests = requests ) for item in batch: pass mock_request.assert_called_with('POST', 'https://graph.facebook.com/', files = {}, data = { 'batch': json.dumps(requests), 'access_token': TEST_USER_ACCESS_TOKEN } )
def test_batch_with_errors(): graph = GraphAPI("<access token>") mock_request.return_value.content = json.dumps( [ { "code": 200, "headers": [{"name": "Content-Type", "value": "text/javascript; charset=UTF-8"}], "body": '{"foo": "bar"}', }, { "code": 500, "headers": [{"name": "Content-Type", "value": "text/javascript; charset=UTF-8"}], "body": '{"error_code": 1, "error_msg": "An unknown error occurred"}', }, ] ) mock_request.return_value.status_code = 200 requests = [{"method": "GET", "relative_url": "me/friends"}, {"method": "GET", "relative_url": "me"}] batch = graph.batch(requests) responses = list(batch) assert isinstance(responses[0], dict) assert isinstance(responses[1], Exception)
def test_batch_with_errors(): graph = GraphAPI('<access token>') mock_request.return_value.content = json.dumps([ { 'code': 200, 'headers': [ {'name': 'Content-Type', 'value': 'text/javascript; charset=UTF-8'} ], 'body': '{"foo": "bar"}' }, { 'code': 500, 'headers': [ {'name': 'Content-Type', 'value': 'text/javascript; charset=UTF-8'} ], 'body': '{"error_code": 1, "error_msg": "An unknown error occurred"}' } ]) requests = [ {'method': 'GET', 'relative_url': 'me/friends'}, {'method': 'GET', 'relative_url': 'me'} ] batch = graph.batch(requests) responses = list(batch) assert isinstance(responses[0], dict) assert isinstance(responses[1], Exception)
def test_batch_over_50_requests(): graph = GraphAPI("<access_token") def side_effect_batch_size(*args, **kwargs): batch_size = len(json.loads(kwargs["data"]["batch"])) if batch_size > 50: return MagicMock( content='{"error":{"message":"Too many requests in batch message. Maximum batch size is 50","type":"GraphBatchException"}}', status_code=200, ) else: return MagicMock( content=json.dumps( [ { "code": 200, "headers": [{"name": "Content-Type", "value": "text/javascript; charset=UTF-8"}], "body": '{"foo": "bar"}', } for i in range(batch_size) ] ), status_code=200, ) mock_request.side_effect = side_effect_batch_size requests = [dict(method="GET", relative_url="me?fields=username") for i in range(60)] batch = graph.batch(requests=requests) responses = list(batch) assert len(responses) == 60
def test_batch_with_empty_responses(): graph = GraphAPI('<access token>') mock_request.return_value.content = json.dumps([ None, { 'code': 200, 'headers': [{ 'name': 'Content-Type', 'value': 'text/javascript; charset=UTF-8' }], 'body': '{"foo": "bar"}' } ]) mock_request.return_value.status_code = 200 requests = [{ 'method': 'GET', 'relative_url': 'me/friends' }, { 'method': 'GET', 'relative_url': 'me/photos' }] batch = graph.batch(requests=requests) assert list(batch) == [None, {'foo': 'bar'}]
def test_batch_over_50_requests(): graph = GraphAPI('<access_token') def side_effect_batch_size(*args, **kwargs): batch_size = len(json.loads(kwargs['data']['batch'])) if batch_size > 50: return MagicMock( content= '{"error":{"message":"Too many requests in batch message. Maximum batch size is 50","type":"GraphBatchException"}}', status_code=200) else: return MagicMock(content=json.dumps([{ 'code': 200, 'headers': [{ 'name': 'Content-Type', 'value': 'text/javascript; charset=UTF-8' }], 'body': '{"foo": "bar"}' } for i in range(batch_size)]), status_code=200) mock_request.side_effect = side_effect_batch_size requests = [ dict(method="GET", relative_url="me?fields=username") for i in range(60) ] batch = graph.batch(requests=requests) responses = list(batch) assert len(responses) == 60
def test_batch_with_empty_responses(): graph = GraphAPI('<access token>') mock_request.return_value.content = json.dumps([ None, { 'code': 200, 'headers': [ {'name': 'Content-Type', 'value': 'text/javascript; charset=UTF-8'} ], 'body': '{"foo": "bar"}' } ]) mock_request.return_value.status_code = 200 requests = [ {'method': 'GET', 'relative_url': 'me/friends'}, {'method': 'GET', 'relative_url': 'me/photos'} ] batch = graph.batch( requests=requests ) assert list(batch) == [None, {'foo': 'bar'}]
def test_batch_over_50_requests(): graph = GraphAPI('<access_token') def side_effect_batch_size(*args, **kwargs): batch_size = len(json.loads(kwargs['data']['batch'])) if batch_size > 50: return MagicMock(content='{"error":{"message":"Too many requests in batch message. Maximum batch size is 50","type":"GraphBatchException"}}') else: return MagicMock(content=json.dumps([ { 'code': 200, 'headers': [ {'name': 'Content-Type', 'value': 'text/javascript; charset=UTF-8'} ], 'body': '{"foo": "bar"}' } for i in range(batch_size) ])) mock_request.side_effect = side_effect_batch_size requests = [dict(method="GET", relative_url="me?fields=username") for i in range(60)] batch = graph.batch( requests=requests ) responses = list(batch) assert len(responses) == 60
def test_batch_with_errors(): graph = GraphAPI('<access token>') mock_request.return_value.content = json.dumps([ { 'code': 200, 'headers': [ { 'name': 'Content-Type', 'value': 'text/javascript; charset=UTF-8' } ], 'body': '{"foo": "bar"}' }, { 'code': 500, 'headers': [ { 'name': 'Content-Type', 'value': 'text/javascript; charset=UTF-8' } ], 'body': '{"error_code": 1, "error_msg": "An unknown error occurred"}' } ]) requests = [ { 'method': 'GET', 'relative_url': 'me/friends' }, { 'method': 'GET', 'relative_url': 'me' } ] batch = graph.batch(requests) responses = list(batch) assert isinstance(responses[0], dict) assert isinstance(responses[1], Exception)
def test_batch(): graph = GraphAPI('<access token>') mock_request.return_value.content = json.dumps([{ 'code': 200, 'headers': [{ 'name': 'Content-Type', 'value': 'text/javascript; charset=UTF-8' }], 'body': '{"foo": "bar"}' }]) mock_request.return_value.status_code = 200 requests = [{ 'method': 'GET', 'relative_url': 'me/friends' }, { 'method': 'GET', 'relative_url': 'me/photos' }, { 'method': 'POST', 'relative_url': 'me/feed', 'body': { 'message': 'Hi me.' } }] batch = graph.batch(requests=requests) for item in batch: pass mock_request.assert_called_with('POST', 'https://graph.facebook.com/', files={}, verify=True, timeout=None, data={ 'batch': json.dumps([{ 'method': 'GET', 'relative_url': 'me/friends' }, { 'method': 'GET', 'relative_url': 'me/photos' }, { 'method': 'POST', 'relative_url': 'me/feed', 'body': 'message=Hi+me.' }]), 'access_token': '<access token>' })
def test_batch(): graph = GraphAPI("<access token>") mock_request.return_value.content = json.dumps( [ { "code": 200, "headers": [{"name": "Content-Type", "value": "text/javascript; charset=UTF-8"}], "body": '{"foo": "bar"}', } ] ) mock_request.return_value.status_code = 200 requests = [ {"method": "GET", "relative_url": "me/friends"}, {"method": "GET", "relative_url": "me/photos"}, {"method": "POST", "relative_url": "me/feed", "body": {"message": "Hi me."}}, ] batch = graph.batch(requests=requests) for item in batch: pass mock_request.assert_called_with( "POST", "https://graph.facebook.com/", files={}, verify=True, timeout=None, data={ "batch": json.dumps( [ {"method": "GET", "relative_url": "me/friends"}, {"method": "GET", "relative_url": "me/photos"}, {"method": "POST", "relative_url": "me/feed", "body": "message=Hi+me."}, ] ), "access_token": "<access token>", }, )
def test_batch(): graph = GraphAPI('<access token>') mock_request.return_value.content = json.dumps([ { 'code': 200, 'headers': [ {'name': 'Content-Type', 'value': 'text/javascript; charset=UTF-8'} ], 'body': '{"foo": "bar"}' } ]) mock_request.return_value.status_code = 200 requests = [ {'method': 'GET', 'relative_url': 'me/friends'}, {'method': 'GET', 'relative_url': 'me/photos'}, {'method': 'POST', 'relative_url': 'me/feed', 'body': {'message': 'Hi me.'}} ] batch = graph.batch( requests=requests ) for item in batch: pass mock_request.assert_called_with( 'POST', 'https://graph.facebook.com/', files={}, verify=True, timeout=None, data={ 'batch': json.dumps([ {'method': 'GET', 'relative_url': 'me/friends'}, {'method': 'GET', 'relative_url': 'me/photos'}, {'method': 'POST', 'relative_url': 'me/feed', 'body': 'message=Hi+me.'} ]), 'access_token': '<access token>' } )
def test_batch_with_empty_responses(): graph = GraphAPI("<access token>") mock_request.return_value.content = json.dumps( [ None, { "code": 200, "headers": [{"name": "Content-Type", "value": "text/javascript; charset=UTF-8"}], "body": '{"foo": "bar"}', }, ] ) mock_request.return_value.status_code = 200 requests = [{"method": "GET", "relative_url": "me/friends"}, {"method": "GET", "relative_url": "me/photos"}] batch = graph.batch(requests=requests) assert list(batch) == [None, {"foo": "bar"}]
def test_batch_with_errors(): graph = GraphAPI(TEST_USER_ACCESS_TOKEN) mock_request.return_value.content = json.dumps([ { 'code': 500, 'headers': [ { 'name': 'Content-Type', 'value': 'text/javascript; charset=UTF-8' } ], 'body': '{"error_code": 1, "error_msg": "An unknown error occurred"}' } ]) requests = [{ 'method': 'GET', 'relative_url': 'me' }] batch = graph.batch(requests) for item in batch: assert isinstance(item, Exception) assert_equal(requests[0], item.request)
def test_batch_error_references_request(): graph = GraphAPI("<access token>") mock_request.return_value.content = json.dumps( [ { "code": 500, "headers": [{"name": "Content-Type", "value": "text/javascript; charset=UTF-8"}], "body": '{"error_code": 1, "error_msg": "An unknown error occurred"}', } ] ) mock_request.return_value.status_code = 200 requests = [{"method": "GET", "relative_url": "me"}] batch = graph.batch(requests) responses = list(batch) assert_equal(responses[0].request, requests[0])
def test_batch_error_references_request(): graph = GraphAPI('<access token>') mock_request.return_value.content = json.dumps([{ 'code': 500, 'headers': [{ 'name': 'Content-Type', 'value': 'text/javascript; charset=UTF-8' }], 'body': '{"error_code": 1, "error_msg": "An unknown error occurred"}' }]) requests = [{'method': 'GET', 'relative_url': 'me'}] batch = graph.batch(requests) responses = list(batch) assert_equal(responses[0].request, requests[0])
def test_batch_error_references_request(): graph = GraphAPI('<access token>') mock_request.return_value.content = json.dumps([ { 'code': 500, 'headers': [ { 'name': 'Content-Type', 'value': 'text/javascript; charset=UTF-8' } ], 'body': '{"error_code": 1, "error_msg": "An unknown error occurred"}' } ]) requests = [ { 'method': 'GET', 'relative_url': 'me' } ] batch = graph.batch(requests) responses = list(batch) assert_equal(responses[0].request, requests[0])
class PageMiner(object): failed_posts = [] post_fields = '?fields=message,name,shares,comments.limit(1).summary(1),created_time,reactions.type(LIKE).limit(0).summary(1).as(LIKE),reactions.type(WOW).limit(0).summary(1).as(WOW),reactions.type(SAD).limit(0).summary(1).as(SAD),reactions.type(LOVE).limit(0).summary(1).as(LOVE),reactions.type(ANGRY).limit(0).summary(1).as(ANGRY),reactions.type(HAHA).limit(0).summary(1).as(HAHA)' comment_fields = '?fields=likes.limit(0).summary(1),message,comment_count,from,attachment' nb_posts = 0 def __init__(self, page, API_KEY, update=True, posts_ids=[]): self.graph = GraphAPI(API_KEY, version='2.9') self.name = page['name'] self.id = page['fb_id'] self.type = page['type'] self.type = page['type'] if update == True: self._from = page['last_update'] else: self._from = '' self.posts_ids = posts_ids if posts_ids != [] else [ u for u in self.get_posts_ids() ] self.nb_posts = len(self.posts_ids) def _mine(self, post_list=[], progress=True): post_list = post_list if post_list != [] else self.posts_ids post_bach = [{ 'method': 'GET', 'relative_url': p_id + self.post_fields } for p_id in post_list] n = len(post_list) posts = self.graph.batch(post_bach) if progress: bar = FillingSquaresBar('Mining %s:' % self.name, max=n) for post in posts: p = self.clean_post(post) if p != None: yield p bar.next() bar.finish() else: for post in posts: p = self.clean_post(post) if p != None: yield p def get_posts_ids(self): if self.type == 'page': posts_url = str(self.id) + '/posts' elif self.type == 'group': posts_url = str(self.id) + '/feed' id_lists = self.graph.get( posts_url, page=True, limit=100, fields='id') if self._from == '' else self.graph.get( posts_url, page=True, limit=100, fields='id', since=self._from) for id_list in id_lists: for d in id_list["data"]: yield d['id'] def clean_post(self, raw_post): post = {'origin': self.id} post['fb_id'] = raw_post['id'] attachment = [t for t in self.attachments(raw_post['id'])] if attachment != []: post['attachments'] = attachment try: name = raw_post['name'] except Exception: pass else: if name[:11] != 'Photos from': post['name'] = name reactions = self.clean_reactions(raw_post) if reactions != None: post['reactions'] = reactions try: post['text'] = raw_post['message'] except: self.failed_posts.append(raw_post['id']) return None post['created_time'] = datetime.strptime(raw_post['created_time'], '%Y-%m-%dT%H:%M:%S+0000') try: post['#shares'] = raw_post['shares']['count'] except: post['#shares'] = 0 comments = [c for c in self.comments(raw_post['id'])] if comments != []: post['comments'] = comments post['#comments'] = len(comments) return (post) def clean_reactions(self, raw_post): try: reactions = {} for k in ['LIKE', 'WOW', 'LOVE', 'SAD', 'ANGRY', 'HAHA']: reactions[k] = raw_post[k]['summary']['total_count'] return reactions except Exception as e: #print(str(e)) return None def comments(self, post_id): comment_bach = [{ 'method': 'GET', 'relative_url': c_id + self.comment_fields } for c_id in self.comments_ids(post_id)] try: raw_comments = self.graph.batch(comment_bach) for raw_comment in raw_comments: c = self.clean_comment(raw_comment) if c != None: yield c except Exception: #print ('Error in comments') return [] def attachments(self, post_id): attachments = [] link_to_attachments = str(post_id) + '/attachments' try: a = self.graph.get(link_to_attachments) for d in a['data'][0]['subattachments']['data']: yield {'url': d['media']['image']['src'], 'type': d['type']} except Exception as e: return [] def clean_comment(self, raw_comment): try: comment = {} comment['author'] = raw_comment['from'] comment['text'] = raw_comment['message'] comment['#replays'] = raw_comment['comment_count'] comment['#likes'] = raw_comment['likes']['summary']['total_count'] try: comment['attachment'] = aw_comment['attachment'] except Exception: pass return comment except Exception as e: #print(str(e)) #print('Error in clean_comment: '+comment_row['id']) return None def comments_ids(self, post_id): link_to_comments = str(post_id) + '/comments' id_lists = self.graph.get(link_to_comments, page=True, limit=100, fields='id') for id_list in id_lists: for d in id_list["data"]: yield d['id']