def run():
indexStart = 0
facebookDb = FacebookDataDatabase()
for raw_url in adDb.selectAdData()[indexStart:]:
global token
token = getToken()
url = modifyUrl(raw_url)
post_number = get_page_id(url)
if not facebookDb.isPageInDb(post_number):
page = Page()
page.metrics = getPageMetrics(url)
page.posts = getPostData(url)
if page.posts:
for post in page.posts:
token = getToken()
set_post_data(post)
StoreInFacebookData(post.id, post.image_url, post.message,
post.share_count, post.comment_count,
page.metrics.fan_count,
page.metrics.rating_count,
page.metrics.talking_about_count,
page.metrics.star_rating, -1)
print("Stored!", post.id)
else:
pass
def getPostData(url):
main_page = requests.get(url)
page_raw = ast.literal_eval(main_page.text)
if "error" in page_raw:
url = url.replace(".limit(1000)", ".limit(100)")
if "message" in page_raw:
if "reduce the amount of data" in page_raw["message"]:
print("Reducing")
main_page = requests.get(url)
page_raw = ast.literal_eval(main_page.text)
else:
print("Error", page_raw, url)
token = getToken()
return getPostData(url)
else:
return
page_data = page_raw["posts"]["data"]
posts = []
for post_raw in page_data:
post = Post()
if "message" in post_raw:
post.message = escape_url(post_raw["message"])
post.id = post_raw["id"]
posts.append(post)
return posts
def get_post_on_comment_url(comment):
if comment.id:
global token
token = getToken()
url = '''https://graph.facebook.com/v2.11/{0}?access_token={1}&debug=all&format=json&method=get&pretty=0&suppress_http_code=1000'''.format(
comment.id, token)
post = requests.get(url)
raw = ast.literal_eval(post.text)
get_key_data(raw, "message",
comment.setMessage) # set the message for the comment
def setGlobals():
global token
global start_url_string
global graph_url_start
global end_token
token = getToken()
start_url_string = r"https://facebook.com/"
graph_url_start = r"https://graph.facebook.com/v2.11/"
end_token = '''?access_token={0}&fields=posts.limit(1000)&debug=all&format=json&method=get&pretty=0&suppress_http_code=1000'''.format(
token)
def run():
index = q.get()
post = data[index]
global token
token = getToken()
post_id = post[0]
comment_url = '''https://graph.facebook.com/v2.11/{0}?fields=comments.limit(1000000)&access_token={1}&debug=all&format=json&method=get&pretty=0&suppress_http_code=1000'''.format(
post_id, token)
post_data = requests.get(comment_url)
try:
post_raw_data = ast.literal_eval(post_data.text)
except:
sys.exit(post_data.text)
if "comments" in post_raw_data:
comment_count = len(post_raw_data["comments"]["data"])
sys.stdout.write("Count: {0}\n".format(comment_count))
to_insert.put(InsertData(comment_count, post_id))
else:
to_insert.put(InsertData(0, post_id))
sys.stdout.write("Storing index {0}\n".format(index))
writeIndex(index)
sys.path.append("../../")
from Notebooks.SearchFbData.GetKeyData import get_key_data
from Notebooks.Token.GenerateToken import getToken
from Notebooks.LinkDatabases.FacebookData import FacebookDataDatabase
from Notebooks.LinkDatabases.PostComments import PostDataDatabase
import requests
import ast
import unicodedata
facebookDb = FacebookDataDatabase()
commentDb = PostDataDatabase()
global token
token = getToken()
# Get/Set post data. These are person posts on the Ad in question
class Comment:
def __init__(self):
self.id = None
self.message = None
self.like_count = None
def setId(self, id):
self.id = id
def setMessage(self, message):
self.message = unicodedata.normalize('NFKD', message).encode(
'ascii', 'ignore').decode('ascii').replace("'", "''")