class Global:
batch_size = 256
epochs = 100
group_size = 100_000
plot_losses = PlotLearning("nlp_nn_sentiment_count")
facebookDb = FacebookDataDatabase()
regularizer_function = None
def run():
indexStart = 0
facebookDb = FacebookDataDatabase()
for raw_url in adDb.selectAdData()[indexStart:]:
global token
token = getToken()
url = modifyUrl(raw_url)
post_number = get_page_id(url)
if not facebookDb.isPageInDb(post_number):
page = Page()
page.metrics = getPageMetrics(url)
page.posts = getPostData(url)
if page.posts:
for post in page.posts:
token = getToken()
set_post_data(post)
StoreInFacebookData(post.id, post.image_url, post.message,
post.share_count, post.comment_count,
page.metrics.fan_count,
page.metrics.rating_count,
page.metrics.talking_about_count,
page.metrics.star_rating, -1)
print("Stored!", post.id)
else:
pass
class MessageGetter:
facebookDb = FacebookDataDatabase()
@staticmethod
def get_columns(remove_columns=True):
columns = MessageGetter.facebookDb.getColumnNames()
columns = list(map(lambda x: x[1], columns))
if remove_columns:
columns.remove("imageId")
columns.remove("imageUrl")
return columns
@staticmethod
def __dict_factory(row, columns):
d = {}
for idx, col in enumerate(columns):
d[col] = row[idx]
return d
@staticmethod
def __get_post():
number_of_posts_to_train_on = len(
MessageGetter.facebookDb.selectFacebookData())
postData = MessageGetter.facebookDb.selectFacebookData()
for i, post in enumerate(postData[:number_of_posts_to_train_on]):
if i % 1000 == 0:
print("gather data percent: ", i / number_of_posts_to_train_on)
post_obj = MessageGetter.__dict_factory(
post, MessageGetter.get_columns(False))
yield post_obj
@staticmethod
def get_post_generator():
post_generator = MessageGetter.__get_post()
return post_generator
class Static:
facebookDb = FacebookDataDatabase()
metric_getter = facebookDb.getShareCount # Set this to change the model type
group_size = 6000
limit = 7000
plot_losses = PlotLearning("combined_keras_model")
batch_size = 1
epochs = 20
class Static:
facebookDb = FacebookDataDatabase()
metric_getter = facebookDb.getCommentCount # Set this to change the model type
limit = 80_000 # len(os.listdir("../Image_CNN/images"))
group_size = limit
plot_losses = PlotLearning("combined_keras_model_comment_count")
batch_size = 512
epochs = 100
class Static:
facebookDb = FacebookDataDatabase()
metric_getter = facebookDb.getShareCount # Set this to change the model type
limit = len(os.listdir("../Image_CNN/images"))
group_size = limit
plot_losses = PlotLearning("combined_keras_model")
batch_size = 1
epochs = 20
class Global:
batch_size = 256
epochs = 100
group_size = 100_000
plot_losses = PlotLearning("nlp_nn_comment_count")
facebookDb = FacebookDataDatabase()
metric_getter = facebookDb.getCommentCount
metric_name = "commentCount"
regularizer_function = None
class Global:
batch_size = 1
epochs = 5
group_size = 6000
plot_losses = PlotLearning("nlp_nn")
facebookDb = FacebookDataDatabase()
metric_getter = facebookDb.getSentiment
metric_name = None
regularizer_function = None
def test_to_vector():
import os
all_files = os.listdir("../Image_CNN/images")
ids = list(map(lambda x: x[:-4], all_files))
from Notebooks.LinkDatabases.FacebookData import FacebookDataDatabase
facebookDb = FacebookDataDatabase()
metric_getter = facebookDb.getCommentCount # Set this to change the model type
rows = list(
map(
lambda x: x[0],
filter(lambda x: x if x else None,
map(lambda x: facebookDb.getRow(x), ids))))
data = list(map(lambda x: (x[0], x[10], x[2], x[3]), rows))
messages = list(map(lambda x: x[1], data))
word_vectors = to_vector(messages)
for word_vector in word_vectors[:100]:
print(word_vector)
import os
import sys
from datetime import datetime
sys.path.append("../../")
from Notebooks.SearchFbData.GetKeyData import get_key_data
from Notebooks.Token.GenerateToken import getToken
from Notebooks.LinkDatabases.FacebookData import FacebookDataDatabase
from Notebooks.LinkDatabases.PostComments import PostDataDatabase
import requests
import ast
import unicodedata
facebookDb = FacebookDataDatabase()
commentDb = PostDataDatabase()
global token
token = getToken()
# Get/Set post data. These are person posts on the Ad in question
class Comment:
def __init__(self):
self.id = None
self.message = None
self.like_count = None
def setId(self, id):
self.id = id
from Notebooks.LinkDatabases.FacebookData import FacebookDataDatabase
import pandas as pd
import numpy as np
fbDatabase = FacebookDataDatabase()
counts = list(map(lambda x: x[0] if x[0] > 0 else 0, fbDatabase.selectColumnData("commentCount")))
for x in counts[:100]:
print(x)
df = pd.DataFrame(counts, columns=["commentCount"])
df.to_csv("comment_counts.csv")
class Static:
facebookDb = FacebookDataDatabase()
metric_getter = facebookDb.getShareCount # Set this to change the model type
limit = 100000 # len(os.listdir("../Image_CNN/images"))
group_size = limit
class Static:
facebookDb = FacebookDataDatabase()
metric_getter = facebookDb.getSentiment # Set this to change the model type
group_size = 40000
limit = 60000
class Static:
facebookDb = FacebookDataDatabase()
metric_getter = facebookDb.getCommentCount # Set this to change the model type
limit = 30000
class Static:
facebookDb = FacebookDataDatabase()
metric_getter = facebookDb.getShareCount # Set this to change the model type
group_size = 150000
limit = 200000
class Static:
facebookDb = FacebookDataDatabase()
metric_getter = facebookDb.getCommentCount # Set this to change the model type
group_size = 10000
limit = 10000
plot_losses = PlotLearning("combined_keras_model")