def get_videos():
db_name = '../data_repository/dataset.db'
channels = 'SELECT * FROM video'
conn = create_connection(db_name)
cursor = conn.cursor()
cursor.execute(channels)
return cursor.fetchall()
def main():
# Disable OAuthlib's HTTPS verification when running locally.
# *DO NOT* leave this option enabled in production.
os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
api_service_name = "youtube"
api_version = "v3"
conn = create_connection("../data_repository/dataset.db")
youtube = googleapiclient.discovery.build(api_service_name,
api_version,
developerKey=DEVELOPER_KEY)
for id in get_video_ids(conn):
request = youtube.commentThreads().list(part="snippet",
maxResults=100,
videoId=id[0])
response = request.execute()
print(response)
comment_list = response["items"]
for comment in comment_list:
row = (
comment["id"], comment["snippet"]["videoId"],
comment["snippet"]["topLevelComment"]["snippet"]
["textOriginal"],
comment["snippet"]["topLevelComment"]["snippet"]["likeCount"])
add_comments(conn, row)
conn.close()
def main():
# Disable OAuthlib's HTTPS verification when running locally.
# *DO NOT* leave this option enabled in production.
os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
api_service_name = "youtube"
api_version = "v3"
credentials = Secrets['apiKey']
youtube = googleapiclient.discovery.build(api_service_name,
api_version,
developerKey=credentials)
request = youtube.channels().list(part="statistics,snippet",
id="UCz97F7dMxBNOfGYu3rx8aCw",
maxResults=50)
response = request.execute()
channel = response['items'][0]
conn = create_connection("../data_repository/dataset.db")
print(response)
add_channel(
conn=conn,
channel_id=channel['id'],
title=channel['snippet']['title'],
thumbnail_url=channel['snippet']['thumbnails']['default']['url'],
videos=channel['statistics']['videoCount'],
subscribers=channel['statistics']['subscriberCount'])
def main():
model = fasttext.FastText.load_model(
'../../Documents/Python Scripts/azure_django_apis/sentiment_api/models/sentiment_model_q.ftz'
)
conn = create_connection('../data_repository/dataset.db')
for item in get_comments(conn):
prediction = model.predict(strip_down_comment(item[2]))
insert_prediction(conn, item[0], prediction[0][0], prediction[1][0])
def execute():
db_name = "./dataset.db"
create_labeled_tweets = """ CREATE TABLE IF NOT EXISTS labeled_tweet (
id integer PRIMARY KEY,
target integer,
classified_tweet text NOT NULL);"""
conn = create_connection(db_name)
if conn is not None:
create_table(conn, create_labeled_tweets)
def execute():
db_name = "./dataset.db"
create_comments_table = """ CREATE TABLE IF NOT EXISTS comment (
comment_id text PRIMARY KEY,
video_id text NOT NULL,
comment text,
like_count integer
);"""
conn = create_connection(db_name)
if conn is not None:
create_table(conn, create_comments_table)
def execute():
db_name = "./dataset.db"
create_comments_table = """ CREATE TABLE IF NOT EXISTS channel (
channel_id text PRIMARY KEY,
title text NOT NULL,
thumbnail_url text,
subscribers integer,
videos integer);"""
conn = create_connection(db_name)
if conn is not None:
create_table(conn, create_comments_table)
def execute():
db_name = './dataset.db'
create_prediction_table = """ CREATE TABLE IF NOT EXISTS prediction (
prediction_id integer PRIMARY KEY,
comment_id text NOT NULL,
label text NOT NULL,
probability text
);"""
conn = create_connection(db_name)
if conn is not None:
create_table(conn, create_prediction_table)
def execute():
db_name = "./dataset.db"
create_video_table = """ CREATE TABLE IF NOT EXISTS video (
video_id text PRIMARY KEY,
channel_id text NOT NULL,
title text NOT NULL,
description text,
thumbnail_url text,
views integer,
likes integer,
dislikes integer);"""
conn = create_connection(db_name)
if conn is not None:
create_table(conn, create_video_table)
def main():
tweet_query = "SELECT * FROM labeled_tweet"
write_file = open('../tweets.txt', 'w', encoding='utf-8')
conn = create_connection("../data_repository/dataset.db")
cur = conn.cursor()
cur.execute(tweet_query)
rows = cur.fetchall()
num_rows = len(rows)
counter = 0
for item in rows:
print("Counter: ", counter)
print(strip_down_comment(item[2]), file=write_file)
print("Percent: ", (counter / num_rows) * 100, "%")
counter += 1
cur.close()
write_file.close()
def main():
# Disable OAuthlib's HTTPS verification when running locally.
# *DO NOT* leave this option enabled in production.
os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
api_service_name = "youtube"
api_version = "v3"
DEVELOPER_KEY = Secrets['apiKey']
videoId = ""
conn = create_connection("../data_repository/dataset.db")
youtube = googleapiclient.discovery.build(
api_service_name, api_version, developerKey = DEVELOPER_KEY)
request = youtube.commentThreads().list(
part="snippet",
maxResults=100,
videoId=videoId
)
while True:
response = request.execute()
print(response)
comment_list = response["items"]
for comment in comment_list:
row = (
comment["snippet"]["videoId"],
comment["snippet"]["topLevelComment"]["snippet"]["textOriginal"])
add_comments(conn, row)
if "nextPageToken" in response.keys():
request = youtube.commentThreads().list(
part="snippet",
pageToken=response['nextPageToken'],
maxResults=100,
videoId=videoId
)
else:
break
conn.close()
def main():
# Disable OAuthlib's HTTPS verification when running locally.
# *DO NOT* leave this option enabled in production.
os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
api_service_name = "youtube"
api_version = "v3"
# client_secrets_file = "YOUR_CLIENT_SECRET_FILE.json"
# Get credentials and create an API client
credentials = Secrets['apiKey']
youtube = googleapiclient.discovery.build(api_service_name,
api_version,
developerKey=credentials)
request = youtube.videos().list(part="statistics,snippet",
id="g4Hbz2jLxvQ")
response = request.execute()
videos = response["items"]
conn = create_connection("../data_repository/dataset.db")
for video in videos:
add_video(
conn=conn,
video_id=video['id'],
channel_id=video['snippet']['channelId'],
title=video['snippet']['title'],
description=video['snippet']['description'],
thumbnail_url=video['snippet']['thumbnails']['default']['url'],
views=video['statistics']['viewCount'],
likes=video['statistics']['likeCount'],
dislikes=video['statistics']['dislikeCount'])
print(response)
conn.commit()
from data_repository.sqlite_functions import create_connection
def get_video_ids(conn):
query = "SELECT video_id FROM video"
cur = conn.cursor()
cur.execute(query)
return cur.fetchall()
if __name__ == '__main__':
conn = create_connection("./data_repository/dataset.db")
rows = get_video_ids(conn)
for i in rows:
print(f'{i[0]} :: {type(i)}')
import fasttext
from data_repository import sqlite_functions
conn = sqlite_functions.create_connection("./data_repository/dataset.db")
insert_values = """INSERT INTO training_results (epoch, learning_rate, word_ngrams, precision, recall)
VALUES (?,?,?,?,?)"""
epoch = [5, 25]
learning_rate = [0.1, 0.5, 1.0]
word_n_grams = [1, 3, 5]
for i in epoch:
for j in learning_rate:
for k in word_n_grams:
model = fasttext.train_supervised(input="yelp.train",
epoch=i,
lr=j,
wordNgrams=k)
model.save_model(
f"./analysis_models/sentiment_model_e{i}_lr{j}_wn{k}.bin")
test_values = model.test("yelp.valid")
print(test_values)
conn.execute(insert_values,
(i, j, k, test_values[1], test_values[2]))
conn.commit()
conn.close()
