def get_videos(): db_name = '../data_repository/dataset.db' channels = 'SELECT * FROM video' conn = create_connection(db_name) cursor = conn.cursor() cursor.execute(channels) return cursor.fetchall()
def main(): # Disable OAuthlib's HTTPS verification when running locally. # *DO NOT* leave this option enabled in production. os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" api_service_name = "youtube" api_version = "v3" conn = create_connection("../data_repository/dataset.db") youtube = googleapiclient.discovery.build(api_service_name, api_version, developerKey=DEVELOPER_KEY) for id in get_video_ids(conn): request = youtube.commentThreads().list(part="snippet", maxResults=100, videoId=id[0]) response = request.execute() print(response) comment_list = response["items"] for comment in comment_list: row = ( comment["id"], comment["snippet"]["videoId"], comment["snippet"]["topLevelComment"]["snippet"] ["textOriginal"], comment["snippet"]["topLevelComment"]["snippet"]["likeCount"]) add_comments(conn, row) conn.close()
def main(): # Disable OAuthlib's HTTPS verification when running locally. # *DO NOT* leave this option enabled in production. os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" api_service_name = "youtube" api_version = "v3" credentials = Secrets['apiKey'] youtube = googleapiclient.discovery.build(api_service_name, api_version, developerKey=credentials) request = youtube.channels().list(part="statistics,snippet", id="UCz97F7dMxBNOfGYu3rx8aCw", maxResults=50) response = request.execute() channel = response['items'][0] conn = create_connection("../data_repository/dataset.db") print(response) add_channel( conn=conn, channel_id=channel['id'], title=channel['snippet']['title'], thumbnail_url=channel['snippet']['thumbnails']['default']['url'], videos=channel['statistics']['videoCount'], subscribers=channel['statistics']['subscriberCount'])
def main(): model = fasttext.FastText.load_model( '../../Documents/Python Scripts/azure_django_apis/sentiment_api/models/sentiment_model_q.ftz' ) conn = create_connection('../data_repository/dataset.db') for item in get_comments(conn): prediction = model.predict(strip_down_comment(item[2])) insert_prediction(conn, item[0], prediction[0][0], prediction[1][0])
def execute(): db_name = "./dataset.db" create_labeled_tweets = """ CREATE TABLE IF NOT EXISTS labeled_tweet ( id integer PRIMARY KEY, target integer, classified_tweet text NOT NULL);""" conn = create_connection(db_name) if conn is not None: create_table(conn, create_labeled_tweets)
def execute(): db_name = "./dataset.db" create_comments_table = """ CREATE TABLE IF NOT EXISTS comment ( comment_id text PRIMARY KEY, video_id text NOT NULL, comment text, like_count integer );""" conn = create_connection(db_name) if conn is not None: create_table(conn, create_comments_table)
def execute(): db_name = "./dataset.db" create_comments_table = """ CREATE TABLE IF NOT EXISTS channel ( channel_id text PRIMARY KEY, title text NOT NULL, thumbnail_url text, subscribers integer, videos integer);""" conn = create_connection(db_name) if conn is not None: create_table(conn, create_comments_table)
def execute(): db_name = './dataset.db' create_prediction_table = """ CREATE TABLE IF NOT EXISTS prediction ( prediction_id integer PRIMARY KEY, comment_id text NOT NULL, label text NOT NULL, probability text );""" conn = create_connection(db_name) if conn is not None: create_table(conn, create_prediction_table)
def execute(): db_name = "./dataset.db" create_video_table = """ CREATE TABLE IF NOT EXISTS video ( video_id text PRIMARY KEY, channel_id text NOT NULL, title text NOT NULL, description text, thumbnail_url text, views integer, likes integer, dislikes integer);""" conn = create_connection(db_name) if conn is not None: create_table(conn, create_video_table)
def main(): tweet_query = "SELECT * FROM labeled_tweet" write_file = open('../tweets.txt', 'w', encoding='utf-8') conn = create_connection("../data_repository/dataset.db") cur = conn.cursor() cur.execute(tweet_query) rows = cur.fetchall() num_rows = len(rows) counter = 0 for item in rows: print("Counter: ", counter) print(strip_down_comment(item[2]), file=write_file) print("Percent: ", (counter / num_rows) * 100, "%") counter += 1 cur.close() write_file.close()
def main(): # Disable OAuthlib's HTTPS verification when running locally. # *DO NOT* leave this option enabled in production. os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" api_service_name = "youtube" api_version = "v3" DEVELOPER_KEY = Secrets['apiKey'] videoId = "" conn = create_connection("../data_repository/dataset.db") youtube = googleapiclient.discovery.build( api_service_name, api_version, developerKey = DEVELOPER_KEY) request = youtube.commentThreads().list( part="snippet", maxResults=100, videoId=videoId ) while True: response = request.execute() print(response) comment_list = response["items"] for comment in comment_list: row = ( comment["snippet"]["videoId"], comment["snippet"]["topLevelComment"]["snippet"]["textOriginal"]) add_comments(conn, row) if "nextPageToken" in response.keys(): request = youtube.commentThreads().list( part="snippet", pageToken=response['nextPageToken'], maxResults=100, videoId=videoId ) else: break conn.close()
def main(): # Disable OAuthlib's HTTPS verification when running locally. # *DO NOT* leave this option enabled in production. os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" api_service_name = "youtube" api_version = "v3" # client_secrets_file = "YOUR_CLIENT_SECRET_FILE.json" # Get credentials and create an API client credentials = Secrets['apiKey'] youtube = googleapiclient.discovery.build(api_service_name, api_version, developerKey=credentials) request = youtube.videos().list(part="statistics,snippet", id="g4Hbz2jLxvQ") response = request.execute() videos = response["items"] conn = create_connection("../data_repository/dataset.db") for video in videos: add_video( conn=conn, video_id=video['id'], channel_id=video['snippet']['channelId'], title=video['snippet']['title'], description=video['snippet']['description'], thumbnail_url=video['snippet']['thumbnails']['default']['url'], views=video['statistics']['viewCount'], likes=video['statistics']['likeCount'], dislikes=video['statistics']['dislikeCount']) print(response) conn.commit()
from data_repository.sqlite_functions import create_connection def get_video_ids(conn): query = "SELECT video_id FROM video" cur = conn.cursor() cur.execute(query) return cur.fetchall() if __name__ == '__main__': conn = create_connection("./data_repository/dataset.db") rows = get_video_ids(conn) for i in rows: print(f'{i[0]} :: {type(i)}')
import fasttext from data_repository import sqlite_functions conn = sqlite_functions.create_connection("./data_repository/dataset.db") insert_values = """INSERT INTO training_results (epoch, learning_rate, word_ngrams, precision, recall) VALUES (?,?,?,?,?)""" epoch = [5, 25] learning_rate = [0.1, 0.5, 1.0] word_n_grams = [1, 3, 5] for i in epoch: for j in learning_rate: for k in word_n_grams: model = fasttext.train_supervised(input="yelp.train", epoch=i, lr=j, wordNgrams=k) model.save_model( f"./analysis_models/sentiment_model_e{i}_lr{j}_wn{k}.bin") test_values = model.test("yelp.valid") print(test_values) conn.execute(insert_values, (i, j, k, test_values[1], test_values[2])) conn.commit() conn.close()