def __init__(self):
self.index = 0
self.youtube_services = []
for api_key in YOUTUBE_API_KEYS:
self.youtube_services.append(
Youtube(api_key, YOUTUBE_API_SERVICE_NAME,
YOUTUBE_API_VERSION))
def initialize():
"""Method for initializing a youtube object, connection to the DB and load comments
get the comments for the youtube videos ids listed in the DB, store the comments in the DB,
train the classifier and print the classifier metrics
Args:
config: dicitionary containing the DB parameters
"""
with app.app_context():
g.config = get_config()
g.yt = Youtube(api_key= g.config ['youtube']['API_KEY'], api_version = g.config ['youtube']['API_VERSION'], service_name=g.config['youtube']['API_SERVICE_NAME'])
video_ids = g.config['youtube_videos_id']
g.comments = []
for video_id in video_ids:
g.comments.extend(g.yt.get_video_comments(video_id))
#db_params = {"user":g.config['db']['user'], "host":g.config['db']["host"], "port":g.config['db']["port"],"database": g.config['db']["database"]}
g.db = DBController(**g.config['db'])
# create table if it does not exist:
g.db.create_comments_table()
# train classifier
g.classifier = Classifier()
g.classifier.train()
# predication of the comments
for comment_item in g.comments:
text = comment_item.text
predication = g.classifier.predict(comment_text)
comment_item.sentiment = predication
# insert the comment in DB
g.db.insert_comments(comment_item)
def fetch_latest_videos():
now_time = datetime.utcnow()
publish_after = now_time - timedelta(seconds=VIDEO_FETCH_INTERVAL)
publish_before = now_time
search = 'corona'
obj = Youtube()
res = obj.search_youtube_videos(search, publish_after, publish_before)
model_obj = []
while True:
next_page_token = res.get('nextPageToken')
items = res.get("items")
for each in items:
published_at = each.get("snippet", {}).get('publishedAt')
title = each.get("snippet", {}).get('title')
description = each.get("snippet", {}).get('description')
url = each.get("snippet", {}).get('thumbnails',
{}).get('default', {}).get("url")
model_obj.append(
YoutubeVideos(title=title,
description=description,
published_at=published_at,
url=url))
if not next_page_token:
break
res = obj.search_youtube_videos(search, publish_after, publish_before,
next_page_token)
if model_obj:
YoutubeVideos.objects.bulk_create(model_obj)
def gen_process() -> list:
event_list = []
if config['youtube']['enable']:
for user_config in config['youtube']['users']:
y = Youtube(user_config)
event_list.append(y)
if config['twitcasting']['enable']:
for user_config in config['twitcasting']['users']:
t = Twitcasting(user_config)
event_list.append(t)
if config['openrec']['enable']:
for user_config in config['openrec']['users']:
o = Openrec(user_config)
event_list.append(o)
if config['mirrativ']['enable']:
for user_config in config['mirrativ']['users']:
m = Mirrativ(user_config)
event_list.append(m)
if config['bilibili']['enable']:
for user_config in config['bilibili']['users']:
b = Bilibili(user_config)
event_list.append(b)
if user_config['article']:
b1 = BilibiliArticle(user_config)
event_list.append(b1)
if user_config['live']:
b2 = BilibiliLive(user_config)
event_list.append(b2)
return event_list
def __init__(self):
self.__youtube = Youtube()
self.__spotify = Spotify()
self.__editor = TagEditor()
self.__last = LastFM()
self.__apple = AppleMusic()
self.__deezer = Deezer()
def test_get_video(mock_request, mock_response):
""" Test getting a YT video """
mock_request.return_value = mock_response
youtube = Youtube("northernlion")
assert youtube.url == (f"https://www.googleapis.com/youtube/v3/search"
f"?key={YOUTUBE_AUTH}&q=northernlion&maxResults"
f"={MAX_REQUESTED_VIDS}&type=video")
assert youtube.get_video(0) == f"{BASE_URL}isaac"
# Test getting a youtube video but no results
mock_response.json.return_value = {"items": []}
youtube = Youtube("northernlion")
with pytest.raises(APIError) as error:
youtube.validate_num_vids()
assert "Sorry, there were no videos" in str(error)
async def populate_query(self, query):
youtube = Youtube()
async for item in youtube.search(query, 500):
self.db[item['id']['videoId']] = query
if len(self.db) % 100 == 0:
print(len(self.db))
self.changed = True
await youtube.close()
def __init__(self, spotify={}, youtube={}):
"""
make a resource to talk to spotify api and youtube api
see spotify.py for params
see youtube.py for params
"""
self.sf = Spotfy(**spotify)
self.yt = Youtube(**youtube_parser)
def get_from_yt(user_input):
youtube = Youtube(yt_resource)
youtube.search_channel(user_input)
youtube.print_channels()
return youtube.select_channel(
int(
input_in_range('Select channel: ', 1,
youtube.total_channels + 1)))
def youtube_thread():
""" Print Youtube Subs Every minute """
yt_instance = Youtube()
while True:
try:
yt_instance.subscribers()
except ConnectionError:
time.sleep(300) # Sleep if error
time.sleep(60)
def form_entry():
if request.method == 'POST':
platform = request.form['platform']
# Form data retrieval from POST
username = request.form['username']
email = request.form['email']
full_name = request.form['full_name']
date_of_birth = request.form['dob']
address = request.form['address']
state = request.form['state']
city = request.form['city']
pincode = request.form['pincode']
crime_type = request.form['crime_type'] # Common in specific cards
if (platform == "twitter"):
link = request.form['tweet_link']
post_content = twitter.get_data_twitter(request.form['tweet_link'],
twapi)
post_content = get_classification(platform, post_content)
elif (platform == "facebook"):
link = request.form['fb_link']
encoded_link = request.form['post_encoded_url']
post_content = fb.get_data_facebook(encoded_link, link)
post_content = get_classification(platform, post_content)
elif (platform == "viraly"):
link = request.form['viraly_post_id']
post_content = viraly.get_data_viraly(
db, request.form['viraly_content-type'],
request.form['viraly_post_id'])
post_content = get_classification(platform, post_content)
elif (platform == "whatsapp"):
link = "http://localhost:3003/static/user-content/" + username + "/" + request.files[
'whatsapp_backup'].filename
post_content = whatsapp.get_data_whatsapp(
username, request.files['whatsapp_backup'],
app.config['UPLOAD_FOLDER'])
post_content = get_classification(platform, post_content)
elif (platform == "youtube"):
yt_link = request.form['youtube_link']
from youtube import Youtube
yt = Youtube()
temp = yt.auto_yt(yt_link, image_model, pretty=True)
return render_template("video-result.html",
temp=temp,
yt_link=yt_link)
elif (platform == "sms"):
post_content = sms.get_data_sms(request.form['sender_number'])
else:
return redirect(url_for('index_main'))
db_push_commons(username, email, full_name, date_of_birth, address,
state, city, pincode, crime_type, platform, link,
post_content)
return redirect(url_for('index_main'))
else:
return redirect(url_for('index_main'))
def add_song(name, list):
startTime = time.time()
ytb = Youtube()
print("ytb " + str(time.time() - startTime))
url = ytb.get_one(name)
print("url " + str(time.time() - startTime))
pafy = ytb.get_audio(url)
print("pafy " + str(time.time() - startTime))
list.add(pafy)
print("list " + str(time.time() - startTime))
def daily_thread():
""" Print daily data every minute """
youtube = Youtube()
portfolio = Portfolio()
while True:
try:
youtube.subscribers()
portfolio.value()
except ConnectionError:
time.sleep(300)
time.sleep(60)
async def populate_random(self):
youtube = Youtube()
async def get_one():
async for item in youtube.random(None, 50):
self.db[item['id']['videoId']] = 'random'
if len(self.db) % 100 == 0:
print(len(self.db))
tasks = [get_one() for i in range(2000)]
await asyncio.gather(*tasks)
await youtube.close()
def setpassword(message, client):
global youtube_instance
global password
global user_id
print("In set password")
with open(os.path.join(sys.path[0], "assets/ussername_pass.json"),
'r+') as passfile:
data = json.load(passfile)
email = data["email"]
passs = data["password"]
credentials = message[9:]
username, password_ = credentials.split(",")
print(username)
print(password_)
if passs == "":
print("Here")
hashpass = hashlib.sha256(str.encode(password_)).hexdigest()
print(hashpass)
data = {"email": username, "password": hashpass}
passfile.seek(0)
json.dump(data, passfile)
passfile.truncate()
youtube_instance = Youtube(username, password_)
password = password_
client.publish("GladOs/messages/raspberry2phone" + user_id,
"Everything OK")
TTS("I am ready to take your commands master!")
else:
if (hashlib.sha256(
str.encode(password_)).hexdigest() == passs):
print("Here")
youtube_instance = Youtube(username, password_)
client.publish("GladOs/messages/raspberry2phone" + user_id,
"Everything OK")
TTS("I am ready to take your commands master!")
else:
client.publish("GladOs/messages/raspberry2phone" + user_id,
"Wrong password enter again!")
def main():
sp = Spotify()
yt = Youtube()
yt_playlist_id = input("Enter youtube playlist id: ")
spotify_playlist_name = input("Enter a name for your spotify playlist: ")
songs = yt.get_songs_from_playlist(yt_playlist_id)
spotify_playlist_id = sp.create_playlist(spotify_playlist_name)
for song in songs:
song_uri = sp.get_song_uri(song.artist, song.title)
was_added = sp.add_song_to_playlist(song_uri, spotify_playlist_id)
if was_added:
print(f'{song.artist} - {song.title} was added to playlist.')
def youtube(*args):
"""
Use the Youtube API to return a youtube video
"""
# Convert from tuple to list so we can modify
try:
idx, args = _get_idx_from_args(list(args))
idx = 0 if idx == -1 else idx
except ValueError as error:
return "text", str(error)
try:
return "text", Youtube(*args).get_video(idx)
except APIError as error:
return "text", str(error)
def get_songs(self):
spotify = spotipy.Spotify(auth=self.token)
results = spotify.search(q=self.genre, type='playlist',limit=10)
for playlist in results['playlists']['items']:
if playlist['tracks']['total'] < self.num_of_vid:
print('tracks less than '+ str(self.num_of_vid) + 'skipping to next playlist')
continue
else:
playlist_tracks = spotify.user_playlist_tracks(playlist['owner']['id'], playlist['id'])
print('playlist has ' +str(playlist_tracks['total'])+ ' songs.')
f_cnt = -1
s_cnt = 0
for playlist_track in playlist_tracks['items']:
f_cnt += 1
if f_cnt == 0:
continue
try:
#song data
song_title = playlist_track['track']['artists'][0]['name'] + ' - ' + playlist_track['track']['name']
song_id = playlist_track['track']['id']
# song features
audio_features = spotify.audio_features(song_id)
# video data
video_url = Youtube(song_title).search_video()
format = pytube.YouTube(video_url).streams.get_by_itag('133')
if s_cnt == self.num_of_vid:
break
# donwload video / save data in db
s_cnt += 1
print('song: '+str(s_cnt)+ ': '+song_title+ ' | ' +video_url+ ' | ' +str(format))
format.download(self.dir)
except:
print('error. continuing to next song.')
continue
break
def main():
playlist_url = input("Youtube playlist url: ")
playlist_name = input("Playlist name: ")
yt = Youtube(playlist_url)
song_titles = yt.get_songs_title()
print(len(song_titles))
songs_info = yt.get_songs_info(song_titles)
print(len(songs_info))
spotify = Spotify()
playlst_id = spotify.create_playlist(playlist_name)
for song_name, artist in songs_info.items():
uri = spotify.get_spotify_uri(artist, song_name)
status = spotify.add_songs_to_playlist(playlst_id, {"uris": [uri]})
if status:
print(f"{artist}-{song_name} was added to playlist.")
else:
print(f"\nERROR!! {artist}-{song_name} could not be added.\n")
def gen_process(self):
if config['youtube']['enable']:
for user_config in config['youtube']['users']:
y = Youtube(user_config)
self.events_multi.append(y)
if config['twitcasting']['enable']:
for user_config in config['twitcasting']['users']:
t = Twitcasting(user_config)
self.events_multi.append(t)
if config['openrec']['enable']:
for user_config in config['openrec']['users']:
o = Openrec(user_config)
self.events_multi.append(o)
if config['mirrativ']['enable']:
for user_config in config['mirrativ']['users']:
m = Mirrativ(user_config)
self.events_multi.append(m)
if config['bilibili']['enable']:
for user_config in config['bilibili']['users']:
b = Bilibili(user_config)
self.events_multi.append(b)
def main():
sp = Spotify()
yt = Youtube()
yt_playlist_id = input("Enter youtube playlist id: ")
spotify_playlist_name = input("Enter a name for your spotify playlist: ")
spotify_playlist_id = sp.create_playlist(spotify_playlist_name)
songs = yt.get_songs_from_playlist(yt_playlist_id)
for song in songs:
song_uri = sp.get_song_uri(song.artist, song.title)
if not song_uri:
print(f"{song.artist} - {song.title} was not found!")
continue
was_added = sp.add_song_to_playlist(song_uri, spotify_playlist_id)
if was_added:
print(f'{song.artist} - {song.title} was added to playlist.')
total_songs_added = sp._num_playlist_songs(spotify_playlist_id)
print(f'Added {total_songs_added} songs out of {len(songs)}')
async def main():
if "youtube.com/channel" in sys.argv[1] or "youtube.com/watch" in sys.argv[
1]:
s = Youtube(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else "")
data = await s.get_play_info()
title = data["title"]
url = data["play_url"]
elif "twitch.tv/" in sys.argv[1]:
s = Twitch(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else "")
data = await s.get_play_info()
title = data["title"]
url = data["play_url"]
elif "douyu.com/" in sys.argv[1]:
s = Douyu(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else "")
data = await s.get_play_info()
title = data["title"]
url = data["play_url"]
elif "huya.com/" in sys.argv[1]:
s = Huya(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else "")
data = await s.get_play_info()
title = data["title"]
url = data["play_url"]
elif "live.bilibili.com/" in sys.argv[1]:
s = Bilibili(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else "")
data = await s.get_play_info()
title = data["title"]
url = data["play_url"]
elif "bilibili.com/" in sys.argv[1]:
s = Bilibili(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else "")
await s.get_play_info_video()
return
else:
await run_ykdl(sys.argv[1])
return
print("title: " + title, flush=True)
print(url, flush=True)
from exalead import Exalead
from quora import Quora
from youtube import Youtube
from parsijoo import Parsijoo
scrapers = {
'g': Google(),
'b': Bing(),
'y': Yahoo(),
'd': Duckduckgo(),
'a': Ask(),
'yd': Yandex(),
'u': Baidu(),
'e': Exalead(),
'q': Quora(),
't': Youtube(),
'p': Parsijoo()
}
def read_in():
lines = sys.stdin.readlines()
return json.loads(lines[0])
def small_test():
assert isinstance(scrapers.google.results_search('fossasia'), list)
def feedgen(query, engine, count=10):
if engine in ['q', 't']:
def start_youtube():
for target_id in channel_id:
y = Youtube(target_id)
y.start()
from youtube_s import Youtube_S
from dang_error import DangError
from helpers import get_text, get_error_text, get_config, config_files_to_env
import logging
logging.basicConfig()
logging.getLogger().setLevel(logging.INFO)
load_dotenv()
config_files_to_env()
DISCORD_TOKEN = os.getenv('DISCORD_TOKEN')
bot = commands.Bot(command_prefix=['dang! ', '!dang ', 'dang!'],
help_command=None)
bot.add_cog(Youtube(bot))
bot.add_cog(Youtube_S(bot))
def get_random_quote(guild):
return choice(get_text(guild.id, "quotes"))
def magic_eight_ball(guild):
return choice(get_text(guild.id, "magic_eight_ball"))
def should_send_random_message(guild=None):
freq = get_config(guild.id, "spam_freq")
if (freq <= 0):
return False
def __init__(self):
self.items = []
self.wiki = Wiki()
self.youtube = Youtube()
self.path = os.getcwd()
self.s = schedule.every().saturday.do(self.job)
data = json.load(user_id_file)
user_id = data["username"]
print(user_id)
if (user_id == ""):
mqtt_thred_to_get_userid = threading.Thread(
target=mqttclient_to_get_userid)
mqtt_thred_to_get_userid.daemon = True
mqtt_thred_to_get_userid.start()
else:
TTS("Welcome back " + user_id)
'''this loop checks whether the username had been passed or not.'''
'''if new username is passed it is written onto disk'''
while (True):
if (len(user_id) > 0):
with open(os.path.join(sys.path[0], "assets/user_id.json"),
'r+') as user_id_file:
data = {"username": user_id}
user_id_file.seek(0)
json.dump(data, user_id_file)
user_id_file.truncate()
break
youtube_instance = Youtube()
'''calling the main message thread from here'''
mqtt_thred = threading.Thread(target=mqttclient)
mqtt_thred.daemon = True
mqtt_thred.start()
while (True):
if active == False:
print("Exiting main thread")
sys.exit()
import asyncio
from aiohttp import web
from aiohttp_session import get_session, new_session
import aiohttp_jinja2
from aiohttp import ClientSession
from bson.objectid import ObjectId
import json
import datetime
from util import routes, get_user, to_objectid
from backend import users, friends, comments, history, shares, playlists, videos
from youtube import Youtube
youtube = Youtube()
# TODO: disable debugging code
@routes.get('/debug:populate')
async def debug_populate(request):
import backend
await backend.clear_all()
# generate random users with personal data
async with ClientSession() as session:
async with session.get(
'https://randomuser.me/api/?results=100&nat=AU') as response:
data = await response.json()
for i, user in enumerate(data['results']):
password = user['login']['password']
email = user['email']
from youtube import Youtube s1omething = Youtube() s1omething.youtube_subscriptions()
def store_media():
youtube = Youtube().get_videos()
giphy = Giphy().get_gifs()
# clear existing results
media.update_one({'name': 'youtube'}, {'$set': {'data': youtube}}, True)
media.update_one({'name': 'giphy'}, {'$set': {'data': giphy}}, True)
