class Build(object):
def __init__(self):
pool = Pool(processes=2)
self.graph = getGraph()
files = findFiles(opts)
self.progressQueue = Queue()
reporter = Process(target=ProgressReport,
args=(self.progressQueue, len(files)))
reporter.start()
result = pool.map(self.cacheFile, enumerate(files), chunksize=5)
self.progressQueue.put('END')
log.info("finished, %s results", len(result))
reporter.join()
def cacheFile(self, (i, filename)):
log.debug("cacheFile %s" % filename)
try:
uri = photoUri(filename)
m = MediaResource(self.graph, uri)
# video extension but not isVideo? where do we correct that?
m.cacheAll()
except Exception:
traceback.print_exc()
self.progressQueue.put("%s" % filename)
def run(self):
in_queue, out_queue = Queue(), Queue()
for i in self.a:
in_queue.put(i)
def f(in_queue, out_queue):
while not in_queue.empty():
time.sleep(1)
out_queue.put(in_queue.get()+1)
pool = Pool(4, f, (in_queue, out_queue))
self.b = []
while len(self.b) < len(self.a):
if not out_queue.empty():
self.b.append(out_queue.get())
pool.terminate()
class FrontServer():
def __init__(self):
self.loginSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.port = settings.port
self.loginQueue = ThreadQue()
def start(self):
self.loginSocket.bind(('localhost',self.port))
self.loginSocket.listen(200)
for x in range(10):
thread = threading.Thread(target=loginWorker, args=(self.loginQueue,))
thread.daemon = True
thread.start()
requestQueue = Queue()
Game.instance().requestQueue = requestQueue
updater = ServerUpdater()
receiver = RequestReader()
cookieMover = CookieMover()
updateThread = threading.Thread(target=updater.run)
updateThread.daemon = True
receiverThread = threading.Thread(target=receiver.run, args=(requestQueue,))
receiverThread.daemon = True
moverThread = threading.Thread(target=cookieMover.run)
moverThread.daemon = True
updateThread.start()
receiverThread.start()
moverThread.start()
try:
print('Server started listening')
while True:
clientSocket, clientAddress = self.loginSocket.accept()
print('Connection made with front server')
self.loginQueue.put(clientSocket)
except (KeyboardInterrupt, SystemExit, socket.error):
self.loginSocket.shutdown(socket.SHUT_RDWR)
self.loginSocket.close()
sys.exit()
class ExThread(Thread):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__status_queue = Queue()
def run(self, *args, **kwargs):
try:
super().run(*args, **kwargs)
except Exception as ex:
#except_type, except_class, tb = sys.exc_info()
self.__status_queue.put(ex)
self.__status_queue.put(None)
def wait_for_exc_info(self):
return self.__status_queue.get()
def join(self):
ex = self.wait_for_exc_info()
if ex is None:
return
else:
raise ex
class Server():
def __init__(self, host, port, basedir):
self.host = host
self.port = port
self.basedir = self.__formatBaseDir(basedir)
self.socket = s.socket(s.AF_INET, s.SOCK_STREAM)
self.q = ThreadQue()
def start(self):
for x in range(100):
thread = threading.Thread(target=responder, args=(self.q,))
thread.daemon = True
thread.start()
try:
self.socket.bind((self.host, self.port))
self.socket.listen(500)
print('Server starting on port '+str(self.port)+' from base directory '+self.basedir)
while True:
clientSocket, clientAddress = self.socket.accept()
self.q.put((clientSocket, self.basedir))
except (KeyboardInterrupt, s.error):
try:
self.socket.shutdown(s.SHUT_RDWR)
self.socket.close()
print('Server socket closed')
sys.exit()
except s.error:
print('Server port busy. Try another port.')
sys.exit()
def __formatBaseDir(self, basedir):
#Trim trailing slash for convenience
if basedir[len(basedir) - 1] is os.path.sep:
basedir = basedir[:-1]
return basedir
class tizyoutubeproxy(object):
"""A class that accesses YouTube, retrieves stream URLs and creates and manages
a playback queue.
"""
def __init__(self):
self.queue = list()
self.queue_index = -1
self.play_queue_order = list()
self.play_modes = TizEnumeration(["NORMAL", "SHUFFLE"])
self.current_play_mode = self.play_modes.NORMAL
self.now_playing_stream = None
# Create multiprocess queues
self.task_queue = Queue()
self.done_queue = Queue()
# Workers
self.workers = list()
def set_play_mode(self, mode):
""" Set the playback mode.
:param mode: current valid values are "NORMAL" and "SHUFFLE"
"""
self.current_play_mode = getattr(self.play_modes, mode)
self.__update_play_queue_order()
def enqueue_audio_stream(self, arg):
"""Add the audio stream of a YouTube video to the
playback queue.
:param arg: a search string
"""
logging.info('arg : %s', arg)
try:
yt_video = pafy.new(arg)
yt_audio = yt_video.getbestaudio(preftype="webm")
if not yt_audio:
raise ValueError(str("No WebM audio stream for : %s" % arg))
yt_info = VideoInfo(ytid=arg, title=yt_audio.title)
self.add_to_playback_queue(audio=yt_audio, video=yt_video, info=yt_info)
self.__update_play_queue_order()
except ValueError:
raise ValueError(str("Video not found : %s" % arg))
def enqueue_audio_playlist(self, arg):
"""Add all audio streams in a YouTube playlist to the playback queue.
:param arg: a YouTube playlist id
"""
logging.info('arg : %s', arg)
try:
count = len(self.queue)
playlist = pafy.get_playlist2(arg)
if len(playlist) > 0:
for yt_video in playlist:
self.add_to_playback_queue(video=yt_video, \
info=VideoInfo(ytid=yt_video.videoid, \
title=yt_video.title))
if count == len(self.queue):
raise ValueError
self.__update_play_queue_order()
except ValueError:
raise ValueError(str("Playlist not found : %s" % arg))
def enqueue_audio_search(self, arg):
"""Search YouTube and add the audio streams to the
playback queue.
:param arg: a search string
"""
logging.info('arg : %s', arg)
try:
query = generate_search_query(arg)
wdata = pafy.call_gdata('search', query)
wdata2 = wdata
count = 0
while True:
for track_info in get_tracks_from_json(wdata2):
self.add_to_playback_queue(info=track_info)
count += 1
if count > 100:
break
if not wdata2.get('nextPageToken'):
break
query['pageToken'] = wdata2['nextPageToken']
wdata2 = pafy.call_gdata('search', query)
self.__update_play_queue_order()
except ValueError:
raise ValueError(str("Could not find any mixes : %s" % arg))
def enqueue_audio_mix(self, arg, feelinglucky=True):
"""Obtain a YouTube mix associated to a given video id or url and add all audio
streams in the mix playlist to the playback queue.
:param arg: a YouTube video id
:param feelinglucky: If True, it will perform another YouTube search to find
alternatives if the original mix cannot be found.
"""
logging.info('arg : %s', arg)
yt_video = None
try:
count = len(self.queue)
yt_video = pafy.new(arg)
playlist = yt_video.mix
if len(playlist) > 0:
for yt_video in playlist:
video_id = yt_video.videoid
video_title = yt_video.title
yt_info = VideoInfo(ytid=video_id, title=video_title)
self.add_to_playback_queue(video=yt_video, info=yt_info)
if count == len(self.queue):
raise ValueError
self.__update_play_queue_order()
except IndexError:
if not feelinglucky:
raise ValueError
else:
print_wrn("[YouTube] Could not find a mix for '{0}'. "\
"Searching YouTube instead. Feeling lucky?." \
.format(arg.encode('utf-8')))
if yt_video.title:
self.enqueue_audio_search(yt_video.title)
else:
self.enqueue_audio_stream(arg)
def enqueue_audio_mix_search(self, arg):
"""Obtain a YouTube mix associated to a given textual search and add all the
audio streams in the mix playlist to the playback queue.
:param arg: a search string
"""
logging.info('arg : %s', arg)
try:
query = generate_search_query(arg)
wdata = pafy.call_gdata('search', query)
wdata2 = wdata
count = len(self.queue)
for track_info in get_tracks_from_json(wdata2):
if track_info and track_info.ytid:
try:
self.enqueue_audio_mix(track_info.ytid, feelinglucky=False)
break
except ValueError:
logging.info('Could not find a mix. Trying another video')
if count == len(self.queue):
raise ValueError
except ValueError:
raise ValueError(str("Could not find any mixes : %s" % arg))
def enqueue_audio_channel_uploads(self, arg):
"""Add all audio streams in a YouTube channel to the playback queue.
:param arg: a YouTube channel url
"""
logging.info('arg : %s', arg)
try:
count = len(self.queue)
channel = pafy.get_channel(arg)
if channel:
for yt_video in channel.uploads:
self.add_to_playback_queue(video=yt_video, \
info=VideoInfo(ytid=yt_video.videoid, \
title=yt_video.title))
if count == len(self.queue):
raise ValueError
self.__update_play_queue_order()
except ValueError:
raise ValueError(str("Channel not found : %s" % arg))
def enqueue_audio_channel_playlist(self, channel_name, playlist_name):
"""Search a playlist within a channel and if found, adds all the audio streams
to the playback queue.
:param arg: a YouTube playlist id
"""
logging.info('args : %s - %s', channel_name, playlist_name)
try:
count = len(self.queue)
channel = pafy.get_channel(channel_name)
if channel:
pl_dict = dict()
pl_titles = list()
pl_name = ''
playlist = None
for pl in channel.playlists:
print_nfo("[YouTube] [Playlist] '{0}'." \
.format(to_ascii(pl.title)))
if fuzz.partial_ratio(playlist_name, pl.title) > 50:
pl_dict[pl.title] = pl
pl_titles.append(pl.title)
if len(pl_titles) > 1:
pl_name = process.extractOne(playlist_name, pl_titles)[0]
playlist = pl_dict[pl_name]
elif len(pl_titles) == 1:
pl_name = pl_titles[0]
playlist = pl_dict[pl_name]
if pl_name:
if pl_name.lower() != playlist_name.lower():
print_wrn("[YouTube] Playlist '{0}' not found. " \
"Playing '{1}' instead." \
.format(to_ascii(playlist_name), \
to_ascii(pl_name)))
for yt_video in playlist:
self.add_to_playback_queue(video=yt_video, \
info=VideoInfo(ytid=yt_video.videoid, \
title=yt_video.title))
if count == len(self.queue):
raise ValueError
self.__update_play_queue_order()
except ValueError:
raise ValueError(str("Channel not found : %s" % channel_name))
def current_audio_stream_title(self):
""" Retrieve the current stream's title.
"""
stream = self.now_playing_stream
title = ''
if stream:
title = to_ascii(stream['a'].title).encode("utf-8")
return title
def current_audio_stream_author(self):
""" Retrieve the current stream's author.
"""
stream = self.now_playing_stream
author = ''
if stream:
author = to_ascii(stream['v'].author).encode("utf-8")
return author
def current_audio_stream_file_size(self):
""" Retrieve the current stream's file size.
"""
stream = self.now_playing_stream
size = 0
if stream:
size = stream['a'].get_filesize()
return size
def current_audio_stream_duration(self):
""" Retrieve the current stream's duration.
"""
stream = self.now_playing_stream
duration = ''
if stream:
duration = to_ascii(stream['v'].duration).encode("utf-8")
return duration
def current_audio_stream_bitrate(self):
""" Retrieve the current stream's bitrate.
"""
stream = self.now_playing_stream
bitrate = ''
if stream:
bitrate = stream['a'].bitrate
return bitrate
def current_audio_stream_view_count(self):
""" Retrieve the current stream's view count.
"""
stream = self.now_playing_stream
viewcount = 0
if stream:
viewcount = stream['v'].viewcount
return viewcount
def current_audio_stream_description(self):
""" Retrieve the current stream's description.
"""
stream = self.now_playing_stream
description = ''
if stream:
description = to_ascii(stream['v'].description).encode("utf-8")
return description
def current_audio_stream_file_extension(self):
""" Retrieve the current stream's file extension.
"""
stream = self.now_playing_stream
file_extension = ''
if stream:
file_extension = to_ascii(stream['a'].extension).encode("utf-8")
return file_extension
def current_audio_stream_video_id(self):
""" Retrieve the current stream's video id.
"""
stream = self.now_playing_stream
video_id = ''
if stream:
video_id = to_ascii(stream['i'].ytid).encode("utf-8")
return video_id
def current_audio_stream_published(self):
""" Retrieve the current stream's upload date and time.
"""
stream = self.now_playing_stream
if stream:
published = to_ascii(stream['v'].published).encode("utf-8")
return published
def current_audio_stream_queue_index_and_queue_length(self):
""" Retrieve index in the queue (starting from 1) of the current stream and the
length of the playback queue.
"""
return self.queue_index + 1, len(self.queue)
def clear_queue(self):
""" Clears the playback queue.
"""
self.queue = list()
self.queue_index = -1
def remove_current_url(self):
"""Remove the currently active url from the playback queue.
"""
logging.info("")
if len(self.queue) and self.queue_index:
stream = self.queue[self.queue_index]
print_nfo("[YouTube] [Stream] '{0}' removed." \
.format(to_ascii(stream['i'].title).encode("utf-8")))
del self.queue[self.queue_index]
self.queue_index -= 1
if self.queue_index < 0:
self.queue_index = 0
self.__update_play_queue_order()
def next_url(self):
""" Retrieve the url of the next stream in the playback queue.
"""
logging.info("")
try:
if len(self.queue):
self.queue_index += 1
if (self.queue_index < len(self.queue)) \
and (self.queue_index >= 0):
next_stream = self.queue[self.play_queue_order \
[self.queue_index]]
return self.__retrieve_stream_url(next_stream, self.queue_index).rstrip()
else:
self.queue_index = -1
return self.next_url()
else:
return ''
except (KeyError, AttributeError):
# TODO: We don't remove this for now
# del self.queue[self.queue_index]
logging.info("KeyError, or AttributeError exception")
return self.next_url()
except (IOError):
# Remove this video
del self.queue[self.queue_index]
logging.info("IOError exception")
return self.next_url()
def prev_url(self):
""" Retrieve the url of the previous stream in the playback queue.
"""
logging.info("")
try:
if len(self.queue):
self.queue_index -= 1
if (self.queue_index < len(self.queue)) \
and (self.queue_index >= 0):
prev_stream = self.queue[self.play_queue_order \
[self.queue_index]]
return self.__retrieve_stream_url(prev_stream, self.queue_index).rstrip()
else:
self.queue_index = len(self.queue)
return self.prev_url()
else:
return ''
except (KeyError, AttributeError):
# TODO: We don't remove this for now
# del self.queue[self.queue_index]
logging.info("exception")
return self.prev_url()
except (IOError):
# Remove this video
del self.queue[self.queue_index]
logging.info("IOError exception")
return self.next_url()
def __update_play_queue_order(self):
""" Update the queue playback order.
A sequential order is applied if the current play mode is "NORMAL" or a
random order if current play mode is "SHUFFLE"
"""
total_streams = len(self.queue)
if total_streams:
if not len(self.play_queue_order):
# Create a sequential play order, if empty
self.play_queue_order = range(total_streams)
if self.current_play_mode == self.play_modes.SHUFFLE:
random.shuffle(self.play_queue_order)
print_nfo("[YouTube] [Streams in queue] '{0}'." \
.format(total_streams))
def __retrieve_stream_url(self, stream, queue_index):
""" Retrieve a stream url
"""
try:
if not len(self.workers):
for _ in range(WORKER_PROCESSES):
proc = Process(target=obtain_stream, \
args=(self.task_queue, \
self.done_queue)).start()
self.workers.append(proc)
while not self.done_queue.empty():
stream = self.done_queue.get()
self.queue[stream['q']] = stream
stream = self.queue[queue_index]
if not stream.get('v') or not stream.get('a'):
logging.info("ytid : %s", stream['i'].ytid)
video = stream.get('v')
if not video:
video = pafy.new(stream['i'].ytid)
audio = video.getbestaudio(preftype="webm")
if not audio:
logging.info("no suitable audio found")
raise AttributeError()
stream.update({'a': audio, 'v': video})
# streams = stream.get('v').audiostreams[::-1]
# pprint.pprint(streams)
# dump_stream_info(streams)
self.now_playing_stream = stream
return stream['a'].url.encode("utf-8")
except AttributeError:
logging.info("Could not retrieve the stream url!")
raise
def add_to_playback_queue(self, audio=None, video=None, info=None):
""" Add to the playback queue. """
if audio:
print_nfo("[YouTube] [Stream] '{0}' [{1}]." \
.format(to_ascii(audio.title).encode("utf-8"), \
to_ascii(audio.extension)))
if info:
print_nfo("[YouTube] [Stream] '{0}'." \
.format(to_ascii(info.title).encode("utf-8")))
queue_index = len(self.queue)
self.task_queue.put(dict(a=audio, v=video, i=info, q=queue_index))
self.queue.append(
dict(a=audio, v=video, i=info, q=queue_index))
class seed(Process):
"""
Consumer thread that takes fuzzable requests from a Queue that's populated
by the crawl plugins and identified vulnerabilities by performing various
requests.
"""
def __init__(self, w3af_core):
"""
:param w3af_core: The w3af core that we'll use for status reporting
"""
super(seed, self).__init__(name='SeedController')
self.name = 'Seed'
self._w3af_core = w3af_core
# See documentation in the property below
self._out_queue = Queue()
def get_result(self, timeout=0.5):
return self._out_queue.get_nowait()
def has_pending_work(self):
return self._out_queue.qsize() != 0
def join(self):
return
def terminate(self):
return
def seed_output_queue(self, target_urls):
"""
Create the first fuzzable request objects based on the targets and put
them in the output Queue.
This will start the whole discovery process, since plugins are going
to consume from that Queue and then put their results in it again in
order to continue discovering.
"""
# We only want to scan pages that are in current scope
in_scope = lambda fr: fr.get_url().get_domain() == url.get_domain()
for url in target_urls:
# batman-fix already done verify, no need below codes
# try:
# #
# # GET the initial target URLs in order to save them
# # in a list and use them as our bootstrap URLs
# #
# response = self._w3af_core.uri_opener.GET(url, cache=True)
# except ScanMustStopException, w3:
# om.out.error('The target server is unreachable. Stopping.')
# raise w3
# except HTTPRequestException, hre:
# msg = 'The target URL: "%s" is unreachable. Exception: "%s".'
# om.out.error(msg % (url, hre))
# except Exception, e:
# msg = 'The target URL: "%s" is unreachable because of an' \
# ' unhandled exception. Error description: "%s". See' \
# ' debug output for more information.\n' \
# 'Traceback for this error:\n%s'
# om.out.error(msg % (url, e, traceback.format_exc()))
# else:
# _seed = FuzzableRequest(response.get_uri())
_seed = FuzzableRequest(url)
if in_scope(_seed):
self._out_queue.put((None, None, _seed))
# Update the set that lives in the KB
kb.kb.add_fuzzable_request(_seed)
self._out_queue.put(POISON_PILL)
# encoding=utf-8
'''
@author: liangzai
pip install multiprocessing
'''
from multiprocessing.dummy import Pool, Queue
import threading
import time
# 队列
tasks = Queue()
for i in range(9):
tasks.put(str(i)*2)
def main(name):
while True:
time.sleep(1)
if name.empty():
print("name is over.")
break
else:
thread = threading.current_thread().getName()
print("线程：%s 打印：%s" % (thread, name.get()))
# 启动四个线程
pool = Pool(4, main, (tasks,))
while True:
time.sleep(5)
def parallel_apply(func,
iterable,
workers,
max_queue_size,
callback=None,
dummy=False):
"""多进程或多线程地将func应用到iterable的每个元素中。
注意这个apply是异步且无序的，也就是说依次输入a,b,c，但是
输出可能是func(c), func(a), func(b)。
参数：
dummy: False是多进程/线性，True则是多线程/线性；
callback: 处理单个输出的回调函数。
"""
if dummy:
from multiprocessing.dummy import Pool, Queue
else:
from multiprocessing import Pool, Queue
in_queue, out_queue = Queue(max_queue_size), Queue()
def worker_step(in_queue, out_queue):
# 单步函数包装成循环执行
while True:
i, d = in_queue.get()
r = func(d)
out_queue.put((i, r))
# 启动多进程/线程
pool = Pool(workers, worker_step, (in_queue, out_queue))
if callback is None:
results = []
# 后处理函数
def process_out_queue():
out_count = 0
for _ in range(out_queue.qsize()):
i, d = out_queue.get()
out_count += 1
if callback is None:
results.append((i, d))
else:
callback(d)
return out_count
# 存入数据，取出结果
in_count, out_count = 0, 0
for i, d in enumerate(iterable):
in_count += 1
while True:
try:
in_queue.put((i, d), block=False)
break
except six.moves.queue.Full:
out_count += process_out_queue()
if in_count % max_queue_size == 0:
out_count += process_out_queue()
while out_count != in_count:
out_count += process_out_queue()
pool.terminate()
if callback is None:
results = sorted(results, key=lambda r: r[0])
return [r[1] for r in results]
class tizyoutubeproxy(object):
"""A class that accesses YouTube, retrieves stream URLs and creates and manages
a playback queue.
"""
def __init__(self, api_key=API_KEY):
self.queue = list()
self.queue_index = -1
self.play_queue_order = list()
self.play_modes = TizEnumeration(["NORMAL", "SHUFFLE"])
self.current_play_mode = self.play_modes.NORMAL
self.now_playing_stream = None
# Create multiprocess queues
self.task_queue = Queue()
self.done_queue = Queue()
# Workers
self.workers = list()
self.api_key = api_key if api_key != "" else API_KEY
pafy.set_api_key(self.api_key)
def set_play_mode(self, mode):
""" Set the playback mode.
:param mode: current valid values are "NORMAL" and "SHUFFLE"
"""
self.current_play_mode = getattr(self.play_modes, mode)
self._update_play_queue_order()
def enqueue_audio_stream(self, arg):
"""Add the audio stream of a YouTube video to the
playback queue.
:param arg: a search string
"""
logging.info("arg : %s", arg)
try:
print_msg("[YouTube] [Audio strean] : '{0}'. ".format(arg))
yt_search = MEMORY.cache(run_youtube_search)
yt_video = yt_search(arg)
yt_audio = yt_video.getbestaudio(preftype="webm")
if not yt_audio:
raise ValueError(str("No WebM audio stream for : %s" % arg))
yt_info = VideoInfo(ytid=arg, title=yt_audio.title)
self._add_to_playback_queue(audio=yt_audio, video=yt_video, info=yt_info)
self._update_play_queue_order()
except ValueError:
raise ValueError(str("Video not found : %s" % arg))
def enqueue_audio_playlist(self, arg):
"""Add all audio streams in a YouTube playlist to the playback queue.
:param arg: a YouTube playlist id
"""
logging.info("arg : %s", arg)
try:
print_msg("[YouTube] [Audio playlist] : '{0}'. ".format(arg))
count = len(self.queue)
yt_pl_search = MEMORY.cache(run_youtube_playlist_search)
playlist = yt_pl_search(arg)
if len(playlist) > 0:
for yt_video in playlist:
self._add_to_playback_queue(
video=yt_video,
info=VideoInfo(ytid=yt_video.videoid, title=yt_video.title),
)
if count == len(self.queue):
raise ValueError
self._update_play_queue_order()
except ValueError:
raise ValueError(str("Playlist not found : %s" % arg))
def enqueue_audio_search(self, arg):
"""Search YouTube and add the audio streams to the
playback queue.
:param arg: a search string
"""
logging.info("arg : %s", arg)
try:
print_msg("[YouTube] [Audio search] : '{0}'. ".format(arg))
yt_dt_search = MEMORY.cache(run_youtube_data_search)
query = generate_search_query(arg, self.api_key)
wdata = yt_dt_search("search", query)
wdata2 = wdata
count = 0
while True:
for track_info in get_tracks_from_json(wdata2):
self._add_to_playback_queue(info=track_info)
count += 1
if count > 100:
break
if not wdata2.get("nextPageToken"):
break
query["pageToken"] = wdata2["nextPageToken"]
wdata2 = yt_dt_search("search", query)
self._update_play_queue_order()
except ValueError:
raise ValueError(str("Could not find any mixes : %s" % arg))
def enqueue_audio_mix(self, arg, feelinglucky=True):
"""Obtain a YouTube mix associated to a given video id or url and add all audio
streams in the mix playlist to the playback queue.
:param arg: a YouTube video id
:param feelinglucky: If True, it will perform another YouTube search to find
alternatives if the original mix cannot be found.
"""
logging.info("arg : %s", arg)
yt_video = None
try:
print_msg("[YouTube] [Audio mix] : '{0}'. ".format(arg))
count = len(self.queue)
yt_search = MEMORY.cache(run_youtube_search)
yt_video = yt_search(arg)
playlist = yt_video.mix
if len(playlist) > 0:
for yt_video in playlist:
video_id = yt_video.videoid
video_title = yt_video.title
yt_info = VideoInfo(ytid=video_id, title=video_title)
self._add_to_playback_queue(video=yt_video, info=yt_info)
if count == len(self.queue):
raise ValueError
self._update_play_queue_order()
except IndexError:
if not feelinglucky:
raise ValueError
else:
print_adv(
"[YouTube] Could not find a mix for '{0}'. "
"Searching YouTube instead. Feeling lucky?.".format(
arg.encode("utf-8")
)
)
if yt_video.title:
self.enqueue_audio_search(yt_video.title)
else:
self.enqueue_audio_stream(arg)
def enqueue_audio_mix_search(self, arg):
"""Obtain a YouTube mix associated to a given textual search and add all the
audio streams in the mix playlist to the playback queue.
:param arg: a search string
"""
logging.info("arg : %s", arg)
try:
print_msg("[YouTube] [Audio mix search] : '{0}'. ".format(arg))
yt_dt_search = MEMORY.cache(run_youtube_data_search)
wdata = yt_dt_search("search", generate_search_query(arg, self.api_key))
wdata2 = wdata
count = len(self.queue)
for track_info in get_tracks_from_json(wdata2):
if track_info and track_info.ytid:
try:
self.enqueue_audio_mix(track_info.ytid, feelinglucky=False)
break
except ValueError:
logging.info("Could not find a mix. Trying another video")
if count == len(self.queue):
raise ValueError
except ValueError:
raise ValueError(str("Could not find any mixes : %s" % arg))
def enqueue_audio_channel_uploads(self, arg):
"""Add all audio streams in a YouTube channel to the playback queue.
:param arg: a YouTube channel url
"""
logging.info("arg : %s", arg)
try:
print_msg("[YouTube] [Audio channel uploads] : '{0}'. ".format(arg))
count = len(self.queue)
yt_ch_search = MEMORY.cache(run_youtube_channel_search)
channel = yt_ch_search(arg)
if channel:
for yt_video in channel.uploads:
self._add_to_playback_queue(
video=yt_video,
info=VideoInfo(ytid=yt_video.videoid, title=yt_video.title),
)
if count == len(self.queue):
raise ValueError
self._update_play_queue_order()
except ValueError:
raise ValueError(str("Channel not found : %s" % arg))
def enqueue_audio_channel_playlist(self, channel_name, playlist_name):
"""Search a playlist within a channel and if found, adds all the audio streams
to the playback queue.
:param arg: a YouTube playlist id
"""
logging.info("args : %s - %s", channel_name, playlist_name)
try:
print_msg(
"[YouTube] [Audio channel playlist] : '{0} - {1}'. ".format(
channel_name, playlist_name
)
)
count = len(self.queue)
yt_ch_search = MEMORY.cache(run_youtube_channel_search)
channel = yt_ch_search(channel_name)
if channel:
pl_dict = dict()
pl_titles = list()
pl_name = ""
playlist = None
for pl in channel.playlists:
print_nfo("[YouTube] [Playlist] '{0}'.".format(to_ascii(pl.title)))
if fuzz.partial_ratio(playlist_name, pl.title) > 50:
pl_dict[pl.title] = pl
pl_titles.append(pl.title)
if len(pl_titles) > 1:
pl_name = process.extractOne(playlist_name, pl_titles)[0]
playlist = pl_dict[pl_name]
elif len(pl_titles) == 1:
pl_name = pl_titles[0]
playlist = pl_dict[pl_name]
if pl_name:
if pl_name.lower() != playlist_name.lower():
print_adv(
"[YouTube] Playlist '{0}' not found. "
"Playing '{1}' instead.".format(
to_ascii(playlist_name), to_ascii(pl_name)
)
)
for yt_video in playlist:
self._add_to_playback_queue(
video=yt_video,
info=VideoInfo(ytid=yt_video.videoid, title=yt_video.title),
)
if count == len(self.queue):
raise ValueError
self._update_play_queue_order()
except ValueError:
raise ValueError(str("Channel not found : %s" % channel_name))
def current_audio_stream_title(self):
""" Retrieve the current stream's title.
"""
stream = self.now_playing_stream
title = ""
if stream:
title = to_ascii(stream["a"].title)
return title
def current_audio_stream_author(self):
""" Retrieve the current stream's author.
"""
stream = self.now_playing_stream
author = ""
if stream:
author = to_ascii(stream["v"].author)
return author
def current_audio_stream_file_size(self):
""" Retrieve the current stream's file size.
"""
stream = self.now_playing_stream
size = 0
if stream:
size = stream["a"].get_filesize()
return size
def current_audio_stream_duration(self):
""" Retrieve the current stream's duration.
"""
stream = self.now_playing_stream
duration = ""
if stream:
duration = to_ascii(stream["v"].duration)
return duration
def current_audio_stream_bitrate(self):
""" Retrieve the current stream's bitrate.
"""
stream = self.now_playing_stream
bitrate = ""
if stream:
bitrate = stream["a"].bitrate
return bitrate
def current_audio_stream_view_count(self):
""" Retrieve the current stream's view count.
"""
stream = self.now_playing_stream
viewcount = 0
if stream:
viewcount = stream["v"].viewcount
return viewcount
def current_audio_stream_description(self):
""" Retrieve the current stream's description.
"""
stream = self.now_playing_stream
description = ""
if stream:
description = to_ascii(stream["v"].description)
return description
def current_audio_stream_file_extension(self):
""" Retrieve the current stream's file extension.
"""
stream = self.now_playing_stream
file_extension = ""
if stream:
file_extension = to_ascii(stream["a"].extension)
return file_extension
def current_audio_stream_video_id(self):
""" Retrieve the current stream's video id.
"""
stream = self.now_playing_stream
video_id = ""
if stream:
video_id = to_ascii(stream["i"].ytid)
return video_id
def current_audio_stream_published(self):
""" Retrieve the current stream's upload date and time.
"""
stream = self.now_playing_stream
if stream:
published = to_ascii(stream["v"].published)
return published
def current_audio_stream_queue_index_and_queue_length(self):
""" Retrieve index in the queue (starting from 1) of the current stream and the
length of the playback queue.
"""
return self.play_queue_order[self.queue_index] + 1, len(self.queue)
def clear_queue(self):
""" Clears the playback queue.
"""
self.queue = list()
self.queue_index = -1
def remove_current_url(self):
"""Remove the currently active url from the playback queue.
"""
logging.info("")
if len(self.queue) and self.queue_index:
stream = self.queue[self.queue_index]
print_nfo(
"[YouTube] [Stream] '{0}' removed.".format(to_ascii(stream["i"].title))
)
del self.queue[self.queue_index]
self.queue_index -= 1
if self.queue_index < 0:
self.queue_index = 0
self._update_play_queue_order()
def next_url(self):
""" Retrieve the url of the next stream in the playback queue.
"""
logging.info("")
try:
if len(self.queue):
self.queue_index += 1
if (self.queue_index < len(self.queue)) and (self.queue_index >= 0):
next_stream = self.queue[self.play_queue_order[self.queue_index]]
return self._retrieve_stream_url(
next_stream, self.play_queue_order[self.queue_index]
).rstrip()
else:
self.queue_index = -1
return self.next_url()
else:
return ""
except (KeyError, AttributeError):
# TODO: We don't remove this for now
# del self.queue[self.queue_index]
logging.info("KeyError, or AttributeError exception")
return self.next_url()
except (IOError):
# Remove this video
del self.queue[self.queue_index]
logging.info("IOError exception")
return self.next_url()
def prev_url(self):
""" Retrieve the url of the previous stream in the playback queue.
"""
logging.info("")
try:
if len(self.queue):
self.queue_index -= 1
if (self.queue_index < len(self.queue)) and (self.queue_index >= 0):
prev_stream = self.queue[self.play_queue_order[self.queue_index]]
return self._retrieve_stream_url(
prev_stream, self.play_queue_order[self.queue_index]
).rstrip()
else:
self.queue_index = len(self.queue)
return self.prev_url()
else:
return ""
except (KeyError, AttributeError):
# TODO: We don't remove this for now
# del self.queue[self.queue_index]
logging.info("exception")
return self.prev_url()
except (IOError):
# Remove this video
del self.queue[self.queue_index]
logging.info("IOError exception")
return self.next_url()
def _update_play_queue_order(self):
""" Update the queue playback order.
A sequential order is applied if the current play mode is "NORMAL" or a
random order if current play mode is "SHUFFLE"
"""
total_streams = len(self.queue)
if total_streams:
if not len(self.play_queue_order):
# Create a sequential play order, if empty
self.play_queue_order = list(range(total_streams))
if self.current_play_mode == self.play_modes.SHUFFLE:
random.shuffle(self.play_queue_order)
print_nfo("[YouTube] [Streams in queue] '{0}'.".format(total_streams))
def _retrieve_stream_url(self, stream, queue_index):
""" Retrieve a stream url
"""
try:
if not len(self.workers):
for _ in range(WORKER_PROCESSES):
proc = Process(
target=obtain_stream, args=(self.task_queue, self.done_queue)
).start()
self.workers.append(proc)
while not self.done_queue.empty():
stream = self.done_queue.get()
self.queue[stream["q"]] = stream
stream = self.queue[queue_index]
if not stream.get("v") or not stream.get("a"):
logging.info("ytid : %s", stream["i"].ytid)
video = stream.get("v")
if not video:
yt_search = MEMORY.cache(run_youtube_search)
video = yt_search(stream["i"].ytid)
audio = video.getbestaudio(preftype="webm")
if not audio:
logging.info("no suitable audio found")
raise AttributeError()
stream.update({"a": audio, "v": video})
# streams = stream.get('v').audiostreams[::-1]
# pprint.pprint(streams)
# dump_stream_info(streams)
self.now_playing_stream = stream
return stream["a"].url
except AttributeError:
logging.info("Could not retrieve the stream url!")
raise
def _add_to_playback_queue(self, audio=None, video=None, info=None):
""" Add to the playback queue. """
if audio:
print_nfo(
"[YouTube] [Stream] '{0}' [{1}].".format(
to_ascii(audio.title), to_ascii(audio.extension)
)
)
if info:
print_nfo("[YouTube] [Stream] '{0}'.".format(to_ascii(info.title)))
queue_index = len(self.queue)
self.task_queue.put(dict(a=audio, v=video, i=info, q=queue_index))
self.queue.append(dict(a=audio, v=video, i=info, q=queue_index))
