def for_genre(self, genre):
"""
Search tracks for a specific genre and add them to playlist
"""
sm_artists = []
# seed data
logger.debug(genre)
tracks = self.lfm.get_station(genre)
random.shuffle(tracks)
for track in tracks:
artist = track.split(' - ')[0]
# check station blacklist
if self.artist_blacklisted_for_genre(artist, genre):
continue
# check history /banned/ blacklist
if TrackNormalizer.is_locally_blacklisted(track):
logger.debug('Track %s is blacklisted using "ban" command', track)
continue
self.playlist_queue.put(track)
for atmp in self.similar_artists(artist, genre):
if not atmp in sm_artists:
# new one
sm_artists.append(atmp)
[self.playlist_queue.put(tr) for tr in self.lfm.get_top_tracks(atmp)[:3]]
# operate on dataset
result = sm_artists
for artist in sm_artists:
tmp = self.similar_artists(artist, genre)
for aname in tmp:
if not aname in sm_artists and not aname in result:
# new one
result.append(aname)
[self.playlist_queue.put(tr) for tr in self.lfm.get_top_tracks(aname)[:3]]
def background_listener(self):
"""
Run loop and wait for commands
TODO: Localize this
"""
msg = 'Vicki is listening'
self.tts.say_put(msg)
# TODO: Fix this using callback or something so that
# we do not record ourselves
time.sleep(3)
logger.warning(msg)
self.wake_up = False
while not self.shutdown:
if self.wake_up:
logger.warning('Wake word!')
run_hooks(None, self.recognition_hooks, 'on_recognition_start')
self.tts.say_put('Yes')
self.wake_up = False
else:
time.sleep(0.01)
continue
volume = self.player.player.get_volume()
self.player.player.set_volume(0)
logger.debug('recog start')
run_hooks(None, self.recognition_hooks, 'on_recognition_progress')
# command goes next
try:
with sr.Microphone() as source:
self.rec.adjust_for_ambient_noise(source)
audio = self.rec.listen(source, timeout=5)
except sr.WaitTimeoutError:
self.player.player.set_volume(volume)
continue
try:
result = self.rec.recognize_google(audio)
except sr.UnknownValueError:
msg = 'I could not understand audio'
self.tts.say_put(msg)
logger.warning(msg)
result = None
except sr.RequestError as e:
msg = 'Recognition error'
self.tts.say_put(msg)
logger.warning('{0}; {1}'.format(msg, e))
result = None
logger.debug('recog end')
run_hooks(None, self.recognition_hooks, 'on_recognition_end')
self.player.player.set_volume(volume)
if result:
result = result.lower() # pylint:disable=no-member
logger.debug('Putting %r into processing queue', repr(result))
# allow commands to be processed by player instance first
self.player.put(result)
# process local cmd
if Command(result).COMMAND == Command.SHUTDOWN:
logger.debug('Vicki.Listener.stop() called...')
self.stop()
logger.debug('Vicki.Listener exit')
def send_and_wait(self, message):
"""
Send message and wait for response
"""
uuid = uuid4()
full_msg = {'expires': int(time.time()) + self.TTL,
'uuid': uuid,
'message': message}
self.queue.put(full_msg)
exit_stamp = int(time.time()) + self.TTL
while int(time.time()) <= exit_stamp and not self.exit:
if not self.queue.empty():
full_msg = self.queue.get()
# purge expired messages
if int(time.time()) >= int(full_msg.get('expires')):
logger.debug('Message %s - %s expired, purging...', int(time.time()), full_msg)
continue
# get/check message
if full_msg.get('uuid') != uuid:
logger.debug('Message %s not for me %s, requeueing...', full_msg, uuid)
self.queue.put(full_msg)
else:
message = full_msg.get('message', '')
break
time.sleep(0.01)
return message
def download_hook(cls, response):
"""
YDL download hook
"""
if response['status'] == 'finished':
logger.debug('Done downloading, now converting ...')
cls.target_filename = response['filename']
def play(self, path, track, block=True):
"""
Play track using libvlc player
"""
try:
media = self.instance.media_new(path)
self.player.set_media(media)
self.player.play()
except Exception as exc:
run_hooks(self.argparser, self.player_hooks, 'on_playback_error', exception=exc)
return False
track_name = None
# allow playback to start
time.sleep(3)
while True:
if self.exit:
break
try:
time.sleep(0.5)
except KeyboardInterrupt:
pass
if not self.player.is_playing() and not self.paused:
break
if self.meta_or_track(track) != track_name:
track_name = self.meta_or_track(track)
self.current_track = track_name
if track_ok(track_name):
run_hooks(self.argparser, self.player_hooks, 'on_playback_start', path=path, track=track_name)
else:
logger.debug('Track %r matches blacklist %s, hooks not ran.', track_name, 'voiceplay.utils.track')
return True
def download(cls, trackname, url):
"""
Run track download, use cache to store data
"""
cache = MixedCache()
template = os.path.join(cls.cfg_data().get('cache_dir'), cache.track_to_hash(trackname)) + '.%(ext)s'
if isinstance(template, str) and sys.version_info.major == 2:
template = template.decode('utf-8')
verbose = logger.level == logging.DEBUG
ydl_opts = {'keepvideo': False, 'verbose': verbose, 'format': 'bestaudio/best',
'quiet': not verbose, 'outtmpl': template,
'postprocessors': [{'preferredcodec': 'mp3', 'preferredquality': '0',
'nopostoverwrites': True, 'key': 'FFmpegExtractAudio'}],
'logger': logger,
'progress_hooks': [cls.download_hook]}
# Try remote HTTP cache first
result = cache.get_from_cache(os.path.join(cls.cfg_data().get('cache_dir'), cache.track_to_hash(trackname)) + '.mp3')
if result:
return result
logger.debug('Using source url %s', url)
with YoutubeDL(ydl_opts) as ydl:
ydl.download([url])
audio_file = re.sub('\.[^\.]+$', '.mp3', cls.target_filename)
cache.copy_to_cache(audio_file)
return audio_file
def download_hook(cls, response):
"""
YDL download hook
"""
if response['status'] == 'finished':
logger.debug('Done downloading, now converting ...')
cls.target_filename = response['filename']
def send_and_wait(self, message):
"""
Send message and wait for response
"""
uuid = uuid4()
full_msg = {
'expires': int(time.time()) + self.TTL,
'uuid': uuid,
'message': message
}
self.queue.put(full_msg)
exit_stamp = int(time.time()) + self.TTL
while int(time.time()) <= exit_stamp and not self.exit:
if not self.queue.empty():
full_msg = self.queue.get()
# purge expired messages
if int(time.time()) >= int(full_msg.get('expires')):
logger.debug('Message %s - %s expired, purging...',
int(time.time()), full_msg)
continue
# get/check message
if full_msg.get('uuid') != uuid:
logger.debug('Message %s not for me %s, requeueing...',
full_msg, uuid)
self.queue.put(full_msg)
else:
message = full_msg.get('message', '')
break
time.sleep(0.01)
return message
def run_player_tasks(self, message):
"""
Run player tasks
"""
ran = False
for task in self.player_tasks:
for regexp in task.__regexp__:
if re.match(regexp, message, re.I) is not None:
start = time.time()
ran = True
task.prefetch_callback = self.add_to_prefetch_q
task.argparser = self._argparser
task.get_exit = self.get_exit
task.player = self.player
task.tts = self.tts
th = threading.Thread(target=task.process,
args=(regexp, message))
th.daemon = True
th.start()
self.task_pool.append(th)
logger.debug('Task %r completed in %ss', task,
'{0:.2f}'.format(int(time.time()) - start))
break
if ran:
break
return ran
def background_listener(self):
"""
Run loop and wait for commands
TODO: Localize this
"""
msg = 'Vicki is listening'
self.tts.say_put(msg)
# TODO: Fix this using callback or something so that
# we do not record ourselves
time.sleep(3)
logger.warning(msg)
self.wake_up = False
while not self.shutdown:
if self.wake_up:
logger.warning('Wake word!')
run_hooks(None, self.recognition_hooks, 'on_recognition_start')
self.tts.say_put('Yes')
self.wake_up = False
else:
time.sleep(0.01)
continue
volume = self.player.player.get_volume()
self.player.player.set_volume(0)
logger.debug('recog start')
run_hooks(None, self.recognition_hooks, 'on_recognition_progress')
# command goes next
try:
with sr.Microphone() as source:
self.rec.adjust_for_ambient_noise(source)
audio = self.rec.listen(source, timeout=5)
except sr.WaitTimeoutError:
self.player.player.set_volume(volume)
continue
try:
result = self.rec.recognize_google(audio)
except sr.UnknownValueError:
msg = 'I could not understand audio'
self.tts.say_put(msg)
logger.warning(msg)
result = None
except sr.RequestError as e:
msg = 'Recognition error'
self.tts.say_put(msg)
logger.warning('{0}; {1}'.format(msg, e))
result = None
logger.debug('recog end')
run_hooks(None, self.recognition_hooks, 'on_recognition_end')
self.player.player.set_volume(volume)
if result:
result = result.lower() # pylint:disable=no-member
logger.debug('Putting %r into processing queue', repr(result))
# allow commands to be processed by player instance first
self.player.put(result)
# process local cmd
if Command(result).COMMAND == Command.SHUTDOWN:
logger.debug('Vicki.Listener.stop() called...')
self.stop()
logger.debug('Vicki.Listener exit')
def start(cls, debug=False):
"""
Start player
"""
logger.debug('VLCPlayer.start.argparser: %r', cls.argparser)
cls.debug = debug
if not cls.player:
cls.player = VLCInstance(argparser=cls.argparser, debug=debug)
def register(self):
"""
Register signal traps
"""
logger.debug('%s: %s: %s', self.__class__.__name__, 'my pid is', os.getpid())
signal.signal(signal.SIGHUP, self.receive_signal)
signal.signal(signal.SIGUSR1, self.receive_signal)
signal.signal(signal.SIGUSR2, self.receive_signal)
def start(cls, debug=False):
"""
Start player
"""
logger.debug('VLCPlayer.start.argparser: %r', cls.argparser)
cls.debug = debug
if not cls.player:
cls.player = VLCInstance(argparser=cls.argparser, debug=debug)
def start(self):
"""
Start TTS as a separate thread
"""
logger.debug('Starting TTS engine...')
self.threads = ThreadGroup()
self.threads.targets = [self.poll_loop]
self.threads.start_all()
def upload(self, file_name):
"""
Upload file to Dropbox
"""
if not self.health_check():
return
fpath = os.path.join(os.path.sep, os.path.basename(file_name))
self.dbx.files_upload(open(file_name, 'r').read(), fpath)
logger.debug('DBox File path: %s', fpath)
def copy_to_cache(cls, target_filename):
"""
Transfer file to cache (if it is not cached, of course)"
"""
is_cached = cls.is_remote_cached(target_filename)
if not is_cached:
cache = cls.CACHE_BACKEND()
cache.upload(target_filename)
logger.debug('File %r was uploaded to %r', target_filename, cls.CACHE_BACKEND)
def upload(self, file_name):
"""
Upload file to Dropbox
"""
if not self.health_check():
return
fpath = os.path.join(os.path.sep, os.path.basename(file_name))
self.dbx.files_upload(open(file_name, 'r').read(), fpath)
logger.debug('DBox File path: %s', fpath)
def search(self, query=''):
"""
Search Dropbox application folder
"""
files = []
for entry in self.dbx.files_list_folder(query).entries:
files.append([entry.name, entry.id])
logger.debug('DBox: found %s files', len(files))
return files
def search(self, query=''):
"""
Search Dropbox application folder
"""
files = []
for entry in self.dbx.files_list_folder(query).entries:
files.append([entry.name, entry.id])
logger.debug('DBox: found %s files', len(files))
return files
def put_message(self, uuid, message):
"""
Put message into queue
"""
full_msg = {'expires': int(time.time()) + self.TTL,
'uuid': uuid,
'message': message}
logger.debug('SQD put: %r', message)
self.queue.put(full_msg)
def register(self):
"""
Register signal traps
"""
logger.debug('%s: %s: %s', self.__class__.__name__, 'my pid is',
os.getpid())
signal.signal(signal.SIGHUP, self.receive_signal)
signal.signal(signal.SIGUSR1, self.receive_signal)
signal.signal(signal.SIGUSR2, self.receive_signal)
def send_traceback(exc_info, fname):
"""
Simple bugtracker submission method
"""
from voiceplay.config import Config
url = Config.cfg_data().get('bugtracker_url')
data = exc2encode(exc_info, fname)
r = requests.post(url, data=data)
logger.debug('Crashlog sent to %s with response code %s', url, r.status_code)
def send_traceback(exc_info, fname):
"""
Simple bugtracker submission method
"""
from voiceplay.config import Config
url = Config.cfg_data().get('bugtracker_url')
data = exc2encode(exc_info, fname)
r = requests.post(url, data=data)
logger.debug('Crashlog sent to %s with response code %s', url,
r.status_code)
def debugself(cls, *args, **kwargs):
"""
Logging wrapper
"""
name = inspect.stack()[1][3]
logger.debug('%s.%s: args: %r kwargs: %r',
cls.__name__,
name,
args,
kwargs)
def set_artist_url(artist, url):
"""
Download and save artists' album art
"""
if not (url and url.startswith('http')):
logger.debug('Broken url %r', url)
return
req = requests.get(url)
if req.status_code == 200:
voiceplaydb.write_artist_image(artist, req.content)
def set_artist_url(artist, url):
"""
Download and save artists' album art
"""
if not (url and url.startswith('http')):
logger.debug('Broken url %r', url)
return
req = requests.get(url)
if req.status_code == 200:
voiceplaydb.write_artist_image(artist, req.content)
def artist_blacklisted_for_genre(self, artist, genre):
"""
Check if artist is blacklisted for specific genre
"""
blacklisted = False
blacklisted_artists = self.artist_genre_blacklist.get(genre.lower(), [])
if artist.encode('utf-8') in blacklisted_artists:
logger.debug('Artist %s is blacklisted for genre %s', artist, genre.lower())
blacklisted = True
return blacklisted
def debug_traceback(exc_info, fname, include_traceback=True, message=None):
"""
Get full traceback, show if debugging is enabled
"""
typ, value, tb = exc_info
trace = ''.join(traceback.format_exception(typ, value, tb))
message = 'Method crashed (see message below), restarting...' if not message else message
if include_traceback:
message += '\n\n%s\n' % trace
send_traceback(exc_info, fname)
logger.debug(message)
def debug_traceback(exc_info, fname, include_traceback=True, message=None):
"""
Get full traceback, show if debugging is enabled
"""
typ, value, tb = exc_info
trace = ''.join(traceback.format_exception(typ, value, tb))
message = 'Method crashed (see message below), restarting...' if not message else message
if include_traceback:
message += '\n\n%s\n' % trace
send_traceback(exc_info, fname)
logger.debug(message)
def put_message(self, uuid, message):
"""
Put message into queue
"""
full_msg = {
'expires': int(time.time()) + self.TTL,
'uuid': uuid,
'message': message
}
logger.debug('SQD put: %r', message)
self.queue.put(full_msg)
def calculate(self, video_metadata, query):
if type(video_metadata) != list or len(video_metadata) != 3:
raise RuntimeError(
'%r %s %r' %
(self.__class__, ' got invalid data (type): ', video_metadata))
score = self.default_score
for rule in self.ruleset:
logger.debug('Entering %s', rule)
result = rule(video_metadata, query)
logger.debug('Exiting %s with result %s', rule, result)
score += result
return score
def artist_blacklisted_for_genre(self, artist, genre):
"""
Check if artist is blacklisted for specific genre
"""
blacklisted = False
blacklisted_artists = self.artist_genre_blacklist.get(
genre.lower(), [])
if artist.encode('utf-8') in blacklisted_artists:
logger.debug('Artist %s is blacklisted for genre %s', artist,
genre.lower())
blacklisted = True
return blacklisted
def notify(cls, *args, **kwargs):
"""
Notification dispatcher
"""
argparser = kwargs.get('argparser', '')
track = kwargs.get('track', '')
if not (track and argparser):
return
if argparser.no_track_save:
logger.debug('TrackHistory disabled for this session...')
return
voiceplaydb.update_played_tracks(track)
def similar_artists(self, artist, genre):
"""
Find similar artists for artist/genre combination
"""
sm_artists = []
similar = self.lfm.get_similar_artists(artist)
iterator = tqdm(similar) if logger.level == logging.DEBUG else similar
for similar_artist in iterator:
if genre.lower() in self.lfm.get_artist_tags(similar_artist) and not similar_artist in sm_artists and not self.artist_blacklisted_for_genre(similar_artist, genre):
logger.debug('Genre match for %s', similar_artist)
sm_artists.append(similar_artist)
return sm_artists
def get_full_message(self):
"""
Get message from queue
"""
message = {}
while not self.exit:
if not self.queue.empty():
message = self.queue.get()
break
time.sleep(0.01)
logger.debug('SQD get: %r', message)
return message
def get_from_cache(cls, target_filename):
"""
Get file from cache
"""
is_cached = cls.is_remote_cached(target_filename)
if is_cached:
cache = cls.CACHE_BACKEND()
cache.download(is_cached, target_filename)
logger.debug('File %r was downloaded from %r', target_filename, cls.CACHE_BACKEND)
else:
target_filename = None
return target_filename
def is_remote_cached(cls, target_filename):
"""
Return remote file ID if it is cached
"""
is_cached = None
cache = cls.CACHE_BACKEND()
for file_name, file_id in cache.search():
if file_name == os.path.basename(target_filename):
is_cached = file_id
logger.debug('File %r already cached at %r', target_filename, cls.CACHE_BACKEND)
break
return is_cached
def notify(cls, *args, **kwargs):
"""
Notification dispatcher
"""
argparser = kwargs.get('argparser', '')
track = kwargs.get('track', '')
if not (track and argparser):
return
if argparser.no_track_save:
logger.debug('TrackHistory disabled for this session...')
return
voiceplaydb.update_played_tracks(track)
def get_full_message(self):
"""
Get message from queue
"""
message = {}
while not self.exit:
if not self.queue.empty():
message = self.queue.get()
break
time.sleep(0.01)
logger.debug('SQD get: %r', message)
return message
def cmd_loop(self):
"""
Run command loop / periodically check for commands from queue
"""
while not self.shutdown_flag:
if not self.queue.empty():
message = self.queue.get()
else:
time.sleep(0.01)
continue
self.play_from_parser(message)
self.queue.task_done()
logger.debug('Vickiplayer.cmd_loop exit')
def run_hooks(argparser, hooks, evt, *args, **kwargs):
"""
Run hooks (player/recognition/etc)
"""
for hook in hooks:
logger.debug('Running hook: %r', hook)
hook.argparser = argparser
method = getattr(hook, evt)
if method:
try:
method(*args, **kwargs)
except Exception as _:
debug_traceback(sys.exc_info(), __file__)
def cmd_loop(self):
"""
Run command loop / periodically check for commands from queue
"""
while not self.shutdown_flag:
if not self.queue.empty():
message = self.queue.get()
else:
time.sleep(0.01)
continue
self.play_from_parser(message)
self.queue.task_done()
logger.debug('Vickiplayer.cmd_loop exit')
def run_hooks(argparser, hooks, evt, *args, **kwargs):
"""
Run hooks (player/recognition/etc)
"""
for hook in hooks:
logger.debug('Running hook: %r', hook)
hook.argparser = argparser
method = getattr(hook, evt)
if method:
try:
method(*args, **kwargs)
except Exception as _:
debug_traceback(sys.exc_info(), __file__)
def __init__(self, debug=False, player_backend='vlc'):
self.debug = debug
self.rec = sr.Recognizer()
self.tts = TextToSpeech()
self.shutdown = False
if self.debug:
logger.setLevel(logging.DEBUG)
logger.debug('Vicki init completed')
self.player = VickiPlayer(tts=self.tts, debug=self.debug, player_backend=player_backend)
self.wakeword_receiver = None
self.listener = None
self.recognition_hooks = sorted(PluginLoader().find_classes('voiceplay.player.hooks', BasePlayerHook),
key=cmp_to_key(lambda x, y: cmp(x.__priority__, y.__priority__)))
def poll_loop(self):
"""
Poll speech queue
"""
while not self.shutdown:
if not self.queue.empty():
message = self.queue.get()
logger.debug('TTS: %r', message)
self.say(message)
self.queue.task_done()
else:
time.sleep(0.01)
logger.debug('TTS poll_loop exit')
def url_hook(cls, *args, **kwargs):
"""
URL hook for youtube_dl
"""
message = args[0]
if not message.startswith('http') and not message.startswith('['):
cls.add_to_list()
cls.title = message
elif message.startswith('http'):
cls.add_to_list()
cls.new_url = message
else:
logger.debug(message)
def get_check_all(self):
"""
Get results, use cache if needed.
"""
result = voiceplaydb.get_cached_service(self.cache_file, expires=7)
if result:
logger.debug('Using %s cached version...', self.cache_file) # pylint:disable=no-member
result = json.loads(result)
else:
logger.debug('Fetching and storing fresh %s version...', self.cache_file) # pylint:disable=no-member
result = self.get_all() # pylint:disable=no-member
if result:
voiceplaydb.set_cached_service(self.cache_file, json.dumps(result))
return result
def purge_cache():
"""
Purge file storage cache
"""
from voiceplay.config import Config
logger.debug('Purging cache...')
cache_dir = Config.cfg_data().get('cache_dir', '')
if os.path.exists(cache_dir) and os.path.isdir(cache_dir):
files = glob(os.path.join(cache_dir, '*'))
for fname in files:
try:
os.remove(fname)
except Exception as exc:
logger.debug('Removal of %r failed, please check permissions', exc)
def upload(self, fname):
"""
Upload file to Google Drive
"""
if not self.health_check():
return
file_metadata = {
'name': os.path.basename(fname),
'parents': ['appDataFolder']
}
media = MediaFileUpload(fname, mimetype='audio/mpeg', resumable=True)
file_obj = self.service.files().create(body=file_metadata,
media_body=media,
fields='id').execute()
logger.debug('File ID: %s', file_obj.get('id'))
def similar_artists(self, artist, genre):
"""
Find similar artists for artist/genre combination
"""
sm_artists = []
similar = self.lfm.get_similar_artists(artist)
iterator = tqdm(similar) if logger.level == logging.DEBUG else similar
for similar_artist in iterator:
if genre.lower() in self.lfm.get_artist_tags(
similar_artist
) and not similar_artist in sm_artists and not self.artist_blacklisted_for_genre(
similar_artist, genre):
logger.debug('Genre match for %s', similar_artist)
sm_artists.append(similar_artist)
return sm_artists
def prefetch_loop(self):
"""
Prefetch loop allows significant speedup in playback as it fetches tracks in background
"""
while not self.shutdown_flag:
if not self.prefetch_q.empty():
trackname = self.prefetch_q.get()
else:
time.sleep(0.01)
continue
logger.debug('prefetch_loop got from queue: %r',
trackname.encode('utf-8'))
start = time.time()
BasePlayerTask.search_full_track(trackname, download=True)
elapsed = round(time.time() - start, 3)
logger.debug('prefetch_loop finished downloading, took %ss',
elapsed)
def __init__(self, debug=False, player_backend='vlc'):
self.debug = debug
self.rec = sr.Recognizer()
self.tts = TextToSpeech()
self.shutdown = False
if self.debug:
logger.setLevel(logging.DEBUG)
logger.debug('Vicki init completed')
self.player = VickiPlayer(tts=self.tts,
debug=self.debug,
player_backend=player_backend)
self.wakeword_receiver = None
self.listener = None
self.recognition_hooks = sorted(
PluginLoader().find_classes('voiceplay.player.hooks',
BasePlayerHook),
key=cmp_to_key(lambda x, y: cmp(x.__priority__, y.__priority__)))
