def __init__(self):
"""Class constructor """
# TODO: Provide token and app name in a file
personalToken = "AdgFCqmDrSlqvvdsFVTyJqpyArvcqPywDvpfgPiI"
self.client = discogs_client.Client("PlaylistMaker/0.1", user_token=personalToken)
self.logging = BaseLogger()
self.indent = " "
def __init__(self): cmd.Cmd.__init__(self) component = self.__class__.__name__ self.items = [] self.chosenItem = [] self.streamer = None self.inspect = None self.streamURI = '' # Creating logger self.logger = BaseLogger(terminal=self) # Parsing data file rootName = 'Streamer' filename = "Conf/Streamer.xml" self.streamConf = ParseXml2Dict(filename, rootName)
class StreamConsole(cmd.Cmd):
"""Simple command processor example."""
prompt = 'StreamPlayer> '
intro = "Welcome to stream player, v.0.1"
doc_header = 'Help commands'
misc_header = 'misc_header'
undoc_header = 'No help description'
def __init__(self):
cmd.Cmd.__init__(self)
component = self.__class__.__name__
self.items = []
self.chosenItem = []
self.streamer = None
self.inspect = None
self.streamURI = ''
# Creating logger
self.logger = BaseLogger(terminal=self)
# Parsing data file
rootName = 'Streamer'
filename = "Conf/Streamer.xml"
self.streamConf = ParseXml2Dict(filename, rootName)
#print "==== streamConf:", json.dumps(self.streamConf, sort_keys=True, indent=4, separators=(',', ': '))
def cmdloop(self, intro=None):
'''Command loop '''
#print 'cmdloop(%s)' % intro
#return cmd.Cmd.cmdloop(self, intro)
try:
return cmd.Cmd.cmdloop(self)
except KeyboardInterrupt as e:
self.do_exit('')
time.sleep(0.5)
def preloop(self):
'''print 'preloop() '''
def postloop(self):
'''print 'postloop() '''
def parseline(self, line):
ret = cmd.Cmd.parseline(self, line)
return ret
def emptyline(self):
'''Method called for empty line '''
#return cmd.Cmd.emptyline(self)
return
def default(self, line):
print 'default(%s)' % line
return cmd.Cmd.default(self, line)
def precmd(self, line):
''' Method called for preprocessing command'''
#print 'precmd(%s)' % line
return cmd.Cmd.precmd(self, line)
def postcmd(self, stop, line):
''' Method called for post processing command'''
#print 'postcmd(%s, %s)' % (stop, line)
return cmd.Cmd.postcmd(self, stop, line)
def do_EOF(self, line):
"Exit"
print ""
if self.streamer is not None:
self.streamer.Stop()
if self.inspect is not None:
self.inspect.Exit()
return True
def do_exit(self, line):
print ""
if self.streamer is not None:
self.streamer.Stop()
self.streamer.Exit()
if self.inspect is not None:
self.inspect.Exit()
return True
# Stream commands
def do_stream(self, line):
'''Provide actions to play/stop/pause and volume up/down in a stream'''
#print "====", line
try:
operations = self.streamConf['StreamOps']['Operations']
if not self.ValidateOps(operations, line):
return
tasks = line.split()
if tasks[0] == "show":
streams = self.streamConf["Streams"]["Stream"]
for i in range(len(streams)):
stream = streams[i]
desc = stream['StreamDescription']
uri = stream['Uri']
self.logger.debug(" ["+str(i+1)+"] "+desc+":\n"+" "+uri)
elif tasks[0] == 'set_uri':
if len(tasks)<2:
self.logger.debug(' Warning: No index given')
return
index = int(tasks[1])
streams = self.streamConf["Streams"]["Stream"]
if index >= len(streams) or index < 0:
self.logger.debug(" Index ["+tasks[1]+"] is out of range")
return
# Setting up stream value from configuration file
self.logger.debug(" Setting URI: "+streams[index]['StreamDescription'])
self.streamURI = streams[index]['Uri']
# Setting up URI in player
if self.streamer is not None and self.streamer.IsRunning():
self.logger.debug(' Passing stream URI...')
self.streamer.SetURI(self.streamURI)
return
elif tasks[0] == "play":
if len(self.streamURI)<1:
self.logger.debug(" No URI has been defined")
return
if self.streamer == None:
self.streamer = StreamPlayer(uri=self.streamURI,
logger=self.logger)
self.emptyline()
elif self.streamer.IsRunning():
self.streamer.Play()
elif tasks[0] == "pause":
if self.streamer is not None:
self.streamer.Pause()
elif tasks[0] == "stop":
if self.streamer is not None:
self.streamer.Stop()
elif tasks[0].startswith("volume"):
volume = self.streamer.Volume()
if tasks[0] == "volume_up":
self.streamer.VolumeUp(volume)
elif tasks[0] == "volume_down":
self.streamer.VolumeDown(volume)
elif tasks[0] == "volume_mute":
self.streamer.VolumeMute()
elif tasks[0] == "inspect":
if self.streamer is None:
self.logger.debug(" Streamer not started yet")
elif self.inspect is None:
if len(tasks)>1 and '://' in tasks[1]:
self.inspect = Inspector(streamer=self.streamer,
logger=self.logger,
console=self,
endpoint=tasks[1])
else:
self.inspect = Inspector(streamer=self.streamer,
console=self,
logger=self.logger)
elif self.inspect.IsRunning():
if len(tasks)>1:
if tasks[1] == 'stop':
self.logger.debug(" Stream inspection stopped")
self.inspect.Exit()
else:
self.logger.debug(" Stream inspection already running")
elif not self.inspect.IsRunning():
self.inspect.Exit()
if len(tasks)>1 and '://' in tasks[1]:
self.inspect = Inspector(streamer=self.streamer,
logger=self.logger,
console=self,
endpoint=tasks[1])
else:
self.logger.debug(" Restarting stream inspection")
self.inspect = Inspector(streamer=self.streamer,
console=self,
logger=self.logger)
elif tasks[0] == 'reset':
self.logger.debug(" Resetting streamer...")
self.streamer.Stop()
self.streamer.Exit()
self.streamer = None
time.sleep(1)
self.do_stream('play')
self.logger.debug(" Setting new streamer for inspecting")
self.inspect.SetStreamer(streamer=self.streamer)
#self.inspect.clone(self.inspect.streamer)
except ValueError:
self.logger.debug(" Incorrect index value: "+line.split()[1])
except Exception as inst:
exc_type, exc_obj, exc_tb = sys.exc_info()
exception_fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
exception_line = str(exc_tb.tb_lineno)
exception_type = str(type(inst))
exception_desc = str(inst)
self.logger.debug( " %s: %s in %s:%s"%(exception_type,
exception_desc,
exception_fname,
exception_line ))
def complete_stream(self, text, line, begidx, endidx):
keys = self.streamConf['StreamOps']['Operations']
if not text:
completions = keys
else:
completions = [ f
for f in keys
if f.startswith(text) ]
return completions
def help_stream(self):
print '\n'.join([ 'stream [play]',
'\tPlays a stream',
'stream [pauses]',
'\tPauses a stream',
'stream [stop]',
'\tVolume up/down',
'stream [volume] [up down]',
'\tSets volume up or down',
'stream [show]',
'\tShows available URI',
'stream [set_uri] [INDEX]',
'\tSets a new URI by using index',
])
def ValidateOps(self, operations, line):
'''Generic validation'''
keys = operations
if len(line) < 1:
self.logger.debug(' Warning: No option given')
return False
tasks = line.split()
if tasks[0] not in operations:
self.logger.debug(' Warning: Invalid option: '+line)
return False
return True
# Report commands
def do_tracks(self, line):
'''Provide actions to play/stop/pause and volume up/down in a stream'''
tasks = line.split()
try:
operations = self.streamConf['StreamReport']['Operations']
if not self.ValidateOps(operations, line):
return
#print '====', tasks
database = MongoHandler('test_database', 'tracks')
if tasks[0] == "all":
self.logger.debug(" Getting tracks from database")
posts = database.Report()
myFormat = '%5s|%8s|%85s|'
sizePosts = posts.count()
for i in range(sizePosts):
post = posts[i]
print myFormat%(str(i),post['State'], post['Track'])
elif tasks[0] == "clean":
self.logger.debug(" Removing repeated tracks")
posts = database.CleanRepeated()
elif tasks[0] == "played":
self.logger.debug(" Getting all played tracks from database")
posts = database.Report('Played')
myFormat = '%5s|%8s|%85s|'
sizePosts = posts.count()
for i in range(sizePosts):
post = posts[i]
print myFormat%(str(i),post['State'], post['Track'])
elif tasks[0] == "download":
''' '''
# Choosing how many tracks should be downloaded in one go
totalTracks = 1
if len(tasks)>1:
totalTracks = int(tasks[1])
posts = database.Report('Played')
sizePosts= posts.count()
self.logger.debug(" Getting %d of %d played track(s) from database"%
(totalTracks, sizePosts))
# Updating the tracks
for i in range(totalTracks):
post = posts[i]
# Searching current track
trackName = post['Track']
self.logger.debug(" %d/%d Searching: [%s] ..."%
(i+1, totalTracks, trackName))
bestMatch, findings = self.SearchSong(trackName)
if findings<1:
self.logger.debug(' No results from online query')
state = 'NotFound'
updated = database.UpdateTrackState(trackName, state)
updatedLog = 'updated' if updated else 'NOT updated'
self.logger.debug(" State in played record was %s with [%s]"%(updatedLog, state))
continue
if len(bestMatch)<1:
self.logger.debug(" Best match for track was not found")
state = 'NotMatched'
updated = database.UpdateTrackState(trackName, state)
updatedLog = 'updated' if updated else 'NOT updated'
self.logger.debug(" State in played record was %s with [%s]"%(updatedLog, state))
continue
# Downloading track
path = self.DownloadTrack(bestMatch)
if len(path)>0:
updated = database.UpdateTrackState(trackName, 'Downloaded', path=path)
updatedLog = 'updated' if updated else 'NOT updated'
self.logger.debug(" State in played record was %s with [%s]"%(updatedLog, 'Downloaded'))
except ValueError as inst:
if tasks[0] == "download":
self.logger.debug(" Required download files is not a NUMBER")
except Exception as inst:
exc_type, exc_obj, exc_tb = sys.exc_info()
exception_fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
exception_line = str(exc_tb.tb_lineno)
exception_type = str(type(inst))
exception_desc = str(inst)
self.logger.debug( " %s: %s in %s:%s"%(exception_type,
exception_desc,
exception_fname,
exception_line ))
def complete_tracks(self, text, line, begidx, endidx):
keys = self.streamConf['StreamReport']['Operations']
if not text:
completions = keys
else:
completions = [ f
for f in keys
if f.startswith(text) ]
return completions
def help_tracks(self):
print '\n'.join([ 'tracks [all]',
'\tDisplay all songs',
'tracks [clean]',
'\tRemoves repeated entries'
'tracks [played]',
'\tDisplay only played tracks',
'tracks [download] [NUMBER]',
'\tDownloads [NUMBER] played tracks, default is 1 track'
])
def SearchSong(self, trackName):
''' '''
try:
# Getting configuration options
maxResults = int(self.streamConf['SearchOptions']['maxResults'])
maxRatio = float(self.streamConf['SearchOptions']['maxRatio'])
maxValue = 0.0
bestMatch = ()
# Preparing Youtube query
options = {'query':trackName, 'maxResults':maxResults}
songFinder = PlaylistGenerator()
items = songFinder.SearchInYouTube(options)
#print "==== items:", len(items['videos'])
videosFound = len(items['videos'])
if len(items['videos'])<1:
return bestMatch, videosFound
for item in items['videos']:
match = get_close_matches(item['title'], [trackName, ''])
if len(match)>0:
#print " =>", item['title']
s = SequenceMatcher(None, item['title'], trackName)
ratio = s.ratio()
#print ratio,"==>", item['title'], ":", maxValue >= ratio
if ratio >= maxValue:
maxValue = ratio
bestMatch = (ratio, item['id'], item['title'])
#else:
#print " ", item['title']
#print "==== BestMatch:", bestMatch
if len(bestMatch)<1:
return bestMatch, videosFound
bestMatchRatio = bestMatch[0]
if bestMatchRatio < maxRatio:
self.logger.debug(' Ratio is lower than maximum allowed of %s'%str(maxRatio))
isGoodTitle = self.WordByWord(bestMatch[2], trackName, bestMatchRatio)
if not isGoodTitle:
self.logger.debug(' Does not matches the entry...')
bestMatch = ()
return bestMatch, videosFound
except Exception as inst:
exc_type, exc_obj, exc_tb = sys.exc_info()
exception_fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
exception_line = str(exc_tb.tb_lineno)
exception_type = str(type(inst))
exception_desc = str(inst)
self.logger.debug( " %s: %s in %s:%s"%(exception_type,
exception_desc,
exception_fname,
exception_line ))
def DownloadTrack(self, bestMatch):
''' '''
downloaded = ''
try:
itemId = bestMatch[1]
itemTitle = bestMatch[2]
url = 'https://www.youtube.com/watch?v='+itemId
ydl_opts = {
u'keepvideo': False,
u'format': u'bestaudio/best',
u'postprocessors': [{u'preferredcodec': u'mp3',
u'preferredquality': u'0',
u'key': u'FFmpegExtractAudio'}]
}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
success = ydl.download([url])
#print "====", success
# Get a nice file name, otherwise it screws the unix file system
# Stating new downloaded file
fileName = itemTitle+"-"+itemId+'.mp3'
if success == 0:
#fileName = re.sub(re.escape(''.join('"')) , '\"', fileName)
fileName = self.ChangeSpecialChars(fileName)
self.logger.debug(' + Stating file: '+fileName)
mode = os.stat(fileName).st_mode
if S_ISREG(mode):
self.logger.debug(' + Successfully downloaded file: '+fileName)
# Creating store path if not exist
destinationPath = os.path.abspath('DownlaodedSongs')
if not os.path.exists(destinationPath):
self.logger.debug(' + Creating directoy '+destinationPath)
os.makedirs(destinationPath)
else:
self.logger.debug(' + Directory '+destinationPath+' exists')
# Moving files to storing directory
source = os.path.abspath('')+'/'+fileName
dest = destinationPath+'/'+fileName
os.rename(source, dest)
self.logger.debug(' + Storing '+fileName)
downloaded = dest
except Exception as inst:
exc_type, exc_obj, exc_tb = sys.exc_info()
exception_fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
exception_line = str(exc_tb.tb_lineno)
exception_type = str(type(inst))
exception_desc = str(inst)
print "MongoHandler.Exists: %s: %s in %s:%s"%(exception_type,
exception_desc,
exception_fname,
exception_line )
finally:
return downloaded
def ChangeSpecialChars(self, str1):
specialChars = '"'
newStr = ''
for c in str1:
if c == '"':
newStr += "\'"
elif c == "'":
newStr += "\'"
else:
newStr += c
return newStr
def TextCleanup(self, text):
new = ""
for i in text:
if i not in '.,|[!@#$%^&*?_~-+/()]':
new += i
return new
def WordByWord(self, str1, str2, bestRatio):
''''''
try:
# Getting best score word-by-word
word1 = str1.split()
word2 = str2.split()
listing = []
for w in word1:
if len(w)>1:
w = self.TextCleanup(w)
highest = 0.0
curr_word = [w, '', highest]
for v in word2:
if len(v)>1:
v = self.TextCleanup(v)
s = SequenceMatcher(None, w, v)
ratio = s.ratio()
#print " - comparing: [",w,"/", v, "]:", ratio
if ratio >= highest:
highest = ratio
curr_word[1] = v
curr_word[2] = ratio
if curr_word[2]>0.0:
#print " ",curr_word
listing.append(curr_word)
#print "="*20
# Checking average of matches
sumed = 0.0
hits = 0.0
length = len(listing)
for word in listing:
sumed += word[2]
if word[2]>=0.8:
hits+=1
average = (sumed/length)
hitsPercentage = (hits/length)
#print "Length:", length
#print "Avg:", average
#print "Hits:", hitsPercentage
msg = " Best match is:\n\t ratio:\t\t"+str(bestRatio)+ \
"\n\t best:\t\t"+str1+ \
"\n\t original:\t"+str2+ \
"\n\t average:\t"+str(average)+ \
"\n\t hits:\t\t"+str(hitsPercentage)
self.logger.debug(msg)
isGoodTitle = average >= ratio or hitsPercentage >= 0.7
return isGoodTitle
except Exception as inst:
exc_type, exc_obj, exc_tb = sys.exc_info()
exception_fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
exception_line = str(exc_tb.tb_lineno)
exception_type = str(type(inst))
exception_desc = str(inst)
self.logger.debug( " %s: %s in %s:%s"%(exception_type,
exception_desc,
exception_fname,
exception_line ))
class DiscogsQuery:
def __init__(self):
"""Class constructor """
# TODO: Provide token and app name in a file
personalToken = "AdgFCqmDrSlqvvdsFVTyJqpyArvcqPywDvpfgPiI"
self.client = discogs_client.Client("PlaylistMaker/0.1", user_token=personalToken)
self.logging = BaseLogger()
self.indent = " "
def SearchSongByRelease(self, videoName):
"""Searches a song in Discogs database """
spc = self.indent
songName = ""
artistName = ""
taggedData = {}
# Looking for style of name separator
separators = [" - ", "- ", " -"]
for s in separators:
if s in videoName:
splitter = s
break
else:
splitter = ""
# Get song and artist name
splitterFonud = True
if len(splitter) > 0:
tSearchName = videoName.split(splitter)
else:
logMsg = spc + "- Could not find splitter between artist and song name"
self.logging.log(LogLevel.CONSOLE, logMsg)
tSearchName = videoName
splitterFonud = False
if len(tSearchName) > 1:
if splitterFonud:
(artistName, songName) = tSearchName
else:
songName = tSearchName
else:
logMsg = spc + "- Incorrect name convention: " + videoName
self.logging.log(LogLevel.CONSOLE, logMsg)
return taggedData
logMsg = spc + "+ Searching for song: [" + videoName + "]"
self.logging.log(LogLevel.CONSOLE, logMsg)
results = self.client.search(videoName, type="release")
pages = results.pages
foundItems = len(results)
logMsg = spc + "+ Found " + str(foundItems) + " page results"
self.logging.log(LogLevel.CONSOLE, logMsg)
if foundItems > 0:
logMsg = spc + "+ Finding release for: " + songName
self.logging.log(LogLevel.CONSOLE, logMsg)
foundSongName, release = self.FindRelease(results, songName)
if foundSongName == [] or release == []:
logMsg = spc + "- No release found for: " + songName
self.logging.log(LogLevel.CONSOLE, logMsg)
else:
# TODO: Requires input mp3 metadata in real file
logMsg = spc + "+ Getting song information for: " + foundSongName
self.logging.log(LogLevel.CONSOLE, logMsg)
taggedData = self.GetTagData(release, foundSongName)
return taggedData
def FindRelease(self, results, songName):
""" Searches a release in dicogs database"""
spc = self.indent
trackTitle = []
for r in range(len(results)):
data = results[r].data
release = self.client.release(data["id"])
for track in release.tracklist:
lClosestMatches = get_close_matches(songName, [track.title, ""])
trackInSong = track.title in songName
if len(lClosestMatches) > 0 or trackInSong:
trackTitle = track.title
logMsg = spc + "+ Found: " + track.title
self.logging.log(LogLevel.CONSOLE, logMsg)
return trackTitle, release
return [], []
def GetTagData(self, release, songName):
"""Looks for MP3 metadata from a release """
# TODO: Give percentage of found items, how complete is the tag
total = 13
itemsFound = [0, total]
# Preparing data encapsulation
dData = {}
data = release.data
keys = data.keys()
# 1-1) Song name
itemsFound[0] += 1
dData["SongName"] = songName
# print "*** 1 SongName",";", songName," ==>",itemsFound[0]
# 2-2) Release ID
if "id" in keys:
itemsFound[0] += 1
dData["ReleaseID"] = data["id"]
# print "*** 2 ReleaseID",":",data['id']," ==>",itemsFound[0]
# 3) Artist information
if "artists" in keys:
artistFound = [0, 0]
dData["Artist"] = {}
for artist in data["artists"]:
artistsKeys = artist.keys()
# 3-3.1) Artist Name
if "name" in artistsKeys:
artistFound[0] = 1
dData["Artist"]["Name"] = artist["name"]
# print "*** 3 ArtistName",":",artist['name']," ==>",artistFound[0]
# 4-Artist ID
if "id" in artistsKeys:
artistFound[1] = 1
dData["Artist"]["ID"] = artist["id"]
# print "*** 4 ArtistID",":",artist['id']," ==>",artistFound[0]
itemsFound[0] += sum(artistFound)
# print "*** Artist",": ==>",itemsFound[0]
# 5-4) Country
if "country" in keys:
itemsFound[0] += 1
dData["Country"] = data["country"]
# print "*** 5 Country",":",data['country'],": ==>",itemsFound[0]
# 6-5) Genre
if "genres" in keys:
genresFound = [0]
dData["Genres"] = []
for genre in data["genres"]:
genresFound[0] = 1
dData["Genres"].append(genre)
# print "*** 6 Genres",":",genre,": ==>",genresFound[0]
itemsFound[0] += sum(genresFound)
# print "*** Genres",": ==>",itemsFound[0]
# 6) Image information
if "images" in keys:
dData["Image"] = {}
imagesFound = [0, 0, 0]
for image in data["images"]:
imageKeys = image.keys()
# 7-6.1) Height
if "height" in imageKeys:
imagesFound[0] = 1
dData["Image"]["Height"] = image["height"]
# print "*** 7 ImageHeight",":",image['height'],": ==>",imagesFound[0]
# 8-6.2) Width
if "width" in imageKeys:
imagesFound[1] = 1
dData["Image"]["Width"] = image["width"]
# print "*** 8 ImageWidth",":",image['width'],": ==>",imagesFound[0]
# 9-6.3) URI
if "uri" in imageKeys:
imagesFound[2] = 1
dData["Image"]["URI"] = image["uri"]
# print "*** 9 ImageURI",":",image['uri'],": ==>",imagesFound[0]
break
itemsFound[0] += sum(imagesFound)
# print "*** Image",": ==>",itemsFound[0]
# 10-7) Style
if "styles" in keys:
stylesFound = [0]
dData["Styles"] = []
for style in data["styles"]:
stylesFound[0] = 1
dData["Styles"].append(style)
# print "*** 10 Styles",":",style,": ==>",stylesFound[0]
itemsFound[0] += sum(stylesFound)
# print "*** Styles",": ==>",itemsFound[0]
# 11-8) Title
if "title" in keys:
itemsFound[0] += 1
dData["Album"] = data["title"]
# print "*** 11 Album",":",data['title'],": ==>",itemsFound[0]
# 12-8) Year
if "year" in keys:
itemsFound[0] += 1
dData["Year"] = data["year"]
# print "*** 12 Year",":",data['year'],": ==>",itemsFound[0]
# 13-9) Label Name
if "labels" in keys:
labelsFound = [0]
dData["Publisher"] = []
for label in data["labels"]:
labelKeys = label.keys()
if "name" in labelKeys:
labelsFound[0] = 1
dData["Publisher"].append(label["name"])
# print "*** 13 Publisher",":",label['name'],": ==>",labelsFound[0]
itemsFound[0] += sum(labelsFound)
# print "*** Publisher",": ==>",itemsFound[0]
dData["Completeness"] = float((itemsFound[0]) / float(itemsFound[1])) * 100.0
# if dData['Completeness']>0:
# print data
return dData
