def savePlaylistAJAX(self):
playlistname = request.params['name']
trackids = simplejson.loads(request.params['trackids'])
trackids = map(lambda x: x.replace('track_',''), trackids)
results = Session.query(Track, MBRecording).join(MBRecording).filter(Track.id.in_(trackids)).all()
results.sort(lambda a,b: cmp(trackids.index(a[0].id), trackids.index(b[0].id)))
recordings = map(itemgetter(1), results)
Session.begin()
user_name = request.environ['repoze.what.credentials']['repoze.what.userid']
user_id = Session.query(User).filter(User.user_name==user_name).one().user_id
playlist = Session.query(Playlist) \
.filter(Playlist.owner_id==user_id) \
.filter(Playlist.name==playlistname) \
.first()
if not trackids:
if playlist is None:
return '{}'
else:
Session.delete(playlist)
else:
if playlist is None:
playlist = Playlist(user_id, playlistname)
Session.add(playlist)
else:
playlist.modified = datetime.now()
playlist.tracks = recordings
Session.commit()
return '{}'
def getLyricsAJAX(self):
trackid = request.params['trackid'].split('_')[1]
track = Session.query(Track).filter_by(id=trackid).one()
if not track.lyrics and \
(track.lastHitLyricWiki is None or \
datetime.now() > track.lastHitLyricWiki + timedelta(days=10)):
track.lastHitLyricWiki = datetime.now()
title = track.id3title
artist = track.id3artist
params = {
'artist' : artist,
'song' : title,
'fmt' : 'json',
}
url = 'http://lyrics.wikia.com/api.php?%s' % urllib.urlencode(params)
html = urllib.urlopen(url).read()
if not "'lyrics':'Not found'" in html:
search = re.search("'url':'(?P<url>.*?)'",html)
lyricurl = urllib.unquote(search.group('url'))
lyrichtml = urllib.urlopen(lyricurl).read()
lyrics = re.search("<div class='lyricbox'>.*?</div>(?P<lyrics>.*?)<!-- \n", lyrichtml).group('lyrics')
lyrics = unescape(lyrics)
track.lyrics = lyrics
Session.begin()
Session.commit()
json = {}
if track.lyrics:
json['lyrics'] = track.lyrics
return simplejson.dumps(json)
def saveLyricsFramesAJAX(self):
trackid = request.params['trackid']
recordedFrames = simplejson.loads(request.params['frames']) # list of (time, lyrics line index) pairs
recordedFrames.sort(key = itemgetter(0)) # (sort by time)
Session.begin()
track = Session.query(Track).filter_by(id=trackid).one()
track.lyricsFrames = recordedFrames
Session.commit()
def clearAlbumArt(self):
id = request.params['id']
Session.begin()
album = Session.query(Album).filter_by(id=id).one()
album.albumArtFilename = None
album.lastHitAlbumArtExchange = None
Session.commit()
return 'Cleared album art for ' + album.artist.name + ' - ' + album.name
def setAlbumArt(self):
id = request.params['id']
url = request.params['url']
Session.begin()
album = Session.query(Album).filter_by(id=id).one()
album.albumArtFilename = albumart._fetchAlbumArt(album.artist.name, album.name, url)
Session.commit()
return 'Set album art for ' + album.artist.name + ' - ' + album.name + ' to ' + url + ', saved to ' + album.albumArtFilename
def invite(self):
if request.POST:
who = request.params['who']
c.code = ''.join(random.choice(string.letters + string.digits) for i in xrange(32))
invite = Invite(who, c.code)
Session.begin()
Session.add(invite)
Session.commit()
return render('/display-invite.html')
else:
return render('/create-invite.html')
def makeRandom(self):
user_name = request.environ['repoze.what.credentials']['repoze.what.userid']
user = Session.query(User).filter(User.user_name==user_name).one()
Session.begin()
for i in range(10):
info = BenKVP(owner = user,
key = 'random_key_{0}'.format(i),
value = 'random_value_{0}'.format(i))
Session.add(info)
Session.commit()
return 'success'
def getLyricsAJAX(self):
trackid = request.params['trackid'].split('_')[1]
track = Session.query(Track).filter_by(id=trackid).one()
ip = request.environ['REMOTE_ADDR']
user_name = request.environ['repoze.what.credentials']['repoze.what.userid']
user = Session.query(User).filter(User.user_name==user_name).one()
Session.begin()
play = TrackPlay(track.mbid, user, ip)
Session.add(play)
Session.commit()
lyrics = lyricsservice.get_lyrics(Session, track)
if lyrics:
return simplejson.dumps({'lyrics' : lyrics})
else:
return '{}'
def getAlbumArtAJAX(self):
trackid = request.params['trackid'].split('_')[1]
track = Session.query(Track).filter_by(id=trackid).one()
if not track.album.albumArtFilename and ( \
track.album.lastHitAlbumArtExchange is None \
or datetime.now() > track.album.lastHitAlbumArtExchange + timedelta(days=10)):
track.album.lastHitAlbumArtExchange = datetime.now()
album = track.album.name
artist = track.album.artist.name
if artist == 'Various Artists':
q = album
else:
q = (artist + ' ' + album)
q = q.replace("'","")
site = 'http://www.albumartexchange.com'
params = {
'grid' : '2x7',
'sort' : 7,
'q' : q,
}
url = site + '/covers.php?%s' % urllib.urlencode(params)
html = urllib.urlopen(url).read()
search = re.search('src="/phputil/scale_image.php\?size=150&src=(?P<src>.*?)"',html)
if search:
image = site + urllib.unquote(search.group('src'))
extension = image.rsplit('.', 1)[1]
delchars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
delchars = delchars.translate(None," ()'&!-+_.")
filename = (artist + ' - ' + album).encode('utf-8').translate(None, delchars) + '.' + extension
urllib.urlretrieve(image, 'scatterbrainz/public/art/' + filename)
albumArt = '/art/' + filename
track.album.albumArtFilename = albumArt
Session.begin()
Session.commit()
json = {}
if track.album.albumArtFilename:
json['albumArtURL'] = track.album.albumArtFilename
return simplejson.dumps(json)
def clearimport(download):
with importlock:
assert not download.isdone, 'download is already done'
assert download.failedimport, 'import isnt failed'
mbid = download.release_group_mbid
dirpath = mbid[:2] + '/' + mbid
cmd = 'rm -rf ' + Config.MUSIC_PATH + dirpath
log.info('running ' + cmd)
os.system(cmd)
if Config.SCP_SHOP_DOWNLOADS:
cmd = 'rm -rf ' + Config.SCP_FOLDER + '/' + download.infohash
log.info('running ' + cmd)
os.system(cmd)
Session.begin()
download.failedimport = False
download.importtrace = None
Session.commit()
def getTrackInfoAJAX(self):
trackid = request.params['trackid'].split('_')[1]
track = Session.query(Track).filter_by(id=trackid).one()
(artistName, albumName, trackName) = (track.id3artist, track.id3album, track.id3title)
log.info(request.remote_addr + ' playing ' + artistName + ' - ' + trackName)
if (not track.artist.mbid or not track.album.mbid or not track.album.asin) and \
(track.album.lastHitMusicbrainz is None \
or datetime.now() > track.album.lastHitMusicbrainz + timedelta(days=10)):
track.album.lastHitMusicbrainz = datetime.now()
album = track.album
artist = album.artist
release = None
if album.mbid:
release = getRelease(album.mbid)
else:
if artist.name == 'Various Artists':
release = searchRelease(None, album.name)
else:
if artist.name.startswith('The') or artist.name.startswith('the') or artist.name.startswith('THE'):
artistSearch = artist.name[4:]
else:
artistSearch = artist.name
release = searchRelease(artistSearch, album.name)
if release and not album.mbid:
album.mbid = release.id.split('/')[-1]
if release and not artist.mbid:
artist.mbid = release.artist.id.split('/')[-1]
if release:
albumName = release.title
asin = release.getAsin()
if asin:
track.album.asin = asin
if release.artist:
artistName = release.artist.name
Session.begin()
Session.commit()
json = {}
json['artist'] = artistName
json['album'] = albumName
json['track'] = trackName
if track.album.asin:
json['asin'] = track.album.asin
return simplejson.dumps(json)
def run(self):
# Only let one worker run via pg advisory lock
acquired = Session.execute(select([func.pg_try_advisory_lock(importlockid)])).fetchone()[0]
threadid = 'PID ' + str(os.getpid()) + ' thread ' + str(threading.current_thread())
if acquired:
log.info('[shop worker] %s acquired lock, starting..' % threadid)
else:
return
pendingdownloads = False
while True:
try:
downloads = Session.query(ShopDownload) \
.filter(ShopDownload.isdone==False) \
.filter(ShopDownload.failedimport==False) \
.all()
pendingdownloads = len(downloads) != 0
if pendingdownloads:
rtorrent = xmlrpclib.ServerProxy(Config.SHOP_RPC_URL)
for download in downloads:
try:
infohash = download.infohash
iscomplete = rtorrent.d.get_complete(infohash) == 1
if iscomplete:
shopservice.importDownload(download)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
importtrace = repr(traceback.format_exception(exc_type, exc_value, exc_traceback))
log.error('[shop worker] caught exception in loop ' + importtrace)
Session.rollback()
Session.begin()
download.failedimport = True
download.importtrace = importtrace
Session.commit()
except Exception as e:
log.error('[shop worker] caught exception out of loop ' + repr(e))
Session.rollback()
if pendingdownloads:
time.sleep(10)
else:
time.sleep(30)
def currentMembersForTrackArtist(self):
trackid = request.params["trackid"]
track = Session.query(Track).filter_by(id=trackid).one()
# search for triples with subject matching current artist
db_uri = u""
for triple in Session.query(RDFTriple).filter_by(artistid=track.artistid):
if triple.subject == ":artist":
if triple.predicate == "hasdbpedia":
db_uri = triple.obj
if db_uri == "":
art_mbid = track.artist.mbid
db_uri = sDB.dbpedia_from_MBID(art_mbid)
db_unicode = unicode(db_uri.__str__())
artist = track.artist
triple = RDFTriple(
subject=u":artist", predicate=u"hasdbpedia", obj=db_unicode, artist=artist, track=None, album=None
)
Session.save(triple)
Session.commit()
g = Graph()
g.parse(db_uri)
pmem_re = re.compile("pastmember", re.I)
cmem_re = re.compile("currentmember", re.I)
out = {}
past_members = []
current_members = []
for s, p, o in g.triples((None, None, None)):
if re.search(pmem_re, p):
past_members.append(getURILabel(o))
if re.search(cmem_re, p):
current_members.append(getURILabel(o))
out["past_members"] = past_members
out["current_members"] = current_members
return sjson.dumps(out)
def create(self):
usr = request.params['login']
if Session.query(User).filter_by(user_name=usr).count() > 0:
return simplejson.dumps({'success':False,'msg':'That username is already taken, sorry.'})
pwd = request.params['pass']
if len(usr) < 3 or len(pwd) < 3:
return simplejson.dumps({'success':False,'msg':'Your username and password must each be at least 3 characters.'})
code = request.params['code']
invite = Session.query(Invite).filter_by(code=code).first()
if invite is None:
return simplejson.dumps({'success':False,'msg':'Your registration code appears to be invalid.'})
user = User()
user.who = invite.who
user.user_name = usr
user.password = pwd
user.registered = datetime.now()
Session.begin()
user.groups = [Session.query(Group).filter_by(group_name='users').one()]
Session.delete(invite)
Session.add(user)
Session.commit()
return simplejson.dumps({'success':True})
def setup_app(command, conf, vars):
"""Place any commands to setup scatterbrainz here"""
load_environment(conf.global_conf, conf.local_conf)
# Create the tables if they don't already exist
log.info("Creating tables")
meta.metadata.create_all(bind=meta.engine)
log.info("Tables created")
session = Session()
session.begin()
loginPerm = Permission()
loginPerm.permission_name = u'login'
adminPerm = Permission()
adminPerm.permission_name = u'admin'
adminGroup = Group()
adminGroup.group_name = u'admins'
adminGroup.permissions = [loginPerm, adminPerm]
userGroup = Group()
userGroup.group_name = u'users'
userGroup.permissions = [loginPerm]
admin = User()
admin.user_name = u'admin'
admin.password = u'default'
admin.who = u'admin'
admin.registered = datetime.now()
admin.groups = [adminGroup]
session.add_all([loginPerm, adminPerm, adminGroup, userGroup, admin])
session.commit()
def index(self):
t = Track('asdf')
Session.save(t)
Session.commit()
def load(self):
commit = 'commit' in request.params and request.params['commit'] == 'true'
s = ''
if commit:
Session.begin()
now = datetime.now()
initialLoad = True #Session.query(AudioFile).count() == 0
if initialLoad:
s = _msg(s, 'Initial track loading!')
else:
s = _msg(s, 'Updating tracks!')
then = now
missing = 0
changed = 0
for track in Session.query(AudioFile):
path = os.path.join(MUSIC, track.filepath)
if os.path.exists(path):
size = os.path.getsize(path)
mtime = datetime.fromtimestamp(os.path.getmtime(path))
if size != track.filesize or mtime != track.filemtime:
changed = changed + 1
s = _msg(s, 'Modified file: ' + path)
if commit:
raise Exception('not implemented!')
else:
s = _msg(s, 'Deleted file: ' + path)
missing = missing + 1
if commit:
Session.delete(track)
s = _msg(s, 'Found ' + str(missing) + ' missing files and ' + str(changed) + ' modified files, took ' + \
str(datetime.now() - then))
then = datetime.now()
filepaths = set(map(lambda t: t.filepath, Session.query(AudioFile)))
s = _msg(s, 'Querying for all filepaths took ' + str(datetime.now() - then))
then = datetime.now()
added = 0
skippedNoMBID = 0
release_groups_added = set()
unknownrelease = set()
unknownrecording = set()
alreadyhaverecordingrelease = set()
alreadyhavereleasegroup = set()
unicodeproblems = set()
fuckedmp3s = set()
for dirname, dirnames, filenames in os.walk(INCOMING, followlinks=True):
for filename in filenames:
if not os.path.splitext(filename)[-1].lower() == '.mp3':
continue
try:
filepath = os.path.join(os.path.relpath(dirname, INCOMING), filename).decode('utf-8')
except UnicodeDecodeError:
log.error('unicode problem ' + os.path.join(os.path.relpath(dirname, INCOMING), filename))
unicodeproblems.add(os.path.join(os.path.relpath(dirname, INCOMING), filename))
continue
if not initialLoad and filepath in filepaths:
continue
if not initialLoad:
s = _msg(s, 'New file: ' + filepath)
if not commit:
continue
# get size, date
fileabspath = os.path.join(dirname,filename)
filesize = os.path.getsize(fileabspath)
filemtime = datetime.fromtimestamp(os.path.getmtime(fileabspath))
# mp3 length, bitrate, etc.
try:
mutagen = MP3(fileabspath)
except:
fuckedmp3s.add(fileabspath)
log.error('f****d mp3 ' + fileabspath)
continue
info = mutagen.info
mp3bitrate = info.bitrate
mp3samplerate = info.sample_rate
mp3length = int(round(info.length))
if info.sketchy:
fuckedmp3s.add(fileabspath)
log.error('sketchy mp3! ' + fileabspath)
continue
# brainz!!
if RECORDING_MBID_KEY not in mutagen:
skippedNoMBID = skippedNoMBID + 1
continue
recordingmbid = mutagen[RECORDING_MBID_KEY].data
if not recordingmbid:
skippedNoMBID = skippedNoMBID + 1
continue
if RELEASE_MBID_KEY not in mutagen:
skippedNoMBID = skippedNoMBID + 1
continue
releasembid = mutagen[RELEASE_MBID_KEY].text[0]
if not releasembid or len(mutagen[RELEASE_MBID_KEY].text) > 1:
skippedNoMBID = skippedNoMBID + 1
continue
release = Session.query(MBRelease).filter(MBRelease.gid==releasembid).first()
if release is None:
release = Session.query(MBRelease) \
.join(MBReleaseGIDRedirect) \
.filter(MBReleaseGIDRedirect.gid==releasembid) \
.first()
if release is None:
if releasembid not in unknownrelease:
log.error('couldnt find release mbid ' + releasembid)
unknownrelease.add(releasembid)
continue
recording = Session.query(MBRecording).filter(MBRecording.gid==recordingmbid).first()
if recording is None:
recording = Session.query(MBRecording) \
.join(MBRecordingGIDRedirect) \
.filter(MBRecordingGIDRedirect.gid==recordingmbid) \
.first()
if recording is None:
if recordingmbid not in unknownrecording:
log.error('couldnt find recording mbid ' + recordingmbid)
unknownrecording.add(recordingmbid)
continue
releasegroupmbid = release.releasegroup.gid
dirs = os.path.join(releasegroupmbid[:2], releasegroupmbid)
newdir = os.path.join(MUSIC, dirs)
if releasegroupmbid not in release_groups_added:
existing = Session.query(Album) \
.filter(Album.mbid==releasegroupmbid) \
.first()
if existing != None:
if releasegroupmbid not in alreadyhavereleasegroup:
log.info('already have release group ' + existing.artistcredit + ' ' + existing.name + ' ' + existing.mbid)
alreadyhavereleasegroup.add(releasegroupmbid)
continue
else:
os.makedirs(newdir)
release_groups_added.add(releasegroupmbid)
existing = Session.query(AudioFile) \
.filter(AudioFile.recordingmbid==recording.gid) \
.filter(AudioFile.releasembid==release.gid) \
.first()
if existing:
if (recording.gid + '-' + release.gid) not in alreadyhaverecordingrelease:
log.error('already existing recording/release combo for file ' + filepath)
alreadyhaverecordingrelease.add(recording.gid + '-' + release.gid)
continue
# check for existing release group, make new release group directory if necessary
# link file into new directory
newfilename = release.gid + '-' + recording.gid + '.mp3'
ln = os.path.join(newdir, newfilename)
os.link(fileabspath, ln)
lnrelpath = os.path.join(dirs, newfilename)
track = AudioFile(filepath=lnrelpath,
filesize=filesize,
filemtime=filemtime,
mp3bitrate=mp3bitrate,
mp3samplerate=mp3samplerate,
mp3length=mp3length,
recording=recording,
release=release,
added=now,
)
Session.add(track)
added = added + 1
if commit:
s = _msg(s, 'Building model for new tracks took ' + str(datetime.now() - then))
then = datetime.now()
Session.commit()
s = _msg(s, """Committed %(added)d new tracks, skipped %(skipped)d""" \
% {'added':added-skippedNoMBID, 'skipped':skippedNoMBID} + ', took ' + str(datetime.now() - then))
log.error("unknownrelease: " + unknownrelease.__str__())
log.error("unknownrecording: " + unknownrecording.__str__())
log.error("alreadyhaverecordingrelease: " + alreadyhaverecordingrelease.__str__())
log.error("alreadyhavereleasegroup: " + alreadyhavereleasegroup.__str__())
log.error("unicodeproblems: " + unicodeproblems.__str__())
log.error("fuckedmp3s: " + fuckedmp3s.__str__())
else:
s = _msg(s, 'Saw ' + str(added) + ' new tracks, took ' + str(datetime.now() - then))
return s
def load(self):
done = False
now = datetime.now()
numFilesSeen = 0
numBadFiles = 0
numSketchy = 0
commitbuffer = []
albums = {}
artists = {}
variousArtists = Artist(name='Various Artists',
mbid='89ad4ac3-39f7-470e-963a-56509c546377',
added=now)
artists['Various Artists'] = variousArtists
Session.save(variousArtists)
for dirname, dirnames, filenames in os.walk('scatterbrainz/public/.music/'):
localAlbums = {}
for filename in filenames:
try:
numFilesSeen = numFilesSeen + 1
# get path, size, date
fileabspath = os.path.join(dirname,filename)
filepath = os.path.join(os.path.relpath(dirname, 'scatterbrainz/public/.music/'), filename)
filesize = os.path.getsize(fileabspath)
filemtime = datetime.fromtimestamp(os.path.getmtime(fileabspath))
ext = os.path.splitext(filename)[-1]
if not ext == '.mp3': continue
# mp3 length, bitrate, etc.
mutagen = MP3(fileabspath, ID3=EasyID3)
info = mutagen.info
mp3bitrate = info.bitrate
mp3samplerate = info.sample_rate
mp3length = int(round(info.length))
if info.sketchy:
mp3['sketchy'] = true
numSketchy = numSketchy + 1
log.warn('sketchy MP3: ' + filename)
# id3
# keys: ['album', 'date', 'version', 'composer', 'title'
# 'genre', 'tracknumber', 'lyricist', 'artist']
id3artist = getid3prop(mutagen, 'artist')
id3album = getid3prop(mutagen, 'album')
id3title = getid3prop(mutagen, 'title')
id3tracknum = getid3prop(mutagen, 'tracknumber')
id3date = getid3prop(mutagen, 'date')
id3composer = getid3prop(mutagen, 'composer')
id3genre = getid3prop(mutagen, 'genre')
id3lyricist = getid3prop(mutagen, 'lyricist')
# additional musicbrainz related keys: At some point,
# should probably switch from easyID3 to ordinary ID3
# class to get extra MB relationship data.
mbartistid = getid3prop(mutagen,'musicbrainz_albumartistid')
mbalbumid = getid3prop(mutagen,'musicbrainz_albumid')
mbtrackid = getid3prop(mutagen,'musicbrainz_trackid')
if not id3artist:
artist = None
elif id3artist in artists:
artist = artists[id3artist]
else:
artist = Artist(name=id3artist,
mbid=mbartistid,
added=now)
Session.save(artist)
artists[id3artist] = artist
if not id3album:
album = None
elif id3album in localAlbums:
album = localAlbums[id3album]
if artist != album.artist:
album.artist = variousArtists
else:
album = Album(name=id3album,
artist=artist,
<<<<<<< HEAD
mbid=mbalbumid,
albumArtURL=None,
=======
>>>>>>> upstream/master
added=now)
Session.save(album)
albums[id3album] = album
localAlbums[id3album] = album
track = Track(artist=artist,
album=album,
filepath=filepath.decode('utf-8'),
filesize=filesize,
filemtime=filemtime,
mp3bitrate=mp3bitrate,
mp3samplerate=mp3samplerate,
mp3length=mp3length,
id3artist=id3artist,
id3album=id3album,
id3title=id3title,
id3tracknum=id3tracknum,
id3date=id3date,
id3composer=id3composer,
id3genre=id3genre,
id3lyricist=id3lyricist,
added=now,
mbid=mbtrackid,
)
Session.save(track)
triple = RDFTriple(subject = u":track",
predicate = u"ison",
obj=u":album",
artist=None,
track=track,
album=album)
Session.save(triple)
except Exception as e:
numBadFiles = numBadFiles + 1
log.error('Could not load file "' + filename + '" due to exception: '
+ e.__class__.__name__ + ': ' + str(e))
if done:
break
Session.commit()
otherNow = datetime.now()
return """Saw %(numFilesSeen)d tracks, %(numArtists)d artists and %(numAlbums)d albums.
%(numBadFiles)d failed, %(numSketchy)d sketchy. Loaded in %(time)s""" \
% {'numFilesSeen':numFilesSeen, 'numBadFiles':numBadFiles,
'numArtists': len(artists), 'numAlbums': len(albums),
'numSketchy' : numSketchy, 'time' : str(otherNow - now)}
def getLyricsAJAX(self):
trackid = request.params['trackid'].split('_')[1]
track = Session.query(Track).filter_by(id=trackid).one()
if not track.lyrics and \
(track.lastHitLyricWiki is None or \
datetime.now() > track.lastHitLyricWiki + timedelta(days=10)):
track.lastHitLyricWiki = datetime.now()
title = track.id3title
artist = track.id3artist
def getLyricsHtml(encoder = lambda s: s):
params = {
'artist' : encoder(artist),
'song' : encoder(title),
'fmt' : 'json',
}
url = 'http://lyrics.wikia.com/api.php?%s' % urllib.urlencode(params)
log.info('[lyric] Hitting ' + url)
html = urllib.urlopen(url).read()
return html if not "'lyrics':'Not found'" in html else None
try:
html = getLyricsHtml()
except UnicodeEncodeError:
html = getLyricsHtml(lambda s: s.encode('utf-8')) or \
getLyricsHtml(lambda s: unicodedata.normalize('NFKD', s).encode('ascii', 'ignore'))
# wikia keys some int'l lyrics on correctly-encoded titles
# and others on normalized (e.g., "el nino") titles
if html:
search = re.search("'url':'(?P<url>.*?)'",html)
lyricurl = urllib.unquote(search.group('url'))
page = urllib.quote(lyricurl.split('/')[-1])
lyricurl = 'http://lyrics.wikia.com/index.php?title='+page+'&action=edit'
log.info('[lyric] Hitting ' + lyricurl)
lyrichtml = urllib.urlopen(lyricurl).read()
lyricREstr = "<lyrics>(?P<lyrics>.*)</lyrics>"
lyricRE = re.search(lyricREstr, lyrichtml, re.S)
if lyricRE:
lyrics = lyricRE.group('lyrics').strip('\n')
if '{{gracenote_takedown}}' in lyrics:
historyurl = 'http://lyrics.wikia.com/index.php?title='+page+'&action=history'
log.info('[lyric] Found gracenote takedown, hitting ' + historyurl)
historyhtml = urllib.urlopen(historyurl).read()
oldidRE = re.search(".*GracenoteBot.*?/index\.php\?title=.*?&oldid=(?P<oldid>\d+)", historyhtml, re.S)
if oldidRE:
oldid = oldidRE.group('oldid')
oldlyricsurl = lyricurl + '&oldid=' + oldid
log.info('[lyric] found pre-takedown lyrics! hitting ' + oldlyricsurl)
oldlyrichtml = urllib.urlopen(oldlyricsurl).read()
lyricRE = re.search(lyricREstr, oldlyrichtml, re.S)
if lyricRE:
lyrics = lyricRE.group('lyrics').strip('\n')
if '{{gracenote_takedown}}' in lyrics:
raise Exception('[lyric] Still found takedown lyrics!')
elif '{{Instrumental}}' in lyrics:
track.lyrics = u'(Instrumental)'
else:
track.lyrics = lyrics.replace('\n','<br />').decode('utf-8')
else:
log.info('[lyric] failed lyrics!')
raise Exception('failed lyrics!')
else:
log.info('[lyric] no pre-takedown lyrics found :(')
elif '{{Instrumental}}' in lyrics:
track.lyrics = u'(Instrumental)'
else:
track.lyrics = lyrics.replace('\n','<br />').decode('utf-8')
else:
log.info('[lyric] failed lyrics!')
raise Exception('failed lyrics!')
else:
log.info('[lyric] No results found')
Session.begin()
Session.commit()
json = {}
if track.lyrics:
json['lyrics'] = track.lyrics
json['trackid'] = track.id
json['frames'] = track.lyricsFrames
return simplejson.dumps(json)
def download(Session, mbid, owner_id):
with shoplock:
now = datetime.now()
existingdownload = Session.query(ShopDownload) \
.filter(ShopDownload.release_group_mbid==mbid) \
.first()
if existingdownload:
return existingdownload.infohash
attempt = Session.query(ShopDownloadAttempt) \
.filter(ShopDownloadAttempt.mbid==mbid) \
.first()
if attempt and now < attempt.tried + timedelta(days=10):
return None
if not attempt:
attempt = ShopDownloadAttempt(mbid, now, True)
attempt.tried = now
log.info('[shop] searching for ' + mbid)
Session.begin()
if not maybeloggedin:
login()
(album, albumname, artistname) = Session.query(MBReleaseGroup, MBReleaseName, MBArtistName) \
.join(MBReleaseName) \
.join(MBReleaseGroup.artistcredit, MBArtistCredit.name) \
.filter(MBReleaseGroup.gid==mbid) \
.one()
searchartist = artistname.name
searchalbum = albumname.name
if searchartist == 'Various Artists':
searchartist = ''
searchartist = _cleanupSearchTerm(searchartist)
searchalbum = _cleanupSearchTerm(searchalbum)
url = searchurl + '?' + urllib.urlencode({
'artistname' : searchartist,
'groupname' : searchalbum,
'action' : 'advanced',
'format' : 'MP3',
'order_by' : 'seeders'
})
log.info('[shop] hitting ' + url)
handle = opener.open(url)
page = lxml.parse(handle).getroot()
html = lxml.tostring(page)
if loginpath in html:
log.warn('[shop] login url found in search result, logging in..')
login()
handle = opener.open(url)
page = lxml.parse(handle).getroot()
html = lxml.tostring(page)
if loginpath in html:
log.error('[shop] couldnt login!')
raise Exception('couldnt login!')
if 'Your search did not match anything' in html:
log.warn('[shop] no results')
attempt.gotsearchresults = False
if attempt not in Session:
Session.add(attempt)
Session.commit()
return None
# Gather up all tracks from all releases
results = Session.query(MBTrack, MBArtistName, MBTrackName, MBRelease, MBMedium) \
.join(MBTrackList, MBMedium, MBRelease, MBReleaseGroup) \
.join(MBTrackName).join(MBTrack.artistcredit, MBArtistCredit.name) \
.filter(MBReleaseGroup.gid==mbid) \
.all()
releases = {}
for (track, artistname, trackname, release, medium) in results:
data = {'id' : track.id, 'num' : track.position, 'disc' : medium.position, 'artist' : artistname.name, 'name' : trackname.name}
mbid = release.gid
if mbid in releases:
releases[mbid].append(data)
else:
releases[mbid] = [data]
for releaseid in releases.keys():
release = releases[releaseid]
release.sort(key=itemgetter('disc', 'num'))
log.info('[shop] release group ' + mbid + ' has ' + str(len(releases)) + ' releases to check against')
# Try to match any of the releases with the first few results
torrent_table = page.cssselect('table#torrent_table')[0]
groups = torrent_table.cssselect('tr.group')
numresults = len(groups)
for torrent in groups[:3]:
torrentpageurl = shopbaseurl + '/' + torrent.cssselect('a[title="View Torrent"]')[0].attrib['href']
log.info('[shop] hitting ' + torrentpageurl)
thandle = opener.open(torrentpageurl)
tpage = lxml.parse(thandle).getroot()
# Gather up information about all downloads for this torrent
ttable = tpage.cssselect('table.torrent_table')[0]
downloads = []
for download in ttable.cssselect('tr.group_torrent[id]'):
torrentid = re.sub('^torrent', '', download.attrib['id'])
torrenttype = download.cssselect('a[onclick]')[0].text.encode('ascii', 'ignore').split('/')[0].strip()
if torrenttype != 'MP3':
continue
downloadurl = download.cssselect('a[title=Download]')[0].attrib['href']
if len(download.cssselect('td')) != 5:
raise Exception('Torrent ' + torrentid + ' has !=5 TD tags at ' + torrentpageurl)
numseeders = int(download.cssselect('td')[3].text.replace(',', ''))
if numseeders < 1:
continue
filestr = ttable.cssselect('div#files_' + torrentid)[0].cssselect('tr')[1:]
filenames = []
for tr in filestr:
filename = tr.cssselect('td')[0].text
if filename.lower().endswith('.mp3'):
filenames.append({'original' : filename,
'compare' : re.sub(' +', '', filename.split('/')[-1][:-4].lower())})
downloads.append({'seeders' : numseeders, 'torrentid' : torrentid, 'url' : downloadurl, 'filenames' : filenames})
if not downloads:
log.info('[shop] no seeded files of correct type found at torrent ' + torrentid)
# See if any of the downloads nicely match any of the releases, trying best seeded first
downloads.sort(key=itemgetter('seeders'), reverse=True)
for download in downloads:
releasescores = []
filenames = download['filenames']
for releaseid in releases.keys():
release = releases[releaseid]
if len(filenames) != len(release):
minscore = 0
avgscore = 0
else:
numtracks = len(release)
minscore = None
minscoreidx = None
sumscore = 0
for i in range(numtracks):
rtrack = release[i]
rtracknum = '%02d' % rtrack['num']
rtartist = rtrack['artist'].lower()
rtname = rtrack['name'].lower()
dname = filenames[i]['compare']
name1 = rtracknum + ' ' + rtname
name2 = rtracknum + ' ' + rtartist + ' ' + rtname
score1 = SequenceMatcher(None, dname, name1).ratio()
score2 = SequenceMatcher(None, dname, name2).ratio()
score = max(score1, score2)
sumscore = sumscore + score
if score < minscore or minscore is None:
minscore = score
minscoreidx = i
avgscore = sumscore * 1.0 / numtracks
log.info('[shop] match avg=' + str(avgscore) + ' min=' + str(minscore) + ' ' + download['torrentid'] + ' -> ' + releaseid)
releasescores.append({'releaseid' : releaseid, 'min' : minscore, 'avg' : avgscore})
releasescores = filter(lambda x: x['min'] > 0.3 and x['avg'] > 0.70, releasescores)
releasescores.sort(key=itemgetter('avg'), reverse=True)
if releasescores:
# Toss torrent over to rtorrent via xml-rpc
releaseid = releasescores[0]['releaseid']
torrenturl = shopbaseurl + '/' + download['url']
torrentdata = opener.open(torrenturl).read()
torrentdecode = bencode.bdecode(torrentdata)
infohash = hashlib.sha1(bencode.bencode(torrentdecode['info'])).hexdigest().upper()
with tempfile.NamedTemporaryFile(delete=False) as torrentfile:
torrentpath = torrentfile.name
torrentfile.write(torrentdata)
# scp torrent over if necessary
if Config.SCP_SHOP_DOWNLOADS:
remotetorrentpath = '/tmp/' + infohash + '.torrent'
cmd = Config.SCP_CMD + ' ' + torrentpath + ' ' + Config.SCP_REMOTE + ':' + remotetorrentpath
log.info('[shop] running ' + cmd)
retval = os.system(cmd)
os.unlink(torrentpath)
if retval != 0:
raise Exception('scp command [' + cmd + '] returned ' + str(retval))
torrentpath = remotetorrentpath
rtorrent = xmlrpclib.ServerProxy(Config.SHOP_RPC_URL)
rtorrent.load_start(torrentpath, "execute=rm," + torrentpath)
log.info('[shop] downloaded ' + torrenturl + ' has ' + str(download['seeders']) +
' seeders, infohash=' + infohash + ', match to album ' + releaseid)
file_json = simplejson.dumps(map(itemgetter('original'), filenames))
minscore = releasescores[0]['min']
avgscore = releasescores[0]['avg']
shopdownload = ShopDownload(releaseid, album.gid, infohash, torrenturl, torrentpageurl, download['torrentid'], download['seeders'], file_json, minscore, avgscore, owner_id)
Session.add(shopdownload)
if attempt in Session:
Session.delete(attempt)
Session.commit()
return infohash
log.info('[shop] no matches, sorry :(')
attempt.gotsearchresults = True
if attempt not in Session:
Session.add(attempt)
Session.commit()
return None
def load(self):
commit = 'commit' in request.params and request.params['commit'] == 'true'
s = ''
now = datetime.now()
albums = {}
artists = {}
if commit:
Session.begin()
variousArtists = Session.query(Artist).filter_by(mbid=u'89ad4ac3-39f7-470e-963a-56509c546377').first()
if variousArtists is None:
variousArtists = Artist(name=u'Various Artists',
mbid=u'89ad4ac3-39f7-470e-963a-56509c546377',
added=now)
Session.save(variousArtists)
s = _msg(s, 'Committed various artists placeholder')
artists['Various Artists'] = variousArtists
initialLoad = Session.query(Track).count() == 0
if initialLoad:
s = _msg(s, 'Initial track loading!')
else:
s = _msg(s, 'Updating tracks!')
then = now
missing = 0
changed = 0
for track in Session.query(Track):
path = os.path.join(BASE, track.filepath)
if os.path.exists(path):
size = os.path.getsize(path)
mtime = datetime.fromtimestamp(os.path.getmtime(path))
if size != track.filesize or mtime != track.filemtime:
changed = changed + 1
s = _msg(s, 'Modified file: ' + path)
if commit:
raise Exception('not implemented!')
else:
s = _msg(s, 'Deleted file: ' + path)
missing = missing + 1
if commit:
Session.delete(track)
s = _msg(s, 'Found ' + str(missing) + ' missing files and ' + str(changed) + ' modified files, took ' + \
str(datetime.now() - then))
then = datetime.now()
filepaths = set(map(lambda t: t.filepath, Session.query(Track)))
s = _msg(s, 'Querying for all filepaths took ' + str(datetime.now() - then))
then = datetime.now()
added = 0
for dirname, dirnames, filenames in os.walk(BASE):
localAlbums = {}
for filename in filenames:
filepath = os.path.join(os.path.relpath(dirname, BASE), filename).decode('utf-8')
if not os.path.splitext(filename)[-1].lower() == '.mp3':
continue
if not initialLoad and filepath in filepaths:
continue
added = added + 1
if not initialLoad:
s = _msg(s, 'New file: ' + filepath)
if not commit:
continue
# get size, date
fileabspath = os.path.join(dirname,filename)
filesize = os.path.getsize(fileabspath)
filemtime = datetime.fromtimestamp(os.path.getmtime(fileabspath))
# mp3 length, bitrate, etc.
mutagen = MP3(fileabspath, ID3=EasyID3)
info = mutagen.info
mp3bitrate = info.bitrate
mp3samplerate = info.sample_rate
mp3length = int(round(info.length))
if info.sketchy:
raise Exception('sketchy mp3! ' + filename)
# id3
# keys: ['album', 'date', 'version', 'composer', 'title'
# 'genre', 'tracknumber', 'lyricist', 'artist']
id3artist = getid3prop(mutagen, 'artist')
id3album = getid3prop(mutagen, 'album')
id3title = getid3prop(mutagen, 'title')
id3tracknum = getid3prop(mutagen, 'tracknumber')
id3date = getid3prop(mutagen, 'date')
id3composer = getid3prop(mutagen, 'composer')
id3genre = getid3prop(mutagen, 'genre')
id3lyricist = getid3prop(mutagen, 'lyricist')
# additional musicbrainz related keys: At some point,
# should probably switch from easyID3 to ordinary ID3
# class to get extra MB relationship data.
mbartistid = getid3prop(mutagen,'musicbrainz_albumartistid')
mbalbumid = getid3prop(mutagen,'musicbrainz_albumid')
mbtrackid = getid3prop(mutagen,'musicbrainz_trackid')
if not id3artist:
artist = None
elif id3artist in artists:
artist = artists[id3artist]
else:
if initialLoad:
artistFromDb = None
else:
artistFromDb = Session.query(Artist).filter_by(name=id3artist).first()
if artistFromDb is None:
artist = Artist(name=id3artist,
mbid=mbartistid,
added=now)
Session.save(artist)
else:
artist = artistFromDb
artists[id3artist] = artist
if not id3album:
album = None
elif id3album in localAlbums:
album = localAlbums[id3album]
if artist != album.artist:
album.artist = variousArtists
else:
album = Album(name=id3album,
artist=artist,
added=now,
mbid=mbalbumid)
Session.save(album)
albums[id3album] = album
localAlbums[id3album] = album
track = Track(artist=artist,
album=album,
filepath=filepath,
filesize=filesize,
filemtime=filemtime,
mp3bitrate=mp3bitrate,
mp3samplerate=mp3samplerate,
mp3length=mp3length,
id3artist=id3artist,
id3album=id3album,
id3title=id3title,
id3tracknum=id3tracknum,
id3date=id3date,
id3composer=id3composer,
id3genre=id3genre,
id3lyricist=id3lyricist,
added=now,
mbid=mbtrackid,
)
Session.save(track)
if commit:
s = _msg(s, 'Building model for new tracks took ' + str(datetime.now() - then))
then = datetime.now()
Session.commit()
s = _msg(s, """Committed %(added)d new tracks, %(numArtists)d new artists, %(numAlbums)d new albums""" \
% {'added':added, 'numArtists': len(artists), 'numAlbums': len(albums)} + \
', took ' + str(datetime.now() - then))
else:
s = _msg(s, 'Saw ' + str(added) + ' new tracks, took ' + str(datetime.now() - then))
return s
def importDownload(shopdownload):
with importlock:
Session.begin()
assert not shopdownload.isdone, 'download is already imported'
assert not shopdownload.failedimport, 'download already failed import'
log.info('[shop] starting to import ' + shopdownload.infohash)
promisedfiles = simplejson.loads(shopdownload.file_json)
rtorrent = xmlrpclib.ServerProxy(Config.SHOP_RPC_URL)
assert rtorrent.d.get_complete(shopdownload.infohash) == 1, 'rtorrent says download isnt done yet'
release_mbid = shopdownload.release_mbid
mbalbum = Session.query(MBReleaseGroup) \
.join(MBRelease) \
.filter(MBRelease.gid==release_mbid) \
.one()
mbartists = Session.query(MBArtist, MBArtistName) \
.join(MBArtistCreditName, MBArtistCredit, MBReleaseGroup) \
.join(MBArtist.name) \
.filter(MBReleaseGroup.gid==mbalbum.gid) \
.all()
artists = Session.query(Artist) \
.filter(Artist.mbid.in_(map(lambda x: x[0].gid, mbartists))) \
.all()
localartistmbids = map(attrgetter('mbid'), artists)
# Add Album, Artists and artist-album relationships
insertmaps = []
for (artist, name) in mbartists:
if artist.gid not in localartistmbids:
a = Artist(name.name, artist.gid)
artists.append(a)
Session.add(a)
albumname = mbalbum.name.name
artistname = mbalbum.artistcredit.name.name
albummeta = mbalbum.meta[0]
album = Album(unicode(mbalbum.gid), unicode(albumname), unicode(artistname), albummeta.year, albummeta.month, albummeta.day, unicode(artistname + ' ' + albumname))
album.artists = artists
# Build up mapping of promised filename -> (Track)
results = Session.query(MBTrack, MBMedium, MBRecording, MBTrackName) \
.join(MBTrackList, MBMedium, MBRelease) \
.join(MBRecording) \
.join(MBTrack.name) \
.filter(MBRelease.gid==release_mbid) \
.all()
tracks = []
for (track, medium, recording, name) in results:
stableid = hashlib.md5(release_mbid + '_' + recording.gid +
'_' + str(track.position) + '_' + str(medium.position)).hexdigest()
track = Track(unicode(stableid), None, recording.gid, mbalbum.gid, unicode(name.name), track.position, medium.position, albumname, artistname)
tracks.append(track)
Session.add(track)
tracks.sort(key=attrgetter('discnum', 'tracknum'))
assert len(promisedfiles) == len(tracks), 'len(promisedfiles=' + promisedfiles.__repr__() + ') != len(tracks=' + tracks.__repr__() + ')'
promisedfilemap = {}
for i in range(len(tracks)):
normalizedfilename = filter(str.isalnum, promisedfiles[i].lower().encode('ascii', 'ignore'))
assert normalizedfilename not in promisedfilemap, 'normalizedfilename='+normalizedfilename + ' in promisedfilemap=' + promisedfilemap.__repr__()
promisedfilemap[normalizedfilename] = tracks[i]
# Build up mapping of absolute track filename -> (Track, AudioFile),
# and link files into their proper library location
trackfiles = []
now = datetime.now()
dirpath = album.mbid[:2] + '/' + album.mbid
if os.path.isdir(Config.MUSIC_PATH + dirpath):
assert False, 'directory ' + Config.MUSIC_PATH + dirpath + ' already exists'
os.mkdir(Config.MUSIC_PATH + dirpath)
torrentdir = rtorrent.d.get_base_path(shopdownload.infohash)
# scp stuff back if necessary
if Config.SCP_SHOP_DOWNLOADS:
remote_dir = pipes.quote(pipes.quote(torrentdir)) # this is awesome
local_dir = Config.SCP_FOLDER + '/' + shopdownload.infohash
cmd = Config.SCP_CMD + ' ' + Config.SCP_REMOTE + ':' + remote_dir + ' ' + local_dir
log.info('[shop] running ' + cmd)
scpstart = time.time()
retval = os.system(cmd)
scpend = time.time()
elapsed = int(round(scpend-scpstart))
log.info('[shop] scp of ' + shopdownload.infohash + ' done, took ' + str(elapsed) + ' seconds')
if retval != 0:
raise Exception('scp command [' + cmd + '] returned ' + str(retval))
torrentdir = local_dir
os.system("find " + torrentdir + " -type f -exec rename -v 's/[^[:ascii:]]//g' {} \;")
for root, dirs, actualfiles in os.walk(torrentdir):
for f in actualfiles:
abspath = os.path.join(root, f)
relpath = os.path.join(os.path.relpath(root, torrentdir), f)
if relpath.startswith('./'):
relpath = relpath[2:]
normalizedfilename = filter(str.isalnum, relpath.lower().encode('ascii', 'ignore'))
if normalizedfilename not in promisedfilemap:
continue
track = promisedfilemap[normalizedfilename]
filepath = dirpath + '/' + release_mbid + '-' + track.mbid + '.mp3'
os.link(abspath, Config.MUSIC_PATH + filepath)
filesize = os.path.getsize(abspath)
filemtime = datetime.fromtimestamp(os.path.getmtime(abspath))
mutagen = MP3(abspath)
info = mutagen.info
mp3bitrate = info.bitrate
mp3samplerate = info.sample_rate
mp3length = int(round(info.length))
audiofile = AudioFile(release_mbid, track.mbid, unicode(filepath), filesize, filemtime, mp3bitrate,
mp3samplerate, mp3length, now)
track.file = audiofile
trackfiles.append({'track' : track, 'file' : audiofile, 'path' : abspath})
Session.add(audiofile)
Session.add(album)
assert len(trackfiles) == len(tracks) == len(promisedfilemap), 'len(trackfiles='+str(len(trackfiles))+') == len(tracks='+str(len(tracks))+') == len(promisedfilemap='+str(len(promisedfilemap))+')'
shopdownload.isdone = True
shopdownload.finished = datetime.now()
Session.commit()
log.info('[shop] done importing ' + shopdownload.infohash)
