def savePlaylistAJAX(self):
playlistname = request.params['name']
trackids = simplejson.loads(request.params['trackids'])
trackids = map(lambda x: x.replace('track_',''), trackids)
results = Session.query(Track, MBRecording).join(MBRecording).filter(Track.id.in_(trackids)).all()
results.sort(lambda a,b: cmp(trackids.index(a[0].id), trackids.index(b[0].id)))
recordings = map(itemgetter(1), results)
Session.begin()
user_name = request.environ['repoze.what.credentials']['repoze.what.userid']
user_id = Session.query(User).filter(User.user_name==user_name).one().user_id
playlist = Session.query(Playlist) \
.filter(Playlist.owner_id==user_id) \
.filter(Playlist.name==playlistname) \
.first()
if not trackids:
if playlist is None:
return '{}'
else:
Session.delete(playlist)
else:
if playlist is None:
playlist = Playlist(user_id, playlistname)
Session.add(playlist)
else:
playlist.modified = datetime.now()
playlist.tracks = recordings
Session.commit()
return '{}'
def create(self):
usr = request.params['login']
if Session.query(User).filter_by(user_name=usr).count() > 0:
return simplejson.dumps({'success':False,'msg':'That username is already taken, sorry.'})
pwd = request.params['pass']
if len(usr) < 3 or len(pwd) < 3:
return simplejson.dumps({'success':False,'msg':'Your username and password must each be at least 3 characters.'})
code = request.params['code']
invite = Session.query(Invite).filter_by(code=code).first()
if invite is None:
return simplejson.dumps({'success':False,'msg':'Your registration code appears to be invalid.'})
user = User()
user.who = invite.who
user.user_name = usr
user.password = pwd
user.registered = datetime.now()
Session.begin()
user.groups = [Session.query(Group).filter_by(group_name='users').one()]
Session.delete(invite)
Session.add(user)
Session.commit()
return simplejson.dumps({'success':True})
def load(self):
commit = 'commit' in request.params and request.params['commit'] == 'true'
s = ''
if commit:
Session.begin()
now = datetime.now()
initialLoad = True #Session.query(AudioFile).count() == 0
if initialLoad:
s = _msg(s, 'Initial track loading!')
else:
s = _msg(s, 'Updating tracks!')
then = now
missing = 0
changed = 0
for track in Session.query(AudioFile):
path = os.path.join(MUSIC, track.filepath)
if os.path.exists(path):
size = os.path.getsize(path)
mtime = datetime.fromtimestamp(os.path.getmtime(path))
if size != track.filesize or mtime != track.filemtime:
changed = changed + 1
s = _msg(s, 'Modified file: ' + path)
if commit:
raise Exception('not implemented!')
else:
s = _msg(s, 'Deleted file: ' + path)
missing = missing + 1
if commit:
Session.delete(track)
s = _msg(s, 'Found ' + str(missing) + ' missing files and ' + str(changed) + ' modified files, took ' + \
str(datetime.now() - then))
then = datetime.now()
filepaths = set(map(lambda t: t.filepath, Session.query(AudioFile)))
s = _msg(s, 'Querying for all filepaths took ' + str(datetime.now() - then))
then = datetime.now()
added = 0
skippedNoMBID = 0
release_groups_added = set()
unknownrelease = set()
unknownrecording = set()
alreadyhaverecordingrelease = set()
alreadyhavereleasegroup = set()
unicodeproblems = set()
fuckedmp3s = set()
for dirname, dirnames, filenames in os.walk(INCOMING, followlinks=True):
for filename in filenames:
if not os.path.splitext(filename)[-1].lower() == '.mp3':
continue
try:
filepath = os.path.join(os.path.relpath(dirname, INCOMING), filename).decode('utf-8')
except UnicodeDecodeError:
log.error('unicode problem ' + os.path.join(os.path.relpath(dirname, INCOMING), filename))
unicodeproblems.add(os.path.join(os.path.relpath(dirname, INCOMING), filename))
continue
if not initialLoad and filepath in filepaths:
continue
if not initialLoad:
s = _msg(s, 'New file: ' + filepath)
if not commit:
continue
# get size, date
fileabspath = os.path.join(dirname,filename)
filesize = os.path.getsize(fileabspath)
filemtime = datetime.fromtimestamp(os.path.getmtime(fileabspath))
# mp3 length, bitrate, etc.
try:
mutagen = MP3(fileabspath)
except:
fuckedmp3s.add(fileabspath)
log.error('f****d mp3 ' + fileabspath)
continue
info = mutagen.info
mp3bitrate = info.bitrate
mp3samplerate = info.sample_rate
mp3length = int(round(info.length))
if info.sketchy:
fuckedmp3s.add(fileabspath)
log.error('sketchy mp3! ' + fileabspath)
continue
# brainz!!
if RECORDING_MBID_KEY not in mutagen:
skippedNoMBID = skippedNoMBID + 1
continue
recordingmbid = mutagen[RECORDING_MBID_KEY].data
if not recordingmbid:
skippedNoMBID = skippedNoMBID + 1
continue
if RELEASE_MBID_KEY not in mutagen:
skippedNoMBID = skippedNoMBID + 1
continue
releasembid = mutagen[RELEASE_MBID_KEY].text[0]
if not releasembid or len(mutagen[RELEASE_MBID_KEY].text) > 1:
skippedNoMBID = skippedNoMBID + 1
continue
release = Session.query(MBRelease).filter(MBRelease.gid==releasembid).first()
if release is None:
release = Session.query(MBRelease) \
.join(MBReleaseGIDRedirect) \
.filter(MBReleaseGIDRedirect.gid==releasembid) \
.first()
if release is None:
if releasembid not in unknownrelease:
log.error('couldnt find release mbid ' + releasembid)
unknownrelease.add(releasembid)
continue
recording = Session.query(MBRecording).filter(MBRecording.gid==recordingmbid).first()
if recording is None:
recording = Session.query(MBRecording) \
.join(MBRecordingGIDRedirect) \
.filter(MBRecordingGIDRedirect.gid==recordingmbid) \
.first()
if recording is None:
if recordingmbid not in unknownrecording:
log.error('couldnt find recording mbid ' + recordingmbid)
unknownrecording.add(recordingmbid)
continue
releasegroupmbid = release.releasegroup.gid
dirs = os.path.join(releasegroupmbid[:2], releasegroupmbid)
newdir = os.path.join(MUSIC, dirs)
if releasegroupmbid not in release_groups_added:
existing = Session.query(Album) \
.filter(Album.mbid==releasegroupmbid) \
.first()
if existing != None:
if releasegroupmbid not in alreadyhavereleasegroup:
log.info('already have release group ' + existing.artistcredit + ' ' + existing.name + ' ' + existing.mbid)
alreadyhavereleasegroup.add(releasegroupmbid)
continue
else:
os.makedirs(newdir)
release_groups_added.add(releasegroupmbid)
existing = Session.query(AudioFile) \
.filter(AudioFile.recordingmbid==recording.gid) \
.filter(AudioFile.releasembid==release.gid) \
.first()
if existing:
if (recording.gid + '-' + release.gid) not in alreadyhaverecordingrelease:
log.error('already existing recording/release combo for file ' + filepath)
alreadyhaverecordingrelease.add(recording.gid + '-' + release.gid)
continue
# check for existing release group, make new release group directory if necessary
# link file into new directory
newfilename = release.gid + '-' + recording.gid + '.mp3'
ln = os.path.join(newdir, newfilename)
os.link(fileabspath, ln)
lnrelpath = os.path.join(dirs, newfilename)
track = AudioFile(filepath=lnrelpath,
filesize=filesize,
filemtime=filemtime,
mp3bitrate=mp3bitrate,
mp3samplerate=mp3samplerate,
mp3length=mp3length,
recording=recording,
release=release,
added=now,
)
Session.add(track)
added = added + 1
if commit:
s = _msg(s, 'Building model for new tracks took ' + str(datetime.now() - then))
then = datetime.now()
Session.commit()
s = _msg(s, """Committed %(added)d new tracks, skipped %(skipped)d""" \
% {'added':added-skippedNoMBID, 'skipped':skippedNoMBID} + ', took ' + str(datetime.now() - then))
log.error("unknownrelease: " + unknownrelease.__str__())
log.error("unknownrecording: " + unknownrecording.__str__())
log.error("alreadyhaverecordingrelease: " + alreadyhaverecordingrelease.__str__())
log.error("alreadyhavereleasegroup: " + alreadyhavereleasegroup.__str__())
log.error("unicodeproblems: " + unicodeproblems.__str__())
log.error("fuckedmp3s: " + fuckedmp3s.__str__())
else:
s = _msg(s, 'Saw ' + str(added) + ' new tracks, took ' + str(datetime.now() - then))
return s
def download(Session, mbid, owner_id):
with shoplock:
now = datetime.now()
existingdownload = Session.query(ShopDownload) \
.filter(ShopDownload.release_group_mbid==mbid) \
.first()
if existingdownload:
return existingdownload.infohash
attempt = Session.query(ShopDownloadAttempt) \
.filter(ShopDownloadAttempt.mbid==mbid) \
.first()
if attempt and now < attempt.tried + timedelta(days=10):
return None
if not attempt:
attempt = ShopDownloadAttempt(mbid, now, True)
attempt.tried = now
log.info('[shop] searching for ' + mbid)
Session.begin()
if not maybeloggedin:
login()
(album, albumname, artistname) = Session.query(MBReleaseGroup, MBReleaseName, MBArtistName) \
.join(MBReleaseName) \
.join(MBReleaseGroup.artistcredit, MBArtistCredit.name) \
.filter(MBReleaseGroup.gid==mbid) \
.one()
searchartist = artistname.name
searchalbum = albumname.name
if searchartist == 'Various Artists':
searchartist = ''
searchartist = _cleanupSearchTerm(searchartist)
searchalbum = _cleanupSearchTerm(searchalbum)
url = searchurl + '?' + urllib.urlencode({
'artistname' : searchartist,
'groupname' : searchalbum,
'action' : 'advanced',
'format' : 'MP3',
'order_by' : 'seeders'
})
log.info('[shop] hitting ' + url)
handle = opener.open(url)
page = lxml.parse(handle).getroot()
html = lxml.tostring(page)
if loginpath in html:
log.warn('[shop] login url found in search result, logging in..')
login()
handle = opener.open(url)
page = lxml.parse(handle).getroot()
html = lxml.tostring(page)
if loginpath in html:
log.error('[shop] couldnt login!')
raise Exception('couldnt login!')
if 'Your search did not match anything' in html:
log.warn('[shop] no results')
attempt.gotsearchresults = False
if attempt not in Session:
Session.add(attempt)
Session.commit()
return None
# Gather up all tracks from all releases
results = Session.query(MBTrack, MBArtistName, MBTrackName, MBRelease, MBMedium) \
.join(MBTrackList, MBMedium, MBRelease, MBReleaseGroup) \
.join(MBTrackName).join(MBTrack.artistcredit, MBArtistCredit.name) \
.filter(MBReleaseGroup.gid==mbid) \
.all()
releases = {}
for (track, artistname, trackname, release, medium) in results:
data = {'id' : track.id, 'num' : track.position, 'disc' : medium.position, 'artist' : artistname.name, 'name' : trackname.name}
mbid = release.gid
if mbid in releases:
releases[mbid].append(data)
else:
releases[mbid] = [data]
for releaseid in releases.keys():
release = releases[releaseid]
release.sort(key=itemgetter('disc', 'num'))
log.info('[shop] release group ' + mbid + ' has ' + str(len(releases)) + ' releases to check against')
# Try to match any of the releases with the first few results
torrent_table = page.cssselect('table#torrent_table')[0]
groups = torrent_table.cssselect('tr.group')
numresults = len(groups)
for torrent in groups[:3]:
torrentpageurl = shopbaseurl + '/' + torrent.cssselect('a[title="View Torrent"]')[0].attrib['href']
log.info('[shop] hitting ' + torrentpageurl)
thandle = opener.open(torrentpageurl)
tpage = lxml.parse(thandle).getroot()
# Gather up information about all downloads for this torrent
ttable = tpage.cssselect('table.torrent_table')[0]
downloads = []
for download in ttable.cssselect('tr.group_torrent[id]'):
torrentid = re.sub('^torrent', '', download.attrib['id'])
torrenttype = download.cssselect('a[onclick]')[0].text.encode('ascii', 'ignore').split('/')[0].strip()
if torrenttype != 'MP3':
continue
downloadurl = download.cssselect('a[title=Download]')[0].attrib['href']
if len(download.cssselect('td')) != 5:
raise Exception('Torrent ' + torrentid + ' has !=5 TD tags at ' + torrentpageurl)
numseeders = int(download.cssselect('td')[3].text.replace(',', ''))
if numseeders < 1:
continue
filestr = ttable.cssselect('div#files_' + torrentid)[0].cssselect('tr')[1:]
filenames = []
for tr in filestr:
filename = tr.cssselect('td')[0].text
if filename.lower().endswith('.mp3'):
filenames.append({'original' : filename,
'compare' : re.sub(' +', '', filename.split('/')[-1][:-4].lower())})
downloads.append({'seeders' : numseeders, 'torrentid' : torrentid, 'url' : downloadurl, 'filenames' : filenames})
if not downloads:
log.info('[shop] no seeded files of correct type found at torrent ' + torrentid)
# See if any of the downloads nicely match any of the releases, trying best seeded first
downloads.sort(key=itemgetter('seeders'), reverse=True)
for download in downloads:
releasescores = []
filenames = download['filenames']
for releaseid in releases.keys():
release = releases[releaseid]
if len(filenames) != len(release):
minscore = 0
avgscore = 0
else:
numtracks = len(release)
minscore = None
minscoreidx = None
sumscore = 0
for i in range(numtracks):
rtrack = release[i]
rtracknum = '%02d' % rtrack['num']
rtartist = rtrack['artist'].lower()
rtname = rtrack['name'].lower()
dname = filenames[i]['compare']
name1 = rtracknum + ' ' + rtname
name2 = rtracknum + ' ' + rtartist + ' ' + rtname
score1 = SequenceMatcher(None, dname, name1).ratio()
score2 = SequenceMatcher(None, dname, name2).ratio()
score = max(score1, score2)
sumscore = sumscore + score
if score < minscore or minscore is None:
minscore = score
minscoreidx = i
avgscore = sumscore * 1.0 / numtracks
log.info('[shop] match avg=' + str(avgscore) + ' min=' + str(minscore) + ' ' + download['torrentid'] + ' -> ' + releaseid)
releasescores.append({'releaseid' : releaseid, 'min' : minscore, 'avg' : avgscore})
releasescores = filter(lambda x: x['min'] > 0.3 and x['avg'] > 0.70, releasescores)
releasescores.sort(key=itemgetter('avg'), reverse=True)
if releasescores:
# Toss torrent over to rtorrent via xml-rpc
releaseid = releasescores[0]['releaseid']
torrenturl = shopbaseurl + '/' + download['url']
torrentdata = opener.open(torrenturl).read()
torrentdecode = bencode.bdecode(torrentdata)
infohash = hashlib.sha1(bencode.bencode(torrentdecode['info'])).hexdigest().upper()
with tempfile.NamedTemporaryFile(delete=False) as torrentfile:
torrentpath = torrentfile.name
torrentfile.write(torrentdata)
# scp torrent over if necessary
if Config.SCP_SHOP_DOWNLOADS:
remotetorrentpath = '/tmp/' + infohash + '.torrent'
cmd = Config.SCP_CMD + ' ' + torrentpath + ' ' + Config.SCP_REMOTE + ':' + remotetorrentpath
log.info('[shop] running ' + cmd)
retval = os.system(cmd)
os.unlink(torrentpath)
if retval != 0:
raise Exception('scp command [' + cmd + '] returned ' + str(retval))
torrentpath = remotetorrentpath
rtorrent = xmlrpclib.ServerProxy(Config.SHOP_RPC_URL)
rtorrent.load_start(torrentpath, "execute=rm," + torrentpath)
log.info('[shop] downloaded ' + torrenturl + ' has ' + str(download['seeders']) +
' seeders, infohash=' + infohash + ', match to album ' + releaseid)
file_json = simplejson.dumps(map(itemgetter('original'), filenames))
minscore = releasescores[0]['min']
avgscore = releasescores[0]['avg']
shopdownload = ShopDownload(releaseid, album.gid, infohash, torrenturl, torrentpageurl, download['torrentid'], download['seeders'], file_json, minscore, avgscore, owner_id)
Session.add(shopdownload)
if attempt in Session:
Session.delete(attempt)
Session.commit()
return infohash
log.info('[shop] no matches, sorry :(')
attempt.gotsearchresults = True
if attempt not in Session:
Session.add(attempt)
Session.commit()
return None
def load(self):
commit = 'commit' in request.params and request.params['commit'] == 'true'
s = ''
now = datetime.now()
albums = {}
artists = {}
if commit:
Session.begin()
variousArtists = Session.query(Artist).filter_by(mbid=u'89ad4ac3-39f7-470e-963a-56509c546377').first()
if variousArtists is None:
variousArtists = Artist(name=u'Various Artists',
mbid=u'89ad4ac3-39f7-470e-963a-56509c546377',
added=now)
Session.save(variousArtists)
s = _msg(s, 'Committed various artists placeholder')
artists['Various Artists'] = variousArtists
initialLoad = Session.query(Track).count() == 0
if initialLoad:
s = _msg(s, 'Initial track loading!')
else:
s = _msg(s, 'Updating tracks!')
then = now
missing = 0
changed = 0
for track in Session.query(Track):
path = os.path.join(BASE, track.filepath)
if os.path.exists(path):
size = os.path.getsize(path)
mtime = datetime.fromtimestamp(os.path.getmtime(path))
if size != track.filesize or mtime != track.filemtime:
changed = changed + 1
s = _msg(s, 'Modified file: ' + path)
if commit:
raise Exception('not implemented!')
else:
s = _msg(s, 'Deleted file: ' + path)
missing = missing + 1
if commit:
Session.delete(track)
s = _msg(s, 'Found ' + str(missing) + ' missing files and ' + str(changed) + ' modified files, took ' + \
str(datetime.now() - then))
then = datetime.now()
filepaths = set(map(lambda t: t.filepath, Session.query(Track)))
s = _msg(s, 'Querying for all filepaths took ' + str(datetime.now() - then))
then = datetime.now()
added = 0
for dirname, dirnames, filenames in os.walk(BASE):
localAlbums = {}
for filename in filenames:
filepath = os.path.join(os.path.relpath(dirname, BASE), filename).decode('utf-8')
if not os.path.splitext(filename)[-1].lower() == '.mp3':
continue
if not initialLoad and filepath in filepaths:
continue
added = added + 1
if not initialLoad:
s = _msg(s, 'New file: ' + filepath)
if not commit:
continue
# get size, date
fileabspath = os.path.join(dirname,filename)
filesize = os.path.getsize(fileabspath)
filemtime = datetime.fromtimestamp(os.path.getmtime(fileabspath))
# mp3 length, bitrate, etc.
mutagen = MP3(fileabspath, ID3=EasyID3)
info = mutagen.info
mp3bitrate = info.bitrate
mp3samplerate = info.sample_rate
mp3length = int(round(info.length))
if info.sketchy:
raise Exception('sketchy mp3! ' + filename)
# id3
# keys: ['album', 'date', 'version', 'composer', 'title'
# 'genre', 'tracknumber', 'lyricist', 'artist']
id3artist = getid3prop(mutagen, 'artist')
id3album = getid3prop(mutagen, 'album')
id3title = getid3prop(mutagen, 'title')
id3tracknum = getid3prop(mutagen, 'tracknumber')
id3date = getid3prop(mutagen, 'date')
id3composer = getid3prop(mutagen, 'composer')
id3genre = getid3prop(mutagen, 'genre')
id3lyricist = getid3prop(mutagen, 'lyricist')
# additional musicbrainz related keys: At some point,
# should probably switch from easyID3 to ordinary ID3
# class to get extra MB relationship data.
mbartistid = getid3prop(mutagen,'musicbrainz_albumartistid')
mbalbumid = getid3prop(mutagen,'musicbrainz_albumid')
mbtrackid = getid3prop(mutagen,'musicbrainz_trackid')
if not id3artist:
artist = None
elif id3artist in artists:
artist = artists[id3artist]
else:
if initialLoad:
artistFromDb = None
else:
artistFromDb = Session.query(Artist).filter_by(name=id3artist).first()
if artistFromDb is None:
artist = Artist(name=id3artist,
mbid=mbartistid,
added=now)
Session.save(artist)
else:
artist = artistFromDb
artists[id3artist] = artist
if not id3album:
album = None
elif id3album in localAlbums:
album = localAlbums[id3album]
if artist != album.artist:
album.artist = variousArtists
else:
album = Album(name=id3album,
artist=artist,
added=now,
mbid=mbalbumid)
Session.save(album)
albums[id3album] = album
localAlbums[id3album] = album
track = Track(artist=artist,
album=album,
filepath=filepath,
filesize=filesize,
filemtime=filemtime,
mp3bitrate=mp3bitrate,
mp3samplerate=mp3samplerate,
mp3length=mp3length,
id3artist=id3artist,
id3album=id3album,
id3title=id3title,
id3tracknum=id3tracknum,
id3date=id3date,
id3composer=id3composer,
id3genre=id3genre,
id3lyricist=id3lyricist,
added=now,
mbid=mbtrackid,
)
Session.save(track)
if commit:
s = _msg(s, 'Building model for new tracks took ' + str(datetime.now() - then))
then = datetime.now()
Session.commit()
s = _msg(s, """Committed %(added)d new tracks, %(numArtists)d new artists, %(numAlbums)d new albums""" \
% {'added':added, 'numArtists': len(artists), 'numAlbums': len(albums)} + \
', took ' + str(datetime.now() - then))
else:
s = _msg(s, 'Saw ' + str(added) + ' new tracks, took ' + str(datetime.now() - then))
return s
