def savePlaylistAJAX(self): playlistname = request.params['name'] trackids = simplejson.loads(request.params['trackids']) trackids = map(lambda x: x.replace('track_',''), trackids) results = Session.query(Track, MBRecording).join(MBRecording).filter(Track.id.in_(trackids)).all() results.sort(lambda a,b: cmp(trackids.index(a[0].id), trackids.index(b[0].id))) recordings = map(itemgetter(1), results) Session.begin() user_name = request.environ['repoze.what.credentials']['repoze.what.userid'] user_id = Session.query(User).filter(User.user_name==user_name).one().user_id playlist = Session.query(Playlist) \ .filter(Playlist.owner_id==user_id) \ .filter(Playlist.name==playlistname) \ .first() if not trackids: if playlist is None: return '{}' else: Session.delete(playlist) else: if playlist is None: playlist = Playlist(user_id, playlistname) Session.add(playlist) else: playlist.modified = datetime.now() playlist.tracks = recordings Session.commit() return '{}'
def create(self): usr = request.params['login'] if Session.query(User).filter_by(user_name=usr).count() > 0: return simplejson.dumps({'success':False,'msg':'That username is already taken, sorry.'}) pwd = request.params['pass'] if len(usr) < 3 or len(pwd) < 3: return simplejson.dumps({'success':False,'msg':'Your username and password must each be at least 3 characters.'}) code = request.params['code'] invite = Session.query(Invite).filter_by(code=code).first() if invite is None: return simplejson.dumps({'success':False,'msg':'Your registration code appears to be invalid.'}) user = User() user.who = invite.who user.user_name = usr user.password = pwd user.registered = datetime.now() Session.begin() user.groups = [Session.query(Group).filter_by(group_name='users').one()] Session.delete(invite) Session.add(user) Session.commit() return simplejson.dumps({'success':True})
def load(self): commit = 'commit' in request.params and request.params['commit'] == 'true' s = '' if commit: Session.begin() now = datetime.now() initialLoad = True #Session.query(AudioFile).count() == 0 if initialLoad: s = _msg(s, 'Initial track loading!') else: s = _msg(s, 'Updating tracks!') then = now missing = 0 changed = 0 for track in Session.query(AudioFile): path = os.path.join(MUSIC, track.filepath) if os.path.exists(path): size = os.path.getsize(path) mtime = datetime.fromtimestamp(os.path.getmtime(path)) if size != track.filesize or mtime != track.filemtime: changed = changed + 1 s = _msg(s, 'Modified file: ' + path) if commit: raise Exception('not implemented!') else: s = _msg(s, 'Deleted file: ' + path) missing = missing + 1 if commit: Session.delete(track) s = _msg(s, 'Found ' + str(missing) + ' missing files and ' + str(changed) + ' modified files, took ' + \ str(datetime.now() - then)) then = datetime.now() filepaths = set(map(lambda t: t.filepath, Session.query(AudioFile))) s = _msg(s, 'Querying for all filepaths took ' + str(datetime.now() - then)) then = datetime.now() added = 0 skippedNoMBID = 0 release_groups_added = set() unknownrelease = set() unknownrecording = set() alreadyhaverecordingrelease = set() alreadyhavereleasegroup = set() unicodeproblems = set() fuckedmp3s = set() for dirname, dirnames, filenames in os.walk(INCOMING, followlinks=True): for filename in filenames: if not os.path.splitext(filename)[-1].lower() == '.mp3': continue try: filepath = os.path.join(os.path.relpath(dirname, INCOMING), filename).decode('utf-8') except UnicodeDecodeError: log.error('unicode problem ' + os.path.join(os.path.relpath(dirname, INCOMING), filename)) unicodeproblems.add(os.path.join(os.path.relpath(dirname, INCOMING), filename)) continue if not initialLoad and filepath in filepaths: continue if not initialLoad: s = _msg(s, 'New file: ' + filepath) if not commit: continue # get size, date fileabspath = os.path.join(dirname,filename) filesize = os.path.getsize(fileabspath) filemtime = datetime.fromtimestamp(os.path.getmtime(fileabspath)) # mp3 length, bitrate, etc. try: mutagen = MP3(fileabspath) except: fuckedmp3s.add(fileabspath) log.error('f****d mp3 ' + fileabspath) continue info = mutagen.info mp3bitrate = info.bitrate mp3samplerate = info.sample_rate mp3length = int(round(info.length)) if info.sketchy: fuckedmp3s.add(fileabspath) log.error('sketchy mp3! ' + fileabspath) continue # brainz!! if RECORDING_MBID_KEY not in mutagen: skippedNoMBID = skippedNoMBID + 1 continue recordingmbid = mutagen[RECORDING_MBID_KEY].data if not recordingmbid: skippedNoMBID = skippedNoMBID + 1 continue if RELEASE_MBID_KEY not in mutagen: skippedNoMBID = skippedNoMBID + 1 continue releasembid = mutagen[RELEASE_MBID_KEY].text[0] if not releasembid or len(mutagen[RELEASE_MBID_KEY].text) > 1: skippedNoMBID = skippedNoMBID + 1 continue release = Session.query(MBRelease).filter(MBRelease.gid==releasembid).first() if release is None: release = Session.query(MBRelease) \ .join(MBReleaseGIDRedirect) \ .filter(MBReleaseGIDRedirect.gid==releasembid) \ .first() if release is None: if releasembid not in unknownrelease: log.error('couldnt find release mbid ' + releasembid) unknownrelease.add(releasembid) continue recording = Session.query(MBRecording).filter(MBRecording.gid==recordingmbid).first() if recording is None: recording = Session.query(MBRecording) \ .join(MBRecordingGIDRedirect) \ .filter(MBRecordingGIDRedirect.gid==recordingmbid) \ .first() if recording is None: if recordingmbid not in unknownrecording: log.error('couldnt find recording mbid ' + recordingmbid) unknownrecording.add(recordingmbid) continue releasegroupmbid = release.releasegroup.gid dirs = os.path.join(releasegroupmbid[:2], releasegroupmbid) newdir = os.path.join(MUSIC, dirs) if releasegroupmbid not in release_groups_added: existing = Session.query(Album) \ .filter(Album.mbid==releasegroupmbid) \ .first() if existing != None: if releasegroupmbid not in alreadyhavereleasegroup: log.info('already have release group ' + existing.artistcredit + ' ' + existing.name + ' ' + existing.mbid) alreadyhavereleasegroup.add(releasegroupmbid) continue else: os.makedirs(newdir) release_groups_added.add(releasegroupmbid) existing = Session.query(AudioFile) \ .filter(AudioFile.recordingmbid==recording.gid) \ .filter(AudioFile.releasembid==release.gid) \ .first() if existing: if (recording.gid + '-' + release.gid) not in alreadyhaverecordingrelease: log.error('already existing recording/release combo for file ' + filepath) alreadyhaverecordingrelease.add(recording.gid + '-' + release.gid) continue # check for existing release group, make new release group directory if necessary # link file into new directory newfilename = release.gid + '-' + recording.gid + '.mp3' ln = os.path.join(newdir, newfilename) os.link(fileabspath, ln) lnrelpath = os.path.join(dirs, newfilename) track = AudioFile(filepath=lnrelpath, filesize=filesize, filemtime=filemtime, mp3bitrate=mp3bitrate, mp3samplerate=mp3samplerate, mp3length=mp3length, recording=recording, release=release, added=now, ) Session.add(track) added = added + 1 if commit: s = _msg(s, 'Building model for new tracks took ' + str(datetime.now() - then)) then = datetime.now() Session.commit() s = _msg(s, """Committed %(added)d new tracks, skipped %(skipped)d""" \ % {'added':added-skippedNoMBID, 'skipped':skippedNoMBID} + ', took ' + str(datetime.now() - then)) log.error("unknownrelease: " + unknownrelease.__str__()) log.error("unknownrecording: " + unknownrecording.__str__()) log.error("alreadyhaverecordingrelease: " + alreadyhaverecordingrelease.__str__()) log.error("alreadyhavereleasegroup: " + alreadyhavereleasegroup.__str__()) log.error("unicodeproblems: " + unicodeproblems.__str__()) log.error("fuckedmp3s: " + fuckedmp3s.__str__()) else: s = _msg(s, 'Saw ' + str(added) + ' new tracks, took ' + str(datetime.now() - then)) return s
def download(Session, mbid, owner_id): with shoplock: now = datetime.now() existingdownload = Session.query(ShopDownload) \ .filter(ShopDownload.release_group_mbid==mbid) \ .first() if existingdownload: return existingdownload.infohash attempt = Session.query(ShopDownloadAttempt) \ .filter(ShopDownloadAttempt.mbid==mbid) \ .first() if attempt and now < attempt.tried + timedelta(days=10): return None if not attempt: attempt = ShopDownloadAttempt(mbid, now, True) attempt.tried = now log.info('[shop] searching for ' + mbid) Session.begin() if not maybeloggedin: login() (album, albumname, artistname) = Session.query(MBReleaseGroup, MBReleaseName, MBArtistName) \ .join(MBReleaseName) \ .join(MBReleaseGroup.artistcredit, MBArtistCredit.name) \ .filter(MBReleaseGroup.gid==mbid) \ .one() searchartist = artistname.name searchalbum = albumname.name if searchartist == 'Various Artists': searchartist = '' searchartist = _cleanupSearchTerm(searchartist) searchalbum = _cleanupSearchTerm(searchalbum) url = searchurl + '?' + urllib.urlencode({ 'artistname' : searchartist, 'groupname' : searchalbum, 'action' : 'advanced', 'format' : 'MP3', 'order_by' : 'seeders' }) log.info('[shop] hitting ' + url) handle = opener.open(url) page = lxml.parse(handle).getroot() html = lxml.tostring(page) if loginpath in html: log.warn('[shop] login url found in search result, logging in..') login() handle = opener.open(url) page = lxml.parse(handle).getroot() html = lxml.tostring(page) if loginpath in html: log.error('[shop] couldnt login!') raise Exception('couldnt login!') if 'Your search did not match anything' in html: log.warn('[shop] no results') attempt.gotsearchresults = False if attempt not in Session: Session.add(attempt) Session.commit() return None # Gather up all tracks from all releases results = Session.query(MBTrack, MBArtistName, MBTrackName, MBRelease, MBMedium) \ .join(MBTrackList, MBMedium, MBRelease, MBReleaseGroup) \ .join(MBTrackName).join(MBTrack.artistcredit, MBArtistCredit.name) \ .filter(MBReleaseGroup.gid==mbid) \ .all() releases = {} for (track, artistname, trackname, release, medium) in results: data = {'id' : track.id, 'num' : track.position, 'disc' : medium.position, 'artist' : artistname.name, 'name' : trackname.name} mbid = release.gid if mbid in releases: releases[mbid].append(data) else: releases[mbid] = [data] for releaseid in releases.keys(): release = releases[releaseid] release.sort(key=itemgetter('disc', 'num')) log.info('[shop] release group ' + mbid + ' has ' + str(len(releases)) + ' releases to check against') # Try to match any of the releases with the first few results torrent_table = page.cssselect('table#torrent_table')[0] groups = torrent_table.cssselect('tr.group') numresults = len(groups) for torrent in groups[:3]: torrentpageurl = shopbaseurl + '/' + torrent.cssselect('a[title="View Torrent"]')[0].attrib['href'] log.info('[shop] hitting ' + torrentpageurl) thandle = opener.open(torrentpageurl) tpage = lxml.parse(thandle).getroot() # Gather up information about all downloads for this torrent ttable = tpage.cssselect('table.torrent_table')[0] downloads = [] for download in ttable.cssselect('tr.group_torrent[id]'): torrentid = re.sub('^torrent', '', download.attrib['id']) torrenttype = download.cssselect('a[onclick]')[0].text.encode('ascii', 'ignore').split('/')[0].strip() if torrenttype != 'MP3': continue downloadurl = download.cssselect('a[title=Download]')[0].attrib['href'] if len(download.cssselect('td')) != 5: raise Exception('Torrent ' + torrentid + ' has !=5 TD tags at ' + torrentpageurl) numseeders = int(download.cssselect('td')[3].text.replace(',', '')) if numseeders < 1: continue filestr = ttable.cssselect('div#files_' + torrentid)[0].cssselect('tr')[1:] filenames = [] for tr in filestr: filename = tr.cssselect('td')[0].text if filename.lower().endswith('.mp3'): filenames.append({'original' : filename, 'compare' : re.sub(' +', '', filename.split('/')[-1][:-4].lower())}) downloads.append({'seeders' : numseeders, 'torrentid' : torrentid, 'url' : downloadurl, 'filenames' : filenames}) if not downloads: log.info('[shop] no seeded files of correct type found at torrent ' + torrentid) # See if any of the downloads nicely match any of the releases, trying best seeded first downloads.sort(key=itemgetter('seeders'), reverse=True) for download in downloads: releasescores = [] filenames = download['filenames'] for releaseid in releases.keys(): release = releases[releaseid] if len(filenames) != len(release): minscore = 0 avgscore = 0 else: numtracks = len(release) minscore = None minscoreidx = None sumscore = 0 for i in range(numtracks): rtrack = release[i] rtracknum = '%02d' % rtrack['num'] rtartist = rtrack['artist'].lower() rtname = rtrack['name'].lower() dname = filenames[i]['compare'] name1 = rtracknum + ' ' + rtname name2 = rtracknum + ' ' + rtartist + ' ' + rtname score1 = SequenceMatcher(None, dname, name1).ratio() score2 = SequenceMatcher(None, dname, name2).ratio() score = max(score1, score2) sumscore = sumscore + score if score < minscore or minscore is None: minscore = score minscoreidx = i avgscore = sumscore * 1.0 / numtracks log.info('[shop] match avg=' + str(avgscore) + ' min=' + str(minscore) + ' ' + download['torrentid'] + ' -> ' + releaseid) releasescores.append({'releaseid' : releaseid, 'min' : minscore, 'avg' : avgscore}) releasescores = filter(lambda x: x['min'] > 0.3 and x['avg'] > 0.70, releasescores) releasescores.sort(key=itemgetter('avg'), reverse=True) if releasescores: # Toss torrent over to rtorrent via xml-rpc releaseid = releasescores[0]['releaseid'] torrenturl = shopbaseurl + '/' + download['url'] torrentdata = opener.open(torrenturl).read() torrentdecode = bencode.bdecode(torrentdata) infohash = hashlib.sha1(bencode.bencode(torrentdecode['info'])).hexdigest().upper() with tempfile.NamedTemporaryFile(delete=False) as torrentfile: torrentpath = torrentfile.name torrentfile.write(torrentdata) # scp torrent over if necessary if Config.SCP_SHOP_DOWNLOADS: remotetorrentpath = '/tmp/' + infohash + '.torrent' cmd = Config.SCP_CMD + ' ' + torrentpath + ' ' + Config.SCP_REMOTE + ':' + remotetorrentpath log.info('[shop] running ' + cmd) retval = os.system(cmd) os.unlink(torrentpath) if retval != 0: raise Exception('scp command [' + cmd + '] returned ' + str(retval)) torrentpath = remotetorrentpath rtorrent = xmlrpclib.ServerProxy(Config.SHOP_RPC_URL) rtorrent.load_start(torrentpath, "execute=rm," + torrentpath) log.info('[shop] downloaded ' + torrenturl + ' has ' + str(download['seeders']) + ' seeders, infohash=' + infohash + ', match to album ' + releaseid) file_json = simplejson.dumps(map(itemgetter('original'), filenames)) minscore = releasescores[0]['min'] avgscore = releasescores[0]['avg'] shopdownload = ShopDownload(releaseid, album.gid, infohash, torrenturl, torrentpageurl, download['torrentid'], download['seeders'], file_json, minscore, avgscore, owner_id) Session.add(shopdownload) if attempt in Session: Session.delete(attempt) Session.commit() return infohash log.info('[shop] no matches, sorry :(') attempt.gotsearchresults = True if attempt not in Session: Session.add(attempt) Session.commit() return None
def load(self): commit = 'commit' in request.params and request.params['commit'] == 'true' s = '' now = datetime.now() albums = {} artists = {} if commit: Session.begin() variousArtists = Session.query(Artist).filter_by(mbid=u'89ad4ac3-39f7-470e-963a-56509c546377').first() if variousArtists is None: variousArtists = Artist(name=u'Various Artists', mbid=u'89ad4ac3-39f7-470e-963a-56509c546377', added=now) Session.save(variousArtists) s = _msg(s, 'Committed various artists placeholder') artists['Various Artists'] = variousArtists initialLoad = Session.query(Track).count() == 0 if initialLoad: s = _msg(s, 'Initial track loading!') else: s = _msg(s, 'Updating tracks!') then = now missing = 0 changed = 0 for track in Session.query(Track): path = os.path.join(BASE, track.filepath) if os.path.exists(path): size = os.path.getsize(path) mtime = datetime.fromtimestamp(os.path.getmtime(path)) if size != track.filesize or mtime != track.filemtime: changed = changed + 1 s = _msg(s, 'Modified file: ' + path) if commit: raise Exception('not implemented!') else: s = _msg(s, 'Deleted file: ' + path) missing = missing + 1 if commit: Session.delete(track) s = _msg(s, 'Found ' + str(missing) + ' missing files and ' + str(changed) + ' modified files, took ' + \ str(datetime.now() - then)) then = datetime.now() filepaths = set(map(lambda t: t.filepath, Session.query(Track))) s = _msg(s, 'Querying for all filepaths took ' + str(datetime.now() - then)) then = datetime.now() added = 0 for dirname, dirnames, filenames in os.walk(BASE): localAlbums = {} for filename in filenames: filepath = os.path.join(os.path.relpath(dirname, BASE), filename).decode('utf-8') if not os.path.splitext(filename)[-1].lower() == '.mp3': continue if not initialLoad and filepath in filepaths: continue added = added + 1 if not initialLoad: s = _msg(s, 'New file: ' + filepath) if not commit: continue # get size, date fileabspath = os.path.join(dirname,filename) filesize = os.path.getsize(fileabspath) filemtime = datetime.fromtimestamp(os.path.getmtime(fileabspath)) # mp3 length, bitrate, etc. mutagen = MP3(fileabspath, ID3=EasyID3) info = mutagen.info mp3bitrate = info.bitrate mp3samplerate = info.sample_rate mp3length = int(round(info.length)) if info.sketchy: raise Exception('sketchy mp3! ' + filename) # id3 # keys: ['album', 'date', 'version', 'composer', 'title' # 'genre', 'tracknumber', 'lyricist', 'artist'] id3artist = getid3prop(mutagen, 'artist') id3album = getid3prop(mutagen, 'album') id3title = getid3prop(mutagen, 'title') id3tracknum = getid3prop(mutagen, 'tracknumber') id3date = getid3prop(mutagen, 'date') id3composer = getid3prop(mutagen, 'composer') id3genre = getid3prop(mutagen, 'genre') id3lyricist = getid3prop(mutagen, 'lyricist') # additional musicbrainz related keys: At some point, # should probably switch from easyID3 to ordinary ID3 # class to get extra MB relationship data. mbartistid = getid3prop(mutagen,'musicbrainz_albumartistid') mbalbumid = getid3prop(mutagen,'musicbrainz_albumid') mbtrackid = getid3prop(mutagen,'musicbrainz_trackid') if not id3artist: artist = None elif id3artist in artists: artist = artists[id3artist] else: if initialLoad: artistFromDb = None else: artistFromDb = Session.query(Artist).filter_by(name=id3artist).first() if artistFromDb is None: artist = Artist(name=id3artist, mbid=mbartistid, added=now) Session.save(artist) else: artist = artistFromDb artists[id3artist] = artist if not id3album: album = None elif id3album in localAlbums: album = localAlbums[id3album] if artist != album.artist: album.artist = variousArtists else: album = Album(name=id3album, artist=artist, added=now, mbid=mbalbumid) Session.save(album) albums[id3album] = album localAlbums[id3album] = album track = Track(artist=artist, album=album, filepath=filepath, filesize=filesize, filemtime=filemtime, mp3bitrate=mp3bitrate, mp3samplerate=mp3samplerate, mp3length=mp3length, id3artist=id3artist, id3album=id3album, id3title=id3title, id3tracknum=id3tracknum, id3date=id3date, id3composer=id3composer, id3genre=id3genre, id3lyricist=id3lyricist, added=now, mbid=mbtrackid, ) Session.save(track) if commit: s = _msg(s, 'Building model for new tracks took ' + str(datetime.now() - then)) then = datetime.now() Session.commit() s = _msg(s, """Committed %(added)d new tracks, %(numArtists)d new artists, %(numAlbums)d new albums""" \ % {'added':added, 'numArtists': len(artists), 'numAlbums': len(albums)} + \ ', took ' + str(datetime.now() - then)) else: s = _msg(s, 'Saw ' + str(added) + ' new tracks, took ' + str(datetime.now() - then)) return s