def savePlaylistAJAX(self): playlistname = request.params['name'] trackids = simplejson.loads(request.params['trackids']) trackids = map(lambda x: x.replace('track_',''), trackids) results = Session.query(Track, MBRecording).join(MBRecording).filter(Track.id.in_(trackids)).all() results.sort(lambda a,b: cmp(trackids.index(a[0].id), trackids.index(b[0].id))) recordings = map(itemgetter(1), results) Session.begin() user_name = request.environ['repoze.what.credentials']['repoze.what.userid'] user_id = Session.query(User).filter(User.user_name==user_name).one().user_id playlist = Session.query(Playlist) \ .filter(Playlist.owner_id==user_id) \ .filter(Playlist.name==playlistname) \ .first() if not trackids: if playlist is None: return '{}' else: Session.delete(playlist) else: if playlist is None: playlist = Playlist(user_id, playlistname) Session.add(playlist) else: playlist.modified = datetime.now() playlist.tracks = recordings Session.commit() return '{}'
def invite(self): if request.POST: who = request.params['who'] c.code = ''.join(random.choice(string.letters + string.digits) for i in xrange(32)) invite = Invite(who, c.code) Session.begin() Session.add(invite) Session.commit() return render('/display-invite.html') else: return render('/create-invite.html')
def makeRandom(self): user_name = request.environ['repoze.what.credentials']['repoze.what.userid'] user = Session.query(User).filter(User.user_name==user_name).one() Session.begin() for i in range(10): info = BenKVP(owner = user, key = 'random_key_{0}'.format(i), value = 'random_value_{0}'.format(i)) Session.add(info) Session.commit() return 'success'
def getLyricsAJAX(self): trackid = request.params['trackid'].split('_')[1] track = Session.query(Track).filter_by(id=trackid).one() ip = request.environ['REMOTE_ADDR'] user_name = request.environ['repoze.what.credentials']['repoze.what.userid'] user = Session.query(User).filter(User.user_name==user_name).one() Session.begin() play = TrackPlay(track.mbid, user, ip) Session.add(play) Session.commit() lyrics = lyricsservice.get_lyrics(Session, track) if lyrics: return simplejson.dumps({'lyrics' : lyrics}) else: return '{}'
def create(self): usr = request.params['login'] if Session.query(User).filter_by(user_name=usr).count() > 0: return simplejson.dumps({'success':False,'msg':'That username is already taken, sorry.'}) pwd = request.params['pass'] if len(usr) < 3 or len(pwd) < 3: return simplejson.dumps({'success':False,'msg':'Your username and password must each be at least 3 characters.'}) code = request.params['code'] invite = Session.query(Invite).filter_by(code=code).first() if invite is None: return simplejson.dumps({'success':False,'msg':'Your registration code appears to be invalid.'}) user = User() user.who = invite.who user.user_name = usr user.password = pwd user.registered = datetime.now() Session.begin() user.groups = [Session.query(Group).filter_by(group_name='users').one()] Session.delete(invite) Session.add(user) Session.commit() return simplejson.dumps({'success':True})
def load(self): commit = 'commit' in request.params and request.params['commit'] == 'true' s = '' if commit: Session.begin() now = datetime.now() initialLoad = True #Session.query(AudioFile).count() == 0 if initialLoad: s = _msg(s, 'Initial track loading!') else: s = _msg(s, 'Updating tracks!') then = now missing = 0 changed = 0 for track in Session.query(AudioFile): path = os.path.join(MUSIC, track.filepath) if os.path.exists(path): size = os.path.getsize(path) mtime = datetime.fromtimestamp(os.path.getmtime(path)) if size != track.filesize or mtime != track.filemtime: changed = changed + 1 s = _msg(s, 'Modified file: ' + path) if commit: raise Exception('not implemented!') else: s = _msg(s, 'Deleted file: ' + path) missing = missing + 1 if commit: Session.delete(track) s = _msg(s, 'Found ' + str(missing) + ' missing files and ' + str(changed) + ' modified files, took ' + \ str(datetime.now() - then)) then = datetime.now() filepaths = set(map(lambda t: t.filepath, Session.query(AudioFile))) s = _msg(s, 'Querying for all filepaths took ' + str(datetime.now() - then)) then = datetime.now() added = 0 skippedNoMBID = 0 release_groups_added = set() unknownrelease = set() unknownrecording = set() alreadyhaverecordingrelease = set() alreadyhavereleasegroup = set() unicodeproblems = set() fuckedmp3s = set() for dirname, dirnames, filenames in os.walk(INCOMING, followlinks=True): for filename in filenames: if not os.path.splitext(filename)[-1].lower() == '.mp3': continue try: filepath = os.path.join(os.path.relpath(dirname, INCOMING), filename).decode('utf-8') except UnicodeDecodeError: log.error('unicode problem ' + os.path.join(os.path.relpath(dirname, INCOMING), filename)) unicodeproblems.add(os.path.join(os.path.relpath(dirname, INCOMING), filename)) continue if not initialLoad and filepath in filepaths: continue if not initialLoad: s = _msg(s, 'New file: ' + filepath) if not commit: continue # get size, date fileabspath = os.path.join(dirname,filename) filesize = os.path.getsize(fileabspath) filemtime = datetime.fromtimestamp(os.path.getmtime(fileabspath)) # mp3 length, bitrate, etc. try: mutagen = MP3(fileabspath) except: fuckedmp3s.add(fileabspath) log.error('f****d mp3 ' + fileabspath) continue info = mutagen.info mp3bitrate = info.bitrate mp3samplerate = info.sample_rate mp3length = int(round(info.length)) if info.sketchy: fuckedmp3s.add(fileabspath) log.error('sketchy mp3! ' + fileabspath) continue # brainz!! if RECORDING_MBID_KEY not in mutagen: skippedNoMBID = skippedNoMBID + 1 continue recordingmbid = mutagen[RECORDING_MBID_KEY].data if not recordingmbid: skippedNoMBID = skippedNoMBID + 1 continue if RELEASE_MBID_KEY not in mutagen: skippedNoMBID = skippedNoMBID + 1 continue releasembid = mutagen[RELEASE_MBID_KEY].text[0] if not releasembid or len(mutagen[RELEASE_MBID_KEY].text) > 1: skippedNoMBID = skippedNoMBID + 1 continue release = Session.query(MBRelease).filter(MBRelease.gid==releasembid).first() if release is None: release = Session.query(MBRelease) \ .join(MBReleaseGIDRedirect) \ .filter(MBReleaseGIDRedirect.gid==releasembid) \ .first() if release is None: if releasembid not in unknownrelease: log.error('couldnt find release mbid ' + releasembid) unknownrelease.add(releasembid) continue recording = Session.query(MBRecording).filter(MBRecording.gid==recordingmbid).first() if recording is None: recording = Session.query(MBRecording) \ .join(MBRecordingGIDRedirect) \ .filter(MBRecordingGIDRedirect.gid==recordingmbid) \ .first() if recording is None: if recordingmbid not in unknownrecording: log.error('couldnt find recording mbid ' + recordingmbid) unknownrecording.add(recordingmbid) continue releasegroupmbid = release.releasegroup.gid dirs = os.path.join(releasegroupmbid[:2], releasegroupmbid) newdir = os.path.join(MUSIC, dirs) if releasegroupmbid not in release_groups_added: existing = Session.query(Album) \ .filter(Album.mbid==releasegroupmbid) \ .first() if existing != None: if releasegroupmbid not in alreadyhavereleasegroup: log.info('already have release group ' + existing.artistcredit + ' ' + existing.name + ' ' + existing.mbid) alreadyhavereleasegroup.add(releasegroupmbid) continue else: os.makedirs(newdir) release_groups_added.add(releasegroupmbid) existing = Session.query(AudioFile) \ .filter(AudioFile.recordingmbid==recording.gid) \ .filter(AudioFile.releasembid==release.gid) \ .first() if existing: if (recording.gid + '-' + release.gid) not in alreadyhaverecordingrelease: log.error('already existing recording/release combo for file ' + filepath) alreadyhaverecordingrelease.add(recording.gid + '-' + release.gid) continue # check for existing release group, make new release group directory if necessary # link file into new directory newfilename = release.gid + '-' + recording.gid + '.mp3' ln = os.path.join(newdir, newfilename) os.link(fileabspath, ln) lnrelpath = os.path.join(dirs, newfilename) track = AudioFile(filepath=lnrelpath, filesize=filesize, filemtime=filemtime, mp3bitrate=mp3bitrate, mp3samplerate=mp3samplerate, mp3length=mp3length, recording=recording, release=release, added=now, ) Session.add(track) added = added + 1 if commit: s = _msg(s, 'Building model for new tracks took ' + str(datetime.now() - then)) then = datetime.now() Session.commit() s = _msg(s, """Committed %(added)d new tracks, skipped %(skipped)d""" \ % {'added':added-skippedNoMBID, 'skipped':skippedNoMBID} + ', took ' + str(datetime.now() - then)) log.error("unknownrelease: " + unknownrelease.__str__()) log.error("unknownrecording: " + unknownrecording.__str__()) log.error("alreadyhaverecordingrelease: " + alreadyhaverecordingrelease.__str__()) log.error("alreadyhavereleasegroup: " + alreadyhavereleasegroup.__str__()) log.error("unicodeproblems: " + unicodeproblems.__str__()) log.error("fuckedmp3s: " + fuckedmp3s.__str__()) else: s = _msg(s, 'Saw ' + str(added) + ' new tracks, took ' + str(datetime.now() - then)) return s
def download(Session, mbid, owner_id): with shoplock: now = datetime.now() existingdownload = Session.query(ShopDownload) \ .filter(ShopDownload.release_group_mbid==mbid) \ .first() if existingdownload: return existingdownload.infohash attempt = Session.query(ShopDownloadAttempt) \ .filter(ShopDownloadAttempt.mbid==mbid) \ .first() if attempt and now < attempt.tried + timedelta(days=10): return None if not attempt: attempt = ShopDownloadAttempt(mbid, now, True) attempt.tried = now log.info('[shop] searching for ' + mbid) Session.begin() if not maybeloggedin: login() (album, albumname, artistname) = Session.query(MBReleaseGroup, MBReleaseName, MBArtistName) \ .join(MBReleaseName) \ .join(MBReleaseGroup.artistcredit, MBArtistCredit.name) \ .filter(MBReleaseGroup.gid==mbid) \ .one() searchartist = artistname.name searchalbum = albumname.name if searchartist == 'Various Artists': searchartist = '' searchartist = _cleanupSearchTerm(searchartist) searchalbum = _cleanupSearchTerm(searchalbum) url = searchurl + '?' + urllib.urlencode({ 'artistname' : searchartist, 'groupname' : searchalbum, 'action' : 'advanced', 'format' : 'MP3', 'order_by' : 'seeders' }) log.info('[shop] hitting ' + url) handle = opener.open(url) page = lxml.parse(handle).getroot() html = lxml.tostring(page) if loginpath in html: log.warn('[shop] login url found in search result, logging in..') login() handle = opener.open(url) page = lxml.parse(handle).getroot() html = lxml.tostring(page) if loginpath in html: log.error('[shop] couldnt login!') raise Exception('couldnt login!') if 'Your search did not match anything' in html: log.warn('[shop] no results') attempt.gotsearchresults = False if attempt not in Session: Session.add(attempt) Session.commit() return None # Gather up all tracks from all releases results = Session.query(MBTrack, MBArtistName, MBTrackName, MBRelease, MBMedium) \ .join(MBTrackList, MBMedium, MBRelease, MBReleaseGroup) \ .join(MBTrackName).join(MBTrack.artistcredit, MBArtistCredit.name) \ .filter(MBReleaseGroup.gid==mbid) \ .all() releases = {} for (track, artistname, trackname, release, medium) in results: data = {'id' : track.id, 'num' : track.position, 'disc' : medium.position, 'artist' : artistname.name, 'name' : trackname.name} mbid = release.gid if mbid in releases: releases[mbid].append(data) else: releases[mbid] = [data] for releaseid in releases.keys(): release = releases[releaseid] release.sort(key=itemgetter('disc', 'num')) log.info('[shop] release group ' + mbid + ' has ' + str(len(releases)) + ' releases to check against') # Try to match any of the releases with the first few results torrent_table = page.cssselect('table#torrent_table')[0] groups = torrent_table.cssselect('tr.group') numresults = len(groups) for torrent in groups[:3]: torrentpageurl = shopbaseurl + '/' + torrent.cssselect('a[title="View Torrent"]')[0].attrib['href'] log.info('[shop] hitting ' + torrentpageurl) thandle = opener.open(torrentpageurl) tpage = lxml.parse(thandle).getroot() # Gather up information about all downloads for this torrent ttable = tpage.cssselect('table.torrent_table')[0] downloads = [] for download in ttable.cssselect('tr.group_torrent[id]'): torrentid = re.sub('^torrent', '', download.attrib['id']) torrenttype = download.cssselect('a[onclick]')[0].text.encode('ascii', 'ignore').split('/')[0].strip() if torrenttype != 'MP3': continue downloadurl = download.cssselect('a[title=Download]')[0].attrib['href'] if len(download.cssselect('td')) != 5: raise Exception('Torrent ' + torrentid + ' has !=5 TD tags at ' + torrentpageurl) numseeders = int(download.cssselect('td')[3].text.replace(',', '')) if numseeders < 1: continue filestr = ttable.cssselect('div#files_' + torrentid)[0].cssselect('tr')[1:] filenames = [] for tr in filestr: filename = tr.cssselect('td')[0].text if filename.lower().endswith('.mp3'): filenames.append({'original' : filename, 'compare' : re.sub(' +', '', filename.split('/')[-1][:-4].lower())}) downloads.append({'seeders' : numseeders, 'torrentid' : torrentid, 'url' : downloadurl, 'filenames' : filenames}) if not downloads: log.info('[shop] no seeded files of correct type found at torrent ' + torrentid) # See if any of the downloads nicely match any of the releases, trying best seeded first downloads.sort(key=itemgetter('seeders'), reverse=True) for download in downloads: releasescores = [] filenames = download['filenames'] for releaseid in releases.keys(): release = releases[releaseid] if len(filenames) != len(release): minscore = 0 avgscore = 0 else: numtracks = len(release) minscore = None minscoreidx = None sumscore = 0 for i in range(numtracks): rtrack = release[i] rtracknum = '%02d' % rtrack['num'] rtartist = rtrack['artist'].lower() rtname = rtrack['name'].lower() dname = filenames[i]['compare'] name1 = rtracknum + ' ' + rtname name2 = rtracknum + ' ' + rtartist + ' ' + rtname score1 = SequenceMatcher(None, dname, name1).ratio() score2 = SequenceMatcher(None, dname, name2).ratio() score = max(score1, score2) sumscore = sumscore + score if score < minscore or minscore is None: minscore = score minscoreidx = i avgscore = sumscore * 1.0 / numtracks log.info('[shop] match avg=' + str(avgscore) + ' min=' + str(minscore) + ' ' + download['torrentid'] + ' -> ' + releaseid) releasescores.append({'releaseid' : releaseid, 'min' : minscore, 'avg' : avgscore}) releasescores = filter(lambda x: x['min'] > 0.3 and x['avg'] > 0.70, releasescores) releasescores.sort(key=itemgetter('avg'), reverse=True) if releasescores: # Toss torrent over to rtorrent via xml-rpc releaseid = releasescores[0]['releaseid'] torrenturl = shopbaseurl + '/' + download['url'] torrentdata = opener.open(torrenturl).read() torrentdecode = bencode.bdecode(torrentdata) infohash = hashlib.sha1(bencode.bencode(torrentdecode['info'])).hexdigest().upper() with tempfile.NamedTemporaryFile(delete=False) as torrentfile: torrentpath = torrentfile.name torrentfile.write(torrentdata) # scp torrent over if necessary if Config.SCP_SHOP_DOWNLOADS: remotetorrentpath = '/tmp/' + infohash + '.torrent' cmd = Config.SCP_CMD + ' ' + torrentpath + ' ' + Config.SCP_REMOTE + ':' + remotetorrentpath log.info('[shop] running ' + cmd) retval = os.system(cmd) os.unlink(torrentpath) if retval != 0: raise Exception('scp command [' + cmd + '] returned ' + str(retval)) torrentpath = remotetorrentpath rtorrent = xmlrpclib.ServerProxy(Config.SHOP_RPC_URL) rtorrent.load_start(torrentpath, "execute=rm," + torrentpath) log.info('[shop] downloaded ' + torrenturl + ' has ' + str(download['seeders']) + ' seeders, infohash=' + infohash + ', match to album ' + releaseid) file_json = simplejson.dumps(map(itemgetter('original'), filenames)) minscore = releasescores[0]['min'] avgscore = releasescores[0]['avg'] shopdownload = ShopDownload(releaseid, album.gid, infohash, torrenturl, torrentpageurl, download['torrentid'], download['seeders'], file_json, minscore, avgscore, owner_id) Session.add(shopdownload) if attempt in Session: Session.delete(attempt) Session.commit() return infohash log.info('[shop] no matches, sorry :(') attempt.gotsearchresults = True if attempt not in Session: Session.add(attempt) Session.commit() return None
def importDownload(shopdownload): with importlock: Session.begin() assert not shopdownload.isdone, 'download is already imported' assert not shopdownload.failedimport, 'download already failed import' log.info('[shop] starting to import ' + shopdownload.infohash) promisedfiles = simplejson.loads(shopdownload.file_json) rtorrent = xmlrpclib.ServerProxy(Config.SHOP_RPC_URL) assert rtorrent.d.get_complete(shopdownload.infohash) == 1, 'rtorrent says download isnt done yet' release_mbid = shopdownload.release_mbid mbalbum = Session.query(MBReleaseGroup) \ .join(MBRelease) \ .filter(MBRelease.gid==release_mbid) \ .one() mbartists = Session.query(MBArtist, MBArtistName) \ .join(MBArtistCreditName, MBArtistCredit, MBReleaseGroup) \ .join(MBArtist.name) \ .filter(MBReleaseGroup.gid==mbalbum.gid) \ .all() artists = Session.query(Artist) \ .filter(Artist.mbid.in_(map(lambda x: x[0].gid, mbartists))) \ .all() localartistmbids = map(attrgetter('mbid'), artists) # Add Album, Artists and artist-album relationships insertmaps = [] for (artist, name) in mbartists: if artist.gid not in localartistmbids: a = Artist(name.name, artist.gid) artists.append(a) Session.add(a) albumname = mbalbum.name.name artistname = mbalbum.artistcredit.name.name albummeta = mbalbum.meta[0] album = Album(unicode(mbalbum.gid), unicode(albumname), unicode(artistname), albummeta.year, albummeta.month, albummeta.day, unicode(artistname + ' ' + albumname)) album.artists = artists # Build up mapping of promised filename -> (Track) results = Session.query(MBTrack, MBMedium, MBRecording, MBTrackName) \ .join(MBTrackList, MBMedium, MBRelease) \ .join(MBRecording) \ .join(MBTrack.name) \ .filter(MBRelease.gid==release_mbid) \ .all() tracks = [] for (track, medium, recording, name) in results: stableid = hashlib.md5(release_mbid + '_' + recording.gid + '_' + str(track.position) + '_' + str(medium.position)).hexdigest() track = Track(unicode(stableid), None, recording.gid, mbalbum.gid, unicode(name.name), track.position, medium.position, albumname, artistname) tracks.append(track) Session.add(track) tracks.sort(key=attrgetter('discnum', 'tracknum')) assert len(promisedfiles) == len(tracks), 'len(promisedfiles=' + promisedfiles.__repr__() + ') != len(tracks=' + tracks.__repr__() + ')' promisedfilemap = {} for i in range(len(tracks)): normalizedfilename = filter(str.isalnum, promisedfiles[i].lower().encode('ascii', 'ignore')) assert normalizedfilename not in promisedfilemap, 'normalizedfilename='+normalizedfilename + ' in promisedfilemap=' + promisedfilemap.__repr__() promisedfilemap[normalizedfilename] = tracks[i] # Build up mapping of absolute track filename -> (Track, AudioFile), # and link files into their proper library location trackfiles = [] now = datetime.now() dirpath = album.mbid[:2] + '/' + album.mbid if os.path.isdir(Config.MUSIC_PATH + dirpath): assert False, 'directory ' + Config.MUSIC_PATH + dirpath + ' already exists' os.mkdir(Config.MUSIC_PATH + dirpath) torrentdir = rtorrent.d.get_base_path(shopdownload.infohash) # scp stuff back if necessary if Config.SCP_SHOP_DOWNLOADS: remote_dir = pipes.quote(pipes.quote(torrentdir)) # this is awesome local_dir = Config.SCP_FOLDER + '/' + shopdownload.infohash cmd = Config.SCP_CMD + ' ' + Config.SCP_REMOTE + ':' + remote_dir + ' ' + local_dir log.info('[shop] running ' + cmd) scpstart = time.time() retval = os.system(cmd) scpend = time.time() elapsed = int(round(scpend-scpstart)) log.info('[shop] scp of ' + shopdownload.infohash + ' done, took ' + str(elapsed) + ' seconds') if retval != 0: raise Exception('scp command [' + cmd + '] returned ' + str(retval)) torrentdir = local_dir os.system("find " + torrentdir + " -type f -exec rename -v 's/[^[:ascii:]]//g' {} \;") for root, dirs, actualfiles in os.walk(torrentdir): for f in actualfiles: abspath = os.path.join(root, f) relpath = os.path.join(os.path.relpath(root, torrentdir), f) if relpath.startswith('./'): relpath = relpath[2:] normalizedfilename = filter(str.isalnum, relpath.lower().encode('ascii', 'ignore')) if normalizedfilename not in promisedfilemap: continue track = promisedfilemap[normalizedfilename] filepath = dirpath + '/' + release_mbid + '-' + track.mbid + '.mp3' os.link(abspath, Config.MUSIC_PATH + filepath) filesize = os.path.getsize(abspath) filemtime = datetime.fromtimestamp(os.path.getmtime(abspath)) mutagen = MP3(abspath) info = mutagen.info mp3bitrate = info.bitrate mp3samplerate = info.sample_rate mp3length = int(round(info.length)) audiofile = AudioFile(release_mbid, track.mbid, unicode(filepath), filesize, filemtime, mp3bitrate, mp3samplerate, mp3length, now) track.file = audiofile trackfiles.append({'track' : track, 'file' : audiofile, 'path' : abspath}) Session.add(audiofile) Session.add(album) assert len(trackfiles) == len(tracks) == len(promisedfilemap), 'len(trackfiles='+str(len(trackfiles))+') == len(tracks='+str(len(tracks))+') == len(promisedfilemap='+str(len(promisedfilemap))+')' shopdownload.isdone = True shopdownload.finished = datetime.now() Session.commit() log.info('[shop] done importing ' + shopdownload.infohash)