def savePlaylistAJAX(self): playlistname = request.params['name'] trackids = simplejson.loads(request.params['trackids']) trackids = map(lambda x: x.replace('track_',''), trackids) results = Session.query(Track, MBRecording).join(MBRecording).filter(Track.id.in_(trackids)).all() results.sort(lambda a,b: cmp(trackids.index(a[0].id), trackids.index(b[0].id))) recordings = map(itemgetter(1), results) Session.begin() user_name = request.environ['repoze.what.credentials']['repoze.what.userid'] user_id = Session.query(User).filter(User.user_name==user_name).one().user_id playlist = Session.query(Playlist) \ .filter(Playlist.owner_id==user_id) \ .filter(Playlist.name==playlistname) \ .first() if not trackids: if playlist is None: return '{}' else: Session.delete(playlist) else: if playlist is None: playlist = Playlist(user_id, playlistname) Session.add(playlist) else: playlist.modified = datetime.now() playlist.tracks = recordings Session.commit() return '{}'
def getLyricsAJAX(self): trackid = request.params['trackid'].split('_')[1] track = Session.query(Track).filter_by(id=trackid).one() if not track.lyrics and \ (track.lastHitLyricWiki is None or \ datetime.now() > track.lastHitLyricWiki + timedelta(days=10)): track.lastHitLyricWiki = datetime.now() title = track.id3title artist = track.id3artist params = { 'artist' : artist, 'song' : title, 'fmt' : 'json', } url = 'http://lyrics.wikia.com/api.php?%s' % urllib.urlencode(params) html = urllib.urlopen(url).read() if not "'lyrics':'Not found'" in html: search = re.search("'url':'(?P<url>.*?)'",html) lyricurl = urllib.unquote(search.group('url')) lyrichtml = urllib.urlopen(lyricurl).read() lyrics = re.search("<div class='lyricbox'>.*?</div>(?P<lyrics>.*?)<!-- \n", lyrichtml).group('lyrics') lyrics = unescape(lyrics) track.lyrics = lyrics Session.begin() Session.commit() json = {} if track.lyrics: json['lyrics'] = track.lyrics return simplejson.dumps(json)
def saveLyricsFramesAJAX(self): trackid = request.params['trackid'] recordedFrames = simplejson.loads(request.params['frames']) # list of (time, lyrics line index) pairs recordedFrames.sort(key = itemgetter(0)) # (sort by time) Session.begin() track = Session.query(Track).filter_by(id=trackid).one() track.lyricsFrames = recordedFrames Session.commit()
def clearAlbumArt(self): id = request.params['id'] Session.begin() album = Session.query(Album).filter_by(id=id).one() album.albumArtFilename = None album.lastHitAlbumArtExchange = None Session.commit() return 'Cleared album art for ' + album.artist.name + ' - ' + album.name
def setAlbumArt(self): id = request.params['id'] url = request.params['url'] Session.begin() album = Session.query(Album).filter_by(id=id).one() album.albumArtFilename = albumart._fetchAlbumArt(album.artist.name, album.name, url) Session.commit() return 'Set album art for ' + album.artist.name + ' - ' + album.name + ' to ' + url + ', saved to ' + album.albumArtFilename
def invite(self): if request.POST: who = request.params['who'] c.code = ''.join(random.choice(string.letters + string.digits) for i in xrange(32)) invite = Invite(who, c.code) Session.begin() Session.add(invite) Session.commit() return render('/display-invite.html') else: return render('/create-invite.html')
def makeRandom(self): user_name = request.environ['repoze.what.credentials']['repoze.what.userid'] user = Session.query(User).filter(User.user_name==user_name).one() Session.begin() for i in range(10): info = BenKVP(owner = user, key = 'random_key_{0}'.format(i), value = 'random_value_{0}'.format(i)) Session.add(info) Session.commit() return 'success'
def getLyricsAJAX(self): trackid = request.params['trackid'].split('_')[1] track = Session.query(Track).filter_by(id=trackid).one() ip = request.environ['REMOTE_ADDR'] user_name = request.environ['repoze.what.credentials']['repoze.what.userid'] user = Session.query(User).filter(User.user_name==user_name).one() Session.begin() play = TrackPlay(track.mbid, user, ip) Session.add(play) Session.commit() lyrics = lyricsservice.get_lyrics(Session, track) if lyrics: return simplejson.dumps({'lyrics' : lyrics}) else: return '{}'
def getAlbumArtAJAX(self): trackid = request.params['trackid'].split('_')[1] track = Session.query(Track).filter_by(id=trackid).one() if not track.album.albumArtFilename and ( \ track.album.lastHitAlbumArtExchange is None \ or datetime.now() > track.album.lastHitAlbumArtExchange + timedelta(days=10)): track.album.lastHitAlbumArtExchange = datetime.now() album = track.album.name artist = track.album.artist.name if artist == 'Various Artists': q = album else: q = (artist + ' ' + album) q = q.replace("'","") site = 'http://www.albumartexchange.com' params = { 'grid' : '2x7', 'sort' : 7, 'q' : q, } url = site + '/covers.php?%s' % urllib.urlencode(params) html = urllib.urlopen(url).read() search = re.search('src="/phputil/scale_image.php\?size=150&src=(?P<src>.*?)"',html) if search: image = site + urllib.unquote(search.group('src')) extension = image.rsplit('.', 1)[1] delchars = ''.join(c for c in map(chr, range(256)) if not c.isalnum()) delchars = delchars.translate(None," ()'&!-+_.") filename = (artist + ' - ' + album).encode('utf-8').translate(None, delchars) + '.' + extension urllib.urlretrieve(image, 'scatterbrainz/public/art/' + filename) albumArt = '/art/' + filename track.album.albumArtFilename = albumArt Session.begin() Session.commit() json = {} if track.album.albumArtFilename: json['albumArtURL'] = track.album.albumArtFilename return simplejson.dumps(json)
def clearimport(download): with importlock: assert not download.isdone, 'download is already done' assert download.failedimport, 'import isnt failed' mbid = download.release_group_mbid dirpath = mbid[:2] + '/' + mbid cmd = 'rm -rf ' + Config.MUSIC_PATH + dirpath log.info('running ' + cmd) os.system(cmd) if Config.SCP_SHOP_DOWNLOADS: cmd = 'rm -rf ' + Config.SCP_FOLDER + '/' + download.infohash log.info('running ' + cmd) os.system(cmd) Session.begin() download.failedimport = False download.importtrace = None Session.commit()
def getTrackInfoAJAX(self): trackid = request.params['trackid'].split('_')[1] track = Session.query(Track).filter_by(id=trackid).one() (artistName, albumName, trackName) = (track.id3artist, track.id3album, track.id3title) log.info(request.remote_addr + ' playing ' + artistName + ' - ' + trackName) if (not track.artist.mbid or not track.album.mbid or not track.album.asin) and \ (track.album.lastHitMusicbrainz is None \ or datetime.now() > track.album.lastHitMusicbrainz + timedelta(days=10)): track.album.lastHitMusicbrainz = datetime.now() album = track.album artist = album.artist release = None if album.mbid: release = getRelease(album.mbid) else: if artist.name == 'Various Artists': release = searchRelease(None, album.name) else: if artist.name.startswith('The') or artist.name.startswith('the') or artist.name.startswith('THE'): artistSearch = artist.name[4:] else: artistSearch = artist.name release = searchRelease(artistSearch, album.name) if release and not album.mbid: album.mbid = release.id.split('/')[-1] if release and not artist.mbid: artist.mbid = release.artist.id.split('/')[-1] if release: albumName = release.title asin = release.getAsin() if asin: track.album.asin = asin if release.artist: artistName = release.artist.name Session.begin() Session.commit() json = {} json['artist'] = artistName json['album'] = albumName json['track'] = trackName if track.album.asin: json['asin'] = track.album.asin return simplejson.dumps(json)
def run(self): # Only let one worker run via pg advisory lock acquired = Session.execute(select([func.pg_try_advisory_lock(importlockid)])).fetchone()[0] threadid = 'PID ' + str(os.getpid()) + ' thread ' + str(threading.current_thread()) if acquired: log.info('[shop worker] %s acquired lock, starting..' % threadid) else: return pendingdownloads = False while True: try: downloads = Session.query(ShopDownload) \ .filter(ShopDownload.isdone==False) \ .filter(ShopDownload.failedimport==False) \ .all() pendingdownloads = len(downloads) != 0 if pendingdownloads: rtorrent = xmlrpclib.ServerProxy(Config.SHOP_RPC_URL) for download in downloads: try: infohash = download.infohash iscomplete = rtorrent.d.get_complete(infohash) == 1 if iscomplete: shopservice.importDownload(download) except: exc_type, exc_value, exc_traceback = sys.exc_info() importtrace = repr(traceback.format_exception(exc_type, exc_value, exc_traceback)) log.error('[shop worker] caught exception in loop ' + importtrace) Session.rollback() Session.begin() download.failedimport = True download.importtrace = importtrace Session.commit() except Exception as e: log.error('[shop worker] caught exception out of loop ' + repr(e)) Session.rollback() if pendingdownloads: time.sleep(10) else: time.sleep(30)
def currentMembersForTrackArtist(self): trackid = request.params["trackid"] track = Session.query(Track).filter_by(id=trackid).one() # search for triples with subject matching current artist db_uri = u"" for triple in Session.query(RDFTriple).filter_by(artistid=track.artistid): if triple.subject == ":artist": if triple.predicate == "hasdbpedia": db_uri = triple.obj if db_uri == "": art_mbid = track.artist.mbid db_uri = sDB.dbpedia_from_MBID(art_mbid) db_unicode = unicode(db_uri.__str__()) artist = track.artist triple = RDFTriple( subject=u":artist", predicate=u"hasdbpedia", obj=db_unicode, artist=artist, track=None, album=None ) Session.save(triple) Session.commit() g = Graph() g.parse(db_uri) pmem_re = re.compile("pastmember", re.I) cmem_re = re.compile("currentmember", re.I) out = {} past_members = [] current_members = [] for s, p, o in g.triples((None, None, None)): if re.search(pmem_re, p): past_members.append(getURILabel(o)) if re.search(cmem_re, p): current_members.append(getURILabel(o)) out["past_members"] = past_members out["current_members"] = current_members return sjson.dumps(out)
def create(self): usr = request.params['login'] if Session.query(User).filter_by(user_name=usr).count() > 0: return simplejson.dumps({'success':False,'msg':'That username is already taken, sorry.'}) pwd = request.params['pass'] if len(usr) < 3 or len(pwd) < 3: return simplejson.dumps({'success':False,'msg':'Your username and password must each be at least 3 characters.'}) code = request.params['code'] invite = Session.query(Invite).filter_by(code=code).first() if invite is None: return simplejson.dumps({'success':False,'msg':'Your registration code appears to be invalid.'}) user = User() user.who = invite.who user.user_name = usr user.password = pwd user.registered = datetime.now() Session.begin() user.groups = [Session.query(Group).filter_by(group_name='users').one()] Session.delete(invite) Session.add(user) Session.commit() return simplejson.dumps({'success':True})
def setup_app(command, conf, vars): """Place any commands to setup scatterbrainz here""" load_environment(conf.global_conf, conf.local_conf) # Create the tables if they don't already exist log.info("Creating tables") meta.metadata.create_all(bind=meta.engine) log.info("Tables created") session = Session() session.begin() loginPerm = Permission() loginPerm.permission_name = u'login' adminPerm = Permission() adminPerm.permission_name = u'admin' adminGroup = Group() adminGroup.group_name = u'admins' adminGroup.permissions = [loginPerm, adminPerm] userGroup = Group() userGroup.group_name = u'users' userGroup.permissions = [loginPerm] admin = User() admin.user_name = u'admin' admin.password = u'default' admin.who = u'admin' admin.registered = datetime.now() admin.groups = [adminGroup] session.add_all([loginPerm, adminPerm, adminGroup, userGroup, admin]) session.commit()
def index(self): t = Track('asdf') Session.save(t) Session.commit()
def load(self): commit = 'commit' in request.params and request.params['commit'] == 'true' s = '' if commit: Session.begin() now = datetime.now() initialLoad = True #Session.query(AudioFile).count() == 0 if initialLoad: s = _msg(s, 'Initial track loading!') else: s = _msg(s, 'Updating tracks!') then = now missing = 0 changed = 0 for track in Session.query(AudioFile): path = os.path.join(MUSIC, track.filepath) if os.path.exists(path): size = os.path.getsize(path) mtime = datetime.fromtimestamp(os.path.getmtime(path)) if size != track.filesize or mtime != track.filemtime: changed = changed + 1 s = _msg(s, 'Modified file: ' + path) if commit: raise Exception('not implemented!') else: s = _msg(s, 'Deleted file: ' + path) missing = missing + 1 if commit: Session.delete(track) s = _msg(s, 'Found ' + str(missing) + ' missing files and ' + str(changed) + ' modified files, took ' + \ str(datetime.now() - then)) then = datetime.now() filepaths = set(map(lambda t: t.filepath, Session.query(AudioFile))) s = _msg(s, 'Querying for all filepaths took ' + str(datetime.now() - then)) then = datetime.now() added = 0 skippedNoMBID = 0 release_groups_added = set() unknownrelease = set() unknownrecording = set() alreadyhaverecordingrelease = set() alreadyhavereleasegroup = set() unicodeproblems = set() fuckedmp3s = set() for dirname, dirnames, filenames in os.walk(INCOMING, followlinks=True): for filename in filenames: if not os.path.splitext(filename)[-1].lower() == '.mp3': continue try: filepath = os.path.join(os.path.relpath(dirname, INCOMING), filename).decode('utf-8') except UnicodeDecodeError: log.error('unicode problem ' + os.path.join(os.path.relpath(dirname, INCOMING), filename)) unicodeproblems.add(os.path.join(os.path.relpath(dirname, INCOMING), filename)) continue if not initialLoad and filepath in filepaths: continue if not initialLoad: s = _msg(s, 'New file: ' + filepath) if not commit: continue # get size, date fileabspath = os.path.join(dirname,filename) filesize = os.path.getsize(fileabspath) filemtime = datetime.fromtimestamp(os.path.getmtime(fileabspath)) # mp3 length, bitrate, etc. try: mutagen = MP3(fileabspath) except: fuckedmp3s.add(fileabspath) log.error('f****d mp3 ' + fileabspath) continue info = mutagen.info mp3bitrate = info.bitrate mp3samplerate = info.sample_rate mp3length = int(round(info.length)) if info.sketchy: fuckedmp3s.add(fileabspath) log.error('sketchy mp3! ' + fileabspath) continue # brainz!! if RECORDING_MBID_KEY not in mutagen: skippedNoMBID = skippedNoMBID + 1 continue recordingmbid = mutagen[RECORDING_MBID_KEY].data if not recordingmbid: skippedNoMBID = skippedNoMBID + 1 continue if RELEASE_MBID_KEY not in mutagen: skippedNoMBID = skippedNoMBID + 1 continue releasembid = mutagen[RELEASE_MBID_KEY].text[0] if not releasembid or len(mutagen[RELEASE_MBID_KEY].text) > 1: skippedNoMBID = skippedNoMBID + 1 continue release = Session.query(MBRelease).filter(MBRelease.gid==releasembid).first() if release is None: release = Session.query(MBRelease) \ .join(MBReleaseGIDRedirect) \ .filter(MBReleaseGIDRedirect.gid==releasembid) \ .first() if release is None: if releasembid not in unknownrelease: log.error('couldnt find release mbid ' + releasembid) unknownrelease.add(releasembid) continue recording = Session.query(MBRecording).filter(MBRecording.gid==recordingmbid).first() if recording is None: recording = Session.query(MBRecording) \ .join(MBRecordingGIDRedirect) \ .filter(MBRecordingGIDRedirect.gid==recordingmbid) \ .first() if recording is None: if recordingmbid not in unknownrecording: log.error('couldnt find recording mbid ' + recordingmbid) unknownrecording.add(recordingmbid) continue releasegroupmbid = release.releasegroup.gid dirs = os.path.join(releasegroupmbid[:2], releasegroupmbid) newdir = os.path.join(MUSIC, dirs) if releasegroupmbid not in release_groups_added: existing = Session.query(Album) \ .filter(Album.mbid==releasegroupmbid) \ .first() if existing != None: if releasegroupmbid not in alreadyhavereleasegroup: log.info('already have release group ' + existing.artistcredit + ' ' + existing.name + ' ' + existing.mbid) alreadyhavereleasegroup.add(releasegroupmbid) continue else: os.makedirs(newdir) release_groups_added.add(releasegroupmbid) existing = Session.query(AudioFile) \ .filter(AudioFile.recordingmbid==recording.gid) \ .filter(AudioFile.releasembid==release.gid) \ .first() if existing: if (recording.gid + '-' + release.gid) not in alreadyhaverecordingrelease: log.error('already existing recording/release combo for file ' + filepath) alreadyhaverecordingrelease.add(recording.gid + '-' + release.gid) continue # check for existing release group, make new release group directory if necessary # link file into new directory newfilename = release.gid + '-' + recording.gid + '.mp3' ln = os.path.join(newdir, newfilename) os.link(fileabspath, ln) lnrelpath = os.path.join(dirs, newfilename) track = AudioFile(filepath=lnrelpath, filesize=filesize, filemtime=filemtime, mp3bitrate=mp3bitrate, mp3samplerate=mp3samplerate, mp3length=mp3length, recording=recording, release=release, added=now, ) Session.add(track) added = added + 1 if commit: s = _msg(s, 'Building model for new tracks took ' + str(datetime.now() - then)) then = datetime.now() Session.commit() s = _msg(s, """Committed %(added)d new tracks, skipped %(skipped)d""" \ % {'added':added-skippedNoMBID, 'skipped':skippedNoMBID} + ', took ' + str(datetime.now() - then)) log.error("unknownrelease: " + unknownrelease.__str__()) log.error("unknownrecording: " + unknownrecording.__str__()) log.error("alreadyhaverecordingrelease: " + alreadyhaverecordingrelease.__str__()) log.error("alreadyhavereleasegroup: " + alreadyhavereleasegroup.__str__()) log.error("unicodeproblems: " + unicodeproblems.__str__()) log.error("fuckedmp3s: " + fuckedmp3s.__str__()) else: s = _msg(s, 'Saw ' + str(added) + ' new tracks, took ' + str(datetime.now() - then)) return s
def load(self): done = False now = datetime.now() numFilesSeen = 0 numBadFiles = 0 numSketchy = 0 commitbuffer = [] albums = {} artists = {} variousArtists = Artist(name='Various Artists', mbid='89ad4ac3-39f7-470e-963a-56509c546377', added=now) artists['Various Artists'] = variousArtists Session.save(variousArtists) for dirname, dirnames, filenames in os.walk('scatterbrainz/public/.music/'): localAlbums = {} for filename in filenames: try: numFilesSeen = numFilesSeen + 1 # get path, size, date fileabspath = os.path.join(dirname,filename) filepath = os.path.join(os.path.relpath(dirname, 'scatterbrainz/public/.music/'), filename) filesize = os.path.getsize(fileabspath) filemtime = datetime.fromtimestamp(os.path.getmtime(fileabspath)) ext = os.path.splitext(filename)[-1] if not ext == '.mp3': continue # mp3 length, bitrate, etc. mutagen = MP3(fileabspath, ID3=EasyID3) info = mutagen.info mp3bitrate = info.bitrate mp3samplerate = info.sample_rate mp3length = int(round(info.length)) if info.sketchy: mp3['sketchy'] = true numSketchy = numSketchy + 1 log.warn('sketchy MP3: ' + filename) # id3 # keys: ['album', 'date', 'version', 'composer', 'title' # 'genre', 'tracknumber', 'lyricist', 'artist'] id3artist = getid3prop(mutagen, 'artist') id3album = getid3prop(mutagen, 'album') id3title = getid3prop(mutagen, 'title') id3tracknum = getid3prop(mutagen, 'tracknumber') id3date = getid3prop(mutagen, 'date') id3composer = getid3prop(mutagen, 'composer') id3genre = getid3prop(mutagen, 'genre') id3lyricist = getid3prop(mutagen, 'lyricist') # additional musicbrainz related keys: At some point, # should probably switch from easyID3 to ordinary ID3 # class to get extra MB relationship data. mbartistid = getid3prop(mutagen,'musicbrainz_albumartistid') mbalbumid = getid3prop(mutagen,'musicbrainz_albumid') mbtrackid = getid3prop(mutagen,'musicbrainz_trackid') if not id3artist: artist = None elif id3artist in artists: artist = artists[id3artist] else: artist = Artist(name=id3artist, mbid=mbartistid, added=now) Session.save(artist) artists[id3artist] = artist if not id3album: album = None elif id3album in localAlbums: album = localAlbums[id3album] if artist != album.artist: album.artist = variousArtists else: album = Album(name=id3album, artist=artist, <<<<<<< HEAD mbid=mbalbumid, albumArtURL=None, ======= >>>>>>> upstream/master added=now) Session.save(album) albums[id3album] = album localAlbums[id3album] = album track = Track(artist=artist, album=album, filepath=filepath.decode('utf-8'), filesize=filesize, filemtime=filemtime, mp3bitrate=mp3bitrate, mp3samplerate=mp3samplerate, mp3length=mp3length, id3artist=id3artist, id3album=id3album, id3title=id3title, id3tracknum=id3tracknum, id3date=id3date, id3composer=id3composer, id3genre=id3genre, id3lyricist=id3lyricist, added=now, mbid=mbtrackid, ) Session.save(track) triple = RDFTriple(subject = u":track", predicate = u"ison", obj=u":album", artist=None, track=track, album=album) Session.save(triple) except Exception as e: numBadFiles = numBadFiles + 1 log.error('Could not load file "' + filename + '" due to exception: ' + e.__class__.__name__ + ': ' + str(e)) if done: break Session.commit() otherNow = datetime.now() return """Saw %(numFilesSeen)d tracks, %(numArtists)d artists and %(numAlbums)d albums. %(numBadFiles)d failed, %(numSketchy)d sketchy. Loaded in %(time)s""" \ % {'numFilesSeen':numFilesSeen, 'numBadFiles':numBadFiles, 'numArtists': len(artists), 'numAlbums': len(albums), 'numSketchy' : numSketchy, 'time' : str(otherNow - now)}
def getLyricsAJAX(self): trackid = request.params['trackid'].split('_')[1] track = Session.query(Track).filter_by(id=trackid).one() if not track.lyrics and \ (track.lastHitLyricWiki is None or \ datetime.now() > track.lastHitLyricWiki + timedelta(days=10)): track.lastHitLyricWiki = datetime.now() title = track.id3title artist = track.id3artist def getLyricsHtml(encoder = lambda s: s): params = { 'artist' : encoder(artist), 'song' : encoder(title), 'fmt' : 'json', } url = 'http://lyrics.wikia.com/api.php?%s' % urllib.urlencode(params) log.info('[lyric] Hitting ' + url) html = urllib.urlopen(url).read() return html if not "'lyrics':'Not found'" in html else None try: html = getLyricsHtml() except UnicodeEncodeError: html = getLyricsHtml(lambda s: s.encode('utf-8')) or \ getLyricsHtml(lambda s: unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')) # wikia keys some int'l lyrics on correctly-encoded titles # and others on normalized (e.g., "el nino") titles if html: search = re.search("'url':'(?P<url>.*?)'",html) lyricurl = urllib.unquote(search.group('url')) page = urllib.quote(lyricurl.split('/')[-1]) lyricurl = 'http://lyrics.wikia.com/index.php?title='+page+'&action=edit' log.info('[lyric] Hitting ' + lyricurl) lyrichtml = urllib.urlopen(lyricurl).read() lyricREstr = "<lyrics>(?P<lyrics>.*)</lyrics>" lyricRE = re.search(lyricREstr, lyrichtml, re.S) if lyricRE: lyrics = lyricRE.group('lyrics').strip('\n') if '{{gracenote_takedown}}' in lyrics: historyurl = 'http://lyrics.wikia.com/index.php?title='+page+'&action=history' log.info('[lyric] Found gracenote takedown, hitting ' + historyurl) historyhtml = urllib.urlopen(historyurl).read() oldidRE = re.search(".*GracenoteBot.*?/index\.php\?title=.*?&oldid=(?P<oldid>\d+)", historyhtml, re.S) if oldidRE: oldid = oldidRE.group('oldid') oldlyricsurl = lyricurl + '&oldid=' + oldid log.info('[lyric] found pre-takedown lyrics! hitting ' + oldlyricsurl) oldlyrichtml = urllib.urlopen(oldlyricsurl).read() lyricRE = re.search(lyricREstr, oldlyrichtml, re.S) if lyricRE: lyrics = lyricRE.group('lyrics').strip('\n') if '{{gracenote_takedown}}' in lyrics: raise Exception('[lyric] Still found takedown lyrics!') elif '{{Instrumental}}' in lyrics: track.lyrics = u'(Instrumental)' else: track.lyrics = lyrics.replace('\n','<br />').decode('utf-8') else: log.info('[lyric] failed lyrics!') raise Exception('failed lyrics!') else: log.info('[lyric] no pre-takedown lyrics found :(') elif '{{Instrumental}}' in lyrics: track.lyrics = u'(Instrumental)' else: track.lyrics = lyrics.replace('\n','<br />').decode('utf-8') else: log.info('[lyric] failed lyrics!') raise Exception('failed lyrics!') else: log.info('[lyric] No results found') Session.begin() Session.commit() json = {} if track.lyrics: json['lyrics'] = track.lyrics json['trackid'] = track.id json['frames'] = track.lyricsFrames return simplejson.dumps(json)
def download(Session, mbid, owner_id): with shoplock: now = datetime.now() existingdownload = Session.query(ShopDownload) \ .filter(ShopDownload.release_group_mbid==mbid) \ .first() if existingdownload: return existingdownload.infohash attempt = Session.query(ShopDownloadAttempt) \ .filter(ShopDownloadAttempt.mbid==mbid) \ .first() if attempt and now < attempt.tried + timedelta(days=10): return None if not attempt: attempt = ShopDownloadAttempt(mbid, now, True) attempt.tried = now log.info('[shop] searching for ' + mbid) Session.begin() if not maybeloggedin: login() (album, albumname, artistname) = Session.query(MBReleaseGroup, MBReleaseName, MBArtistName) \ .join(MBReleaseName) \ .join(MBReleaseGroup.artistcredit, MBArtistCredit.name) \ .filter(MBReleaseGroup.gid==mbid) \ .one() searchartist = artistname.name searchalbum = albumname.name if searchartist == 'Various Artists': searchartist = '' searchartist = _cleanupSearchTerm(searchartist) searchalbum = _cleanupSearchTerm(searchalbum) url = searchurl + '?' + urllib.urlencode({ 'artistname' : searchartist, 'groupname' : searchalbum, 'action' : 'advanced', 'format' : 'MP3', 'order_by' : 'seeders' }) log.info('[shop] hitting ' + url) handle = opener.open(url) page = lxml.parse(handle).getroot() html = lxml.tostring(page) if loginpath in html: log.warn('[shop] login url found in search result, logging in..') login() handle = opener.open(url) page = lxml.parse(handle).getroot() html = lxml.tostring(page) if loginpath in html: log.error('[shop] couldnt login!') raise Exception('couldnt login!') if 'Your search did not match anything' in html: log.warn('[shop] no results') attempt.gotsearchresults = False if attempt not in Session: Session.add(attempt) Session.commit() return None # Gather up all tracks from all releases results = Session.query(MBTrack, MBArtistName, MBTrackName, MBRelease, MBMedium) \ .join(MBTrackList, MBMedium, MBRelease, MBReleaseGroup) \ .join(MBTrackName).join(MBTrack.artistcredit, MBArtistCredit.name) \ .filter(MBReleaseGroup.gid==mbid) \ .all() releases = {} for (track, artistname, trackname, release, medium) in results: data = {'id' : track.id, 'num' : track.position, 'disc' : medium.position, 'artist' : artistname.name, 'name' : trackname.name} mbid = release.gid if mbid in releases: releases[mbid].append(data) else: releases[mbid] = [data] for releaseid in releases.keys(): release = releases[releaseid] release.sort(key=itemgetter('disc', 'num')) log.info('[shop] release group ' + mbid + ' has ' + str(len(releases)) + ' releases to check against') # Try to match any of the releases with the first few results torrent_table = page.cssselect('table#torrent_table')[0] groups = torrent_table.cssselect('tr.group') numresults = len(groups) for torrent in groups[:3]: torrentpageurl = shopbaseurl + '/' + torrent.cssselect('a[title="View Torrent"]')[0].attrib['href'] log.info('[shop] hitting ' + torrentpageurl) thandle = opener.open(torrentpageurl) tpage = lxml.parse(thandle).getroot() # Gather up information about all downloads for this torrent ttable = tpage.cssselect('table.torrent_table')[0] downloads = [] for download in ttable.cssselect('tr.group_torrent[id]'): torrentid = re.sub('^torrent', '', download.attrib['id']) torrenttype = download.cssselect('a[onclick]')[0].text.encode('ascii', 'ignore').split('/')[0].strip() if torrenttype != 'MP3': continue downloadurl = download.cssselect('a[title=Download]')[0].attrib['href'] if len(download.cssselect('td')) != 5: raise Exception('Torrent ' + torrentid + ' has !=5 TD tags at ' + torrentpageurl) numseeders = int(download.cssselect('td')[3].text.replace(',', '')) if numseeders < 1: continue filestr = ttable.cssselect('div#files_' + torrentid)[0].cssselect('tr')[1:] filenames = [] for tr in filestr: filename = tr.cssselect('td')[0].text if filename.lower().endswith('.mp3'): filenames.append({'original' : filename, 'compare' : re.sub(' +', '', filename.split('/')[-1][:-4].lower())}) downloads.append({'seeders' : numseeders, 'torrentid' : torrentid, 'url' : downloadurl, 'filenames' : filenames}) if not downloads: log.info('[shop] no seeded files of correct type found at torrent ' + torrentid) # See if any of the downloads nicely match any of the releases, trying best seeded first downloads.sort(key=itemgetter('seeders'), reverse=True) for download in downloads: releasescores = [] filenames = download['filenames'] for releaseid in releases.keys(): release = releases[releaseid] if len(filenames) != len(release): minscore = 0 avgscore = 0 else: numtracks = len(release) minscore = None minscoreidx = None sumscore = 0 for i in range(numtracks): rtrack = release[i] rtracknum = '%02d' % rtrack['num'] rtartist = rtrack['artist'].lower() rtname = rtrack['name'].lower() dname = filenames[i]['compare'] name1 = rtracknum + ' ' + rtname name2 = rtracknum + ' ' + rtartist + ' ' + rtname score1 = SequenceMatcher(None, dname, name1).ratio() score2 = SequenceMatcher(None, dname, name2).ratio() score = max(score1, score2) sumscore = sumscore + score if score < minscore or minscore is None: minscore = score minscoreidx = i avgscore = sumscore * 1.0 / numtracks log.info('[shop] match avg=' + str(avgscore) + ' min=' + str(minscore) + ' ' + download['torrentid'] + ' -> ' + releaseid) releasescores.append({'releaseid' : releaseid, 'min' : minscore, 'avg' : avgscore}) releasescores = filter(lambda x: x['min'] > 0.3 and x['avg'] > 0.70, releasescores) releasescores.sort(key=itemgetter('avg'), reverse=True) if releasescores: # Toss torrent over to rtorrent via xml-rpc releaseid = releasescores[0]['releaseid'] torrenturl = shopbaseurl + '/' + download['url'] torrentdata = opener.open(torrenturl).read() torrentdecode = bencode.bdecode(torrentdata) infohash = hashlib.sha1(bencode.bencode(torrentdecode['info'])).hexdigest().upper() with tempfile.NamedTemporaryFile(delete=False) as torrentfile: torrentpath = torrentfile.name torrentfile.write(torrentdata) # scp torrent over if necessary if Config.SCP_SHOP_DOWNLOADS: remotetorrentpath = '/tmp/' + infohash + '.torrent' cmd = Config.SCP_CMD + ' ' + torrentpath + ' ' + Config.SCP_REMOTE + ':' + remotetorrentpath log.info('[shop] running ' + cmd) retval = os.system(cmd) os.unlink(torrentpath) if retval != 0: raise Exception('scp command [' + cmd + '] returned ' + str(retval)) torrentpath = remotetorrentpath rtorrent = xmlrpclib.ServerProxy(Config.SHOP_RPC_URL) rtorrent.load_start(torrentpath, "execute=rm," + torrentpath) log.info('[shop] downloaded ' + torrenturl + ' has ' + str(download['seeders']) + ' seeders, infohash=' + infohash + ', match to album ' + releaseid) file_json = simplejson.dumps(map(itemgetter('original'), filenames)) minscore = releasescores[0]['min'] avgscore = releasescores[0]['avg'] shopdownload = ShopDownload(releaseid, album.gid, infohash, torrenturl, torrentpageurl, download['torrentid'], download['seeders'], file_json, minscore, avgscore, owner_id) Session.add(shopdownload) if attempt in Session: Session.delete(attempt) Session.commit() return infohash log.info('[shop] no matches, sorry :(') attempt.gotsearchresults = True if attempt not in Session: Session.add(attempt) Session.commit() return None
def load(self): commit = 'commit' in request.params and request.params['commit'] == 'true' s = '' now = datetime.now() albums = {} artists = {} if commit: Session.begin() variousArtists = Session.query(Artist).filter_by(mbid=u'89ad4ac3-39f7-470e-963a-56509c546377').first() if variousArtists is None: variousArtists = Artist(name=u'Various Artists', mbid=u'89ad4ac3-39f7-470e-963a-56509c546377', added=now) Session.save(variousArtists) s = _msg(s, 'Committed various artists placeholder') artists['Various Artists'] = variousArtists initialLoad = Session.query(Track).count() == 0 if initialLoad: s = _msg(s, 'Initial track loading!') else: s = _msg(s, 'Updating tracks!') then = now missing = 0 changed = 0 for track in Session.query(Track): path = os.path.join(BASE, track.filepath) if os.path.exists(path): size = os.path.getsize(path) mtime = datetime.fromtimestamp(os.path.getmtime(path)) if size != track.filesize or mtime != track.filemtime: changed = changed + 1 s = _msg(s, 'Modified file: ' + path) if commit: raise Exception('not implemented!') else: s = _msg(s, 'Deleted file: ' + path) missing = missing + 1 if commit: Session.delete(track) s = _msg(s, 'Found ' + str(missing) + ' missing files and ' + str(changed) + ' modified files, took ' + \ str(datetime.now() - then)) then = datetime.now() filepaths = set(map(lambda t: t.filepath, Session.query(Track))) s = _msg(s, 'Querying for all filepaths took ' + str(datetime.now() - then)) then = datetime.now() added = 0 for dirname, dirnames, filenames in os.walk(BASE): localAlbums = {} for filename in filenames: filepath = os.path.join(os.path.relpath(dirname, BASE), filename).decode('utf-8') if not os.path.splitext(filename)[-1].lower() == '.mp3': continue if not initialLoad and filepath in filepaths: continue added = added + 1 if not initialLoad: s = _msg(s, 'New file: ' + filepath) if not commit: continue # get size, date fileabspath = os.path.join(dirname,filename) filesize = os.path.getsize(fileabspath) filemtime = datetime.fromtimestamp(os.path.getmtime(fileabspath)) # mp3 length, bitrate, etc. mutagen = MP3(fileabspath, ID3=EasyID3) info = mutagen.info mp3bitrate = info.bitrate mp3samplerate = info.sample_rate mp3length = int(round(info.length)) if info.sketchy: raise Exception('sketchy mp3! ' + filename) # id3 # keys: ['album', 'date', 'version', 'composer', 'title' # 'genre', 'tracknumber', 'lyricist', 'artist'] id3artist = getid3prop(mutagen, 'artist') id3album = getid3prop(mutagen, 'album') id3title = getid3prop(mutagen, 'title') id3tracknum = getid3prop(mutagen, 'tracknumber') id3date = getid3prop(mutagen, 'date') id3composer = getid3prop(mutagen, 'composer') id3genre = getid3prop(mutagen, 'genre') id3lyricist = getid3prop(mutagen, 'lyricist') # additional musicbrainz related keys: At some point, # should probably switch from easyID3 to ordinary ID3 # class to get extra MB relationship data. mbartistid = getid3prop(mutagen,'musicbrainz_albumartistid') mbalbumid = getid3prop(mutagen,'musicbrainz_albumid') mbtrackid = getid3prop(mutagen,'musicbrainz_trackid') if not id3artist: artist = None elif id3artist in artists: artist = artists[id3artist] else: if initialLoad: artistFromDb = None else: artistFromDb = Session.query(Artist).filter_by(name=id3artist).first() if artistFromDb is None: artist = Artist(name=id3artist, mbid=mbartistid, added=now) Session.save(artist) else: artist = artistFromDb artists[id3artist] = artist if not id3album: album = None elif id3album in localAlbums: album = localAlbums[id3album] if artist != album.artist: album.artist = variousArtists else: album = Album(name=id3album, artist=artist, added=now, mbid=mbalbumid) Session.save(album) albums[id3album] = album localAlbums[id3album] = album track = Track(artist=artist, album=album, filepath=filepath, filesize=filesize, filemtime=filemtime, mp3bitrate=mp3bitrate, mp3samplerate=mp3samplerate, mp3length=mp3length, id3artist=id3artist, id3album=id3album, id3title=id3title, id3tracknum=id3tracknum, id3date=id3date, id3composer=id3composer, id3genre=id3genre, id3lyricist=id3lyricist, added=now, mbid=mbtrackid, ) Session.save(track) if commit: s = _msg(s, 'Building model for new tracks took ' + str(datetime.now() - then)) then = datetime.now() Session.commit() s = _msg(s, """Committed %(added)d new tracks, %(numArtists)d new artists, %(numAlbums)d new albums""" \ % {'added':added, 'numArtists': len(artists), 'numAlbums': len(albums)} + \ ', took ' + str(datetime.now() - then)) else: s = _msg(s, 'Saw ' + str(added) + ' new tracks, took ' + str(datetime.now() - then)) return s
def importDownload(shopdownload): with importlock: Session.begin() assert not shopdownload.isdone, 'download is already imported' assert not shopdownload.failedimport, 'download already failed import' log.info('[shop] starting to import ' + shopdownload.infohash) promisedfiles = simplejson.loads(shopdownload.file_json) rtorrent = xmlrpclib.ServerProxy(Config.SHOP_RPC_URL) assert rtorrent.d.get_complete(shopdownload.infohash) == 1, 'rtorrent says download isnt done yet' release_mbid = shopdownload.release_mbid mbalbum = Session.query(MBReleaseGroup) \ .join(MBRelease) \ .filter(MBRelease.gid==release_mbid) \ .one() mbartists = Session.query(MBArtist, MBArtistName) \ .join(MBArtistCreditName, MBArtistCredit, MBReleaseGroup) \ .join(MBArtist.name) \ .filter(MBReleaseGroup.gid==mbalbum.gid) \ .all() artists = Session.query(Artist) \ .filter(Artist.mbid.in_(map(lambda x: x[0].gid, mbartists))) \ .all() localartistmbids = map(attrgetter('mbid'), artists) # Add Album, Artists and artist-album relationships insertmaps = [] for (artist, name) in mbartists: if artist.gid not in localartistmbids: a = Artist(name.name, artist.gid) artists.append(a) Session.add(a) albumname = mbalbum.name.name artistname = mbalbum.artistcredit.name.name albummeta = mbalbum.meta[0] album = Album(unicode(mbalbum.gid), unicode(albumname), unicode(artistname), albummeta.year, albummeta.month, albummeta.day, unicode(artistname + ' ' + albumname)) album.artists = artists # Build up mapping of promised filename -> (Track) results = Session.query(MBTrack, MBMedium, MBRecording, MBTrackName) \ .join(MBTrackList, MBMedium, MBRelease) \ .join(MBRecording) \ .join(MBTrack.name) \ .filter(MBRelease.gid==release_mbid) \ .all() tracks = [] for (track, medium, recording, name) in results: stableid = hashlib.md5(release_mbid + '_' + recording.gid + '_' + str(track.position) + '_' + str(medium.position)).hexdigest() track = Track(unicode(stableid), None, recording.gid, mbalbum.gid, unicode(name.name), track.position, medium.position, albumname, artistname) tracks.append(track) Session.add(track) tracks.sort(key=attrgetter('discnum', 'tracknum')) assert len(promisedfiles) == len(tracks), 'len(promisedfiles=' + promisedfiles.__repr__() + ') != len(tracks=' + tracks.__repr__() + ')' promisedfilemap = {} for i in range(len(tracks)): normalizedfilename = filter(str.isalnum, promisedfiles[i].lower().encode('ascii', 'ignore')) assert normalizedfilename not in promisedfilemap, 'normalizedfilename='+normalizedfilename + ' in promisedfilemap=' + promisedfilemap.__repr__() promisedfilemap[normalizedfilename] = tracks[i] # Build up mapping of absolute track filename -> (Track, AudioFile), # and link files into their proper library location trackfiles = [] now = datetime.now() dirpath = album.mbid[:2] + '/' + album.mbid if os.path.isdir(Config.MUSIC_PATH + dirpath): assert False, 'directory ' + Config.MUSIC_PATH + dirpath + ' already exists' os.mkdir(Config.MUSIC_PATH + dirpath) torrentdir = rtorrent.d.get_base_path(shopdownload.infohash) # scp stuff back if necessary if Config.SCP_SHOP_DOWNLOADS: remote_dir = pipes.quote(pipes.quote(torrentdir)) # this is awesome local_dir = Config.SCP_FOLDER + '/' + shopdownload.infohash cmd = Config.SCP_CMD + ' ' + Config.SCP_REMOTE + ':' + remote_dir + ' ' + local_dir log.info('[shop] running ' + cmd) scpstart = time.time() retval = os.system(cmd) scpend = time.time() elapsed = int(round(scpend-scpstart)) log.info('[shop] scp of ' + shopdownload.infohash + ' done, took ' + str(elapsed) + ' seconds') if retval != 0: raise Exception('scp command [' + cmd + '] returned ' + str(retval)) torrentdir = local_dir os.system("find " + torrentdir + " -type f -exec rename -v 's/[^[:ascii:]]//g' {} \;") for root, dirs, actualfiles in os.walk(torrentdir): for f in actualfiles: abspath = os.path.join(root, f) relpath = os.path.join(os.path.relpath(root, torrentdir), f) if relpath.startswith('./'): relpath = relpath[2:] normalizedfilename = filter(str.isalnum, relpath.lower().encode('ascii', 'ignore')) if normalizedfilename not in promisedfilemap: continue track = promisedfilemap[normalizedfilename] filepath = dirpath + '/' + release_mbid + '-' + track.mbid + '.mp3' os.link(abspath, Config.MUSIC_PATH + filepath) filesize = os.path.getsize(abspath) filemtime = datetime.fromtimestamp(os.path.getmtime(abspath)) mutagen = MP3(abspath) info = mutagen.info mp3bitrate = info.bitrate mp3samplerate = info.sample_rate mp3length = int(round(info.length)) audiofile = AudioFile(release_mbid, track.mbid, unicode(filepath), filesize, filemtime, mp3bitrate, mp3samplerate, mp3length, now) track.file = audiofile trackfiles.append({'track' : track, 'file' : audiofile, 'path' : abspath}) Session.add(audiofile) Session.add(album) assert len(trackfiles) == len(tracks) == len(promisedfilemap), 'len(trackfiles='+str(len(trackfiles))+') == len(tracks='+str(len(tracks))+') == len(promisedfilemap='+str(len(promisedfilemap))+')' shopdownload.isdone = True shopdownload.finished = datetime.now() Session.commit() log.info('[shop] done importing ' + shopdownload.infohash)