class FMCrawler(): def __init__(self): self.m_dbPro=DoubanProtocol() self.m_sids=[] self.db = DBHelper() def crawl(self,data): for i in data.keys(): if(i%3 == 0): continue tag = i channels=data[i] for j in channels.keys(): k = 0 c = 0 count = channels[j] while(k <= 1000 ): k+=1 if(len(self.m_sids) != 0): t_sid = self.m_sids[random.randint(0,len(self.m_sids)-1)] jslist = self.m_dbPro.getNewPlayList(j,sid=t_sid) else: jslist=self.m_dbPro.getNewPlayList(j) if len(jslist) != 0: songs=self.parseSong(jslist,tag) new_add = self.save_to_DB(songs) c+=new_add if count and c>=count: break def parseSong(self,list,tag): if len(list) == 0 : return [] else: self.m_sids = [] songs = [] for i in range(len(list)): try: sid=int(list[i].get('sid'),10) except Exception,e: continue title=list[i].get('title') artist=list[i].get('artist') self.m_sids.append(sid) song = Song(sid,title,artist,tag) songs.append(song) return songs
class DoubanPlayList: def __init__(self, channel = 1): self.channel = channel self.length = 0 self.currentSongIndex = 0 # protocol to get the list self.douban = DoubanProtocol() self.playlist = self.nextPlayList() #pass def nextPlayList(self): playlist = self.douban.getNewPlayList(self.channel) self.length = len(playlist) self.currentSongIndex = 0 return playlist # def nextSong(self): """ get next song in the playlist if current list is exhaused, auto fetch next playlist return a song object """ if self.currentSongIndex >= self.length: self.playlist = self.nextPlayList() song = self.playlist[self.currentSongIndex] self.currentSongIndex += 1 return song def currentSong(self): #print self.currentSongIndex, self.length if self.currentSongIndex == 0: return None song = self.playlist[self.currentSongIndex-1] return song def previousSong(self): # not allowed now pass def changeChannel(self, channel): self.channel = channel self.playlist = self.nextPlayList()