def upgradeTorrents(): # fetch some un-inserted torrents to put into the InvertedIndex sql = """ SELECT torrent_id, name, torrent_file_name FROM Torrent WHERE torrent_id NOT IN (SELECT DISTINCT torrent_id FROM InvertedIndex) AND torrent_file_name IS NOT NULL LIMIT 20""" records = self.fetchall(sql) if len(records) == 0: # upgradation is complete and hence delete the temp file os.remove(tmpfilename) if DEBUG: print >> sys.stderr, time.asctime(), "-", "DB Upgradation: temp-file deleted", tmpfilename return for torrent_id, name, torrent_file_name in records: try: abs_filename = os.path.join(session.get_torrent_collecting_dir(), torrent_file_name) if not os.path.exists(abs_filename): raise RuntimeError(".torrent file not found. Use fallback.") torrentdef = TorrentDef.load(abs_filename) torrent_name = torrentdef.get_name_as_unicode() keywords = Set(split_into_keywords(torrent_name)) for filename in torrentdef.get_files_as_unicode(): keywords.update(split_into_keywords(filename)) except: # failure... most likely the .torrent file # is invalid # use keywords from the torrent name # stored in the database torrent_name = dunno2unicode(name) keywords = Set(split_into_keywords(torrent_name)) # store the keywords in the InvertedIndex # table in the database if len(keywords) > 0: values = [(keyword, torrent_id) for keyword in keywords] self.executemany(u"INSERT OR REPLACE INTO InvertedIndex VALUES(?, ?)", values, commit=False) if DEBUG: print >> sys.stderr, time.asctime(), "-", "DB Upgradation: Extending the InvertedIndex table with", len( values ), "new keywords for", torrent_name # now commit, after parsing the batch of torrents self.commit() # upgradation not yet complete; comeback after 5 sec tqueue.add_task(upgradeTorrents, 5)
def process_query(self, permid, d, selversion): hits = None p = None sendtorrents = False netwq = d['q'] if netwq.startswith("SIMPLE"): # remote query # Format: 'SIMPLE '+string of space separated keywords or # 'SIMPLE+METADATA' +string of space separated keywords # # In the future we could support full SQL queries: # SELECT infohash,torrent_name FROM torrent_db WHERE status = ALIVE if netwq.startswith('SIMPLE+METADATA'): q = d['q'][len('SIMPLE+METADATA '):] sendtorrents = True else: q = d['q'][len('SIMPLE '):] uq = self.clean_netwq(q) kws = split_into_keywords(uq) hits = self.search_torrents(kws, maxhits=MAX_RESULTS,sendtorrents=sendtorrents) p = self.create_remote_query_reply(d['id'],hits,selversion) elif netwq.startswith("CHANNEL"): # channel query if DEBUG: print>>sys.stderr, "Incoming channel query", d['q'] q = d['q'][len('CHANNEL '):] uq = self.clean_netwq(q,channelquery=True) hits = self.channelcast_db.searchChannels(uq) p = self.create_channel_query_reply(d['id'],hits,selversion) # log incoming query, if logfile is set if self.logfile: self.log(permid, q) m = QUERY_REPLY+p if self.overlay_log: nqueries = self.get_peer_nqueries(permid) # RECV_MSG PERMID OVERSION NUM_QUERIES MSG self.overlay_log('RECV_QRY', show_permid(permid), selversion, nqueries, repr(d)) # RPLY_QRY PERMID NUM_HITS MSG self.overlay_log('RPLY_QRY', show_permid(permid), len(hits), repr(p)) self.overlay_bridge.send(permid, m, self.send_callback) self.inc_peer_nqueries(permid)