Esempio n. 1
0
            def upgradeTorrents():
                # fetch some un-inserted torrents to put into the InvertedIndex
                sql = """
                SELECT torrent_id, name, torrent_file_name
                FROM Torrent
                WHERE torrent_id NOT IN (SELECT DISTINCT torrent_id FROM InvertedIndex)
                AND torrent_file_name IS NOT NULL
                LIMIT 20"""
                records = self.fetchall(sql)

                if len(records) == 0:
                    # upgradation is complete and hence delete the temp file
                    os.remove(tmpfilename)
                    if DEBUG:
                        print >> sys.stderr, time.asctime(), "-", "DB Upgradation: temp-file deleted", tmpfilename
                    return

                for torrent_id, name, torrent_file_name in records:
                    try:
                        abs_filename = os.path.join(session.get_torrent_collecting_dir(), torrent_file_name)
                        if not os.path.exists(abs_filename):
                            raise RuntimeError(".torrent file not found. Use fallback.")
                        torrentdef = TorrentDef.load(abs_filename)
                        torrent_name = torrentdef.get_name_as_unicode()
                        keywords = Set(split_into_keywords(torrent_name))
                        for filename in torrentdef.get_files_as_unicode():
                            keywords.update(split_into_keywords(filename))

                    except:
                        # failure... most likely the .torrent file
                        # is invalid

                        # use keywords from the torrent name
                        # stored in the database
                        torrent_name = dunno2unicode(name)
                        keywords = Set(split_into_keywords(torrent_name))

                    # store the keywords in the InvertedIndex
                    # table in the database
                    if len(keywords) > 0:
                        values = [(keyword, torrent_id) for keyword in keywords]
                        self.executemany(u"INSERT OR REPLACE INTO InvertedIndex VALUES(?, ?)", values, commit=False)
                        if DEBUG:
                            print >> sys.stderr, time.asctime(), "-", "DB Upgradation: Extending the InvertedIndex table with", len(
                                values
                            ), "new keywords for", torrent_name

                # now commit, after parsing the batch of torrents
                self.commit()

                # upgradation not yet complete; comeback after 5 sec
                tqueue.add_task(upgradeTorrents, 5)
Esempio n. 2
0
    def process_query(self, permid, d, selversion):
        hits = None
        p = None
        sendtorrents = False

        netwq = d['q']
        if netwq.startswith("SIMPLE"): # remote query
            # Format: 'SIMPLE '+string of space separated keywords or
            #         'SIMPLE+METADATA' +string of space separated keywords
            #
            # In the future we could support full SQL queries:
            # SELECT infohash,torrent_name FROM torrent_db WHERE status = ALIVE
            
            if netwq.startswith('SIMPLE+METADATA'):
                q = d['q'][len('SIMPLE+METADATA '):]
                sendtorrents = True
            else:
                q = d['q'][len('SIMPLE '):]
                    
            uq = self.clean_netwq(q)
            kws = split_into_keywords(uq)
            hits = self.search_torrents(kws, maxhits=MAX_RESULTS,sendtorrents=sendtorrents)
            p = self.create_remote_query_reply(d['id'],hits,selversion)
            
        elif netwq.startswith("CHANNEL"): # channel query
            if DEBUG:
                print>>sys.stderr, "Incoming channel query", d['q']
            q = d['q'][len('CHANNEL '):]
            uq = self.clean_netwq(q,channelquery=True)
            hits = self.channelcast_db.searchChannels(uq)
            p = self.create_channel_query_reply(d['id'],hits,selversion)

        # log incoming query, if logfile is set
        if self.logfile:
            self.log(permid, q)        
     
        m = QUERY_REPLY+p

        if self.overlay_log:
            nqueries = self.get_peer_nqueries(permid)
            # RECV_MSG PERMID OVERSION NUM_QUERIES MSG
            self.overlay_log('RECV_QRY', show_permid(permid), selversion, nqueries, repr(d))

            # RPLY_QRY PERMID NUM_HITS MSG
            self.overlay_log('RPLY_QRY', show_permid(permid), len(hits), repr(p))

        self.overlay_bridge.send(permid, m, self.send_callback)
        
        self.inc_peer_nqueries(permid)