def isValidRemoteVal(d, selversion):
    if not isinstance(d, dict):
        if DEBUG:
            print >> sys.stderr, 'rqmh: reply: a: value not dict'
        return False
    if selversion >= OLPROTO_VER_TWELFTH:
        if not ('content_name' in d and 'length' in d and 'leecher' in d
                and 'seeder' in d and 'category' in d and 'torrent_size' in d
                and 'channel_permid' in d and 'channel_name' in d):
            if DEBUG:
                print >> sys.stderr, 'rqmh: reply: torrentrec12: key missing, got', d.keys(
                )
            return False
        if 'metatype' in d and 'metadata' in d:
            try:
                metatype = d['metatype']
                metadata = d['metadata']
                if metatype == URL_MIME_TYPE:
                    tdef = TorrentDef.load_from_url(metadata)
                else:
                    metainfo = bdecode(metadata)
                    tdef = TorrentDef.load_from_dict(metainfo)
            except:
                if DEBUG:
                    print >> sys.stderr, 'rqmh: reply: torrentrec12: metadata invalid'
                    print_exc()
                return False

    elif selversion >= OLPROTO_VER_ELEVENTH:
        if not ('content_name' in d and 'length' in d and 'leecher' in d
                and 'seeder' in d and 'category' in d and 'torrent_size' in d
                and 'channel_permid' in d and 'channel_name' in d):
            if DEBUG:
                print >> sys.stderr, 'rqmh: reply: torrentrec11: key missing, got', d.keys(
                )
            return False
    elif selversion >= OLPROTO_VER_NINETH:
        if not ('content_name' in d and 'length' in d and 'leecher' in d
                and 'seeder' in d and 'category' in d and 'torrent_size' in d):
            if DEBUG:
                print >> sys.stderr, 'rqmh: reply: torrentrec9: key missing, got', d.keys(
                )
            return False
    elif not ('content_name' in d and 'length' in d and 'leecher' in d
              and 'seeder' in d and 'category' in d):
        if DEBUG:
            print >> sys.stderr, 'rqmh: reply: torrentrec6: key missing, got', d.keys(
            )
        return False
    return True
    def recv_query_reply(self, permid, message, selversion):
        if selversion < OLPROTO_VER_SIXTH:
            return False
        try:
            d = bdecode(message[1:])
        except:
            if DEBUG:
                print >> sys.stderr, 'rquery: Cannot bdecode QUERY_REPLY message', selversion
            return False

        if not isValidQueryReply(d, selversion):
            if DEBUG:
                print >> sys.stderr, 'rquery: not valid QUERY_REPLY message', selversion
            return False
        queryrec = self.is_registered_query_id(d['id'])
        if not queryrec:
            if DEBUG:
                print >> sys.stderr, 'rquery: QUERY_REPLY has unknown query ID', selversion
            return False
        if selversion >= OLPROTO_VER_TWELFTH:
            if queryrec['query'].startswith('SIMPLE+METADATA'):
                for infohash, torrentrec in d['a'].iteritems():
                    if 'metatype' not in torrentrec:
                        if DEBUG:
                            print >> sys.stderr, 'rquery: QUERY_REPLY has no metatype field', selversion
                        return False
                    if 'metadata' not in torrentrec:
                        if DEBUG:
                            print >> sys.stderr, 'rquery: QUERY_REPLY has no metadata field', selversion
                        return False
                    if torrentrec['torrent_size'] != len(
                            torrentrec['metadata']):
                        if DEBUG:
                            print >> sys.stderr, 'rquery: QUERY_REPLY torrent_size != len metadata', selversion
                        return False
                    try:
                        if torrentrec['metatype'] == URL_MIME_TYPE:
                            tdef = TorrentDef.load_from_url(
                                torrentrec['metadata'])
                        else:
                            metainfo = bdecode(torrentrec['metadata'])
                            tdef = TorrentDef.load_from_dict(metainfo)
                    except:
                        if DEBUG:
                            print_exc()
                        return False

        self.process_query_reply(permid, queryrec['query'],
                                 queryrec['usercallback'], d)
        return True
    def search_torrents(self, kws, maxhits=None, sendtorrents=False):
        if DEBUG:
            print >> sys.stderr, 'rquery: search for torrents matching', ` kws `
        allhits = self.torrent_db.searchNames(kws, local=False)
        if maxhits is None:
            hits = allhits
        else:
            hits = allhits[:maxhits]
        colltorrdir = self.session.get_torrent_collecting_dir()
        if sendtorrents:
            print >> sys.stderr, 'rqmh: search_torrents: adding torrents'
            for hit in hits:
                filename = os.path.join(colltorrdir, hit['torrent_file_name'])
                try:
                    tdef = TorrentDef.load(filename)
                    if tdef.get_url_compat():
                        metatype = URL_MIME_TYPE
                        metadata = tdef.get_url()
                    else:
                        metatype = TSTREAM_MIME_TYPE
                        metadata = bencode(tdef.get_metainfo())
                except:
                    print_exc()
                    metadata = None

                hit['metatype'] = metatype
                hit['metadata'] = metadata

            newhits = []
            for hit in hits:
                if hit['metadata'] is not None:
                    newhits.append(hit)

            hits = newhits
        return hits
    def search_torrents(self, kws, maxhits = None, sendtorrents = False):
        if DEBUG:
            print >> sys.stderr, 'rquery: search for torrents matching', `kws`
        allhits = self.torrent_db.searchNames(kws, local=False)
        if maxhits is None:
            hits = allhits
        else:
            hits = allhits[:maxhits]
        colltorrdir = self.session.get_torrent_collecting_dir()
        if sendtorrents:
            print >> sys.stderr, 'rqmh: search_torrents: adding torrents'
            for hit in hits:
                filename = os.path.join(colltorrdir, hit['torrent_file_name'])
                try:
                    tdef = TorrentDef.load(filename)
                    if tdef.get_url_compat():
                        metatype = URL_MIME_TYPE
                        metadata = tdef.get_url()
                    else:
                        metatype = TSTREAM_MIME_TYPE
                        metadata = bencode(tdef.get_metainfo())
                except:
                    print_exc()
                    metadata = None

                hit['metatype'] = metatype
                hit['metadata'] = metadata

            newhits = []
            for hit in hits:
                if hit['metadata'] is not None:
                    newhits.append(hit)

            hits = newhits
        return hits
 def addTorrentToDB(self,
                    filename,
                    torrent_hash,
                    metadata,
                    source='BC',
                    extra_info={},
                    hack=False):
     torrentdef = TorrentDef.load(filename)
     if 'filename' not in extra_info:
         extra_info['filename'] = filename
     torrent = self.torrent_db.addExternalTorrent(torrentdef, source,
                                                  extra_info)
     if torrent is None:
         return
     self.launchmany.set_activity(NTFY_ACT_GOT_METADATA,
                                  unicode('"' + torrent['name'] + '"'),
                                  torrent['category'])
     if self.initialized:
         self.num_torrents += 1
         if not extra_info:
             self.refreshTrackerStatus(torrent)
         if len(self.recently_collected_torrents) < 50:
             self.recently_collected_torrents.append(torrent_hash)
         else:
             self.recently_collected_torrents.pop(0)
             self.recently_collected_torrents.append(torrent_hash)
    def recv_query_reply(self, permid, message, selversion):
        if selversion < OLPROTO_VER_SIXTH:
            return False
        try:
            d = bdecode(message[1:])
        except:
            if DEBUG:
                print >> sys.stderr, 'rquery: Cannot bdecode QUERY_REPLY message', selversion
            return False

        if not isValidQueryReply(d, selversion):
            if DEBUG:
                print >> sys.stderr, 'rquery: not valid QUERY_REPLY message', selversion
            return False
        queryrec = self.is_registered_query_id(d['id'])
        if not queryrec:
            if DEBUG:
                print >> sys.stderr, 'rquery: QUERY_REPLY has unknown query ID', selversion
            return False
        if selversion >= OLPROTO_VER_TWELFTH:
            if queryrec['query'].startswith('SIMPLE+METADATA'):
                for infohash, torrentrec in d['a'].iteritems():
                    if 'metatype' not in torrentrec:
                        if DEBUG:
                            print >> sys.stderr, 'rquery: QUERY_REPLY has no metatype field', selversion
                        return False
                    if 'metadata' not in torrentrec:
                        if DEBUG:
                            print >> sys.stderr, 'rquery: QUERY_REPLY has no metadata field', selversion
                        return False
                    if torrentrec['torrent_size'] != len(torrentrec['metadata']):
                        if DEBUG:
                            print >> sys.stderr, 'rquery: QUERY_REPLY torrent_size != len metadata', selversion
                        return False
                    try:
                        if torrentrec['metatype'] == URL_MIME_TYPE:
                            tdef = TorrentDef.load_from_url(torrentrec['metadata'])
                        else:
                            metainfo = bdecode(torrentrec['metadata'])
                            tdef = TorrentDef.load_from_dict(metainfo)
                    except:
                        if DEBUG:
                            print_exc()
                        return False

        self.process_query_reply(permid, queryrec['query'], queryrec['usercallback'], d)
        return True
 def valid_metadata(self, infohash, metadata):
     try:
         metainfo = bdecode(metadata)
         tdef = TorrentDef.load_from_dict(metainfo)
         got_infohash = tdef.get_infohash()
         if infohash != got_infohash:
             print >> sys.stderr, "metadata: infohash doesn't match the torrent " + 'hash. Required: ' + ` infohash ` + ', but got: ' + ` got_infohash `
             return False
         return True
     except:
         print_exc()
         return False
 def valid_metadata(self, infohash, metadata):
     try:
         metainfo = bdecode(metadata)
         tdef = TorrentDef.load_from_dict(metainfo)
         got_infohash = tdef.get_infohash()
         if infohash != got_infohash:
             print >> sys.stderr, "metadata: infohash doesn't match the torrent " + 'hash. Required: ' + `infohash` + ', but got: ' + `got_infohash`
             return False
         return True
     except:
         print_exc()
         return False
def isValidRemoteVal(d, selversion):
    if not isinstance(d, dict):
        if DEBUG:
            print >> sys.stderr, 'rqmh: reply: a: value not dict'
        return False
    if selversion >= OLPROTO_VER_TWELFTH:
        if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d and 'channel_permid' in d and 'channel_name' in d):
            if DEBUG:
                print >> sys.stderr, 'rqmh: reply: torrentrec12: key missing, got', d.keys()
            return False
        if 'metatype' in d and 'metadata' in d:
            try:
                metatype = d['metatype']
                metadata = d['metadata']
                if metatype == URL_MIME_TYPE:
                    tdef = TorrentDef.load_from_url(metadata)
                else:
                    metainfo = bdecode(metadata)
                    tdef = TorrentDef.load_from_dict(metainfo)
            except:
                if DEBUG:
                    print >> sys.stderr, 'rqmh: reply: torrentrec12: metadata invalid'
                    print_exc()
                return False

    elif selversion >= OLPROTO_VER_ELEVENTH:
        if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d and 'channel_permid' in d and 'channel_name' in d):
            if DEBUG:
                print >> sys.stderr, 'rqmh: reply: torrentrec11: key missing, got', d.keys()
            return False
    elif selversion >= OLPROTO_VER_NINETH:
        if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d):
            if DEBUG:
                print >> sys.stderr, 'rqmh: reply: torrentrec9: key missing, got', d.keys()
            return False
    elif not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d):
        if DEBUG:
            print >> sys.stderr, 'rqmh: reply: torrentrec6: key missing, got', d.keys()
        return False
    return True
    def read_and_send_metadata(self, permid, infohash, torrent_path,
                               selversion):
        torrent_data = self.read_torrent(torrent_path)
        if torrent_data:
            try:
                metainfo = bdecode(torrent_data)
                if 'info' in metainfo and 'private' in metainfo[
                        'info'] and metainfo['info']['private']:
                    if DEBUG:
                        print >> sys.stderr, 'metadata: Not sending torrent', ` torrent_path `, 'because it is private'
                    return 0
            except:
                print_exc()
                return 0

            if DEBUG:
                print >> sys.stderr, 'metadata: sending torrent', ` torrent_path `, len(
                    torrent_data)
            torrent = {}
            torrent['torrent_hash'] = infohash
            tdef = TorrentDef.load_from_dict(metainfo)
            if selversion >= OLPROTO_VER_ELEVENTH and tdef.get_url_compat():
                torrent['metatype'] = URL_MIME_TYPE
                torrent['metadata'] = tdef.get_url()
            else:
                torrent['metatype'] = TSTREAM_MIME_TYPE
                torrent['metadata'] = torrent_data
            if selversion >= OLPROTO_VER_FOURTH:
                data = self.torrent_db.getTorrent(infohash)
                if data is None:
                    return 0
                nleechers = data.get('leecher', -1)
                nseeders = data.get('seeder', -1)
                last_check_ago = int(time()) - data.get('last_check_time', 0)
                if last_check_ago < 0:
                    last_check_ago = 0
                status = data.get('status', 'unknown')
                torrent.update({
                    'leecher': nleechers,
                    'seeder': nseeders,
                    'last_check_time': last_check_ago,
                    'status': status
                })
            return self.do_send_metadata(permid, torrent, selversion)
        else:
            self.torrent_db.deleteTorrent(infohash,
                                          delete_file=True,
                                          commit=True)
            if DEBUG:
                print >> sys.stderr, 'metadata: GET_METADATA: no torrent data to send'
            return 0
    def __requestMagnet(self):
        try:
            if len(self.requestedInfohashes) < self.MAX_CONCURRENT:
                while True:
                    if len(self.list) == 0:
                        return
                    prio, infohash = self.list.pop(0)
                    if infohash in self.requestedInfohashes:
                        if DEBUG:
                            print >> sys.stderr, 'magnetrequester: magnet already requested', bin2str(infohash)
                        continue
                    torrent = self.torrent_db.getTorrent(infohash, ['torrent_file_name'], include_mypref=False)
                    if torrent.get('torrent_file_name', False):
                        torrent_filename = os.path.join(self.metadatahandler.torrent_dir, torrent['torrent_file_name'])
                    else:
                        torrent_filename = None
                    torrent_alt_filename = os.path.join(self.metadatahandler.torrent_dir, get_collected_torrent_filename(infohash))
                    if os.path.isfile(torrent_filename) or os.path.isfile(torrent_alt_filename):
                        if DEBUG:
                            print >> sys.stderr, 'magnetrequester: magnet already on disk', bin2str(infohash)
                    else:
                        break

            else:
                return
        except:
            print_exc()

        magnetlink = 'magnet:?xt=urn:btih:' + hexlify(infohash)
        if DEBUG:
            print >> sys.stderr, 'magnetrequester: requesting magnet', bin2str(infohash), prio, magnetlink
        self.requestedInfohashes.add(infohash)
        TorrentDef.retrieve_from_magnet(magnetlink, self.__torrentdef_retrieved, self.MAGNET_RETRIEVE_TIMEOUT)
        self.overlay_bridge.add_task(lambda : self.__torrentdef_failed(infohash), self.MAGNET_RETRIEVE_TIMEOUT, infohash)
        if len(self.requestedInfohashes) < self.MAX_CONCURRENT:
            self.overlay_bridge.add_task(self.__requestMagnet, self.REQUEST_INTERVAL)
 def addTorrentToDB(self, filename, torrent_hash, metadata, source = 'BC', extra_info = {}, hack = False):
     torrentdef = TorrentDef.load(filename)
     if 'filename' not in extra_info:
         extra_info['filename'] = filename
     torrent = self.torrent_db.addExternalTorrent(torrentdef, source, extra_info)
     if torrent is None:
         return
     self.launchmany.set_activity(NTFY_ACT_GOT_METADATA, unicode('"' + torrent['name'] + '"'), torrent['category'])
     if self.initialized:
         self.num_torrents += 1
         if not extra_info:
             self.refreshTrackerStatus(torrent)
         if len(self.recently_collected_torrents) < 50:
             self.recently_collected_torrents.append(torrent_hash)
         else:
             self.recently_collected_torrents.pop(0)
             self.recently_collected_torrents.append(torrent_hash)
    def read_and_send_metadata(self, permid, infohash, torrent_path, selversion):
        torrent_data = self.read_torrent(torrent_path)
        if torrent_data:
            try:
                metainfo = bdecode(torrent_data)
                if 'info' in metainfo and 'private' in metainfo['info'] and metainfo['info']['private']:
                    if DEBUG:
                        print >> sys.stderr, 'metadata: Not sending torrent', `torrent_path`, 'because it is private'
                    return 0
            except:
                print_exc()
                return 0

            if DEBUG:
                print >> sys.stderr, 'metadata: sending torrent', `torrent_path`, len(torrent_data)
            torrent = {}
            torrent['torrent_hash'] = infohash
            tdef = TorrentDef.load_from_dict(metainfo)
            if selversion >= OLPROTO_VER_ELEVENTH and tdef.get_url_compat():
                torrent['metatype'] = URL_MIME_TYPE
                torrent['metadata'] = tdef.get_url()
            else:
                torrent['metatype'] = TSTREAM_MIME_TYPE
                torrent['metadata'] = torrent_data
            if selversion >= OLPROTO_VER_FOURTH:
                data = self.torrent_db.getTorrent(infohash)
                if data is None:
                    return 0
                nleechers = data.get('leecher', -1)
                nseeders = data.get('seeder', -1)
                last_check_ago = int(time()) - data.get('last_check_time', 0)
                if last_check_ago < 0:
                    last_check_ago = 0
                status = data.get('status', 'unknown')
                torrent.update({'leecher': nleechers,
                 'seeder': nseeders,
                 'last_check_time': last_check_ago,
                 'status': status})
            return self.do_send_metadata(permid, torrent, selversion)
        else:
            self.torrent_db.deleteTorrent(infohash, delete_file=True, commit=True)
            if DEBUG:
                print >> sys.stderr, 'metadata: GET_METADATA: no torrent data to send'
            return 0
Пример #14
0
    def resume_download(self, dltype, filename, initialdlstatus = None):
        try:
            pstate = self.load_download_pstate(filename)
            if DEBUG:
                log('lm::resume_download: dltype', dltype, 'filename', filename, 'dlconfig', pstate['dlconfig'])
            if DEBUG:
                log('lm::resume_download: status', dlstatus_strings[pstate['dlstate']['status']], 'progress', pstate['dlstate']['progress'])
                if pstate['engineresumedata'] is None:
                    log('lm::resume_download: resumedata None')
                else:
                    log('lm::resume_download: resumedata len', len(pstate['engineresumedata']))
            dscfg = DownloadStartupConfig(dlconfig=pstate['dlconfig'])
            if dltype == DLTYPE_TORRENT:
                tdef = TorrentDef.load_from_dict(pstate['metainfo'])
                d = self.add(tdef, dscfg, pstate, initialdlstatus)
            elif dltype == DLTYPE_DIRECT:
                main_url = pstate['url']
                d = self.add_direct_download(main_url, dscfg, pstate, initialdlstatus)
            if initialdlstatus == DLSTATUS_STOPPED:
                dest_files = d.get_dest_files(get_all=True)
                if DEBUG:
                    log('lm::resume_download: check dest files: dest_files', dest_files)
                got_existing_file = False
                for filename, savepath in dest_files:
                    if os.path.exists(savepath):
                        got_existing_file = True
                        break

                if not got_existing_file:
                    if DEBUG:
                        log('lm::resume_download: none of the files exists, remove this download')
                    self.remove(d, removecontent=True)
        except Exception as e:
            if DEBUG:
                log('lm::resume_download: failed to load checkpoint: filename', filename)
                print_exc()
            try:
                if os.access(filename, os.F_OK):
                    os.remove(filename)
            except:
                print_exc()
    def got_metadata(self, permid, message, selversion):
        try:
            message = bdecode(message[1:])
        except:
            print_exc()
            return False

        if not isinstance(message, dict):
            return False
        try:
            infohash = message['torrent_hash']
            if not isValidInfohash(infohash):
                return False
            if infohash not in self.requested_torrents:
                return True
            if self.torrent_db.hasMetaData(infohash):
                return True
            goturl = False
            if selversion >= OLPROTO_VER_ELEVENTH:
                if 'metatype' in message and message['metatype'] == URL_MIME_TYPE:
                    try:
                        tdef = TorrentDef.load_from_url(message['metadata'])
                        metainfo = tdef.get_metainfo()
                        metadata = bencode(metainfo)
                        goturl = True
                    except:
                        print_exc()
                        return False

                else:
                    metadata = message['metadata']
            else:
                metadata = message['metadata']
            if not self.valid_metadata(infohash, metadata):
                return False
            if DEBUG:
                torrent_size = len(metadata)
                if goturl:
                    mdt = 'URL'
                else:
                    mdt = 'torrent'
                print >> sys.stderr, 'metadata: Recvd', mdt, `infohash`, sha(infohash).hexdigest(), torrent_size
            extra_info = {}
            if selversion >= OLPROTO_VER_FOURTH:
                try:
                    extra_info = {'leecher': message.get('leecher', -1),
                     'seeder': message.get('seeder', -1),
                     'last_check_time': message.get('last_check_time', -1),
                     'status': message.get('status', 'unknown')}
                except Exception as msg:
                    print_exc()
                    print >> sys.stderr, 'metadata: wrong extra info in msg - ', message
                    extra_info = {}

            filename = self.save_torrent(infohash, metadata, extra_info=extra_info)
            self.requested_torrents.remove(infohash)
            if filename is not None:
                self.notify_torrent_is_in(infohash, metadata, filename)
            if permid is not None and BARTERCAST_TORRENTS:
                self.overlay_bridge.add_task(lambda : self.olthread_bartercast_torrentexchange(permid, 'downloaded'), 0)
        except Exception as e:
            print_exc()
            print >> sys.stderr, 'metadata: Received metadata is broken', e, message.keys()
            return False

        return True
Пример #16
0
def parsedir(
    directory, parsed, files, blocked, exts=[".torrent", TRIBLER_TORRENT_EXT], return_metainfo=False, errfunc=_errfunc
):
    if DEBUG:
        errfunc("checking dir")
    dirs_to_check = [directory]
    new_files = {}
    new_blocked = {}
    torrent_type = {}
    while dirs_to_check:
        directory = dirs_to_check.pop()
        newtorrents = False
        for f in os.listdir(directory):
            newtorrent = None
            for ext in exts:
                if f.endswith(ext):
                    newtorrent = ext[1:]
                    break

            if newtorrent:
                newtorrents = True
                p = os.path.join(directory, f)
                new_files[p] = [(int(os.path.getmtime(p)), os.path.getsize(p)), 0]
                torrent_type[p] = newtorrent

        if not newtorrents:
            for f in os.listdir(directory):
                p = os.path.join(directory, f)
                if os.path.isdir(p):
                    dirs_to_check.append(p)

    new_parsed = {}
    to_add = []
    added = {}
    removed = {}
    for p, v in new_files.items():
        oldval = files.get(p)
        if not oldval:
            to_add.append(p)
            continue
        h = oldval[1]
        if oldval[0] == v[0]:
            if h:
                if blocked.has_key(p):
                    to_add.append(p)
                else:
                    new_parsed[h] = parsed[h]
                new_files[p] = oldval
            else:
                new_blocked[p] = 1
            continue
        if parsed.has_key(h) and not blocked.has_key(p):
            if DEBUG:
                errfunc("removing " + p + " (will re-add)")
            removed[h] = parsed[h]
        to_add.append(p)

    to_add.sort()
    for p in to_add:
        new_file = new_files[p]
        v, h = new_file
        if new_parsed.has_key(h):
            if not blocked.has_key(p) or files[p][0] != v:
                errfunc("**warning** " + p + " is a duplicate torrent for " + new_parsed[h]["path"])
            new_blocked[p] = 1
            continue
        if DEBUG:
            errfunc("adding " + p)
        try:
            tdef = TorrentDef.load(p)
            h = tdef.get_infohash()
            d = tdef.get_metainfo()
            new_file[1] = h
            if new_parsed.has_key(h):
                errfunc("**warning** " + p + " is a duplicate torrent for " + new_parsed[h]["path"])
                new_blocked[p] = 1
                continue
            a = {}
            a["path"] = p
            f = os.path.basename(p)
            a["file"] = f
            a["type"] = torrent_type[p]
            if tdef.get_url_compat():
                a["url"] = tdef.get_url()
            i = d["info"]
            l = 0
            nf = 0
            if i.has_key("length"):
                l = i.get("length", 0)
                nf = 1
            elif i.has_key("files"):
                for li in i["files"]:
                    nf += 1
                    if li.has_key("length"):
                        l += li["length"]

            a["numfiles"] = nf
            a["length"] = l
            a["name"] = i.get("name", f)

            def setkey(k, d=d, a=a):
                if d.has_key(k):
                    a[k] = d[k]

            setkey("failure reason")
            setkey("warning message")
            setkey("announce-list")
            if tdef.get_urllist() is not None:
                httpseedhashes = []
                for url in tdef.get_urllist():
                    urlhash = sha(url).digest()
                    httpseedhashes.append(urlhash)

                a["url-hash-list"] = httpseedhashes
            if return_metainfo:
                a["metainfo"] = d
        except:
            print_exc()
            errfunc("**warning** " + p + " has errors")
            new_blocked[p] = 1
            continue

        if DEBUG:
            errfunc("... successful")
        new_parsed[h] = a
        added[h] = a

    for p, v in files.items():
        if not new_files.has_key(p) and not blocked.has_key(p):
            if DEBUG:
                errfunc("removing " + p)
            removed[v[1]] = parsed[v[1]]

    if DEBUG:
        errfunc("done checking")
    return (new_parsed, new_files, new_blocked, added, removed)
    def got_metadata(self, permid, message, selversion):
        try:
            message = bdecode(message[1:])
        except:
            print_exc()
            return False

        if not isinstance(message, dict):
            return False
        try:
            infohash = message['torrent_hash']
            if not isValidInfohash(infohash):
                return False
            if infohash not in self.requested_torrents:
                return True
            if self.torrent_db.hasMetaData(infohash):
                return True
            goturl = False
            if selversion >= OLPROTO_VER_ELEVENTH:
                if 'metatype' in message and message[
                        'metatype'] == URL_MIME_TYPE:
                    try:
                        tdef = TorrentDef.load_from_url(message['metadata'])
                        metainfo = tdef.get_metainfo()
                        metadata = bencode(metainfo)
                        goturl = True
                    except:
                        print_exc()
                        return False

                else:
                    metadata = message['metadata']
            else:
                metadata = message['metadata']
            if not self.valid_metadata(infohash, metadata):
                return False
            if DEBUG:
                torrent_size = len(metadata)
                if goturl:
                    mdt = 'URL'
                else:
                    mdt = 'torrent'
                print >> sys.stderr, 'metadata: Recvd', mdt, ` infohash `, sha(
                    infohash).hexdigest(), torrent_size
            extra_info = {}
            if selversion >= OLPROTO_VER_FOURTH:
                try:
                    extra_info = {
                        'leecher': message.get('leecher', -1),
                        'seeder': message.get('seeder', -1),
                        'last_check_time': message.get('last_check_time', -1),
                        'status': message.get('status', 'unknown')
                    }
                except Exception as msg:
                    print_exc()
                    print >> sys.stderr, 'metadata: wrong extra info in msg - ', message
                    extra_info = {}

            filename = self.save_torrent(infohash,
                                         metadata,
                                         extra_info=extra_info)
            self.requested_torrents.remove(infohash)
            if filename is not None:
                self.notify_torrent_is_in(infohash, metadata, filename)
            if permid is not None and BARTERCAST_TORRENTS:
                self.overlay_bridge.add_task(
                    lambda: self.olthread_bartercast_torrentexchange(
                        permid, 'downloaded'), 0)
        except Exception as e:
            print_exc()
            print >> sys.stderr, 'metadata: Received metadata is broken', e, message.keys(
            )
            return False

        return True
Пример #18
0
def parsedir(directory,
             parsed,
             files,
             blocked,
             exts=['.torrent', TRIBLER_TORRENT_EXT],
             return_metainfo=False,
             errfunc=_errfunc):
    if DEBUG:
        errfunc('checking dir')
    dirs_to_check = [directory]
    new_files = {}
    new_blocked = {}
    torrent_type = {}
    while dirs_to_check:
        directory = dirs_to_check.pop()
        newtorrents = False
        for f in os.listdir(directory):
            newtorrent = None
            for ext in exts:
                if f.endswith(ext):
                    newtorrent = ext[1:]
                    break

            if newtorrent:
                newtorrents = True
                p = os.path.join(directory, f)
                new_files[p] = [(int(os.path.getmtime(p)), os.path.getsize(p)),
                                0]
                torrent_type[p] = newtorrent

        if not newtorrents:
            for f in os.listdir(directory):
                p = os.path.join(directory, f)
                if os.path.isdir(p):
                    dirs_to_check.append(p)

    new_parsed = {}
    to_add = []
    added = {}
    removed = {}
    for p, v in new_files.items():
        oldval = files.get(p)
        if not oldval:
            to_add.append(p)
            continue
        h = oldval[1]
        if oldval[0] == v[0]:
            if h:
                if blocked.has_key(p):
                    to_add.append(p)
                else:
                    new_parsed[h] = parsed[h]
                new_files[p] = oldval
            else:
                new_blocked[p] = 1
            continue
        if parsed.has_key(h) and not blocked.has_key(p):
            if DEBUG:
                errfunc('removing ' + p + ' (will re-add)')
            removed[h] = parsed[h]
        to_add.append(p)

    to_add.sort()
    for p in to_add:
        new_file = new_files[p]
        v, h = new_file
        if new_parsed.has_key(h):
            if not blocked.has_key(p) or files[p][0] != v:
                errfunc('**warning** ' + p + ' is a duplicate torrent for ' +
                        new_parsed[h]['path'])
            new_blocked[p] = 1
            continue
        if DEBUG:
            errfunc('adding ' + p)
        try:
            tdef = TorrentDef.load(p)
            h = tdef.get_infohash()
            d = tdef.get_metainfo()
            new_file[1] = h
            if new_parsed.has_key(h):
                errfunc('**warning** ' + p + ' is a duplicate torrent for ' +
                        new_parsed[h]['path'])
                new_blocked[p] = 1
                continue
            a = {}
            a['path'] = p
            f = os.path.basename(p)
            a['file'] = f
            a['type'] = torrent_type[p]
            if tdef.get_url_compat():
                a['url'] = tdef.get_url()
            i = d['info']
            l = 0
            nf = 0
            if i.has_key('length'):
                l = i.get('length', 0)
                nf = 1
            elif i.has_key('files'):
                for li in i['files']:
                    nf += 1
                    if li.has_key('length'):
                        l += li['length']

            a['numfiles'] = nf
            a['length'] = l
            a['name'] = i.get('name', f)

            def setkey(k, d=d, a=a):
                if d.has_key(k):
                    a[k] = d[k]

            setkey('failure reason')
            setkey('warning message')
            setkey('announce-list')
            if tdef.get_urllist() is not None:
                httpseedhashes = []
                for url in tdef.get_urllist():
                    urlhash = sha(url).digest()
                    httpseedhashes.append(urlhash)

                a['url-hash-list'] = httpseedhashes
            if return_metainfo:
                a['metainfo'] = d
        except:
            print_exc()
            errfunc('**warning** ' + p + ' has errors')
            new_blocked[p] = 1
            continue

        if DEBUG:
            errfunc('... successful')
        new_parsed[h] = a
        added[h] = a

    for p, v in files.items():
        if not new_files.has_key(p) and not blocked.has_key(p):
            if DEBUG:
                errfunc('removing ' + p)
            removed[v[1]] = parsed[v[1]]

    if DEBUG:
        errfunc('done checking')
    return (new_parsed, new_files, new_blocked, added, removed)