def isValidRemoteVal(d, selversion):
if not isinstance(d, dict):
if DEBUG:
print >> sys.stderr, 'rqmh: reply: a: value not dict'
return False
if selversion >= OLPROTO_VER_TWELFTH:
if not ('content_name' in d and 'length' in d and 'leecher' in d
and 'seeder' in d and 'category' in d and 'torrent_size' in d
and 'channel_permid' in d and 'channel_name' in d):
if DEBUG:
print >> sys.stderr, 'rqmh: reply: torrentrec12: key missing, got', d.keys(
)
return False
if 'metatype' in d and 'metadata' in d:
try:
metatype = d['metatype']
metadata = d['metadata']
if metatype == URL_MIME_TYPE:
tdef = TorrentDef.load_from_url(metadata)
else:
metainfo = bdecode(metadata)
tdef = TorrentDef.load_from_dict(metainfo)
except:
if DEBUG:
print >> sys.stderr, 'rqmh: reply: torrentrec12: metadata invalid'
print_exc()
return False
elif selversion >= OLPROTO_VER_ELEVENTH:
if not ('content_name' in d and 'length' in d and 'leecher' in d
and 'seeder' in d and 'category' in d and 'torrent_size' in d
and 'channel_permid' in d and 'channel_name' in d):
if DEBUG:
print >> sys.stderr, 'rqmh: reply: torrentrec11: key missing, got', d.keys(
)
return False
elif selversion >= OLPROTO_VER_NINETH:
if not ('content_name' in d and 'length' in d and 'leecher' in d
and 'seeder' in d and 'category' in d and 'torrent_size' in d):
if DEBUG:
print >> sys.stderr, 'rqmh: reply: torrentrec9: key missing, got', d.keys(
)
return False
elif not ('content_name' in d and 'length' in d and 'leecher' in d
and 'seeder' in d and 'category' in d):
if DEBUG:
print >> sys.stderr, 'rqmh: reply: torrentrec6: key missing, got', d.keys(
)
return False
return True
def recv_query_reply(self, permid, message, selversion):
if selversion < OLPROTO_VER_SIXTH:
return False
try:
d = bdecode(message[1:])
except:
if DEBUG:
print >> sys.stderr, 'rquery: Cannot bdecode QUERY_REPLY message', selversion
return False
if not isValidQueryReply(d, selversion):
if DEBUG:
print >> sys.stderr, 'rquery: not valid QUERY_REPLY message', selversion
return False
queryrec = self.is_registered_query_id(d['id'])
if not queryrec:
if DEBUG:
print >> sys.stderr, 'rquery: QUERY_REPLY has unknown query ID', selversion
return False
if selversion >= OLPROTO_VER_TWELFTH:
if queryrec['query'].startswith('SIMPLE+METADATA'):
for infohash, torrentrec in d['a'].iteritems():
if 'metatype' not in torrentrec:
if DEBUG:
print >> sys.stderr, 'rquery: QUERY_REPLY has no metatype field', selversion
return False
if 'metadata' not in torrentrec:
if DEBUG:
print >> sys.stderr, 'rquery: QUERY_REPLY has no metadata field', selversion
return False
if torrentrec['torrent_size'] != len(
torrentrec['metadata']):
if DEBUG:
print >> sys.stderr, 'rquery: QUERY_REPLY torrent_size != len metadata', selversion
return False
try:
if torrentrec['metatype'] == URL_MIME_TYPE:
tdef = TorrentDef.load_from_url(
torrentrec['metadata'])
else:
metainfo = bdecode(torrentrec['metadata'])
tdef = TorrentDef.load_from_dict(metainfo)
except:
if DEBUG:
print_exc()
return False
self.process_query_reply(permid, queryrec['query'],
queryrec['usercallback'], d)
return True
def search_torrents(self, kws, maxhits=None, sendtorrents=False):
if DEBUG:
print >> sys.stderr, 'rquery: search for torrents matching', ` kws `
allhits = self.torrent_db.searchNames(kws, local=False)
if maxhits is None:
hits = allhits
else:
hits = allhits[:maxhits]
colltorrdir = self.session.get_torrent_collecting_dir()
if sendtorrents:
print >> sys.stderr, 'rqmh: search_torrents: adding torrents'
for hit in hits:
filename = os.path.join(colltorrdir, hit['torrent_file_name'])
try:
tdef = TorrentDef.load(filename)
if tdef.get_url_compat():
metatype = URL_MIME_TYPE
metadata = tdef.get_url()
else:
metatype = TSTREAM_MIME_TYPE
metadata = bencode(tdef.get_metainfo())
except:
print_exc()
metadata = None
hit['metatype'] = metatype
hit['metadata'] = metadata
newhits = []
for hit in hits:
if hit['metadata'] is not None:
newhits.append(hit)
hits = newhits
return hits
def search_torrents(self, kws, maxhits = None, sendtorrents = False):
if DEBUG:
print >> sys.stderr, 'rquery: search for torrents matching', `kws`
allhits = self.torrent_db.searchNames(kws, local=False)
if maxhits is None:
hits = allhits
else:
hits = allhits[:maxhits]
colltorrdir = self.session.get_torrent_collecting_dir()
if sendtorrents:
print >> sys.stderr, 'rqmh: search_torrents: adding torrents'
for hit in hits:
filename = os.path.join(colltorrdir, hit['torrent_file_name'])
try:
tdef = TorrentDef.load(filename)
if tdef.get_url_compat():
metatype = URL_MIME_TYPE
metadata = tdef.get_url()
else:
metatype = TSTREAM_MIME_TYPE
metadata = bencode(tdef.get_metainfo())
except:
print_exc()
metadata = None
hit['metatype'] = metatype
hit['metadata'] = metadata
newhits = []
for hit in hits:
if hit['metadata'] is not None:
newhits.append(hit)
hits = newhits
return hits
def addTorrentToDB(self,
filename,
torrent_hash,
metadata,
source='BC',
extra_info={},
hack=False):
torrentdef = TorrentDef.load(filename)
if 'filename' not in extra_info:
extra_info['filename'] = filename
torrent = self.torrent_db.addExternalTorrent(torrentdef, source,
extra_info)
if torrent is None:
return
self.launchmany.set_activity(NTFY_ACT_GOT_METADATA,
unicode('"' + torrent['name'] + '"'),
torrent['category'])
if self.initialized:
self.num_torrents += 1
if not extra_info:
self.refreshTrackerStatus(torrent)
if len(self.recently_collected_torrents) < 50:
self.recently_collected_torrents.append(torrent_hash)
else:
self.recently_collected_torrents.pop(0)
self.recently_collected_torrents.append(torrent_hash)
def recv_query_reply(self, permid, message, selversion):
if selversion < OLPROTO_VER_SIXTH:
return False
try:
d = bdecode(message[1:])
except:
if DEBUG:
print >> sys.stderr, 'rquery: Cannot bdecode QUERY_REPLY message', selversion
return False
if not isValidQueryReply(d, selversion):
if DEBUG:
print >> sys.stderr, 'rquery: not valid QUERY_REPLY message', selversion
return False
queryrec = self.is_registered_query_id(d['id'])
if not queryrec:
if DEBUG:
print >> sys.stderr, 'rquery: QUERY_REPLY has unknown query ID', selversion
return False
if selversion >= OLPROTO_VER_TWELFTH:
if queryrec['query'].startswith('SIMPLE+METADATA'):
for infohash, torrentrec in d['a'].iteritems():
if 'metatype' not in torrentrec:
if DEBUG:
print >> sys.stderr, 'rquery: QUERY_REPLY has no metatype field', selversion
return False
if 'metadata' not in torrentrec:
if DEBUG:
print >> sys.stderr, 'rquery: QUERY_REPLY has no metadata field', selversion
return False
if torrentrec['torrent_size'] != len(torrentrec['metadata']):
if DEBUG:
print >> sys.stderr, 'rquery: QUERY_REPLY torrent_size != len metadata', selversion
return False
try:
if torrentrec['metatype'] == URL_MIME_TYPE:
tdef = TorrentDef.load_from_url(torrentrec['metadata'])
else:
metainfo = bdecode(torrentrec['metadata'])
tdef = TorrentDef.load_from_dict(metainfo)
except:
if DEBUG:
print_exc()
return False
self.process_query_reply(permid, queryrec['query'], queryrec['usercallback'], d)
return True
def valid_metadata(self, infohash, metadata):
try:
metainfo = bdecode(metadata)
tdef = TorrentDef.load_from_dict(metainfo)
got_infohash = tdef.get_infohash()
if infohash != got_infohash:
print >> sys.stderr, "metadata: infohash doesn't match the torrent " + 'hash. Required: ' + ` infohash ` + ', but got: ' + ` got_infohash `
return False
return True
except:
print_exc()
return False
def valid_metadata(self, infohash, metadata):
try:
metainfo = bdecode(metadata)
tdef = TorrentDef.load_from_dict(metainfo)
got_infohash = tdef.get_infohash()
if infohash != got_infohash:
print >> sys.stderr, "metadata: infohash doesn't match the torrent " + 'hash. Required: ' + `infohash` + ', but got: ' + `got_infohash`
return False
return True
except:
print_exc()
return False
def isValidRemoteVal(d, selversion):
if not isinstance(d, dict):
if DEBUG:
print >> sys.stderr, 'rqmh: reply: a: value not dict'
return False
if selversion >= OLPROTO_VER_TWELFTH:
if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d and 'channel_permid' in d and 'channel_name' in d):
if DEBUG:
print >> sys.stderr, 'rqmh: reply: torrentrec12: key missing, got', d.keys()
return False
if 'metatype' in d and 'metadata' in d:
try:
metatype = d['metatype']
metadata = d['metadata']
if metatype == URL_MIME_TYPE:
tdef = TorrentDef.load_from_url(metadata)
else:
metainfo = bdecode(metadata)
tdef = TorrentDef.load_from_dict(metainfo)
except:
if DEBUG:
print >> sys.stderr, 'rqmh: reply: torrentrec12: metadata invalid'
print_exc()
return False
elif selversion >= OLPROTO_VER_ELEVENTH:
if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d and 'channel_permid' in d and 'channel_name' in d):
if DEBUG:
print >> sys.stderr, 'rqmh: reply: torrentrec11: key missing, got', d.keys()
return False
elif selversion >= OLPROTO_VER_NINETH:
if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d):
if DEBUG:
print >> sys.stderr, 'rqmh: reply: torrentrec9: key missing, got', d.keys()
return False
elif not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d):
if DEBUG:
print >> sys.stderr, 'rqmh: reply: torrentrec6: key missing, got', d.keys()
return False
return True
def read_and_send_metadata(self, permid, infohash, torrent_path,
selversion):
torrent_data = self.read_torrent(torrent_path)
if torrent_data:
try:
metainfo = bdecode(torrent_data)
if 'info' in metainfo and 'private' in metainfo[
'info'] and metainfo['info']['private']:
if DEBUG:
print >> sys.stderr, 'metadata: Not sending torrent', ` torrent_path `, 'because it is private'
return 0
except:
print_exc()
return 0
if DEBUG:
print >> sys.stderr, 'metadata: sending torrent', ` torrent_path `, len(
torrent_data)
torrent = {}
torrent['torrent_hash'] = infohash
tdef = TorrentDef.load_from_dict(metainfo)
if selversion >= OLPROTO_VER_ELEVENTH and tdef.get_url_compat():
torrent['metatype'] = URL_MIME_TYPE
torrent['metadata'] = tdef.get_url()
else:
torrent['metatype'] = TSTREAM_MIME_TYPE
torrent['metadata'] = torrent_data
if selversion >= OLPROTO_VER_FOURTH:
data = self.torrent_db.getTorrent(infohash)
if data is None:
return 0
nleechers = data.get('leecher', -1)
nseeders = data.get('seeder', -1)
last_check_ago = int(time()) - data.get('last_check_time', 0)
if last_check_ago < 0:
last_check_ago = 0
status = data.get('status', 'unknown')
torrent.update({
'leecher': nleechers,
'seeder': nseeders,
'last_check_time': last_check_ago,
'status': status
})
return self.do_send_metadata(permid, torrent, selversion)
else:
self.torrent_db.deleteTorrent(infohash,
delete_file=True,
commit=True)
if DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: no torrent data to send'
return 0
def __requestMagnet(self):
try:
if len(self.requestedInfohashes) < self.MAX_CONCURRENT:
while True:
if len(self.list) == 0:
return
prio, infohash = self.list.pop(0)
if infohash in self.requestedInfohashes:
if DEBUG:
print >> sys.stderr, 'magnetrequester: magnet already requested', bin2str(infohash)
continue
torrent = self.torrent_db.getTorrent(infohash, ['torrent_file_name'], include_mypref=False)
if torrent.get('torrent_file_name', False):
torrent_filename = os.path.join(self.metadatahandler.torrent_dir, torrent['torrent_file_name'])
else:
torrent_filename = None
torrent_alt_filename = os.path.join(self.metadatahandler.torrent_dir, get_collected_torrent_filename(infohash))
if os.path.isfile(torrent_filename) or os.path.isfile(torrent_alt_filename):
if DEBUG:
print >> sys.stderr, 'magnetrequester: magnet already on disk', bin2str(infohash)
else:
break
else:
return
except:
print_exc()
magnetlink = 'magnet:?xt=urn:btih:' + hexlify(infohash)
if DEBUG:
print >> sys.stderr, 'magnetrequester: requesting magnet', bin2str(infohash), prio, magnetlink
self.requestedInfohashes.add(infohash)
TorrentDef.retrieve_from_magnet(magnetlink, self.__torrentdef_retrieved, self.MAGNET_RETRIEVE_TIMEOUT)
self.overlay_bridge.add_task(lambda : self.__torrentdef_failed(infohash), self.MAGNET_RETRIEVE_TIMEOUT, infohash)
if len(self.requestedInfohashes) < self.MAX_CONCURRENT:
self.overlay_bridge.add_task(self.__requestMagnet, self.REQUEST_INTERVAL)
def addTorrentToDB(self, filename, torrent_hash, metadata, source = 'BC', extra_info = {}, hack = False):
torrentdef = TorrentDef.load(filename)
if 'filename' not in extra_info:
extra_info['filename'] = filename
torrent = self.torrent_db.addExternalTorrent(torrentdef, source, extra_info)
if torrent is None:
return
self.launchmany.set_activity(NTFY_ACT_GOT_METADATA, unicode('"' + torrent['name'] + '"'), torrent['category'])
if self.initialized:
self.num_torrents += 1
if not extra_info:
self.refreshTrackerStatus(torrent)
if len(self.recently_collected_torrents) < 50:
self.recently_collected_torrents.append(torrent_hash)
else:
self.recently_collected_torrents.pop(0)
self.recently_collected_torrents.append(torrent_hash)
def read_and_send_metadata(self, permid, infohash, torrent_path, selversion):
torrent_data = self.read_torrent(torrent_path)
if torrent_data:
try:
metainfo = bdecode(torrent_data)
if 'info' in metainfo and 'private' in metainfo['info'] and metainfo['info']['private']:
if DEBUG:
print >> sys.stderr, 'metadata: Not sending torrent', `torrent_path`, 'because it is private'
return 0
except:
print_exc()
return 0
if DEBUG:
print >> sys.stderr, 'metadata: sending torrent', `torrent_path`, len(torrent_data)
torrent = {}
torrent['torrent_hash'] = infohash
tdef = TorrentDef.load_from_dict(metainfo)
if selversion >= OLPROTO_VER_ELEVENTH and tdef.get_url_compat():
torrent['metatype'] = URL_MIME_TYPE
torrent['metadata'] = tdef.get_url()
else:
torrent['metatype'] = TSTREAM_MIME_TYPE
torrent['metadata'] = torrent_data
if selversion >= OLPROTO_VER_FOURTH:
data = self.torrent_db.getTorrent(infohash)
if data is None:
return 0
nleechers = data.get('leecher', -1)
nseeders = data.get('seeder', -1)
last_check_ago = int(time()) - data.get('last_check_time', 0)
if last_check_ago < 0:
last_check_ago = 0
status = data.get('status', 'unknown')
torrent.update({'leecher': nleechers,
'seeder': nseeders,
'last_check_time': last_check_ago,
'status': status})
return self.do_send_metadata(permid, torrent, selversion)
else:
self.torrent_db.deleteTorrent(infohash, delete_file=True, commit=True)
if DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: no torrent data to send'
return 0
def resume_download(self, dltype, filename, initialdlstatus = None):
try:
pstate = self.load_download_pstate(filename)
if DEBUG:
log('lm::resume_download: dltype', dltype, 'filename', filename, 'dlconfig', pstate['dlconfig'])
if DEBUG:
log('lm::resume_download: status', dlstatus_strings[pstate['dlstate']['status']], 'progress', pstate['dlstate']['progress'])
if pstate['engineresumedata'] is None:
log('lm::resume_download: resumedata None')
else:
log('lm::resume_download: resumedata len', len(pstate['engineresumedata']))
dscfg = DownloadStartupConfig(dlconfig=pstate['dlconfig'])
if dltype == DLTYPE_TORRENT:
tdef = TorrentDef.load_from_dict(pstate['metainfo'])
d = self.add(tdef, dscfg, pstate, initialdlstatus)
elif dltype == DLTYPE_DIRECT:
main_url = pstate['url']
d = self.add_direct_download(main_url, dscfg, pstate, initialdlstatus)
if initialdlstatus == DLSTATUS_STOPPED:
dest_files = d.get_dest_files(get_all=True)
if DEBUG:
log('lm::resume_download: check dest files: dest_files', dest_files)
got_existing_file = False
for filename, savepath in dest_files:
if os.path.exists(savepath):
got_existing_file = True
break
if not got_existing_file:
if DEBUG:
log('lm::resume_download: none of the files exists, remove this download')
self.remove(d, removecontent=True)
except Exception as e:
if DEBUG:
log('lm::resume_download: failed to load checkpoint: filename', filename)
print_exc()
try:
if os.access(filename, os.F_OK):
os.remove(filename)
except:
print_exc()
def got_metadata(self, permid, message, selversion):
try:
message = bdecode(message[1:])
except:
print_exc()
return False
if not isinstance(message, dict):
return False
try:
infohash = message['torrent_hash']
if not isValidInfohash(infohash):
return False
if infohash not in self.requested_torrents:
return True
if self.torrent_db.hasMetaData(infohash):
return True
goturl = False
if selversion >= OLPROTO_VER_ELEVENTH:
if 'metatype' in message and message['metatype'] == URL_MIME_TYPE:
try:
tdef = TorrentDef.load_from_url(message['metadata'])
metainfo = tdef.get_metainfo()
metadata = bencode(metainfo)
goturl = True
except:
print_exc()
return False
else:
metadata = message['metadata']
else:
metadata = message['metadata']
if not self.valid_metadata(infohash, metadata):
return False
if DEBUG:
torrent_size = len(metadata)
if goturl:
mdt = 'URL'
else:
mdt = 'torrent'
print >> sys.stderr, 'metadata: Recvd', mdt, `infohash`, sha(infohash).hexdigest(), torrent_size
extra_info = {}
if selversion >= OLPROTO_VER_FOURTH:
try:
extra_info = {'leecher': message.get('leecher', -1),
'seeder': message.get('seeder', -1),
'last_check_time': message.get('last_check_time', -1),
'status': message.get('status', 'unknown')}
except Exception as msg:
print_exc()
print >> sys.stderr, 'metadata: wrong extra info in msg - ', message
extra_info = {}
filename = self.save_torrent(infohash, metadata, extra_info=extra_info)
self.requested_torrents.remove(infohash)
if filename is not None:
self.notify_torrent_is_in(infohash, metadata, filename)
if permid is not None and BARTERCAST_TORRENTS:
self.overlay_bridge.add_task(lambda : self.olthread_bartercast_torrentexchange(permid, 'downloaded'), 0)
except Exception as e:
print_exc()
print >> sys.stderr, 'metadata: Received metadata is broken', e, message.keys()
return False
return True
def parsedir(
directory, parsed, files, blocked, exts=[".torrent", TRIBLER_TORRENT_EXT], return_metainfo=False, errfunc=_errfunc
):
if DEBUG:
errfunc("checking dir")
dirs_to_check = [directory]
new_files = {}
new_blocked = {}
torrent_type = {}
while dirs_to_check:
directory = dirs_to_check.pop()
newtorrents = False
for f in os.listdir(directory):
newtorrent = None
for ext in exts:
if f.endswith(ext):
newtorrent = ext[1:]
break
if newtorrent:
newtorrents = True
p = os.path.join(directory, f)
new_files[p] = [(int(os.path.getmtime(p)), os.path.getsize(p)), 0]
torrent_type[p] = newtorrent
if not newtorrents:
for f in os.listdir(directory):
p = os.path.join(directory, f)
if os.path.isdir(p):
dirs_to_check.append(p)
new_parsed = {}
to_add = []
added = {}
removed = {}
for p, v in new_files.items():
oldval = files.get(p)
if not oldval:
to_add.append(p)
continue
h = oldval[1]
if oldval[0] == v[0]:
if h:
if blocked.has_key(p):
to_add.append(p)
else:
new_parsed[h] = parsed[h]
new_files[p] = oldval
else:
new_blocked[p] = 1
continue
if parsed.has_key(h) and not blocked.has_key(p):
if DEBUG:
errfunc("removing " + p + " (will re-add)")
removed[h] = parsed[h]
to_add.append(p)
to_add.sort()
for p in to_add:
new_file = new_files[p]
v, h = new_file
if new_parsed.has_key(h):
if not blocked.has_key(p) or files[p][0] != v:
errfunc("**warning** " + p + " is a duplicate torrent for " + new_parsed[h]["path"])
new_blocked[p] = 1
continue
if DEBUG:
errfunc("adding " + p)
try:
tdef = TorrentDef.load(p)
h = tdef.get_infohash()
d = tdef.get_metainfo()
new_file[1] = h
if new_parsed.has_key(h):
errfunc("**warning** " + p + " is a duplicate torrent for " + new_parsed[h]["path"])
new_blocked[p] = 1
continue
a = {}
a["path"] = p
f = os.path.basename(p)
a["file"] = f
a["type"] = torrent_type[p]
if tdef.get_url_compat():
a["url"] = tdef.get_url()
i = d["info"]
l = 0
nf = 0
if i.has_key("length"):
l = i.get("length", 0)
nf = 1
elif i.has_key("files"):
for li in i["files"]:
nf += 1
if li.has_key("length"):
l += li["length"]
a["numfiles"] = nf
a["length"] = l
a["name"] = i.get("name", f)
def setkey(k, d=d, a=a):
if d.has_key(k):
a[k] = d[k]
setkey("failure reason")
setkey("warning message")
setkey("announce-list")
if tdef.get_urllist() is not None:
httpseedhashes = []
for url in tdef.get_urllist():
urlhash = sha(url).digest()
httpseedhashes.append(urlhash)
a["url-hash-list"] = httpseedhashes
if return_metainfo:
a["metainfo"] = d
except:
print_exc()
errfunc("**warning** " + p + " has errors")
new_blocked[p] = 1
continue
if DEBUG:
errfunc("... successful")
new_parsed[h] = a
added[h] = a
for p, v in files.items():
if not new_files.has_key(p) and not blocked.has_key(p):
if DEBUG:
errfunc("removing " + p)
removed[v[1]] = parsed[v[1]]
if DEBUG:
errfunc("done checking")
return (new_parsed, new_files, new_blocked, added, removed)
def got_metadata(self, permid, message, selversion):
try:
message = bdecode(message[1:])
except:
print_exc()
return False
if not isinstance(message, dict):
return False
try:
infohash = message['torrent_hash']
if not isValidInfohash(infohash):
return False
if infohash not in self.requested_torrents:
return True
if self.torrent_db.hasMetaData(infohash):
return True
goturl = False
if selversion >= OLPROTO_VER_ELEVENTH:
if 'metatype' in message and message[
'metatype'] == URL_MIME_TYPE:
try:
tdef = TorrentDef.load_from_url(message['metadata'])
metainfo = tdef.get_metainfo()
metadata = bencode(metainfo)
goturl = True
except:
print_exc()
return False
else:
metadata = message['metadata']
else:
metadata = message['metadata']
if not self.valid_metadata(infohash, metadata):
return False
if DEBUG:
torrent_size = len(metadata)
if goturl:
mdt = 'URL'
else:
mdt = 'torrent'
print >> sys.stderr, 'metadata: Recvd', mdt, ` infohash `, sha(
infohash).hexdigest(), torrent_size
extra_info = {}
if selversion >= OLPROTO_VER_FOURTH:
try:
extra_info = {
'leecher': message.get('leecher', -1),
'seeder': message.get('seeder', -1),
'last_check_time': message.get('last_check_time', -1),
'status': message.get('status', 'unknown')
}
except Exception as msg:
print_exc()
print >> sys.stderr, 'metadata: wrong extra info in msg - ', message
extra_info = {}
filename = self.save_torrent(infohash,
metadata,
extra_info=extra_info)
self.requested_torrents.remove(infohash)
if filename is not None:
self.notify_torrent_is_in(infohash, metadata, filename)
if permid is not None and BARTERCAST_TORRENTS:
self.overlay_bridge.add_task(
lambda: self.olthread_bartercast_torrentexchange(
permid, 'downloaded'), 0)
except Exception as e:
print_exc()
print >> sys.stderr, 'metadata: Received metadata is broken', e, message.keys(
)
return False
return True
def parsedir(directory,
parsed,
files,
blocked,
exts=['.torrent', TRIBLER_TORRENT_EXT],
return_metainfo=False,
errfunc=_errfunc):
if DEBUG:
errfunc('checking dir')
dirs_to_check = [directory]
new_files = {}
new_blocked = {}
torrent_type = {}
while dirs_to_check:
directory = dirs_to_check.pop()
newtorrents = False
for f in os.listdir(directory):
newtorrent = None
for ext in exts:
if f.endswith(ext):
newtorrent = ext[1:]
break
if newtorrent:
newtorrents = True
p = os.path.join(directory, f)
new_files[p] = [(int(os.path.getmtime(p)), os.path.getsize(p)),
0]
torrent_type[p] = newtorrent
if not newtorrents:
for f in os.listdir(directory):
p = os.path.join(directory, f)
if os.path.isdir(p):
dirs_to_check.append(p)
new_parsed = {}
to_add = []
added = {}
removed = {}
for p, v in new_files.items():
oldval = files.get(p)
if not oldval:
to_add.append(p)
continue
h = oldval[1]
if oldval[0] == v[0]:
if h:
if blocked.has_key(p):
to_add.append(p)
else:
new_parsed[h] = parsed[h]
new_files[p] = oldval
else:
new_blocked[p] = 1
continue
if parsed.has_key(h) and not blocked.has_key(p):
if DEBUG:
errfunc('removing ' + p + ' (will re-add)')
removed[h] = parsed[h]
to_add.append(p)
to_add.sort()
for p in to_add:
new_file = new_files[p]
v, h = new_file
if new_parsed.has_key(h):
if not blocked.has_key(p) or files[p][0] != v:
errfunc('**warning** ' + p + ' is a duplicate torrent for ' +
new_parsed[h]['path'])
new_blocked[p] = 1
continue
if DEBUG:
errfunc('adding ' + p)
try:
tdef = TorrentDef.load(p)
h = tdef.get_infohash()
d = tdef.get_metainfo()
new_file[1] = h
if new_parsed.has_key(h):
errfunc('**warning** ' + p + ' is a duplicate torrent for ' +
new_parsed[h]['path'])
new_blocked[p] = 1
continue
a = {}
a['path'] = p
f = os.path.basename(p)
a['file'] = f
a['type'] = torrent_type[p]
if tdef.get_url_compat():
a['url'] = tdef.get_url()
i = d['info']
l = 0
nf = 0
if i.has_key('length'):
l = i.get('length', 0)
nf = 1
elif i.has_key('files'):
for li in i['files']:
nf += 1
if li.has_key('length'):
l += li['length']
a['numfiles'] = nf
a['length'] = l
a['name'] = i.get('name', f)
def setkey(k, d=d, a=a):
if d.has_key(k):
a[k] = d[k]
setkey('failure reason')
setkey('warning message')
setkey('announce-list')
if tdef.get_urllist() is not None:
httpseedhashes = []
for url in tdef.get_urllist():
urlhash = sha(url).digest()
httpseedhashes.append(urlhash)
a['url-hash-list'] = httpseedhashes
if return_metainfo:
a['metainfo'] = d
except:
print_exc()
errfunc('**warning** ' + p + ' has errors')
new_blocked[p] = 1
continue
if DEBUG:
errfunc('... successful')
new_parsed[h] = a
added[h] = a
for p, v in files.items():
if not new_files.has_key(p) and not blocked.has_key(p):
if DEBUG:
errfunc('removing ' + p)
removed[v[1]] = parsed[v[1]]
if DEBUG:
errfunc('done checking')
return (new_parsed, new_files, new_blocked, added, removed)