def get_new_torrent():
files = os.listdir(config.torrents_new)
if not files:
return
progress = progressbar.ProgressBar(widgets=widget("new torrents"))
for f in progress(files):
f = "%s/%s" % (config.torrents_new, f)
try:
torrent = utils.bdecode(open(f, 'rb').read())
real_hash = hashlib.sha1(utils.bencode(torrent[b'info'])).hexdigest()
hash_to_ignore.add(real_hash.decode("hex"))
os.rename(f, "%s/%s.torrent" % (config.torrents_dir, real_hash))
except utils.BcodeError as e:
pass
def get_new_torrent():
files = os.listdir(config.torrents_new)
if not files:
return
progress = progressbar.ProgressBar(widgets=widget("new torrents"))
for f in progress(files):
f = "%s/%s" % (config.torrents_new, f)
try:
torrent = utils.bdecode(open(f, 'rb').read())
real_hash = hashlib.sha1(utils.bencode(torrent[b'info'])).hexdigest()
hash_to_ignore.add(real_hash.decode("hex"))
os.rename(f, "%s/%s.torrent" % (config.torrents_dir, real_hash))
except utils.BcodeError as e:
pass
def get_torrent(db, hash, base_path=config.torrents_dir):
global downloading, failed
if hash in failed:
return None
if os.path.isfile("%s/%s.torrent" % (base_path, hash)):
torrent = open("%s/%s.torrent" % (base_path, hash), 'rb').read()
try:
torrent = utils.bdecode(torrent)
if not b'info' in torrent:
return {b'info':torrent}
else:
return torrent
except utils.BcodeError as e:
print("FAILED %s: %r" % (hash, e))
failed.add(hash)
else:
return
def get_torrent(db, hash, base_path=config.torrents_dir):
global downloading, failed
if hash in failed:
return None
if os.path.isfile("%s/%s.torrent" % (base_path, hash)):
torrent = open("%s/%s.torrent" % (base_path, hash), 'rb').read()
try:
torrent = utils.bdecode(torrent)
if not b'info' in torrent:
return {b'info':torrent}
else:
return torrent
except utils.BcodeError as e:
print("FAILED %s: %r" % (hash, e))
failed.add(hash)
else:
return
def scrape_http(parsed_tracker, hashes):
print("Scraping HTTP: %s for %s hashes" % (parsed_tracker.geturl(), len(hashes)))
qs = []
for hash in hashes:
url_param = binascii.a2b_hex(hash)
qs.append(("info_hash", url_param))
qs = urllib.urlencode(qs)
pt = parsed_tracker
url = urlunsplit((pt.scheme, pt.netloc, pt.path, qs, pt.fragment))
handle = urllib.urlopen(url)
if handle.getcode() is not 200:
raise RuntimeError("%s status code returned" % handle.getcode())
decoded = bdecode(handle.read())
ret = {}
for hash, stats in decoded["files"].iteritems():
nice_hash = binascii.b2a_hex(hash)
s = stats["complete"]
p = stats["incomplete"]
c = stats["downloaded"]
ret[nice_hash] = {"seeds": s, "peers": p, "complete": c}
return ret
def scrape_http(parsed_tracker, hashes):
print("Scraping HTTP: %s for %s hashes" %
(parsed_tracker.geturl(), len(hashes)))
qs = []
for hash in hashes:
url_param = binascii.a2b_hex(hash)
qs.append(("info_hash", url_param))
qs = urllib.urlencode(qs)
pt = parsed_tracker
url = urlunsplit((pt.scheme, pt.netloc, pt.path, qs, pt.fragment))
handle = urllib.urlopen(url)
if handle.getcode() is not 200:
raise RuntimeError("%s status code returned" % handle.getcode())
decoded = bdecode(handle.read())
ret = {}
for hash, stats in decoded['files'].iteritems():
nice_hash = binascii.b2a_hex(hash)
s = stats["complete"]
p = stats["incomplete"]
c = stats["downloaded"]
ret[nice_hash] = {"seeds": s, "peers": p, "complete": c}
return ret
def handle_20(self, s, payload): # extension type
msg_typ = ord(payload[0])
msg = payload[1:]
if msg_typ == 0:
hash = self._socket_hash[s]
msg = bdecode(msg)
if 'metadata_size' in msg:
if msg['metadata_size'] > 8192000 or msg['metadata_size'] < 1: # plus de 8000ko or less thant 1o
self.clean_socket(s)
if not msg['metadata_size'] in self._metadata_size_qorum[hash]:
self._metadata_size_qorum[hash][msg['metadata_size']] = 1
else:
self._metadata_size_qorum[hash][msg['metadata_size']]+=1
self._peer_metadata[s] = msg['m']['ut_metadata']
if not hash in self._metadata_size:
self._metadata_size[hash] = msg['metadata_size']
self._metadata_pieces_nb[hash] = int(math.ceil(self._metadata_size[hash]/(16.0*1024)))
self._metadata_pieces[hash] = [None] * self._metadata_pieces_nb[hash]
self._metadata_pieces_received[hash] = 0
else:
mps = self.most_probably_size(hash)
if self._metadata_size[hash] < mps:
piece_nb = int(math.ceil(mps/(16.0*1024)))
self._metadata_pieces[hash] = self._metadata_pieces[hash] + ([None] * (piece_nb - self._metadata_pieces_nb[hash]))
self._metadata_size[hash] = mps
self._metadata_pieces_nb[hash] = piece_nb
elif self._metadata_size[hash] > mps:
self._metadata_size[hash] = mps
self._metadata_pieces_nb[hash] = int(math.ceil(mps/(16.0*1024)))
self._metadata_pieces[hash] = self._metadata_pieces[hash][0:self._metadata_pieces_nb[hash]]
self._metadata_pieces_received[hash] = len([i for i in self._metadata_pieces[hash] if i is not None])
self.interested(s)
elif msg_typ == self._am_metadata:
hash = self._socket_hash[s]
msg, data = _bdecode(msg)
if msg['msg_type'] == 0:
if self._metadata_pieces[hash] and msg['piece'] < self._metadata_pieces_nb[hash]:
self.metadata_data(s, msg['piece'])
else:
self.metadata_reject(s, msg['piece'])
elif msg['msg_type'] == 1:
try:
if msg['piece'] < self._metadata_pieces_nb[hash] and self._metadata_pieces[hash][msg['piece']] is None:
self._metadata_pieces[hash][msg['piece']] = data
self._metadata_pieces_received[hash] += 1
if self._metadata_pieces_received[hash] == self._metadata_pieces_nb[hash]:
metadata = "".join(self._metadata_pieces[hash])
if hashlib.sha1(metadata).digest() == hash:
self.meta_data[hash] = metadata
self.clean_hash(hash)
if self.debug:
print "metadata complete"
else:
self.init_hash(hash)
if self.debug:
print "bad metadata %s != %s" % (hashlib.sha1(metadata).hexdigest(), hash.encode("hex"))
except (IndexError, TypeError):
pass
elif msg['msg_type'] == 2:
pass
else:
pass
def handle_20(self, s, payload): # extension type
msg_typ = ord(payload[0])
msg = payload[1:]
if msg_typ == 0:
hash = self._socket_hash[s]
msg = bdecode(msg)
if 'metadata_size' in msg:
if msg['metadata_size'] > 8192000 or msg[
'metadata_size'] < 1: # plus de 8000ko or less thant 1o
self.clean_socket(s)
if not msg['metadata_size'] in self._metadata_size_qorum[hash]:
self._metadata_size_qorum[hash][msg['metadata_size']] = 1
else:
self._metadata_size_qorum[hash][msg['metadata_size']] += 1
self._peer_metadata[s] = msg['m']['ut_metadata']
if not hash in self._metadata_size:
self._metadata_size[hash] = msg['metadata_size']
self._metadata_pieces_nb[hash] = int(
math.ceil(self._metadata_size[hash] / (16.0 * 1024)))
self._metadata_pieces[hash] = [
None
] * self._metadata_pieces_nb[hash]
self._metadata_pieces_received[hash] = 0
else:
mps = self.most_probably_size(hash)
if self._metadata_size[hash] < mps:
piece_nb = int(math.ceil(mps / (16.0 * 1024)))
self._metadata_pieces[
hash] = self._metadata_pieces[hash] + (
[None] *
(piece_nb - self._metadata_pieces_nb[hash]))
self._metadata_size[hash] = mps
self._metadata_pieces_nb[hash] = piece_nb
elif self._metadata_size[hash] > mps:
self._metadata_size[hash] = mps
self._metadata_pieces_nb[hash] = int(
math.ceil(mps / (16.0 * 1024)))
self._metadata_pieces[hash] = self._metadata_pieces[
hash][0:self._metadata_pieces_nb[hash]]
self._metadata_pieces_received[hash] = len([
i for i in self._metadata_pieces[hash]
if i is not None
])
self.interested(s)
elif msg_typ == self._am_metadata:
hash = self._socket_hash[s]
msg, data = _bdecode(msg)
if msg['msg_type'] == 0:
if self._metadata_pieces[hash] and msg[
'piece'] < self._metadata_pieces_nb[hash]:
self.metadata_data(s, msg['piece'])
else:
self.metadata_reject(s, msg['piece'])
elif msg['msg_type'] == 1:
try:
if msg['piece'] < self._metadata_pieces_nb[
hash] and self._metadata_pieces[hash][
msg['piece']] is None:
self._metadata_pieces[hash][msg['piece']] = data
self._metadata_pieces_received[hash] += 1
if self._metadata_pieces_received[
hash] == self._metadata_pieces_nb[hash]:
metadata = "".join(self._metadata_pieces[hash])
if hashlib.sha1(metadata).digest() == hash:
self.meta_data[hash] = metadata
self.clean_hash(hash)
if self.debug:
print "metadata complete"
else:
self.init_hash(hash)
if self.debug:
print "bad metadata %s != %s" % (
hashlib.sha1(metadata).hexdigest(),
hash.encode("hex"))
except (IndexError, TypeError):
pass
elif msg['msg_type'] == 2:
pass
else:
pass
