def get_new_torrent(): files = os.listdir(config.torrents_new) if not files: return progress = progressbar.ProgressBar(widgets=widget("new torrents")) for f in progress(files): f = "%s/%s" % (config.torrents_new, f) try: torrent = utils.bdecode(open(f, 'rb').read()) real_hash = hashlib.sha1(utils.bencode(torrent[b'info'])).hexdigest() hash_to_ignore.add(real_hash.decode("hex")) os.rename(f, "%s/%s.torrent" % (config.torrents_dir, real_hash)) except utils.BcodeError as e: pass
def get_torrent(db, hash, base_path=config.torrents_dir): global downloading, failed if hash in failed: return None if os.path.isfile("%s/%s.torrent" % (base_path, hash)): torrent = open("%s/%s.torrent" % (base_path, hash), 'rb').read() try: torrent = utils.bdecode(torrent) if not b'info' in torrent: return {b'info':torrent} else: return torrent except utils.BcodeError as e: print("FAILED %s: %r" % (hash, e)) failed.add(hash) else: return
def scrape_http(parsed_tracker, hashes): print("Scraping HTTP: %s for %s hashes" % (parsed_tracker.geturl(), len(hashes))) qs = [] for hash in hashes: url_param = binascii.a2b_hex(hash) qs.append(("info_hash", url_param)) qs = urllib.urlencode(qs) pt = parsed_tracker url = urlunsplit((pt.scheme, pt.netloc, pt.path, qs, pt.fragment)) handle = urllib.urlopen(url) if handle.getcode() is not 200: raise RuntimeError("%s status code returned" % handle.getcode()) decoded = bdecode(handle.read()) ret = {} for hash, stats in decoded["files"].iteritems(): nice_hash = binascii.b2a_hex(hash) s = stats["complete"] p = stats["incomplete"] c = stats["downloaded"] ret[nice_hash] = {"seeds": s, "peers": p, "complete": c} return ret
def scrape_http(parsed_tracker, hashes): print("Scraping HTTP: %s for %s hashes" % (parsed_tracker.geturl(), len(hashes))) qs = [] for hash in hashes: url_param = binascii.a2b_hex(hash) qs.append(("info_hash", url_param)) qs = urllib.urlencode(qs) pt = parsed_tracker url = urlunsplit((pt.scheme, pt.netloc, pt.path, qs, pt.fragment)) handle = urllib.urlopen(url) if handle.getcode() is not 200: raise RuntimeError("%s status code returned" % handle.getcode()) decoded = bdecode(handle.read()) ret = {} for hash, stats in decoded['files'].iteritems(): nice_hash = binascii.b2a_hex(hash) s = stats["complete"] p = stats["incomplete"] c = stats["downloaded"] ret[nice_hash] = {"seeds": s, "peers": p, "complete": c} return ret
def handle_20(self, s, payload): # extension type msg_typ = ord(payload[0]) msg = payload[1:] if msg_typ == 0: hash = self._socket_hash[s] msg = bdecode(msg) if 'metadata_size' in msg: if msg['metadata_size'] > 8192000 or msg['metadata_size'] < 1: # plus de 8000ko or less thant 1o self.clean_socket(s) if not msg['metadata_size'] in self._metadata_size_qorum[hash]: self._metadata_size_qorum[hash][msg['metadata_size']] = 1 else: self._metadata_size_qorum[hash][msg['metadata_size']]+=1 self._peer_metadata[s] = msg['m']['ut_metadata'] if not hash in self._metadata_size: self._metadata_size[hash] = msg['metadata_size'] self._metadata_pieces_nb[hash] = int(math.ceil(self._metadata_size[hash]/(16.0*1024))) self._metadata_pieces[hash] = [None] * self._metadata_pieces_nb[hash] self._metadata_pieces_received[hash] = 0 else: mps = self.most_probably_size(hash) if self._metadata_size[hash] < mps: piece_nb = int(math.ceil(mps/(16.0*1024))) self._metadata_pieces[hash] = self._metadata_pieces[hash] + ([None] * (piece_nb - self._metadata_pieces_nb[hash])) self._metadata_size[hash] = mps self._metadata_pieces_nb[hash] = piece_nb elif self._metadata_size[hash] > mps: self._metadata_size[hash] = mps self._metadata_pieces_nb[hash] = int(math.ceil(mps/(16.0*1024))) self._metadata_pieces[hash] = self._metadata_pieces[hash][0:self._metadata_pieces_nb[hash]] self._metadata_pieces_received[hash] = len([i for i in self._metadata_pieces[hash] if i is not None]) self.interested(s) elif msg_typ == self._am_metadata: hash = self._socket_hash[s] msg, data = _bdecode(msg) if msg['msg_type'] == 0: if self._metadata_pieces[hash] and msg['piece'] < self._metadata_pieces_nb[hash]: self.metadata_data(s, msg['piece']) else: self.metadata_reject(s, msg['piece']) elif msg['msg_type'] == 1: try: if msg['piece'] < self._metadata_pieces_nb[hash] and self._metadata_pieces[hash][msg['piece']] is None: self._metadata_pieces[hash][msg['piece']] = data self._metadata_pieces_received[hash] += 1 if self._metadata_pieces_received[hash] == self._metadata_pieces_nb[hash]: metadata = "".join(self._metadata_pieces[hash]) if hashlib.sha1(metadata).digest() == hash: self.meta_data[hash] = metadata self.clean_hash(hash) if self.debug: print "metadata complete" else: self.init_hash(hash) if self.debug: print "bad metadata %s != %s" % (hashlib.sha1(metadata).hexdigest(), hash.encode("hex")) except (IndexError, TypeError): pass elif msg['msg_type'] == 2: pass else: pass
def handle_20(self, s, payload): # extension type msg_typ = ord(payload[0]) msg = payload[1:] if msg_typ == 0: hash = self._socket_hash[s] msg = bdecode(msg) if 'metadata_size' in msg: if msg['metadata_size'] > 8192000 or msg[ 'metadata_size'] < 1: # plus de 8000ko or less thant 1o self.clean_socket(s) if not msg['metadata_size'] in self._metadata_size_qorum[hash]: self._metadata_size_qorum[hash][msg['metadata_size']] = 1 else: self._metadata_size_qorum[hash][msg['metadata_size']] += 1 self._peer_metadata[s] = msg['m']['ut_metadata'] if not hash in self._metadata_size: self._metadata_size[hash] = msg['metadata_size'] self._metadata_pieces_nb[hash] = int( math.ceil(self._metadata_size[hash] / (16.0 * 1024))) self._metadata_pieces[hash] = [ None ] * self._metadata_pieces_nb[hash] self._metadata_pieces_received[hash] = 0 else: mps = self.most_probably_size(hash) if self._metadata_size[hash] < mps: piece_nb = int(math.ceil(mps / (16.0 * 1024))) self._metadata_pieces[ hash] = self._metadata_pieces[hash] + ( [None] * (piece_nb - self._metadata_pieces_nb[hash])) self._metadata_size[hash] = mps self._metadata_pieces_nb[hash] = piece_nb elif self._metadata_size[hash] > mps: self._metadata_size[hash] = mps self._metadata_pieces_nb[hash] = int( math.ceil(mps / (16.0 * 1024))) self._metadata_pieces[hash] = self._metadata_pieces[ hash][0:self._metadata_pieces_nb[hash]] self._metadata_pieces_received[hash] = len([ i for i in self._metadata_pieces[hash] if i is not None ]) self.interested(s) elif msg_typ == self._am_metadata: hash = self._socket_hash[s] msg, data = _bdecode(msg) if msg['msg_type'] == 0: if self._metadata_pieces[hash] and msg[ 'piece'] < self._metadata_pieces_nb[hash]: self.metadata_data(s, msg['piece']) else: self.metadata_reject(s, msg['piece']) elif msg['msg_type'] == 1: try: if msg['piece'] < self._metadata_pieces_nb[ hash] and self._metadata_pieces[hash][ msg['piece']] is None: self._metadata_pieces[hash][msg['piece']] = data self._metadata_pieces_received[hash] += 1 if self._metadata_pieces_received[ hash] == self._metadata_pieces_nb[hash]: metadata = "".join(self._metadata_pieces[hash]) if hashlib.sha1(metadata).digest() == hash: self.meta_data[hash] = metadata self.clean_hash(hash) if self.debug: print "metadata complete" else: self.init_hash(hash) if self.debug: print "bad metadata %s != %s" % ( hashlib.sha1(metadata).hexdigest(), hash.encode("hex")) except (IndexError, TypeError): pass elif msg['msg_type'] == 2: pass else: pass