def __scrape_tracker_http(tracker, magnets, timeout):
start_time = timeit.default_timer()
results = {}
for magnet in magnets:
results[magnet] = {'seeds': 0, 'peers': 0, 'completed': 0}
query_string = []
for magnet in magnets:
query_string.append(("info_hash", binascii.a2b_hex(magnet.info_hash)))
query_string = urllib.urlencode(query_string)
url = urlparse.urlunsplit((tracker.scheme, tracker.netloc, tracker.path,
query_string, tracker.fragment))
try:
http_data = network.http_get(url, timeout=timeout, logging=False)
if http_data:
decoded_response = bcode.bdecode(http_data)
for info_hash, stats in decoded_response['files'].iteritems():
info_hash = binascii.b2a_hex(info_hash).upper()
for magnet in results:
if magnet.info_hash == info_hash:
results[magnet] = {
'seeds': stats['complete'],
'peers': stats['incomplete'],
'completed': stats['downloaded']
}
#sys.stdout.write('{0} : {1:3.1f}s : SCR : {2}://{3} {4}\n'.format('NET:OK', timeit.default_timer() - start_time, tracker.scheme, tracker.netloc, magnet.info_hash))
break
except Exception:
pass
return results
def __scrape_tracker_http(tracker, magnets, timeout):
start_time = timeit.default_timer()
results = {}
for magnet in magnets:
results[magnet] = { 'seeds': 0, 'peers': 0, 'completed': 0 }
query_string = []
for magnet in magnets:
query_string.append(("info_hash", binascii.a2b_hex(magnet.info_hash)))
query_string = urllib.urlencode(query_string)
url = urlparse.urlunsplit((tracker.scheme, tracker.netloc, tracker.path, query_string, tracker.fragment))
try:
http_data = network.http_get(url, timeout=timeout, logging=False)
if http_data:
decoded_response = bcode.bdecode(http_data)
for info_hash, stats in decoded_response['files'].iteritems():
info_hash = binascii.b2a_hex(info_hash).upper()
for magnet in results:
if magnet.info_hash == info_hash:
results[magnet] = { 'seeds': stats['complete'], 'peers': stats['incomplete'], 'completed': stats['downloaded'] }
#sys.stdout.write('{0} : {1:3.1f}s : SCR : {2}://{3} {4}\n'.format('NET:OK', timeit.default_timer() - start_time, tracker.scheme, tracker.netloc, magnet.info_hash))
break
except Exception:
pass
return results
def scrape_http(parsed_tracker, hashes, info):
logging.info("[%d/%d] Scraping HTTP: %s for %s hashes" %
(info[0], info[1], parsed_tracker.geturl(), len(hashes)))
qs = []
for hash in hashes:
url_param = binascii.a2b_hex(hash)
qs.append(("info_hash", url_param))
qs = urllib.parse.urlencode(qs)
pt = parsed_tracker
url = urlunsplit((pt.scheme, pt.netloc, pt.path, qs, pt.fragment))
try:
handle = urllib.urlopen(url, timeout=8)
if handle.getcode() is not 200:
logging.exception("%s status code returned" % handle.getcode())
return {}
ret = {}
decoded = bdecode(handle.read())
for hash, stats in decoded['files'].iteritems():
nice_hash = binascii.b2a_hex(hash)
s = stats["complete"]
p = stats["incomplete"]
c = stats["downloaded"]
ret[nice_hash] = {"seeds": s, "peers": p, "complete": c}
except:
ret = {}
logging.info(ret)
return ret
def getInfo():
path = os.getcwd()
sep = os.sep
torrentSet = [filename for filename in os.listdir(path) if filename.split('.')[-1] == 'torrent']
for i in torrentSet:
with open(path + sep + i, 'rb') as f:
c = bdecode(str(f.read())[2:-1])
print(c.keys())
# print(str(f.read())[2:-1])
break
def _perform_data_stitching(this):
while len(this._buffer) > 0:
try:
temp = bcode.bdecode(this._buffer)
except ValueError:
temp = None
if temp == None:
return
temp_len = len(bcode.bencode(temp))
this._buffer = this._buffer[temp_len:]
map(lambda f: f(temp), this._cb)
def get_torrent_size(data):
if not data:
return 0
t_data = bdecode(data)
t_size = 0
if 'files' in t_data['info']:
files = t_data['info']['files']
for f in files:
t_size += int(f['length'])
else:
t_size += int(t_data['info']['length'])
return t_size
def _perform_data_stitching(this):
while len(this._buffer) > 0:
try:
temp = bcode.bdecode(this._buffer)
except ValueError:
temp = None
if temp == None:
return
temp_len = len(bcode.bencode(temp))
this._buffer = this._buffer[temp_len:]
map(lambda f: f(temp), this._cb)
def getInfo():
path = os.getcwd()
sep = os.sep
torrentSet = [
filename for filename in os.listdir(path)
if filename.split('.')[-1] == 'torrent'
]
for i in torrentSet:
with open(path + sep + i, 'rb') as f:
c = bdecode(str(f.read())[2:-1])
print(c.keys())
# print(str(f.read())[2:-1])
break
def extract(torrent_file, dest=sys.stdout): with open(torrent_file) as f: data = f.read() decoded = bcode.bdecode(data) if not 'url-list' in decoded: print >>sys.stderr, "No webseeds found!" sys.exit(2) wseed = decoded['url-list'][0] infoh = decoded['info'] tname = infoh['name'] if 'length' in infoh: #single file torrent print >>dest, posixpath.join(wseed, urllib.pathname2url(tname)) else: #multi file torrent files = map(lambda x: posixpath.join(*(x['path'])), infoh['files']) for fi in files: fpath = urllib.pathname2url(posixpath.join(tname, fi)) print >>dest, posixpath.join(wseed, fpath)
def scrape_http(parsed_tracker, hashes):
qs = []
for hash in hashes:
url_param = binascii.a2b_hex(hash)
qs.append(("info_hash", url_param))
qs = urllib.urlencode(qs)
pt = parsed_tracker
url = urlunsplit((pt.scheme, pt.netloc, pt.path, qs, pt.fragment))
handle = urllib.urlopen(url)
if handle.getcode() is not 200:
raise RuntimeError("%s status code returned" % handle.getcode())
decoded = bdecode(handle.read())
ret = {}
for hash, stats in decoded['files'].iteritems():
nice_hash = binascii.b2a_hex(hash)
s = stats["complete"]
p = stats["incomplete"]
c = stats["downloaded"]
ret[nice_hash] = {"seeds": s, "peers": p, "complete": c}
return ret
def compromatFunction(ip):
connection1 = Connection('127.0.0.1', 27017)
db1 = connection1.ips
temp = db1.test.find_one({ "_id" : ip })
if temp == None:
return []
#print temp
connection2 = Connection('144.76.168.108', 27017)
connection2.metadata.authenticate('mipt', 'mipt')
db2 = connection2.metadata
item = db2.bcoded_metadata.find({"_id" : { "$in" :temp['hashes'][:15]}})
if item == None:
return []
temp = []
for it in item:
meta = bcode.bdecode(it['bcoded_metadata'])
if meta['name'].strip() != "" and meta['name'] is not(None):
temp.append([ it["_id"], meta['name'] ])
return temp
def scrape_http(parsed_tracker, hashes):
#print "Scraping HTTP: %s for %s hashes" % (parsed_tracker.geturl(), len(hashes))
qs = []
for hash in hashes:
url_param = binascii.a2b_hex(hash)
qs.append(("info_hash", url_param))
qs = urllib.urlencode(qs)
pt = parsed_tracker
url = urlunsplit((pt.scheme, pt.netloc, pt.path, qs, pt.fragment))
handle = urllib.urlopen(url);
if handle.getcode() is not 200:
raise RuntimeError("%s status code returned" % handle.getcode())
decoded = bdecode(handle.read())
ret = {}
for hash, stats in decoded['files'].iteritems():
nice_hash = binascii.b2a_hex(hash)
s = stats["complete"]
p = stats["incomplete"]
c = stats["downloaded"]
ret[nice_hash] = { "seeds" : s, "peers" : p, "complete" : c}
return ret
def test_decode_complex_1(self):
self.assertEqual(bcode.bdecode('d4:teamld4:name3:bob3:agei30e6:skillsl6:python4:htmleed4:name5:jimmy3:agei32e6:skillsleeee'),
{'team': [{'skills': ['python', 'html'], 'age': 30, 'name': 'bob'}, {'skills': [], 'age': 32, 'name': 'jimmy'}]})
def test_decode_single_float(self):
self.assertEqual(bcode.bdecode('3:3.3'), '3.3')
def test_decode_single_list(self):
self.assertEqual(bcode.bdecode('l4:spam4:eggse'), ['spam', 'eggs'])
def test_decode_short_string(self):
self.assertEqual(' egg', bcode.bdecode('4: egg'))
def test_decode_single_unicode_string(self):
self.assertEqual(u'pão'.encode('utf-8'), bcode.bdecode('4:p\xc3\xa3o'))
def test_decode_single_string_2(self):
self.assertEqual(bcode.bdecode('13:spam and eggs'), 'spam and eggs')
def test_decode_single_string_1(self):
self.assertEqual(bcode.bdecode('4:spam'), 'spam')
def test_decode_dict_in_dict(self):
self.assertEqual(bcode.bdecode('d4:spamd1:a1:bee'), {'spam': {'a': 'b'}})
def test_decode_list_in_dict(self):
self.assertEqual(bcode.bdecode('d4:spaml1:a1:bee'), {'spam': ['a', 'b']})
def decode(self, arg=None):
return bcode.bdecode(arg)
def test_decode_list_in_list(self):
self.assertEqual(bcode.bdecode('l4:spaml1:a1:bee'), ['spam', ['a', 'b']])
def test_decode_single_integer(self):
self.assertEqual(bcode.bdecode('i3e'), 3)
def test_decode_dicts_in_list(self):
self.assertEqual(bcode.bdecode('ld1:ai1e1:bi2eed1:ci3e1:di4eee'),
[{'a': 1, 'b': 2}, {'c': 3, 'd': 4}])
def test_decode_single_dict(self):
self.assertEqual(bcode.bdecode('d3:cow3:moo4:spam4:eggse'),
{'cow': 'moo', 'spam': 'eggs'})
