def main():
db = database.get_cursor()
db.execute("SELECT torrent_id, tracker_id, hash, tracker_url, last_scrape FROM vw_scrape")
all_trackers = {}
for x in range(0, db.rowcount):
row = db.fetchone()
turl = row["tracker_url"]
hash = row["hash"]
if turl in all_trackers.iterkeys():
if hash not in all_trackers[turl]:
all_trackers[turl].append(hash)
else:
all_trackers[turl] = [hash]
for tracker, hashes in all_trackers.iteritems():
try:
result = scrape(tracker, hashes)
if result:
for hash, stats in result.iteritems():
db.execute("""UPDATE tracker SET seeds=%s, leechers=%s, completed=%s,
last_scrape=CURRENT_TIMESTAMP, scrape_error=NULL
WHERE torrent_id=(SELECT id FROM torrent WHERE hash=%s) AND tracker_url=%s""",
(stats["seeds"], stats["peers"], stats["complete"], hash, tracker))
db.commit()
except (RuntimeError, NameError, ValueError, socket.timeout) as e:
print "Error: %s" % e
format_strings = ','.join(["'%s'"] * len(hashes)) % tuple(hashes)
db.execute("""UPDATE tracker SET last_scrape=CURRENT_TIMESTAMP, scrape_error=%s
WHERE tracker_url=%s AND torrent_id IN (SELECT id FROM torrent WHERE hash IN (""" + format_strings + "))", (e, tracker))
db.commit()
db.close()
def index():
db = database.get_cursor()
db.execute(
"SELECT hash, name, total_size_bytes FROM torrent WHERE retrieving_data=0 ORDER BY create_date DESC LIMIT 10"
)
lastten = db.fetchall()
db.close()
return template("index.html", lastten=lastten)
def add_to_database(hash, full_magnet_uri=None, already_exists=False, fetch_metadata=True):
db = database.get_cursor()
if not already_exists:
db.execute("INSERT INTO torrent(hash, retrieving_data) VALUES (%s, 1)", (hash))
db.commit()
if fetch_metadata:
magnet_uri = full_magnet_uri if full_magnet_uri else get_magnet_uri(hash)
thread.start_new_thread(fetch_magnet, (magnet_uri,))
db.close()
def get_base64_metadata(hash, decode=False):
db = database.get_cursor()
db.execute("SELECT base64_metadata FROM torrent WHERE hash=%s", hash)
if (db.rowcount > 0):
data = db.fetchone()
db.close()
metadata = data['base64_metadata']
return base64.b64decode(metadata) if decode else metadata
db.close()
return None
def get_base64_metadata(hash, decode=False):
db = database.get_cursor()
db.execute("SELECT base64_metadata FROM torrent WHERE hash=%s", hash)
if (db.rowcount > 0):
data = db.fetchone()
db.close()
metadata = data['base64_metadata']
return base64.b64decode(metadata) if decode else metadata
db.close()
return None
def add_to_database(hash,
full_magnet_uri=None,
already_exists=False,
fetch_metadata=True):
db = database.get_cursor()
if not already_exists:
db.execute("INSERT INTO torrent(hash, retrieving_data) VALUES (%s, 1)",
(hash))
db.commit()
if fetch_metadata:
magnet_uri = full_magnet_uri if full_magnet_uri else get_magnet_uri(
hash)
thread.start_new_thread(fetch_magnet, (magnet_uri, ))
db.close()
def scrape_trackers(hash, tracker_list):
db = database.get_cursor()
for url in tracker_list:
try:
result = scrape(url, [hash])
for hash, stats in result.iteritems():
db.execute("""UPDATE tracker SET seeds=%s, leechers=%s, completed=%s,
last_scrape=CURRENT_TIMESTAMP, scrape_error=NULL
WHERE torrent_id=(SELECT id FROM torrent WHERE hash=%s) AND tracker_url=%s""",
(stats["seeds"], stats["peers"], stats["complete"], hash, url))
db.commit()
except (RuntimeError, NameError, ValueError, socket.timeout) as e:
db.execute("""UPDATE tracker SET last_scrape=CURRENT_TIMESTAMP, scrape_error=%s
WHERE tracker_url=%s AND torrent_id=(SELECT id FROM torrent WHERE hash=%s)""", (e, url, hash))
db.commit()
db.close()
def add_from_torrent_info(info, torrent_metadata=None):
db = database.get_cursor()
torrent_hash = str(info.info_hash())
if is_in_database(torrent_hash):
db.execute("DELETE FROM torrent WHERE hash=%s", (torrent_hash))
total_size = sum([f.size for f in info.files()])
db.execute("INSERT INTO torrent(hash, name, total_size_bytes, retrieving_data, base64_metadata) VALUES (%s, %s, %s, 0, %s)",
(torrent_hash, info.name(), total_size, base64.b64encode(torrent_metadata)))
torrent_id = db.lastrowid
for f in info.files():
db.execute("INSERT INTO file(torrent_id, name, full_location, length_bytes) VALUES (%s, %s, %s, %s)",
(torrent_id, os.path.basename(f.path), f.path, f.size))
for t in info.trackers():
db.execute("INSERT INTO tracker(torrent_id, tracker_url) VALUES (%s, %s)",
(torrent_id, t.url))
db.commit()
db.close()
scrape_trackers(torrent_hash, [t.url for t in info.trackers()])
def scrape_trackers(hash, tracker_list):
db = database.get_cursor()
for url in tracker_list:
try:
result = scrape(url, [hash])
for hash, stats in result.iteritems():
db.execute(
"""UPDATE tracker SET seeds=%s, leechers=%s, completed=%s,
last_scrape=CURRENT_TIMESTAMP, scrape_error=NULL
WHERE torrent_id=(SELECT id FROM torrent WHERE hash=%s) AND tracker_url=%s""",
(stats["seeds"], stats["peers"], stats["complete"], hash,
url))
db.commit()
except (RuntimeError, NameError, ValueError, socket.timeout) as e:
db.execute(
"""UPDATE tracker SET last_scrape=CURRENT_TIMESTAMP, scrape_error=%s
WHERE tracker_url=%s AND torrent_id=(SELECT id FROM torrent WHERE hash=%s)""",
(e, url, hash))
db.commit()
db.close()
def add_from_torrent_info(info, torrent_metadata=None):
db = database.get_cursor()
torrent_hash = str(info.info_hash())
if is_in_database(torrent_hash):
db.execute("DELETE FROM torrent WHERE hash=%s", (torrent_hash))
total_size = sum([f.size for f in info.files()])
db.execute(
"INSERT INTO torrent(hash, name, total_size_bytes, retrieving_data, base64_metadata) VALUES (%s, %s, %s, 0, %s)",
(torrent_hash, info.name(), total_size,
base64.b64encode(torrent_metadata)))
torrent_id = db.lastrowid
for f in info.files():
db.execute(
"INSERT INTO file(torrent_id, name, full_location, length_bytes) VALUES (%s, %s, %s, %s)",
(torrent_id, os.path.basename(f.path), f.path, f.size))
for t in info.trackers():
db.execute(
"INSERT INTO tracker(torrent_id, tracker_url) VALUES (%s, %s)",
(torrent_id, t.url))
db.commit()
db.close()
scrape_trackers(torrent_hash, [t.url for t in info.trackers()])
def main():
db = database.get_cursor()
db.execute(
"SELECT torrent_id, tracker_id, hash, tracker_url, last_scrape FROM vw_scrape"
)
all_trackers = {}
for x in range(0, db.rowcount):
row = db.fetchone()
turl = row["tracker_url"]
hash = row["hash"]
if turl in all_trackers.iterkeys():
if hash not in all_trackers[turl]:
all_trackers[turl].append(hash)
else:
all_trackers[turl] = [hash]
for tracker, hashes in all_trackers.iteritems():
try:
result = scrape(tracker, hashes)
if result:
for hash, stats in result.iteritems():
db.execute(
"""UPDATE tracker SET seeds=%s, leechers=%s, completed=%s,
last_scrape=CURRENT_TIMESTAMP, scrape_error=NULL
WHERE torrent_id=(SELECT id FROM torrent WHERE hash=%s) AND tracker_url=%s""",
(stats["seeds"], stats["peers"], stats["complete"],
hash, tracker))
db.commit()
except (RuntimeError, NameError, ValueError, socket.timeout) as e:
print "Error: %s" % e
format_strings = ','.join(["'%s'"] * len(hashes)) % tuple(hashes)
db.execute(
"""UPDATE tracker SET last_scrape=CURRENT_TIMESTAMP, scrape_error=%s
WHERE tracker_url=%s AND torrent_id IN (SELECT id FROM torrent WHERE hash IN ("""
+ format_strings + "))", (e, tracker))
db.commit()
db.close()
def is_in_database(hash):
db = database.get_cursor()
db.execute("SELECT id FROM torrent WHERE hash=%s", (hash,))
ret = db.rowcount > 0
db.close()
return ret
def info(hash=None):
"""
<h5>Description:</h5>
<p>
Checks the information we have of a torrent that has been added.
If the torrent metadata is still being retrieved, 'torrent.retrieving_data' will be 1.
If the torrent metadata has been retrieved, 'torrent.download_link' will have a value
</p>
<h5>Parameters:</h5>
<ul>
<li><strong>hash</strong> - The hash returned from the /api/upload call</li>
</ul>
<h5>Returns:</h5>
<p>
On success (metadata is still downloading):
<pre>
{
"data": {
"torrent": {
"retrieving_data": 1,
"hash": "ddceab34ac388ca56b0cdbc6eb726ca1844233c6"
}
},
"success": true
}
</pre>
On success (metadata has downloaded):
<pre>
{
"data": {
"files": [{
"size_bytes": 392560640,
"full_location": "Pioneer.One.S01E03.Xvid-VODO/Pioneer.One.S01E03.Xvid-VODO.avi",
"name": "Pioneer.One.S01E03.Xvid-VODO.avi"
}],
"torrent": {
"hash": "ddceab34ac388ca56b0cdbc6eb726ca1844233c5",
"name": "Pioneer.One.S01E03.Xvid-VODO",
"download_link": "/api/metadata/ddceab34ac388ca56b0cdbc6eb726ca1844233c5.torrent",
"total_size_bytes": 402477447
},
"trackers": [{
"leechers": 0,
"completed": 0,
"seeds": 0,
"tracker_url": "udp://tracker.openbittorrent.com:80"
}]
},
"success": true
}
</pre>
On error:
<pre>
{
"message": "Torrent ddceab34ac388ca56b0cdbc6eb726ca1844233c6 is not in database! Call /api/upload to add it",
"success": false
}
</pre>
</p>
<h5>Example:</h5>
<p>
<strong>Check the info of torrent 'ddceab34ac388ca56b0cdbc6eb726ca1844233c6'</strong><br />
/api/info/ddceab34ac388ca56b0cdbc6eb726ca1844233c6
</p>
"""
if not hash or not is_hash(hash):
return api_error("%s is not a valid hash" % hash)
db = database.get_cursor()
db.execute(
"SELECT id, hash, name, total_size_bytes, retrieving_data FROM torrent WHERE hash=%s",
hash)
torrent = db.fetchone()
if not torrent:
return api_error(
"Torrent %s is not in database! Call /api/upload to add it" % hash)
id = torrent["id"]
del torrent["id"]
if torrent["retrieving_data"]:
del torrent["total_size_bytes"]
del torrent["name"]
return api_success({"torrent": torrent})
else:
del torrent["retrieving_data"]
torrent["download_link"] = get_url(
"/api/metadata/<hash:re:[a-fA-F0-9]{40}>.torrent", hash=hash)
torrent["nice_size"] = size(torrent["total_size_bytes"])
db.execute(
"SELECT tracker_url, seeds, leechers, completed, scrape_error FROM tracker WHERE torrent_id=%s",
id)
trackers = db.fetchall()
for tracker in trackers:
if not tracker["scrape_error"]:
del tracker["scrape_error"]
db.execute(
"SELECT name, full_location, length_bytes as size_bytes FROM file WHERE torrent_id=%s",
id)
files = db.fetchall()
for file in files:
file["nice_size"] = size(file["size_bytes"])
db.close()
return api_success({
"torrent": torrent,
"files": files,
"trackers": trackers
})
def get_magnet_uri(hash):
#use these public trackers to help find the torrent via magnet, otherwise itll never be found
public_trackers = "tr=udp://tracker.openbittorrent.com:80&tr=udp://tracker.publicbt.com:80&tr=udp://tracker.istole.it:6969&tr=udp://tracker.ccc.de:80"
return "magnet:?xt=urn:btih:%s&%s" % (hash, public_trackers)
def scrape_trackers(hash, tracker_list):
db = database.get_cursor()
for url in tracker_list:
try:
result = scrape(url, [hash])
for hash, stats in result.iteritems():
db.execute("""UPDATE tracker SET seeds=%s, leechers=%s, completed=%s,
last_scrape=CURRENT_TIMESTAMP, scrape_error=NULL
WHERE torrent_id=(SELECT id FROM torrent WHERE hash=%s) AND tracker_url=%s""",
(stats["seeds"], stats["peers"], stats["complete"], hash, url))
db.commit()
except (RuntimeError, NameError, ValueError, socket.timeout) as e:
db.execute("""UPDATE tracker SET last_scrape=CURRENT_TIMESTAMP, scrape_error=%s
WHERE tracker_url=%s AND torrent_id=(SELECT id FROM torrent WHERE hash=%s)""", (e, url, hash))
db.commit()
db.close()
#get all torrents that were still trying to retrieve metadata and re-add them
db = database.get_cursor()
db.execute("SELECT hash FROM torrent WHERE retrieving_data = 1")
data = db.fetchall()
for item in data:
logger.debug("Reloading '%s' since its metadata hasnt been retrieved yet" % item["hash"])
add_to_database(item["hash"], get_magnet_uri(item["hash"]), True)
db.close()
def is_in_database(hash):
db = database.get_cursor()
db.execute("SELECT id FROM torrent WHERE hash=%s", (hash, ))
ret = db.rowcount > 0
db.close()
return ret
def info(hash=None):
"""
<h5>Description:</h5>
<p>
Checks the information we have of a torrent that has been added.
If the torrent metadata is still being retrieved, 'torrent.retrieving_data' will be 1.
If the torrent metadata has been retrieved, 'torrent.download_link' will have a value
</p>
<h5>Parameters:</h5>
<ul>
<li><strong>hash</strong> - The hash returned from the /api/upload call</li>
</ul>
<h5>Returns:</h5>
<p>
On success (metadata is still downloading):
<pre>
{
"data": {
"torrent": {
"retrieving_data": 1,
"hash": "ddceab34ac388ca56b0cdbc6eb726ca1844233c6"
}
},
"success": true
}
</pre>
On success (metadata has downloaded):
<pre>
{
"data": {
"files": [{
"size_bytes": 392560640,
"full_location": "Pioneer.One.S01E03.Xvid-VODO/Pioneer.One.S01E03.Xvid-VODO.avi",
"name": "Pioneer.One.S01E03.Xvid-VODO.avi"
}],
"torrent": {
"hash": "ddceab34ac388ca56b0cdbc6eb726ca1844233c5",
"name": "Pioneer.One.S01E03.Xvid-VODO",
"download_link": "/api/metadata/ddceab34ac388ca56b0cdbc6eb726ca1844233c5.torrent",
"total_size_bytes": 402477447
},
"trackers": [{
"leechers": 0,
"completed": 0,
"seeds": 0,
"tracker_url": "udp://tracker.openbittorrent.com:80"
}]
},
"success": true
}
</pre>
On error:
<pre>
{
"message": "Torrent ddceab34ac388ca56b0cdbc6eb726ca1844233c6 is not in database! Call /api/upload to add it",
"success": false
}
</pre>
</p>
<h5>Example:</h5>
<p>
<strong>Check the info of torrent 'ddceab34ac388ca56b0cdbc6eb726ca1844233c6'</strong><br />
/api/info/ddceab34ac388ca56b0cdbc6eb726ca1844233c6
</p>
"""
if not hash or not is_hash(hash):
return api_error("%s is not a valid hash" % hash)
db = database.get_cursor()
db.execute("SELECT id, hash, name, total_size_bytes, retrieving_data FROM torrent WHERE hash=%s", hash)
torrent = db.fetchone()
if not torrent:
return api_error("Torrent %s is not in database! Call /api/upload to add it" % hash)
id = torrent["id"]
del torrent["id"]
if torrent["retrieving_data"]:
del torrent["total_size_bytes"]
del torrent["name"]
return api_success({
"torrent" : torrent
})
else:
del torrent["retrieving_data"]
torrent["download_link"] = get_url("/api/metadata/<hash:re:[a-fA-F0-9]{40}>.torrent", hash=hash)
torrent["nice_size"] = size(torrent["total_size_bytes"])
db.execute("SELECT tracker_url, seeds, leechers, completed, scrape_error FROM tracker WHERE torrent_id=%s", id)
trackers = db.fetchall()
for tracker in trackers:
if not tracker["scrape_error"]:
del tracker["scrape_error"]
db.execute("SELECT name, full_location, length_bytes as size_bytes FROM file WHERE torrent_id=%s", id)
files = db.fetchall()
for file in files:
file["nice_size"] = size(file["size_bytes"])
db.close()
return api_success({
"torrent" : torrent, "files" : files, "trackers" : trackers
})
def scrape_trackers(hash, tracker_list):
db = database.get_cursor()
for url in tracker_list:
try:
result = scrape(url, [hash])
for hash, stats in result.iteritems():
db.execute(
"""UPDATE tracker SET seeds=%s, leechers=%s, completed=%s,
last_scrape=CURRENT_TIMESTAMP, scrape_error=NULL
WHERE torrent_id=(SELECT id FROM torrent WHERE hash=%s) AND tracker_url=%s""",
(stats["seeds"], stats["peers"], stats["complete"], hash,
url))
db.commit()
except (RuntimeError, NameError, ValueError, socket.timeout) as e:
db.execute(
"""UPDATE tracker SET last_scrape=CURRENT_TIMESTAMP, scrape_error=%s
WHERE tracker_url=%s AND torrent_id=(SELECT id FROM torrent WHERE hash=%s)""",
(e, url, hash))
db.commit()
db.close()
#get all torrents that were still trying to retrieve metadata and re-add them
db = database.get_cursor()
db.execute("SELECT hash FROM torrent WHERE retrieving_data = 1")
data = db.fetchall()
for item in data:
logger.debug("Reloading '%s' since its metadata hasnt been retrieved yet" %
item["hash"])
add_to_database(item["hash"], get_magnet_uri(item["hash"]), True)
db.close()
