def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False):
''' Stores a torrent to the database.
May throw TorrentExtraValidationException if the form/torrent fails
post-WTForm validation! Exception messages will also be added to their
relevant fields on the given form. '''
torrent_data = upload_form.torrent_file.parsed_data
# Anonymous uploaders and non-trusted uploaders
no_or_new_account = (not uploading_user
or (uploading_user.age < app.config['RATELIMIT_ACCOUNT_AGE']
and not uploading_user.is_trusted))
if app.config['RATELIMIT_UPLOADS'] and no_or_new_account:
now, torrent_count, next_time = check_uploader_ratelimit(uploading_user)
if next_time > now:
# This will flag the dialog in upload.html red and tell API users what's wrong
upload_form.ratelimit.errors = ["You've gone over the upload ratelimit."]
raise TorrentExtraValidationException()
# Delete exisiting torrent which is marked as deleted
if torrent_data.db_id is not None:
models.Torrent.query.filter_by(id=torrent_data.db_id).delete()
db.session.commit()
_delete_cached_torrent_file(torrent_data.db_id)
# The torrent has been validated and is safe to access with ['foo'] etc - all relevant
# keys and values have been checked for (see UploadForm in forms.py for details)
info_dict = torrent_data.torrent_dict['info']
changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict)
# Use uploader-given name or grab it from the torrent
display_name = upload_form.display_name.data.strip() or info_dict['name'].decode('utf8').strip()
information = (upload_form.information.data or '').strip()
description = (upload_form.description.data or '').strip()
torrent_filesize = info_dict.get('length') or sum(
f['length'] for f in info_dict.get('files'))
# In case no encoding, assume UTF-8.
torrent_encoding = torrent_data.torrent_dict.get('encoding', b'utf-8').decode('utf-8')
torrent = models.Torrent(id=torrent_data.db_id,
info_hash=torrent_data.info_hash,
display_name=display_name,
torrent_name=torrent_data.filename,
information=information,
description=description,
encoding=torrent_encoding,
filesize=torrent_filesize,
user=uploading_user,
uploader_ip=ip_address(flask.request.remote_addr).packed)
# Store bencoded info_dict
torrent.info = models.TorrentInfo(info_dict=torrent_data.bencoded_info_dict)
torrent.stats = models.Statistic()
torrent.has_torrent = True
# Fields with default value will be None before first commit, so set .flags
torrent.flags = 0
torrent.anonymous = upload_form.is_anonymous.data if uploading_user else True
torrent.hidden = upload_form.is_hidden.data
torrent.remake = upload_form.is_remake.data
torrent.complete = upload_form.is_complete.data
# Copy trusted status from user if possible
can_mark_trusted = uploading_user and uploading_user.is_trusted
# To do, automatically mark trusted if user is trusted unless user specifies otherwise
torrent.trusted = upload_form.is_trusted.data if can_mark_trusted else False
# Set category ids
torrent.main_category_id, torrent.sub_category_id = \
upload_form.category.parsed_data.get_category_ids()
# To simplify parsing the filelist, turn single-file torrent into a list
torrent_filelist = info_dict.get('files')
used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding
parsed_file_tree = dict()
if not torrent_filelist:
# If single-file, the root will be the file-tree (no directory)
file_tree_root = parsed_file_tree
torrent_filelist = [{'length': torrent_filesize, 'path': [info_dict['name']]}]
else:
# If multi-file, use the directory name as root for files
file_tree_root = parsed_file_tree.setdefault(
info_dict['name'].decode(used_path_encoding), {})
# Parse file dicts into a tree
for file_dict in torrent_filelist:
# Decode path parts from utf8-bytes
path_parts = [path_part.decode(used_path_encoding) for path_part in file_dict['path']]
filename = path_parts.pop()
current_directory = file_tree_root
for directory in path_parts:
current_directory = current_directory.setdefault(directory, {})
# Don't add empty filenames (BitComet directory)
if filename:
current_directory[filename] = file_dict['length']
parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)
json_bytes = json.dumps(parsed_file_tree, separators=(',', ':')).encode('utf8')
torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)
db.session.add(torrent)
db.session.flush()
# Store the users trackers
trackers = OrderedSet()
announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii')
if announce:
trackers.add(announce)
# List of lists with single item
announce_list = torrent_data.torrent_dict.get('announce-list', [])
for announce in announce_list:
trackers.add(announce[0].decode('ascii'))
# Store webseeds
# qBittorrent doesn't omit url-list but sets it as '' even when there are no webseeds
webseed_list = torrent_data.torrent_dict.get('url-list') or []
if isinstance(webseed_list, bytes):
webseed_list = [webseed_list] # qB doesn't contain a sole url in a list
webseeds = OrderedSet(webseed.decode('utf-8') for webseed in webseed_list)
# Remove our trackers, maybe? TODO ?
# Search for/Add trackers in DB
db_trackers = OrderedSet()
for announce in trackers:
tracker = models.Trackers.by_uri(announce)
# Insert new tracker if not found
if not tracker:
tracker = models.Trackers(uri=announce)
db.session.add(tracker)
db.session.flush()
elif tracker.is_webseed:
# If we have an announce marked webseed (user error, malicy?), reset it.
# Better to have "bad" announces than "hiding" proper announces in webseeds/url-list.
tracker.is_webseed = False
db.session.flush()
db_trackers.add(tracker)
# Same for webseeds
for webseed_url in webseeds:
webseed = models.Trackers.by_uri(webseed_url)
if not webseed:
webseed = models.Trackers(uri=webseed_url, is_webseed=True)
db.session.add(webseed)
db.session.flush()
# Don't add trackers into webseeds
if webseed.is_webseed:
db_trackers.add(webseed)
# Store tracker refs in DB
for order, tracker in enumerate(db_trackers):
torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
tracker_id=tracker.id, order=order)
db.session.add(torrent_tracker)
# Before final commit, validate the torrent again
validate_torrent_post_upload(torrent, upload_form)
db.session.commit()
# Store the actual torrent file as well
torrent_file = upload_form.torrent_file.data
if app.config.get('BACKUP_TORRENT_FOLDER'):
torrent_file.seek(0, 0)
torrent_dir = app.config['BACKUP_TORRENT_FOLDER']
if not os.path.exists(torrent_dir):
os.makedirs(torrent_dir)
torrent_path = os.path.join(torrent_dir, '{}.{}'.format(
torrent.id, secure_filename(torrent_file.filename)))
torrent_file.save(torrent_path)
torrent_file.close()
return torrent
def ghetto_import():
if flask.request.remote_addr != '127.0.0.1':
return flask.error(403)
torrent_file = flask.request.files.get('torrent')
try:
torrent_dict = bencode.decode(torrent_file)
# field.data.close()
except (bencode.MalformedBencodeException, UnicodeError):
return 'Malformed torrent file', 500
try:
forms._validate_torrent_metadata(torrent_dict)
except AssertionError as e:
return 'Malformed torrent metadata ({})'.format(e.args[0]), 500
try:
tracker_found = forms._validate_trackers(torrent_dict)
except AssertionError as e:
return 'Malformed torrent trackers ({})'.format(e.args[0]), 500
bencoded_info_dict = bencode.encode(torrent_dict['info'])
info_hash = utils.sha1_hash(bencoded_info_dict)
# Check if the info_hash exists already in the database
torrent = models.Torrent.by_info_hash(info_hash)
if not torrent:
return 'This torrent does not exists', 500
if torrent.has_torrent:
return 'This torrent already has_torrent', 500
# Torrent is legit, pass original filename and dict along
torrent_data = forms.TorrentFileData(filename=os.path.basename(
torrent_file.filename),
torrent_dict=torrent_dict,
info_hash=info_hash,
bencoded_info_dict=bencoded_info_dict)
# The torrent has been validated and is safe to access with ['foo'] etc - all relevant
# keys and values have been checked for (see UploadForm in forms.py for details)
info_dict = torrent_data.torrent_dict['info']
changed_to_utf8 = backend._replace_utf8_values(torrent_data.torrent_dict)
torrent_filesize = info_dict.get('length') or sum(
f['length'] for f in info_dict.get('files'))
# In case no encoding, assume UTF-8.
torrent_encoding = torrent_data.torrent_dict.get('encoding',
b'utf-8').decode('utf-8')
# Store bencoded info_dict
torrent.info = models.TorrentInfo(
info_dict=torrent_data.bencoded_info_dict)
torrent.has_torrent = True
# To simplify parsing the filelist, turn single-file torrent into a list
torrent_filelist = info_dict.get('files')
used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding
parsed_file_tree = dict()
if not torrent_filelist:
# If single-file, the root will be the file-tree (no directory)
file_tree_root = parsed_file_tree
torrent_filelist = [{
'length': torrent_filesize,
'path': [info_dict['name']]
}]
else:
# If multi-file, use the directory name as root for files
file_tree_root = parsed_file_tree.setdefault(
info_dict['name'].decode(used_path_encoding), {})
# Parse file dicts into a tree
for file_dict in torrent_filelist:
# Decode path parts from utf8-bytes
path_parts = [
path_part.decode(used_path_encoding)
for path_part in file_dict['path']
]
filename = path_parts.pop()
current_directory = file_tree_root
for directory in path_parts:
current_directory = current_directory.setdefault(directory, {})
# Don't add empty filenames (BitComet directory)
if filename:
current_directory[filename] = file_dict['length']
parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)
json_bytes = json.dumps(parsed_file_tree,
separators=(',', ':')).encode('utf8')
torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)
db.session.add(torrent)
db.session.flush()
# Store the users trackers
trackers = OrderedSet()
announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii')
if announce:
trackers.add(announce)
# List of lists with single item
announce_list = torrent_data.torrent_dict.get('announce-list', [])
for announce in announce_list:
trackers.add(announce[0].decode('ascii'))
# Remove our trackers, maybe? TODO ?
# Search for/Add trackers in DB
db_trackers = OrderedSet()
for announce in trackers:
tracker = models.Trackers.by_uri(announce)
# Insert new tracker if not found
if not tracker:
tracker = models.Trackers(uri=announce)
db.session.add(tracker)
db.session.flush()
db_trackers.add(tracker)
# Store tracker refs in DB
for order, tracker in enumerate(db_trackers):
torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
tracker_id=tracker.id,
order=order)
db.session.add(torrent_tracker)
db.session.commit()
return 'success'
def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False):
torrent_data = upload_form.torrent_file.parsed_data
# The torrent has been validated and is safe to access with ['foo'] etc - all relevant
# keys and values have been checked for (see UploadForm in forms.py for details)
info_dict = torrent_data.torrent_dict['info']
changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict)
# Use uploader-given name or grab it from the torrent
display_name = upload_form.display_name.data.strip(
) or info_dict['name'].decode('utf8').strip()
information = (upload_form.information.data or '').strip()
description = (upload_form.description.data or '').strip()
torrent_filesize = info_dict.get('length') or sum(
f['length'] for f in info_dict.get('files'))
# In case no encoding, assume UTF-8.
torrent_encoding = torrent_data.torrent_dict.get('encoding',
b'utf-8').decode('utf-8')
torrent = models.Torrent(info_hash=torrent_data.info_hash,
display_name=display_name,
torrent_name=torrent_data.filename,
information=information,
description=description,
encoding=torrent_encoding,
filesize=torrent_filesize,
user=uploading_user)
# Store bencoded info_dict
torrent.info = models.TorrentInfo(
info_dict=torrent_data.bencoded_info_dict)
torrent.stats = models.Statistic()
torrent.has_torrent = True
# Fields with default value will be None before first commit, so set .flags
torrent.flags = 0
torrent.anonymous = upload_form.is_anonymous.data if uploading_user else True
torrent.hidden = upload_form.is_hidden.data
torrent.remake = upload_form.is_remake.data
torrent.complete = upload_form.is_complete.data
# Copy trusted status from user if possible
torrent.trusted = (uploading_user.level >= models.UserLevelType.TRUSTED
) if uploading_user else False
# Set category ids
torrent.main_category_id, torrent.sub_category_id = upload_form.category.parsed_data.get_category_ids(
)
# To simplify parsing the filelist, turn single-file torrent into a list
torrent_filelist = info_dict.get('files')
used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding
parsed_file_tree = dict()
if not torrent_filelist:
# If single-file, the root will be the file-tree (no directory)
file_tree_root = parsed_file_tree
torrent_filelist = [{
'length': torrent_filesize,
'path': [info_dict['name']]
}]
else:
# If multi-file, use the directory name as root for files
file_tree_root = parsed_file_tree.setdefault(
info_dict['name'].decode(used_path_encoding), {})
# Parse file dicts into a tree
for file_dict in torrent_filelist:
# Decode path parts from utf8-bytes
path_parts = [
path_part.decode(used_path_encoding)
for path_part in file_dict['path']
]
filename = path_parts.pop()
current_directory = file_tree_root
for directory in path_parts:
current_directory = current_directory.setdefault(directory, {})
current_directory[filename] = file_dict['length']
parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)
json_bytes = json.dumps(parsed_file_tree,
separators=(',', ':')).encode('utf8')
torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)
db.session.add(torrent)
db.session.flush()
# Store the users trackers
trackers = OrderedSet()
announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii')
if announce:
trackers.add(announce)
# List of lists with single item
announce_list = torrent_data.torrent_dict.get('announce-list', [])
for announce in announce_list:
trackers.add(announce[0].decode('ascii'))
# Remove our trackers, maybe? TODO ?
# Search for/Add trackers in DB
db_trackers = OrderedSet()
for announce in trackers:
tracker = models.Trackers.by_uri(announce)
# Insert new tracker if not found
if not tracker:
tracker = models.Trackers(uri=announce)
db.session.add(tracker)
db_trackers.add(tracker)
db.session.flush()
# Store tracker refs in DB
for order, tracker in enumerate(db_trackers):
torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
tracker_id=tracker.id,
order=order)
db.session.add(torrent_tracker)
db.session.commit()
# Store the actual torrent file as well
torrent_file = upload_form.torrent_file.data
if app.config.get('BACKUP_TORRENT_FOLDER'):
torrent_file.seek(0, 0)
torrent_dir = app.config['BACKUP_TORRENT_FOLDER']
if not os.path.exists(torrent_dir):
os.makedirs(torrent_dir)
torrent_path = os.path.join(
torrent_dir,
'{}.{}'.format(torrent.id, secure_filename(torrent_file.filename)))
torrent_file.save(torrent_path)
torrent_file.close()
return torrent
def api_upload(upload_request):
if upload_request.method == 'POST':
j = None
torrent_file = None
try:
if 'json' in upload_request.files:
f = upload_request.files['json']
j = json.loads(f.read().decode('utf-8'))
if DEBUG_API:
print(json.dumps(j, indent=4))
_json_keys = [
'username', 'password', 'display_name', 'main_cat',
'sub_cat', 'flags'
] # 'information' and 'description' are not required
# Check that required fields are present
for _k in _json_keys:
if _k not in j.keys():
return flask.make_response(
flask.jsonify({
"Error":
"Missing JSON field: {0}.".format(_k)
}), 400)
# Check that no extra fields are present
for k in j.keys():
if k not in [
'username', 'password', 'display_name', 'main_cat',
'sub_cat', 'information', 'description', 'flags'
]:
return flask.make_response(
flask.jsonify(
{"Error": "Incorrect JSON field(s)."}), 400)
else:
return flask.make_response(
flask.jsonify({"Error": "No metadata."}), 400)
if 'torrent' in upload_request.files:
f = upload_request.files['torrent']
if DEBUG_API:
print(f.filename)
torrent_file = f
# print(f.read())
else:
return flask.make_response(
flask.jsonify({"Error": "No torrent file."}), 400)
# 'username' and 'password' must have been provided as they are part of j.keys()
username = j['username']
password = j['password']
# Validate that the provided username and password belong to a valid user
user = models.User.by_username(username)
if not user:
user = models.User.by_email(username)
if not user or password != user.password_hash or user.status == models.UserStatusType.INACTIVE:
return flask.make_response(
flask.jsonify({"Error": "Incorrect username or password"}),
403)
current_user = user
display_name = j['display_name']
if (len(display_name) < 3) or (len(display_name) > 1024):
return flask.make_response(
flask.jsonify({
"Error":
"Torrent name must be between 3 and 1024 characters."
}), 400)
main_cat_name = j['main_cat']
sub_cat_name = j['sub_cat']
cat_subcat_status, cat_id, sub_cat_id = validate_main_sub_cat(
main_cat_name, sub_cat_name)
if not cat_subcat_status:
return flask.make_response(
flask.jsonify(
{"Error": "Incorrect Category / Sub-Category."}), 400)
# TODO Sanitize information
information = None
try:
information = j['information']
if len(information) > 255:
return flask.make_response(
flask.jsonify({
"Error":
"Information is limited to 255 characters."
}), 400)
except Exception as e:
information = ''
# TODO Sanitize description
description = None
try:
description = j['description']
if len(description) > (10 * 1024):
return flask.make_response(
flask.jsonify({
"Error":
"Description is limited to {0} characters.".format(
10 * 1024)
}), 403)
except Exception as e:
description = ''
v_flags = validate_torrent_flags(j['flags'])
if v_flags:
torrent_flags = j['flags']
else:
return flask.make_response(
flask.jsonify({"Error": "Incorrect torrent flags."}), 400)
torrent_status, torrent_data = validate_torrent_file(
torrent_file.filename, torrent_file.read()) # Needs validation
if not torrent_status:
return flask.make_response(
flask.jsonify(
{"Error": "Invalid or Duplicate torrent file."}), 400)
# The torrent has been validated and is safe to access with ['foo'] etc - all relevant
# keys and values have been checked for (see UploadForm in forms.py for details)
info_dict = torrent_data.torrent_dict['info']
changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict)
torrent_filesize = info_dict.get('length') or sum(
f['length'] for f in info_dict.get('files'))
# In case no encoding, assume UTF-8.
torrent_encoding = torrent_data.torrent_dict.get(
'encoding', b'utf-8').decode('utf-8')
torrent = models.Torrent(info_hash=torrent_data.info_hash,
display_name=display_name,
torrent_name=torrent_data.filename,
information=information,
description=description,
encoding=torrent_encoding,
filesize=torrent_filesize,
user=current_user)
# Store bencoded info_dict
torrent.info = models.TorrentInfo(
info_dict=torrent_data.bencoded_info_dict)
torrent.stats = models.Statistic()
torrent.has_torrent = True
# Fields with default value will be None before first commit, so set .flags
torrent.flags = 0
torrent.anonymous = True if torrent_flags[0] else False
torrent.hidden = True if torrent_flags[1] else False
torrent.remake = True if torrent_flags[2] else False
torrent.complete = True if torrent_flags[3] else False
# Copy trusted status from user if possible
torrent.trusted = (
current_user.level >= models.UserLevelType.TRUSTED
) if current_user else False
# Set category ids
torrent.main_category_id = cat_id
torrent.sub_category_id = sub_cat_id
# To simplify parsing the filelist, turn single-file torrent into a list
torrent_filelist = info_dict.get('files')
used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding
parsed_file_tree = dict()
if not torrent_filelist:
# If single-file, the root will be the file-tree (no directory)
file_tree_root = parsed_file_tree
torrent_filelist = [{
'length': torrent_filesize,
'path': [info_dict['name']]
}]
else:
# If multi-file, use the directory name as root for files
file_tree_root = parsed_file_tree.setdefault(
info_dict['name'].decode(used_path_encoding), {})
# Parse file dicts into a tree
for file_dict in torrent_filelist:
# Decode path parts from utf8-bytes
path_parts = [
path_part.decode(used_path_encoding)
for path_part in file_dict['path']
]
filename = path_parts.pop()
current_directory = file_tree_root
for directory in path_parts:
current_directory = current_directory.setdefault(
directory, {})
current_directory[filename] = file_dict['length']
parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)
json_bytes = json.dumps(parsed_file_tree,
separators=(',', ':')).encode('utf8')
torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)
db.session.add(torrent)
db.session.flush()
# Store the users trackers
trackers = OrderedSet()
announce = torrent_data.torrent_dict.get('announce',
b'').decode('ascii')
if announce:
trackers.add(announce)
# List of lists with single item
announce_list = torrent_data.torrent_dict.get('announce-list', [])
for announce in announce_list:
trackers.add(announce[0].decode('ascii'))
# Remove our trackers, maybe? TODO ?
# Search for/Add trackers in DB
db_trackers = OrderedSet()
for announce in trackers:
tracker = models.Trackers.by_uri(announce)
# Insert new tracker if not found
if not tracker:
tracker = models.Trackers(uri=announce)
db.session.add(tracker)
db_trackers.add(tracker)
db.session.flush()
# Store tracker refs in DB
for order, tracker in enumerate(db_trackers):
torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
tracker_id=tracker.id,
order=order)
db.session.add(torrent_tracker)
db.session.commit()
if app.config.get('BACKUP_TORRENT_FOLDER'):
torrent_file.seek(0, 0)
torrent_path = os.path.join(
app.config['BACKUP_TORRENT_FOLDER'],
'{}.{}'.format(torrent.id,
secure_filename(torrent_file.filename)))
torrent_file.save(torrent_path)
torrent_file.close()
#print('Success? {0}'.format(torrent.id))
return flask.make_response(
flask.jsonify({
"Success":
"Request was processed {0}".format(torrent.id)
}), 200)
except Exception as e:
print('Exception: {0}'.format(e))
return flask.make_response(
flask.jsonify({
"Error":
"Incorrect JSON. Please see HELP page for examples."
}), 400)
else:
return flask.make_response(flask.jsonify({"Error": "Bad request"}),
400)
