예제 #1
0
파일: backend.py 프로젝트: cybort/nyaa-1
def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False):
    torrent_data = upload_form.torrent_file.parsed_data

    # The torrent has been  validated and is safe to access with ['foo'] etc - all relevant
    # keys and values have been checked for (see UploadForm in forms.py for details)
    info_dict = torrent_data.torrent_dict['info']

    changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict)

    # Use uploader-given name or grab it from the torrent
    display_name = upload_form.display_name.data.strip(
    ) or info_dict['name'].decode('utf8').strip()
    information = (upload_form.information.data or '').strip()
    description = (upload_form.description.data or '').strip()

    torrent_filesize = info_dict.get('length') or sum(
        f['length'] for f in info_dict.get('files'))

    # In case no encoding, assume UTF-8.
    torrent_encoding = torrent_data.torrent_dict.get('encoding',
                                                     b'utf-8').decode('utf-8')

    torrent = models.Torrent(info_hash=torrent_data.info_hash,
                             display_name=display_name,
                             torrent_name=torrent_data.filename,
                             information=information,
                             description=description,
                             encoding=torrent_encoding,
                             filesize=torrent_filesize,
                             user=uploading_user)

    # Store bencoded info_dict
    torrent.info = models.TorrentInfo(
        info_dict=torrent_data.bencoded_info_dict)
    torrent.stats = models.Statistic()
    torrent.has_torrent = True

    # Fields with default value will be None before first commit, so set .flags
    torrent.flags = 0

    torrent.anonymous = upload_form.is_anonymous.data if uploading_user else True
    torrent.hidden = upload_form.is_hidden.data
    torrent.remake = upload_form.is_remake.data
    torrent.complete = upload_form.is_complete.data
    # Copy trusted status from user if possible
    torrent.trusted = (uploading_user.level >= models.UserLevelType.TRUSTED
                       ) if uploading_user else False
    # Set category ids
    torrent.main_category_id, torrent.sub_category_id = upload_form.category.parsed_data.get_category_ids(
    )

    # To simplify parsing the filelist, turn single-file torrent into a list
    torrent_filelist = info_dict.get('files')

    used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding

    parsed_file_tree = dict()
    if not torrent_filelist:
        # If single-file, the root will be the file-tree (no directory)
        file_tree_root = parsed_file_tree
        torrent_filelist = [{
            'length': torrent_filesize,
            'path': [info_dict['name']]
        }]
    else:
        # If multi-file, use the directory name as root for files
        file_tree_root = parsed_file_tree.setdefault(
            info_dict['name'].decode(used_path_encoding), {})

    # Parse file dicts into a tree
    for file_dict in torrent_filelist:
        # Decode path parts from utf8-bytes
        path_parts = [
            path_part.decode(used_path_encoding)
            for path_part in file_dict['path']
        ]

        filename = path_parts.pop()
        current_directory = file_tree_root

        for directory in path_parts:
            current_directory = current_directory.setdefault(directory, {})

        current_directory[filename] = file_dict['length']

    parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)

    json_bytes = json.dumps(parsed_file_tree,
                            separators=(',', ':')).encode('utf8')
    torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)

    db.session.add(torrent)
    db.session.flush()

    # Store the users trackers
    trackers = OrderedSet()
    announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii')
    if announce:
        trackers.add(announce)

    # List of lists with single item
    announce_list = torrent_data.torrent_dict.get('announce-list', [])
    for announce in announce_list:
        trackers.add(announce[0].decode('ascii'))

    # Remove our trackers, maybe? TODO ?

    # Search for/Add trackers in DB
    db_trackers = OrderedSet()
    for announce in trackers:
        tracker = models.Trackers.by_uri(announce)

        # Insert new tracker if not found
        if not tracker:
            tracker = models.Trackers(uri=announce)
            db.session.add(tracker)

        db_trackers.add(tracker)

    db.session.flush()

    # Store tracker refs in DB
    for order, tracker in enumerate(db_trackers):
        torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
                                                 tracker_id=tracker.id,
                                                 order=order)
        db.session.add(torrent_tracker)

    db.session.commit()

    # Store the actual torrent file as well
    torrent_file = upload_form.torrent_file.data
    if app.config.get('BACKUP_TORRENT_FOLDER'):
        torrent_file.seek(0, 0)

        torrent_dir = app.config['BACKUP_TORRENT_FOLDER']
        if not os.path.exists(torrent_dir):
            os.makedirs(torrent_dir)

        torrent_path = os.path.join(
            torrent_dir,
            '{}.{}'.format(torrent.id, secure_filename(torrent_file.filename)))
        torrent_file.save(torrent_path)
    torrent_file.close()

    return torrent
예제 #2
0
파일: backend.py 프로젝트: zzdespair/nyaa
def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False):
    ''' Stores a torrent to the database.
        May throw TorrentExtraValidationException if the form/torrent fails
        post-WTForm validation! Exception messages will also be added to their
        relevant fields on the given form. '''
    torrent_data = upload_form.torrent_file.parsed_data

    # Anonymous uploaders and non-trusted uploaders
    no_or_new_account = (not uploading_user
                         or (uploading_user.age < app.config['RATELIMIT_ACCOUNT_AGE']
                             and not uploading_user.is_trusted))

    if app.config['RATELIMIT_UPLOADS'] and no_or_new_account:
        now, torrent_count, next_time = check_uploader_ratelimit(uploading_user)
        if next_time > now:
            # This will flag the dialog in upload.html red and tell API users what's wrong
            upload_form.ratelimit.errors = ["You've gone over the upload ratelimit."]
            raise TorrentExtraValidationException()

    # Delete exisiting torrent which is marked as deleted
    if torrent_data.db_id is not None:
        models.Torrent.query.filter_by(id=torrent_data.db_id).delete()
        db.session.commit()
        _delete_cached_torrent_file(torrent_data.db_id)

    # The torrent has been  validated and is safe to access with ['foo'] etc - all relevant
    # keys and values have been checked for (see UploadForm in forms.py for details)
    info_dict = torrent_data.torrent_dict['info']

    changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict)

    # Use uploader-given name or grab it from the torrent
    display_name = upload_form.display_name.data.strip() or info_dict['name'].decode('utf8').strip()
    information = (upload_form.information.data or '').strip()
    description = (upload_form.description.data or '').strip()

    torrent_filesize = info_dict.get('length') or sum(
        f['length'] for f in info_dict.get('files'))

    # In case no encoding, assume UTF-8.
    torrent_encoding = torrent_data.torrent_dict.get('encoding', b'utf-8').decode('utf-8')

    torrent = models.Torrent(id=torrent_data.db_id,
                             info_hash=torrent_data.info_hash,
                             display_name=display_name,
                             torrent_name=torrent_data.filename,
                             information=information,
                             description=description,
                             encoding=torrent_encoding,
                             filesize=torrent_filesize,
                             user=uploading_user,
                             uploader_ip=ip_address(flask.request.remote_addr).packed)

    # Store bencoded info_dict
    torrent.info = models.TorrentInfo(info_dict=torrent_data.bencoded_info_dict)
    torrent.stats = models.Statistic()
    torrent.has_torrent = True

    # Fields with default value will be None before first commit, so set .flags
    torrent.flags = 0

    torrent.anonymous = upload_form.is_anonymous.data if uploading_user else True
    torrent.hidden = upload_form.is_hidden.data
    torrent.remake = upload_form.is_remake.data
    torrent.complete = upload_form.is_complete.data
    # Copy trusted status from user if possible
    can_mark_trusted = uploading_user and uploading_user.is_trusted
    # To do, automatically mark trusted if user is trusted unless user specifies otherwise
    torrent.trusted = upload_form.is_trusted.data if can_mark_trusted else False

    # Set category ids
    torrent.main_category_id, torrent.sub_category_id = \
        upload_form.category.parsed_data.get_category_ids()

    # To simplify parsing the filelist, turn single-file torrent into a list
    torrent_filelist = info_dict.get('files')

    used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding

    parsed_file_tree = dict()
    if not torrent_filelist:
        # If single-file, the root will be the file-tree (no directory)
        file_tree_root = parsed_file_tree
        torrent_filelist = [{'length': torrent_filesize, 'path': [info_dict['name']]}]
    else:
        # If multi-file, use the directory name as root for files
        file_tree_root = parsed_file_tree.setdefault(
            info_dict['name'].decode(used_path_encoding), {})

    # Parse file dicts into a tree
    for file_dict in torrent_filelist:
        # Decode path parts from utf8-bytes
        path_parts = [path_part.decode(used_path_encoding) for path_part in file_dict['path']]

        filename = path_parts.pop()
        current_directory = file_tree_root

        for directory in path_parts:
            current_directory = current_directory.setdefault(directory, {})

        # Don't add empty filenames (BitComet directory)
        if filename:
            current_directory[filename] = file_dict['length']

    parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)

    json_bytes = json.dumps(parsed_file_tree, separators=(',', ':')).encode('utf8')
    torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)

    db.session.add(torrent)
    db.session.flush()

    # Store the users trackers
    trackers = OrderedSet()
    announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii')
    if announce:
        trackers.add(announce)

    # List of lists with single item
    announce_list = torrent_data.torrent_dict.get('announce-list', [])
    for announce in announce_list:
        trackers.add(announce[0].decode('ascii'))

    # Store webseeds
    # qBittorrent doesn't omit url-list but sets it as '' even when there are no webseeds
    webseed_list = torrent_data.torrent_dict.get('url-list') or []
    if isinstance(webseed_list, bytes):
        webseed_list = [webseed_list]  # qB doesn't contain a sole url in a list
    webseeds = OrderedSet(webseed.decode('utf-8') for webseed in webseed_list)

    # Remove our trackers, maybe? TODO ?

    # Search for/Add trackers in DB
    db_trackers = OrderedSet()
    for announce in trackers:
        tracker = models.Trackers.by_uri(announce)

        # Insert new tracker if not found
        if not tracker:
            tracker = models.Trackers(uri=announce)
            db.session.add(tracker)
            db.session.flush()
        elif tracker.is_webseed:
            # If we have an announce marked webseed (user error, malicy?), reset it.
            # Better to have "bad" announces than "hiding" proper announces in webseeds/url-list.
            tracker.is_webseed = False
            db.session.flush()

        db_trackers.add(tracker)

    # Same for webseeds
    for webseed_url in webseeds:
        webseed = models.Trackers.by_uri(webseed_url)

        if not webseed:
            webseed = models.Trackers(uri=webseed_url, is_webseed=True)
            db.session.add(webseed)
            db.session.flush()

        # Don't add trackers into webseeds
        if webseed.is_webseed:
            db_trackers.add(webseed)

    # Store tracker refs in DB
    for order, tracker in enumerate(db_trackers):
        torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
                                                 tracker_id=tracker.id, order=order)
        db.session.add(torrent_tracker)

    # Before final commit, validate the torrent again
    validate_torrent_post_upload(torrent, upload_form)

    db.session.commit()

    # Store the actual torrent file as well
    torrent_file = upload_form.torrent_file.data
    if app.config.get('BACKUP_TORRENT_FOLDER'):
        torrent_file.seek(0, 0)

        torrent_dir = app.config['BACKUP_TORRENT_FOLDER']
        if not os.path.exists(torrent_dir):
            os.makedirs(torrent_dir)

        torrent_path = os.path.join(torrent_dir, '{}.{}'.format(
            torrent.id, secure_filename(torrent_file.filename)))
        torrent_file.save(torrent_path)
    torrent_file.close()

    return torrent
예제 #3
0
def api_upload(upload_request):
    if upload_request.method == 'POST':
        j = None
        torrent_file = None
        try:
            if 'json' in upload_request.files:
                f = upload_request.files['json']
                j = json.loads(f.read().decode('utf-8'))
                if DEBUG_API:
                    print(json.dumps(j, indent=4))

                _json_keys = [
                    'username', 'password', 'display_name', 'main_cat',
                    'sub_cat', 'flags'
                ]  # 'information' and 'description' are not required
                # Check that required fields are present
                for _k in _json_keys:
                    if _k not in j.keys():
                        return flask.make_response(
                            flask.jsonify({
                                "Error":
                                "Missing JSON field: {0}.".format(_k)
                            }), 400)
                # Check that no extra fields are present
                for k in j.keys():
                    if k not in [
                            'username', 'password', 'display_name', 'main_cat',
                            'sub_cat', 'information', 'description', 'flags'
                    ]:
                        return flask.make_response(
                            flask.jsonify(
                                {"Error": "Incorrect JSON field(s)."}), 400)
            else:
                return flask.make_response(
                    flask.jsonify({"Error": "No metadata."}), 400)
            if 'torrent' in upload_request.files:
                f = upload_request.files['torrent']
                if DEBUG_API:
                    print(f.filename)
                torrent_file = f
                # print(f.read())
            else:
                return flask.make_response(
                    flask.jsonify({"Error": "No torrent file."}), 400)

            # 'username' and 'password' must have been provided as they are part of j.keys()
            username = j['username']
            password = j['password']
            # Validate that the provided username and password belong to a valid user
            user = models.User.by_username(username)

            if not user:
                user = models.User.by_email(username)

            if not user or password != user.password_hash or user.status == models.UserStatusType.INACTIVE:
                return flask.make_response(
                    flask.jsonify({"Error": "Incorrect username or password"}),
                    403)

            current_user = user

            display_name = j['display_name']
            if (len(display_name) < 3) or (len(display_name) > 1024):
                return flask.make_response(
                    flask.jsonify({
                        "Error":
                        "Torrent name must be between 3 and 1024 characters."
                    }), 400)

            main_cat_name = j['main_cat']
            sub_cat_name = j['sub_cat']

            cat_subcat_status, cat_id, sub_cat_id = validate_main_sub_cat(
                main_cat_name, sub_cat_name)
            if not cat_subcat_status:
                return flask.make_response(
                    flask.jsonify(
                        {"Error": "Incorrect Category / Sub-Category."}), 400)

            # TODO Sanitize information
            information = None
            try:
                information = j['information']
                if len(information) > 255:
                    return flask.make_response(
                        flask.jsonify({
                            "Error":
                            "Information is limited to 255 characters."
                        }), 400)
            except Exception as e:
                information = ''

            # TODO Sanitize description
            description = None
            try:
                description = j['description']
                if len(description) > (10 * 1024):
                    return flask.make_response(
                        flask.jsonify({
                            "Error":
                            "Description is limited to {0} characters.".format(
                                10 * 1024)
                        }), 403)
            except Exception as e:
                description = ''

            v_flags = validate_torrent_flags(j['flags'])
            if v_flags:
                torrent_flags = j['flags']
            else:
                return flask.make_response(
                    flask.jsonify({"Error": "Incorrect torrent flags."}), 400)

            torrent_status, torrent_data = validate_torrent_file(
                torrent_file.filename, torrent_file.read())  # Needs validation

            if not torrent_status:
                return flask.make_response(
                    flask.jsonify(
                        {"Error": "Invalid or Duplicate torrent file."}), 400)

            # The torrent has been  validated and is safe to access with ['foo'] etc - all relevant
            # keys and values have been checked for (see UploadForm in forms.py for details)
            info_dict = torrent_data.torrent_dict['info']

            changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict)

            torrent_filesize = info_dict.get('length') or sum(
                f['length'] for f in info_dict.get('files'))

            # In case no encoding, assume UTF-8.
            torrent_encoding = torrent_data.torrent_dict.get(
                'encoding', b'utf-8').decode('utf-8')

            torrent = models.Torrent(info_hash=torrent_data.info_hash,
                                     display_name=display_name,
                                     torrent_name=torrent_data.filename,
                                     information=information,
                                     description=description,
                                     encoding=torrent_encoding,
                                     filesize=torrent_filesize,
                                     user=current_user)

            # Store bencoded info_dict
            torrent.info = models.TorrentInfo(
                info_dict=torrent_data.bencoded_info_dict)
            torrent.stats = models.Statistic()
            torrent.has_torrent = True

            # Fields with default value will be None before first commit, so set .flags
            torrent.flags = 0

            torrent.anonymous = True if torrent_flags[0] else False
            torrent.hidden = True if torrent_flags[1] else False
            torrent.remake = True if torrent_flags[2] else False
            torrent.complete = True if torrent_flags[3] else False
            # Copy trusted status from user if possible
            torrent.trusted = (
                current_user.level >= models.UserLevelType.TRUSTED
            ) if current_user else False

            # Set category ids
            torrent.main_category_id = cat_id
            torrent.sub_category_id = sub_cat_id
            # To simplify parsing the filelist, turn single-file torrent into a list
            torrent_filelist = info_dict.get('files')

            used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding

            parsed_file_tree = dict()
            if not torrent_filelist:
                # If single-file, the root will be the file-tree (no directory)
                file_tree_root = parsed_file_tree
                torrent_filelist = [{
                    'length': torrent_filesize,
                    'path': [info_dict['name']]
                }]
            else:
                # If multi-file, use the directory name as root for files
                file_tree_root = parsed_file_tree.setdefault(
                    info_dict['name'].decode(used_path_encoding), {})

            # Parse file dicts into a tree
            for file_dict in torrent_filelist:
                # Decode path parts from utf8-bytes
                path_parts = [
                    path_part.decode(used_path_encoding)
                    for path_part in file_dict['path']
                ]

                filename = path_parts.pop()
                current_directory = file_tree_root

                for directory in path_parts:
                    current_directory = current_directory.setdefault(
                        directory, {})

                current_directory[filename] = file_dict['length']

            parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)

            json_bytes = json.dumps(parsed_file_tree,
                                    separators=(',', ':')).encode('utf8')
            torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)

            db.session.add(torrent)
            db.session.flush()

            # Store the users trackers
            trackers = OrderedSet()
            announce = torrent_data.torrent_dict.get('announce',
                                                     b'').decode('ascii')
            if announce:
                trackers.add(announce)

            # List of lists with single item
            announce_list = torrent_data.torrent_dict.get('announce-list', [])
            for announce in announce_list:
                trackers.add(announce[0].decode('ascii'))

            # Remove our trackers, maybe? TODO ?

            # Search for/Add trackers in DB
            db_trackers = OrderedSet()
            for announce in trackers:
                tracker = models.Trackers.by_uri(announce)

                # Insert new tracker if not found
                if not tracker:
                    tracker = models.Trackers(uri=announce)
                    db.session.add(tracker)

                db_trackers.add(tracker)

            db.session.flush()

            # Store tracker refs in DB
            for order, tracker in enumerate(db_trackers):
                torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
                                                         tracker_id=tracker.id,
                                                         order=order)
                db.session.add(torrent_tracker)

            db.session.commit()

            if app.config.get('BACKUP_TORRENT_FOLDER'):
                torrent_file.seek(0, 0)
                torrent_path = os.path.join(
                    app.config['BACKUP_TORRENT_FOLDER'],
                    '{}.{}'.format(torrent.id,
                                   secure_filename(torrent_file.filename)))
                torrent_file.save(torrent_path)
            torrent_file.close()

            #print('Success? {0}'.format(torrent.id))
            return flask.make_response(
                flask.jsonify({
                    "Success":
                    "Request was processed {0}".format(torrent.id)
                }), 200)
        except Exception as e:
            print('Exception: {0}'.format(e))
            return flask.make_response(
                flask.jsonify({
                    "Error":
                    "Incorrect JSON. Please see HELP page for examples."
                }), 400)
    else:
        return flask.make_response(flask.jsonify({"Error": "Bad request"}),
                                   400)