def test_changing_max_length(): """Ensure changing the max_length argument works.""" test_string = "Just some text to try" # should normally validate assert Markdown()._validate(test_string) is None # but fails if you set a too-short max_length with raises(ValidationError): Markdown(max_length=len(test_string) - 1)._validate(test_string)
class MessageReplySchema(Schema): """Marshmallow schema for message replies.""" reply_id36 = ID36() markdown = Markdown() rendered_html = String(dump_only=True) created_time = DateTime(dump_only=True)
class MessageConversationSchema(Schema): """Marshmallow schema for message conversations.""" conversation_id36 = ID36() subject = SimpleString(max_length=SUBJECT_MAX_LENGTH) markdown = Markdown() rendered_html = String(dump_only=True) created_time = DateTime(dump_only=True)
class CommentSchema(Schema): """Marshmallow schema for comments.""" markdown = Markdown() parent_comment_id36 = ID36() class Meta: """Always use strict checking so error handlers are invoked.""" strict = True
class GroupWikiPageSchema(Schema): """Marshmallow schema for group wiki pages.""" page_name = SimpleString(max_length=PAGE_NAME_MAX_LENGTH) markdown = Markdown(max_length=100_000) class Meta: """Always use strict checking so error handlers are invoked.""" strict = True
class MessageReplySchema(Schema): """Marshmallow schema for message replies.""" reply_id36 = ID36() markdown = Markdown() rendered_html = String(dump_only=True) created_time = DateTime(dump_only=True) class Meta: """Always use strict checking so error handlers are invoked.""" strict = True
class GroupSchema(Schema): """Marshmallow schema for groups.""" path = Ltree(required=True) created_time = DateTime(dump_only=True) short_description = SimpleString( max_length=SHORT_DESCRIPTION_MAX_LENGTH, allow_none=True ) sidebar_markdown = Markdown(allow_none=True) @pre_load def prepare_path(self, data: dict, many: bool, partial: Any) -> dict: """Prepare the path value before it's validated.""" # pylint: disable=unused-argument if not self.context.get("fix_path_capitalization"): return data if "path" not in data or not isinstance(data["path"], str): return data new_data = data.copy() new_data["path"] = new_data["path"].lower() return new_data @validates("path") def validate_path(self, value: sqlalchemy_utils.Ltree) -> None: """Validate the path field, raising an error if an issue exists.""" # check each element for length and against validity regex path_elements = value.path.split(".") for element in path_elements: if len(element) > 256: raise ValidationError("Path element %s is too long" % element) if not GROUP_PATH_ELEMENT_VALID_REGEX.match(element): raise ValidationError("Path element %s is invalid" % element) @pre_load def prepare_sidebar_markdown(self, data: dict, many: bool, partial: Any) -> dict: """Prepare the sidebar_markdown value before it's validated.""" # pylint: disable=unused-argument if "sidebar_markdown" not in data: return data new_data = data.copy() # if the value is empty, convert it to None if not new_data["sidebar_markdown"] or new_data["sidebar_markdown"].isspace(): new_data["sidebar_markdown"] = None return new_data
class GroupSchema(Schema): """Marshmallow schema for groups.""" path = Ltree(required=True, load_from="group_path") created_time = DateTime(dump_only=True) short_description = SimpleString(max_length=SHORT_DESCRIPTION_MAX_LENGTH, allow_none=True) sidebar_markdown = Markdown(allow_none=True) @pre_load def prepare_path(self, data: dict) -> dict: """Prepare the path value before it's validated.""" if not self.context.get("fix_path_capitalization"): return data # path can also be loaded from group_path, so we need to check both keys = ("path", "group_path") for key in keys: if key in data and isinstance(data[key], str): data[key] = data[key].lower() return data @validates("path") def validate_path(self, value: sqlalchemy_utils.Ltree) -> None: """Validate the path field, raising an error if an issue exists.""" # check each element for length and against validity regex path_elements = value.path.split(".") for element in path_elements: if len(element) > 256: raise ValidationError("Path element %s is too long" % element) if not GROUP_PATH_ELEMENT_VALID_REGEX.match(element): raise ValidationError("Path element %s is invalid" % element) @pre_load def prepare_sidebar_markdown(self, data: dict) -> dict: """Prepare the sidebar_markdown value before it's validated.""" if "sidebar_markdown" not in data: return data # if the value is empty, convert it to None if not data["sidebar_markdown"] or data["sidebar_markdown"].isspace(): data["sidebar_markdown"] = None return data class Meta: """Always use strict checking so error handlers are invoked.""" strict = True
class MessageConversationSchema(Schema): """Marshmallow schema for message conversations.""" conversation_id36 = ID36() subject = SimpleString(max_length=SUBJECT_MAX_LENGTH) markdown = Markdown() rendered_html = String(dump_only=True) created_time = DateTime(dump_only=True) class Meta: """Always use strict checking so error handlers are invoked.""" strict = True
class UserSchema(Schema): """Marshmallow schema for users.""" username = String( validate=( Length(min=USERNAME_MIN_LENGTH, max=USERNAME_MAX_LENGTH), Regexp(USERNAME_VALID_REGEX), ), required=True, ) password = String(validate=Length(min=PASSWORD_MIN_LENGTH), required=True, load_only=True) email_address = Email(allow_none=True, load_only=True) email_address_note = String(validate=Length( max=EMAIL_ADDRESS_NOTE_MAX_LENGTH)) created_time = DateTime(dump_only=True) bio_markdown = Markdown(max_length=BIO_MAX_LENGTH, allow_none=True) @post_dump def anonymize_username(self, data: dict, many: bool) -> dict: """Hide the username if the dumping context specifies to do so.""" # pylint: disable=unused-argument if "username" in data and self.context.get("hide_username"): data["username"] = "******" return data @validates_schema def username_pass_not_substrings(self, data: dict, many: bool, partial: Any) -> None: """Ensure the username isn't in the password and vice versa.""" # pylint: disable=unused-argument username = data.get("username") password = data.get("password") if not (username and password): return username = username.lower() password = password.lower() if username in password: raise ValidationError("Password cannot contain username") if password in username: raise ValidationError("Username cannot contain password") @validates("password") def password_not_breached(self, value: str) -> None: """Validate that the password is not in the breached-passwords list. Requires check_breached_passwords be True in the schema's context. """ if not self.context.get("check_breached_passwords"): return if is_breached_password(value): raise ValidationError( "That password exists in a data breach (see sidebar)") @pre_load def username_trim_whitespace(self, data: dict, many: bool, partial: Any) -> dict: """Trim leading/trailing whitespace around the username. Requires username_trim_whitespace be True in the schema's context. """ # pylint: disable=unused-argument if not self.context.get("username_trim_whitespace"): return data if "username" not in data: return data data["username"] = data["username"].strip() return data @pre_load def prepare_email_address(self, data: dict, many: bool, partial: Any) -> dict: """Prepare the email address value before it's validated.""" # pylint: disable=unused-argument if "email_address" not in data: return data # remove any leading/trailing whitespace data["email_address"] = data["email_address"].strip() # if the value is empty, convert it to None if not data["email_address"] or data["email_address"].isspace(): data["email_address"] = None return data @pre_load def prepare_bio_markdown(self, data: dict, many: bool, partial: Any) -> dict: """Prepare the bio_markdown value before it's validated.""" # pylint: disable=unused-argument if "bio_markdown" not in data: return data # if the value is empty, convert it to None if not data["bio_markdown"] or data["bio_markdown"].isspace(): data["bio_markdown"] = None return data
class TopicSchema(Schema): """Marshmallow schema for topics.""" topic_id36 = ID36() title = SimpleString(max_length=TITLE_MAX_LENGTH) topic_type = Enum(dump_only=True) markdown = Markdown(allow_none=True) rendered_html = String(dump_only=True) link = URL(schemes={"http", "https"}, allow_none=True) created_time = DateTime(dump_only=True) tags = List(String()) user = Nested(UserSchema, dump_only=True) group = Nested(GroupSchema, dump_only=True) @pre_load def prepare_title(self, data: dict, many: bool, partial: Any) -> dict: """Prepare the title before it's validated.""" # pylint: disable=unused-argument if "title" not in data: return data new_data = data.copy() split_title = re.split("[.?!]+", new_data["title"]) # the last string in the list will be empty if it ended with punctuation num_sentences = len([piece for piece in split_title if piece]) # strip trailing periods off single-sentence titles if num_sentences == 1: new_data["title"] = new_data["title"].rstrip(".") return new_data @pre_load def prepare_tags(self, data: dict, many: bool, partial: Any) -> dict: """Prepare the tags before they're validated.""" # pylint: disable=unused-argument if "tags" not in data: return data new_data = data.copy() tags: list[str] = [] for tag in new_data["tags"]: tag = tag.lower() # replace underscores with spaces tag = tag.replace("_", " ") # remove any consecutive spaces tag = re.sub(" {2,}", " ", tag) # remove any leading/trailing spaces tag = tag.strip(" ") # drop any empty tags if not tag or tag.isspace(): continue # handle synonyms for name, synonyms in TAG_SYNONYMS.items(): if tag in synonyms: tag = name # skip any duplicate tags if tag in tags: continue tags.append(tag) new_data["tags"] = tags return new_data @validates("tags") def validate_tags(self, value: list[str]) -> None: """Validate the tags field, raising an error if an issue exists. Note that tags are validated by ensuring that each tag would be a valid group path. This is definitely mixing concerns, but it's deliberate in this case. It will allow for some interesting possibilities by ensuring naming "compatibility" between groups and tags. For example, a popular tag in a group could be converted into a sub-group easily. """ group_schema = GroupSchema(partial=True) for tag in value: try: group_schema.load({"path": tag}) except ValidationError as exc: raise ValidationError("Tag %s is invalid" % tag) from exc @pre_load def prepare_markdown(self, data: dict, many: bool, partial: Any) -> dict: """Prepare the markdown value before it's validated.""" # pylint: disable=unused-argument if "markdown" not in data: return data new_data = data.copy() # if the value is empty, convert it to None if not new_data["markdown"] or new_data["markdown"].isspace(): new_data["markdown"] = None return new_data @pre_load def prepare_link(self, data: dict, many: bool, partial: Any) -> dict: """Prepare the link value before it's validated.""" # pylint: disable=unused-argument if "link" not in data: return data new_data = data.copy() # remove leading/trailing whitespace new_data["link"] = new_data["link"].strip() # if the value is empty, convert it to None if not new_data["link"]: new_data["link"] = None return new_data # prepend http:// to the link if it doesn't have a scheme parsed = urlparse(new_data["link"]) if not parsed.scheme: new_data["link"] = "http://" + new_data["link"] # run the link through the url-transformation process new_data["link"] = apply_url_transformations(new_data["link"]) return new_data @validates_schema def link_or_markdown(self, data: dict, many: bool, partial: Any) -> None: """Fail validation unless at least one of link or markdown were set.""" # pylint: disable=unused-argument if "link" not in data and "markdown" not in data: return link = data.get("link") markdown = data.get("markdown") if not (markdown or link): raise ValidationError("Topics must have either markdown or a link.")
class MarkdownFieldTestSchema(Schema): """Simple schema class with a standard Markdown field.""" markdown = Markdown()
class GroupWikiPageSchema(Schema): """Marshmallow schema for group wiki pages.""" page_name = SimpleString(max_length=PAGE_NAME_MAX_LENGTH) markdown = Markdown(max_length=1_000_000)
class CommentSchema(Schema): """Marshmallow schema for comments.""" comment_id36 = ID36() markdown = Markdown() parent_comment_id36 = ID36()
class TopicSchema(Schema): """Marshmallow schema for topics.""" topic_id36 = ID36() title = SimpleString(max_length=TITLE_MAX_LENGTH) topic_type = Enum(dump_only=True) markdown = Markdown(allow_none=True) rendered_html = String(dump_only=True) link = URL(schemes={'http', 'https'}, allow_none=True) created_time = DateTime(dump_only=True) tags = List(Ltree()) user = Nested(UserSchema, dump_only=True) group = Nested(GroupSchema, dump_only=True) @pre_load def prepare_tags(self, data: dict) -> dict: """Prepare the tags before they're validated.""" if 'tags' not in data: return data tags: typing.List[str] = [] for tag in data['tags']: tag = tag.lower() # replace spaces with underscores tag = tag.replace(' ', '_') # remove any consecutive underscores tag = re.sub('_{2,}', '_', tag) # remove any leading/trailing underscores tag = tag.strip('_') # drop any empty tags if not tag or tag.isspace(): continue # skip any duplicate tags if tag in tags: continue tags.append(tag) data['tags'] = tags return data @validates('tags') def validate_tags( self, value: typing.List[sqlalchemy_utils.Ltree], ) -> None: """Validate the tags field, raising an error if an issue exists. Note that tags are validated by ensuring that each tag would be a valid group path. This is definitely mixing concerns, but it's deliberate in this case. It will allow for some interesting possibilities by ensuring naming "compatibility" between groups and tags. For example, a popular tag in a group could be converted into a sub-group easily. """ group_schema = GroupSchema(partial=True) for tag in value: try: group_schema.validate({'path': tag}) except ValidationError: raise ValidationError('Tag %s is invalid' % tag) @pre_load def prepare_markdown(self, data: dict) -> dict: """Prepare the markdown value before it's validated.""" if 'markdown' not in data: return data # if the value is empty, convert it to None if not data['markdown'] or data['markdown'].isspace(): data['markdown'] = None return data @pre_load def prepare_link(self, data: dict) -> dict: """Prepare the link value before it's validated.""" if 'link' not in data: return data # if the value is empty, convert it to None if not data['link'] or data['link'].isspace(): data['link'] = None return data # prepend http:// to the link if it doesn't have a scheme parsed = urlparse(data['link']) if not parsed.scheme: data['link'] = 'http://' + data['link'] return data @validates_schema def link_or_markdown(self, data: dict) -> None: """Fail validation unless at least one of link or markdown were set.""" if 'link' not in data and 'markdown' not in data: return link = data.get('link') markdown = data.get('markdown') if not (markdown or link): raise ValidationError( 'Topics must have either markdown or a link.') class Meta: """Always use strict checking so error handlers are invoked.""" strict = True
class TopicSchema(Schema): """Marshmallow schema for topics.""" topic_id36 = ID36() title = SimpleString(max_length=TITLE_MAX_LENGTH) topic_type = Enum(dump_only=True) markdown = Markdown(allow_none=True) rendered_html = String(dump_only=True) link = URL(schemes={"http", "https"}, allow_none=True) created_time = DateTime(dump_only=True) tags = List(Ltree()) user = Nested(UserSchema, dump_only=True) group = Nested(GroupSchema, dump_only=True) @pre_load def prepare_tags(self, data: dict) -> dict: """Prepare the tags before they're validated.""" if "tags" not in data: return data tags: typing.List[str] = [] for tag in data["tags"]: tag = tag.lower() # replace spaces with underscores tag = tag.replace(" ", "_") # remove any consecutive underscores tag = re.sub("_{2,}", "_", tag) # remove any leading/trailing underscores tag = tag.strip("_") # drop any empty tags if not tag or tag.isspace(): continue # handle synonyms for name, synonyms in TAG_SYNONYMS.items(): if tag in synonyms: tag = name # skip any duplicate tags if tag in tags: continue tags.append(tag) data["tags"] = tags return data @validates("tags") def validate_tags(self, value: typing.List[sqlalchemy_utils.Ltree]) -> None: """Validate the tags field, raising an error if an issue exists. Note that tags are validated by ensuring that each tag would be a valid group path. This is definitely mixing concerns, but it's deliberate in this case. It will allow for some interesting possibilities by ensuring naming "compatibility" between groups and tags. For example, a popular tag in a group could be converted into a sub-group easily. """ group_schema = GroupSchema(partial=True) for tag in value: try: group_schema.validate({"path": str(tag)}) except ValidationError: raise ValidationError("Tag %s is invalid" % tag) @pre_load def prepare_markdown(self, data: dict) -> dict: """Prepare the markdown value before it's validated.""" if "markdown" not in data: return data # if the value is empty, convert it to None if not data["markdown"] or data["markdown"].isspace(): data["markdown"] = None return data @pre_load def prepare_link(self, data: dict) -> dict: """Prepare the link value before it's validated.""" if "link" not in data: return data # if the value is empty, convert it to None if not data["link"] or data["link"].isspace(): data["link"] = None return data # prepend http:// to the link if it doesn't have a scheme parsed = urlparse(data["link"]) if not parsed.scheme: data["link"] = "http://" + data["link"] # run the link through the url-transformation process data["link"] = apply_url_transformations(data["link"]) return data @validates_schema def link_or_markdown(self, data: dict) -> None: """Fail validation unless at least one of link or markdown were set.""" if "link" not in data and "markdown" not in data: return link = data.get("link") markdown = data.get("markdown") if not (markdown or link): raise ValidationError("Topics must have either markdown or a link.") class Meta: """Always use strict checking so error handlers are invoked.""" strict = True