def test_render_markdown_with_code_blocks(self):
"""Testing render_markdown with code blocks"""
if version_info[:2] >= (3, 2):
# Markdown 3.2 adds <code> around each line of code. See
# https://python-markdown.github.io/change_log/release-3.2/
expected_html1 = ('<p></p>\n'
'<div class="codehilite"><pre><span></span>'
'<code>here is a generic code block\n'
'</code></pre></div>')
expected_html2 = (
'<p></p>\n'
'<div class="codehilite"><pre><span></span>'
'<code><span class="c1"># Here is a Python code block</span>\n'
'</code></pre></div>')
else:
expected_html1 = ('<p></p>\n'
'<div class="codehilite"><pre><span></span>'
'here is a generic code block\n'
'</pre></div>')
expected_html2 = (
'<p></p>\n'
'<div class="codehilite"><pre><span></span>'
'<span class="c1"># Here is a Python code block</span>\n'
'</pre></div>')
self.assertEqual(
render_markdown('```\n'
'here is a generic code block\n'
'```\n'), expected_html1)
self.assertEqual(
render_markdown('```python\n'
'# Here is a Python code block\n'
'```\n'), expected_html2)
def test_render_markdown_with_bold_italic(self):
"""Testing render_markdown with bold and italic"""
self.assertEqual(render_markdown('*this is a __test__.*'),
'<p><em>this is a <strong>test</strong>.</em></p>')
self.assertEqual(render_markdown('_**test**_'),
'<p><em><strong>test</strong></em></p>')
def test_render_markdown_with_lists_unordered(self):
"""Testing render_markdown with unordered lists"""
self.assertEqual(
render_markdown(
'* Item 1\n'
'* Item 2\n'
' * Item 2.1\n'
'* Item 3\n'
),
'<ul>\n'
'<li>Item 1</li>\n'
'<li>Item 2<ul>\n'
'<li>Item 2.1</li>\n'
'</ul>\n'
'</li>\n'
'<li>Item 3</li>\n'
'</ul>')
self.assertEqual(
render_markdown(
'- Item 1\n'
'- Item 2\n'
' - Item 2.1\n'
'- Item 3\n'
),
'<ul>\n'
'<li>Item 1</li>\n'
'<li>Item 2<ul>\n'
'<li>Item 2.1</li>\n'
'</ul>\n'
'</li>\n'
'<li>Item 3</li>\n'
'</ul>')
def test_render_markdown_with_italic(self):
"""Testing render_markdown with italic"""
self.assertEqual(render_markdown('*italic*'), '<p><em>italic</em></p>')
self.assertEqual(render_markdown('_italic_'), '<p><em>italic</em></p>')
self.assertEqual(render_markdown('mid*italic*word'),
'<p>mid<em>italic</em>word</p>')
self.assertEqual(render_markdown('mid_notitalic_word'),
'<p>mid_notitalic_word</p>')
def test_render_markdown_with_bold(self):
"""Testing render_markdown with bold"""
self.assertEqual(render_markdown('**bold**'),
'<p><strong>bold</strong></p>')
self.assertEqual(render_markdown('__bold__'),
'<p><strong>bold</strong></p>')
self.assertEqual(render_markdown('mid**bold**word'),
'<p>mid<strong>bold</strong>word</p>')
self.assertEqual(render_markdown('mid__notbold__word'),
'<p>mid__notbold__word</p>')
def test_render_markdown_preserves_self_closed_tags(self):
"""Testing render_markdown preserves self-closed tags"""
self.assertEqual(render_markdown('line1\n'
'line2\n'
'---'), '<p>line1<br />\n'
'line2</p>\n'
'<hr />')
def test_render_markdown_sanitizes_images(self):
"""Testing render_markdown sanitizes XSS in images"""
# This list courtesy of the cheat sheet at
# https://github.com/cujanovic/Markdown-XSS-Payloads/
#
# To ensure clarity, blank lines have been added between each XSS
# test/result.
self.assertEqual(
render_markdown('\n\n'.join([
r')\\',
r'\\',
r'\\',
r'![XSS4](https://example.com/image.png"onload="alert(1))',
r'![XSS5]("onerror="alert(1))',
])).split('\n'),
[
r'<p><img alt="XSS1" />)\</p>',
r'<p><img alt="XSS2" />\</p>',
r'<p><img alt="XSS3\'"`onerror=prompt(document.cookie)" '
r'src="x" />\</p>',
r'<p><img alt="XSS4" src="https://example.com/image.png"'
r'onload="alert(1" />)</p>',
r'<p><img alt="XSS5" src=""onerror="alert(1" />'
r')</p>',
])
def test_render_markdown_with_tables(self):
"""Testing render_markdown with tables"""
self.assertEqual(
render_markdown(
'| Header | Header |\n'
'|--------|--------|\n'
'| Cell | Cell |\n'
'| Cell | Cell |'
),
'<table>\n'
'<thead>\n'
'<tr>\n'
'<th>Header</th>\n'
'<th>Header</th>\n'
'</tr>\n'
'</thead>\n'
'<tbody>\n'
'<tr>\n'
'<td>Cell</td>\n'
'<td>Cell</td>\n'
'</tr>\n'
'<tr>\n'
'<td>Cell</td>\n'
'<td>Cell</td>\n'
'</tr>\n'
'</tbody>\n'
'</table>'
)
def test_render_markdown_with_emojis(self):
"""Testing render_markdown with emojis"""
self.assertEqual(
render_markdown(':thumbsup:'),
'<p><img alt="\U0001f44d" class="emoji" '
'src="https://github.githubassets.com/images/icons/emoji/'
'unicode/1f44d.png" title=":thumbsup:" /></p>')
def test_render_markdown_with_links(self):
"""Testing render_markdown with links"""
self.assertEqual(
render_markdown('[my link](https://www.reviewboard.org/)'),
'<p><a href="https://www.reviewboard.org/">my link</a></p>')
self.assertEqual(
render_markdown('[my link](http://www.reviewboard.org/)'),
'<p><a href="http://www.reviewboard.org/">my link</a></p>')
self.assertEqual(
render_markdown('[my link](mailto:[email protected])'),
'<p><a href="mailto:[email protected]">my link</a></p>')
# Anything else is filtered out.
self.assertEqual(render_markdown('[my link](ftp://ftp.example.com)'),
'<p><a>my link</a></p>')
self.assertEqual(render_markdown('custom://example.com'),
'<p>custom://example.com</p>')
def _generate_preview_html(self, data_string):
"""Return the HTML for a thumbnail preview for a Markdown file.
Returns:
django.utils.safestring.SafeText:
The resulting HTML-safe thumbnail content.
"""
return mark_safe(render_markdown(force_unicode(data_string)))
def test_render_markdown_with_links_and_setting(self):
"""Testing render_markdown with links and
settings.ALLOWED_MARKDOWN_URL_PROTOCOLS
"""
self.assertEqual(
render_markdown('[my link](https://www.reviewboard.org/)'),
'<p><a href="https://www.reviewboard.org/">my link</a></p>')
self.assertEqual(
render_markdown('[my link](http://www.reviewboard.org/)'),
'<p><a href="http://www.reviewboard.org/">my link</a></p>')
self.assertEqual(
render_markdown('[my link](mailto:[email protected])'),
'<p><a href="mailto:[email protected]">my link</a></p>')
self.assertEqual(
render_markdown('[my link](ftp://ftp.example.com)'),
'<p><a href="ftp://ftp.example.com">my link</a></p>')
self.assertEqual(
render_markdown('[my link](custom://ftp.example.com)'),
'<p><a href="custom://ftp.example.com">my link</a></p>')
# Anything else is filtered out.
self.assertEqual(
render_markdown('[my link](other://example.com/)'),
'<p><a>my link</a></p>')
self.assertEqual(
render_markdown('custom2://example.com'),
'<p>custom2://example.com</p>')
def render_change_entry_html(self, info):
old_value = ''
new_value = ''
if 'old' in info:
old_value = info['old'][0] or ''
if 'new' in info:
new_value = info['new'][0] or ''
old_value = render_markdown(old_value)
new_value = render_markdown(new_value)
old_lines = list(iter_markdown_lines(old_value))
new_lines = list(iter_markdown_lines(new_value))
differ = MyersDiffer(old_lines, new_lines)
return ('<table class="diffed-text-area">%s</table>' % ''.join(
self._render_all_change_lines(differ, old_lines, new_lines)))
def test_render_markdown_with_blockquotes(self):
"""Testing render_markdown with blockquotes"""
self.assertEqual(
render_markdown('> here is a line\n'
'> and another\n'), '<p></p>\n'
'<p></p>\n'
'<blockquote>\n'
'<p>here is a line<br />\n'
'and another</p>\n'
'</blockquote>')
def render_change_entry_html(self, info):
old_value = ''
new_value = ''
if 'old' in info:
old_value = info['old'][0] or ''
if 'new' in info:
new_value = info['new'][0] or ''
old_value = render_markdown(old_value)
new_value = render_markdown(new_value)
old_lines = list(iter_markdown_lines(old_value))
new_lines = list(iter_markdown_lines(new_value))
differ = MyersDiffer(old_lines, new_lines)
return ('<table class="diffed-text-area">%s</table>'
% ''.join(self._render_all_change_lines(differ, old_lines,
new_lines)))
def render_value(self, text):
"""Returns the value of the field.
If Markdown is enabled, and the text is not in Markdown format,
the text will be escaped.
"""
text = text or ''
if self.should_render_as_markdown(text):
return render_markdown(text)
else:
return escape(text)
def render_value(self, text):
"""Returns the value of the field.
If Markdown is enabled, and the text is not in Markdown format,
the text will be escaped.
"""
text = text or ''
if self.should_render_as_markdown(text):
return render_markdown(text)
else:
return escape(text)
def render_change_entry_html(self, info):
"""Render a change entry to HTML.
This will render a diff of the changed text.
This function is expected to return safe, valid HTML. Any values
coming from a field or any other form of user input must be
properly escaped.
Args:
info (dict):
A dictionary describing how the field has changed. This is
guaranteed to have ``new`` and ``old`` keys, but may also
contain ``added`` and ``removed`` keys as well.
Returns:
unicode:
The HTML representation of the change entry.
"""
old_value = ''
new_value = ''
if 'old' in info:
old_value = info['old'][0] or ''
if 'new' in info:
new_value = info['new'][0] or ''
old_value = render_markdown(old_value)
new_value = render_markdown(new_value)
old_lines = list(iter_markdown_lines(old_value))
new_lines = list(iter_markdown_lines(new_value))
differ = MyersDiffer(old_lines, new_lines)
return ('<table class="diffed-text-area">%s</table>'
% ''.join(self._render_all_change_lines(differ, old_lines,
new_lines)))
def generate_render(self):
with self.obj.file as f:
f.open()
rendered = render_markdown(f.read())
try:
for line in iter_markdown_lines(rendered):
yield line
except Exception as e:
logger.error('Failed to parse resulting Markdown XHTML for '
'file attachment %d: %s',
self.obj.pk, e,
exc_info=True)
yield _('Error while rendering Markdown content: %s') % e
def test_render_markdown_with_lists_ordered(self):
"""Testing render_markdown with ordered lists"""
self.assertEqual(
render_markdown('1. Item 1\n'
'2. Item 2\n'
' 1. Item 2.1\n'
'3. Item 3\n'), '<ol>\n'
'<li>Item 1</li>\n'
'<li>Item 2<ol>\n'
'<li>Item 2.1</li>\n'
'</ol>\n'
'</li>\n'
'<li>Item 3</li>\n'
'</ol>')
def get_bug_info_uncached(self, repository, bug_id):
"""Return the bug info from the server.
Args:
repository (reviewboard.scmtools.model.Repository):
The repository that is using Splat as a bug tracker.
bug_id (unicode):
The bug identifier.
Returns:
dict:
A dictionary of the bug information.
"""
result = {
'summary': '',
'description': '',
'status': '',
}
url = (
'https://hellosplat.com/api/orgs/%s/tickets/%s/?only-fields=status'
',summary,text,text_format'
% (repository.extra_data['bug_tracker-splat_org_name'], bug_id)
)
try:
rsp = self.client.json_get(url)[0]
ticket = rsp['ticket']
except Exception as e:
logging.warning('Unable to fetch Splat data from %s: %s',
url, e, exc_info=True)
else:
text = ticket['text']
if ticket['text_format'] == 'markdown':
text = render_markdown(text)
result = {
'description': strip_tags(text),
'status': ticket['status'],
'summary': ticket['summary'],
}
return result
def _normalize_text(self, obj, text, request=None, **kwargs):
"""Normalizes text to the proper format.
This considers the requested text format, and whether or not the
object is set for having rich text.
"""
text_type = self._get_requested_text_type(obj, request)
if text_type == self.TEXT_TYPE_PLAIN and obj.rich_text:
text = markdown_unescape(text)
elif text_type == self.TEXT_TYPE_MARKDOWN and not obj.rich_text:
text = markdown_escape(text)
elif text_type == self.TEXT_TYPE_HTML:
if obj.rich_text:
text = render_markdown(text)
else:
text = escape(text)
return text
def _normalize_text(self, text, field_is_rich_text, force_text_type):
"""Normalizes text to the proper format.
This considers the requested text format, and whether or not the
value should be set for rich text.
"""
assert force_text_type
if text is not None:
if force_text_type == self.TEXT_TYPE_PLAIN and field_is_rich_text:
text = markdown_unescape(text)
elif (force_text_type == self.TEXT_TYPE_MARKDOWN and
not field_is_rich_text):
text = markdown_escape(text)
elif force_text_type == self.TEXT_TYPE_HTML:
if field_is_rich_text:
text = render_markdown(text)
else:
text = escape(text)
return text
def test_render_markdown_with_headers(self):
"""Testing render_markdown with headers"""
self.assertEqual(
render_markdown('Header\n'
'======\n'
'\n'
'# Header\n'
'\n'
'Subheader\n'
'---------\n'
'\n'
'## Subheader\n'
'\n'
'### Sub-subheader\n'
'\n'
'#### Sub-sub-subheader\n'), '<h1>Header</h1>\n'
'<h1>Header</h1>\n'
'<h2>Subheader</h2>\n'
'<h2>Subheader</h2>\n'
'<h3>Sub-subheader</h3>\n'
'<h4>Sub-sub-subheader</h4>')
def _normalize_text(self, text, field_is_rich_text, force_text_type):
"""Normalizes text to the proper format.
This considers the requested text format, and whether or not the
value should be set for rich text.
"""
assert force_text_type
if text is not None:
if force_text_type == self.TEXT_TYPE_PLAIN and field_is_rich_text:
text = markdown_unescape(text)
elif (force_text_type == self.TEXT_TYPE_MARKDOWN and
not field_is_rich_text):
text = markdown_escape(text)
elif force_text_type == self.TEXT_TYPE_HTML:
if field_is_rich_text:
text = render_markdown(text)
else:
text = escape(text)
return text
def test_render_markdown_with_strikethrough(self):
"""Testing render_markdown with strikethrough"""
self.assertEqual(
render_markdown('~~strike~~'),
'<p><del>strike</del></p>')
def _render_markdown(text, is_rich_text):
if is_rich_text:
return mark_safe(render_markdown(text))
else:
return text
def test_render_markdown_with_images(self):
"""Testing render_markdown with images"""
self.assertEqual(
render_markdown(''),
'<p><img alt="my image" src="https://example.com/logo.png" />'
'</p>')
def test_render_markdown_with_inline_code(self):
"""Testing render_markdown with inline code"""
self.assertEqual(
render_markdown('here is ``inline code``'),
'<p>here is <code>inline code</code></p>')
def _render_markdown(text, is_rich_text):
if is_rich_text:
return mark_safe(render_markdown(text))
else:
return text
def test_render_markdown_sanitizes_links(self):
"""Testing render_markdown sanitizes XSS in links"""
# This list courtesy of the cheat sheet at
# https://github.com/cujanovic/Markdown-XSS-Payloads/
#
# To ensure clarity, blank lines have been added between each XSS
# test/result.
self.assertEqual(
render_markdown('\n\n'.join([
r'[XSS1](javascript:alert("oh no"))',
r'[XSS2](jAvascript:alert("oh no"))',
r'[XSS3](j a v a s c r i p t:alert("oh no"))',
r'[XSS4](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8'
r'L3NjcmlwdD4K)',
r'[XSS5](javascript'
r':alert('XSS'
r''))',
r'[XSS6](javascript:window.onerror=alert;throw%20'
r'document.cookie)',
r'[XSS7](javascript://%0d%0aprompt(1))',
r'[XSS8](javascript://%0d%0aprompt(1);com)',
r'[XSS9](javascript:window.onerror=alert;throw%20'
r'document.cookie)',
r'[XSS10](javascript://%0d%0awindow.onerror=alert;'
r'throw%20document.cookie)',
r'[XSS11](data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8'
r'L3NjcmlwdD4K)',
r'[XSS12](vbscript:alert(document.domain))',
r'[XSS13](https://example.com " [@bad](/bad) ")',
r'[XSS14](javascript:this,alert(1))',
r'[XSS15](javascript:this,alert(1)))',
r'[XSS16](javascript:this;alert(1))',
r'[XSS17](Javascript:alert(1))',
r'[XSS18](Javas%26%2399;ript:alert(1))',
r'[XSS19](javascript:alert(1))',
r'[XSS20](javascript:confirm(1)',
r'[XSS21](javascript://example.com%0Aprompt(1))',
r'[XSS22](javascript://%0d%0aconfirm(1);com)',
r'[XSS23](javascript:window.onerror=confirm;throw%201)',
r'[XSS24](javascript:alert(document.domain))',
r'[XSS25](javascript://example.com%0Aalert(1))',
r'[XSS26](\'javascript:alert("1")\')',
r'[XSS27](JaVaScRiPt:alert(1))',
r'[XSS28](.alert(1);)',
r'XSS29: [ ](https://a.de?p=[[/data-x=. style='
r'background-color:#000000;z-index:999;width:100%;'
r'position:fixed;top:0;left:0;right:0;bottom:0; data-y=.]])',
r'XSS30: [ ](http://a?p=[[/onclick=alert(0) .]])',
r'[XSS31](javascript:new%20Function`al\ert\`1\``;)',
r'[XSS32](javascript:new%20Function`al\ert\`1\``;)',
r'XSS33: <http://\<meta\ http-equiv=\"refresh\"\ '
r'content=\"0;\ url=http://example.com/\"\>>',
r'XSS33: </http://<?php\><\h1\><script:script>confirm(2)',
r'XSS34: <javascript:prompt(document.cookie)>',
r'XSS35: <javascrip'
r't:alert('XS'
r'S')>',
r'XSS36: _http://[email protected] style=background-image:url('
r'data:image/png;base64,ABCABCABCABC==);'
r'background-repeat:no-repeat;display:block;width:100%;'
r'height:100px; onclick=alert(unescape(/Oh%20No!/.source));'
r'return(false);//',
'XSS37:\n[cite]: (javascript:prompt(document.cookie))',
])).split('\n'),
[
r'<p><a>XSS1</a></p>',
r'<p><a href="j&#X41vascript:alert("oh no")">'
r'XSS2</a></p>',
r'<p><a>XSS3</a></p>',
r'<p><a>XSS4</a></p>',
r'<p><a href="&#x6A&#x61&#x76&#x61&#x73'
r'&#x63&#x72&#x69&#x70&#x74&#x3A'
r'&#x61&#x6C&#x65&#x72&#x74&#x28'
r'&#x27&#x58&#x53&#x53&#x27&#x29">'
r'XSS5</a></p>',
r'<p><a>XSS6</a></p>',
r'<p><a>XSS7</a></p>',
r'<p><a>XSS8</a></p>',
r'<p><a>XSS9</a></p>',
r'<p><a>XSS10</a></p>',
r'<p><a>XSS11</a></p>',
r'<p><a>XSS12</a></p>',
r'<p><a href="https://example.com"'
r' title=" [@bad](/bad) ">XSS13</a></p>',
r'<p><a>XSS14</a></p>',
r'<p><a>XSS15</a></p>',
r'<p>[XSS16](javascript[HTML_REMOVED]alert(1[HTML_REMOVED])'
r'</p>',
r'<p>[XSS17](Javas[HTML_REMOVED]ript:alert(1[HTML_REMOVED])'
r'</p>',
r'<p>[XSS18](Javas%26%2399;ript:alert(1[HTML_REMOVED])</p>',
r'<p>[XSS19](javascript:alert[HTML_REMOVED](1[HTML_REMOVED])'
r'</p>',
r'<p>[XSS20](javascript:confirm(1)</p>',
r'<p><a>XSS21</a></p>',
r'<p><a>XSS22</a></p>',
r'<p><a>XSS23</a></p>',
r'<p>[XSS24](javascript:alert(document.domain[HTML_REMOVED])'
r'</p>',
r'<p><a>XSS25</a></p>',
r'<p><a href="\" title="javascript:alert("1")\">'
r'XSS26</a></p>',
r'<p><a>XSS27</a></p>',
r'<p><a href=".alert(1);">XSS28</a></p>',
r'<p>XSS29: <a href="https://a.de?p=[[/data-x=. style='
r'background-color:#000000;z-index:999;width:100%;'
r'position:fixed;top:0;left:0;right:0;bottom:0; data-y=.]]"> '
r'</a></p>',
r'<p>XSS30: <a href="http://a?p=[[/onclick=alert(0) .]]"> '
r'</a></p>',
r'<p><a>XSS31</a></p>',
r'<p><a>XSS32</a></p>',
r'<p>XSS33: <a href="http://\<meta http-equiv=\"'
r'refresh\" content=\"0; url=http://example.com/'
r'\">">http://\<meta http-equiv=\"refresh\" '
r'content=\"0; url=http://example.com/\"></a></p>',
r'<p>XSS33: </http://<?php><\h1><'
r'script:script>confirm(2)</p>',
r'<p>XSS34: <javascript:prompt(document.cookie)></p>',
r'<p>XSS35: <&#x6A&#x61&#x76&#x61&#x73'
r'&#x63&#x72&#x69&#x70&#x74&#x3A'
r'&#x61&#x6C&#x65&#x72&#x74&#x28'
r'&#x27&#x58&#x53&#x53&#x27&#x29>'
r'</p>',
r'<p>XSS36: <em>http://example</em>@.1 style='
r'background-image:url(data:image/png;base64,ABCABCABCABC==);'
r'background-repeat:no-repeat;display:block;width:100%;'
r'height:100px; onclick=alert(unescape(/Oh%20No!/.source));'
r'return(false);//</p>',
r'<p>XSS37:</p>',
])
