def resources_include_url(name):
env = self.environment
mime_type, encoding = mimetypes.guess_type(name)
try:
# we try to load via the jinja loader, but that tries to load
# as (encoded) text
data = env.loader.get_source(env, name)[0].encode("utf8")
except UnicodeDecodeError:
# if that fails (for instance a binary file, png or ttf)
# we mimic jinja2
pieces = split_template_path(name)
for searchpath in self.template_paths:
filename = os.path.join(searchpath, *pieces)
if os.path.exists(filename):
with open(filename, "rb") as f:
data = f.read()
break
else:
raise ValueError(
f"No file {name!r} found in {searchpath!r}")
data = base64.b64encode(data)
data = data.replace(b"\n", b"").decode("ascii")
src = f"data:{mime_type};base64,{data}"
return markupsafe.Markup(src)
def get_outgoing_url(url):
"""
Bounce a URL off an outgoing URL redirector, such as
outgoing.prod.mozaws.net.
"""
if not settings.REDIRECT_URL:
return url
# django.utils.http._urlparse is a copy of python's urlparse()
# "but uses fixed urlsplit() function".
parsed_url = django_urlparse(url)
url_netloc = parsed_url.netloc
# This prevents a link like javascript://addons.mozilla.org...
# being returned unchanged since the netloc matches the
# safe list see bug 1251023
if parsed_url.scheme not in ['http', 'https']:
return '/'
# No double-escaping, and some domain names are excluded.
allowed = settings.REDIRECT_URL_ALLOW_LIST + [
django_urlparse(settings.EXTERNAL_SITE_URL).netloc
]
if (url_netloc == django_urlparse(settings.REDIRECT_URL).netloc
or url_netloc in allowed):
return url
url = force_bytes(markupsafe.Markup(url).unescape())
sig = hmac.new(force_bytes(settings.REDIRECT_SECRET_KEY),
msg=url,
digestmod=hashlib.sha256).hexdigest()
# Let '&=' through so query params aren't escaped. We probably shouldn't
# bother to quote the query part at all.
return '/'.join(
[settings.REDIRECT_URL.rstrip('/'), sig,
quote(url, safe='/&=')])
def __call__(self, field, **kwargs):
kwargs.setdefault("id", field.id)
if self.multiple:
kwargs["multiple"] = True
if "required" not in kwargs and "required" in getattr(
field, "flags", []):
kwargs["required"] = True
if field.size:
kwargs.setdefault("size", field.size)
html = [
"<select %s>" % wtf_widgets.html_params(name=field.name, **kwargs)
]
for group_label, group_options in field.choices:
html.append("<optgroup %s>" %
wtf_widgets.html_params(label=group_label))
for value, label in group_options:
if self.multiple:
selected = field.coerce(value) in (field.data or [])
else:
selected = field.coerce(value) == field.data
html.append(self.render_option(value, label, selected))
html.append("</optgroup>")
html.append("</select>")
return markupsafe.Markup("".join(html))
def load_docs(request):
URL = app.config.get('DATAGREPPER_BASE_URL', request.url_root)
docs = htmldocs[request.endpoint]
docs = jinja2.Template(docs).render(URL=URL)
return markupsafe.Markup(docs)
def f(self, *args, **kw):
"""Calls SafeFormatter.format and returns a Markup string."""
# SafeFormatter escapes everything so this is safe.
return markupsafe.Markup(self.formatter.format(*args, **kw))
def escapejs(value):
"""Hex encodes characters for use in JavaScript strings."""
value = unicode(value)
for bad, good in _js_escapes:
value = value.replace(bad, good)
return markupsafe.Markup(value)
def index(): # type: ignore
svg = str(self.svg_poker.base_svg)
return flask.render_template("index.html",
svg=markupsafe.Markup(svg))
def render_field(field, attrs):
add_attrs = attrs or {}
soup = BeautifulSoup(str(field), "html.parser")
soup.contents[0].attrs.update(add_attrs)
return markupsafe.Markup(soup)
def markdown(text):
return markupsafe.Markup(
hoedown.html(text,
extensions=MARKDOWN_EXTENSIONS,
render_flags=MARKDOWN_RENDER_FLAGS))
async def display_columns_and_rows(
datasette,
database_name,
table_name,
description,
rows,
link_column=False,
truncate_cells=0,
sortable_columns=None,
):
"""Returns columns, rows for specified table - including fancy foreign key treatment"""
sortable_columns = sortable_columns or set()
db = datasette.databases[database_name]
table_metadata = datasette.table_metadata(database_name, table_name)
column_descriptions = table_metadata.get("columns") or {}
column_details = {
col.name: col
for col in await db.table_column_details(table_name)
}
pks = await db.primary_keys(table_name)
pks_for_display = pks
if not pks_for_display:
pks_for_display = ["rowid"]
columns = []
for r in description:
if r[0] == "rowid" and "rowid" not in column_details:
type_ = "integer"
notnull = 0
else:
type_ = column_details[r[0]].type
notnull = column_details[r[0]].notnull
columns.append({
"name": r[0],
"sortable": r[0] in sortable_columns,
"is_pk": r[0] in pks_for_display,
"type": type_,
"notnull": notnull,
"description": column_descriptions.get(r[0]),
})
column_to_foreign_key_table = {
fk["column"]: fk["other_table"]
for fk in await db.foreign_keys_for_table(table_name)
}
cell_rows = []
base_url = datasette.setting("base_url")
for row in rows:
cells = []
# Unless we are a view, the first column is a link - either to the rowid
# or to the simple or compound primary key
if link_column:
is_special_link_column = len(pks) != 1
pk_path = path_from_row_pks(row, pks, not pks, False)
cells.append({
"column":
pks[0] if len(pks) == 1 else "Link",
"value_type":
"pk",
"is_special_link_column":
is_special_link_column,
"raw":
pk_path,
"value":
markupsafe.Markup(
'<a href="{table_path}/{flat_pks_quoted}">{flat_pks}</a>'.
format(
base_url=base_url,
table_path=datasette.urls.table(
database_name, table_name),
flat_pks=str(markupsafe.escape(pk_path)),
flat_pks_quoted=path_from_row_pks(row, pks, not pks),
)),
})
for value, column_dict in zip(row, columns):
column = column_dict["name"]
if link_column and len(pks) == 1 and column == pks[0]:
# If there's a simple primary key, don't repeat the value as it's
# already shown in the link column.
continue
# First let the plugins have a go
# pylint: disable=no-member
plugin_display_value = None
for candidate in pm.hook.render_cell(
value=value,
column=column,
table=table_name,
database=database_name,
datasette=datasette,
):
candidate = await await_me_maybe(candidate)
if candidate is not None:
plugin_display_value = candidate
break
if plugin_display_value:
display_value = plugin_display_value
elif isinstance(value, bytes):
formatted = format_bytes(len(value))
display_value = markupsafe.Markup(
'<a class="blob-download" href="{}"{}><Binary: {:,} byte{}></a>'
.format(
datasette.urls.row_blob(
database_name,
table_name,
path_from_row_pks(row, pks, not pks),
column,
),
' title="{}"'.format(formatted)
if "bytes" not in formatted else "",
len(value),
"" if len(value) == 1 else "s",
))
elif isinstance(value, dict):
# It's an expanded foreign key - display link to other row
label = value["label"]
value = value["value"]
# The table we link to depends on the column
other_table = column_to_foreign_key_table[column]
link_template = LINK_WITH_LABEL if (
label != value) else LINK_WITH_VALUE
display_value = markupsafe.Markup(
link_template.format(
database=database_name,
base_url=base_url,
table=tilde_encode(other_table),
link_id=tilde_encode(str(value)),
id=str(markupsafe.escape(value)),
label=str(markupsafe.escape(label)) or "-",
))
elif value in ("", None):
display_value = markupsafe.Markup(" ")
elif is_url(str(value).strip()):
display_value = markupsafe.Markup(
'<a href="{url}">{url}</a>'.format(
url=markupsafe.escape(value.strip())))
elif column in table_metadata.get("units", {}) and value != "":
# Interpret units using pint
value = value * ureg(table_metadata["units"][column])
# Pint uses floating point which sometimes introduces errors in the compact
# representation, which we have to round off to avoid ugliness. In the vast
# majority of cases this rounding will be inconsequential. I hope.
value = round(value.to_compact(), 6)
display_value = markupsafe.Markup(f"{value:~P}".replace(
" ", " "))
else:
display_value = str(value)
if truncate_cells and len(display_value) > truncate_cells:
display_value = display_value[:truncate_cells] + "\u2026"
cells.append({
"column":
column,
"value":
display_value,
"raw":
value,
"value_type":
"none" if value is None else str(type(value).__name__),
})
cell_rows.append(Row(cells))
if link_column:
# Add the link column header.
# If it's a simple primary key, we have to remove and re-add that column name at
# the beginning of the header row.
first_column = None
if len(pks) == 1:
columns = [col for col in columns if col["name"] != pks[0]]
first_column = {
"name": pks[0],
"sortable": len(pks) == 1,
"is_pk": True,
"type": column_details[pks[0]].type,
"notnull": column_details[pks[0]].notnull,
}
else:
first_column = {
"name": "Link",
"sortable": False,
"is_pk": False,
"type": "",
"notnull": 0,
}
columns = [first_column] + columns
return columns, cell_rows
def convert_markdown(text, inline=False):
result = markupsafe.Markup(md.convert(text))
if inline and result[:3] == '<p>' and result[-4:] == '</p>':
result = result[3:-4]
return result
def load_docs(request):
URL = fedmsg_config.get('fmn.base_url', request.url_root)
docs = htmldocs[request.endpoint]
docs = jinja2.Template(docs).render(URL=URL)
return markupsafe.Markup(docs)
def user_link(user):
if not user:
return ''
return markupsafe.Markup(_user_link(user))
def linebreaks(text):
return text.replace('\n', markupsafe.Markup('<br/>'))
def load_docs(request):
URL = request.url_root
docs = htmldocs[request.endpoint]
docs = jinja2.Template(docs).render(URL=URL)
return markupsafe.Markup(docs)
def spacer(self):
return markupsafe.Markup(
'<img src="spacer.gif" width="1" height="1" alt="" />')
def HTML(html): # pylint: disable=invalid-name
return markupsafe.Markup(html)
def urlencode_filter(s):
if type(s) == 'Markup':
s = s.unescape()
s = s.encode('utf8')
s = flasktools.quote_plus(s)
return markupsafe.Markup(s)
def md(s: str) -> str:
html = markdown_code_blocks.highlight(s, Renderer=Renderer)
# manually bless the highlighted output.
return markupsafe.Markup(html)
def safe_fn(*args, **kwargs):
return markupsafe.Markup(fn(*args, **kwargs))
async def extra_template():
display_rows = []
for row in results.rows if results else []:
display_row = []
for column, value in zip(results.columns, row):
display_value = value
# Let the plugins have a go
# pylint: disable=no-member
plugin_display_value = None
for candidate in pm.hook.render_cell(
value=value,
column=column,
table=None,
database=database,
datasette=self.ds,
):
candidate = await await_me_maybe(candidate)
if candidate is not None:
plugin_display_value = candidate
break
if plugin_display_value is not None:
display_value = plugin_display_value
else:
if value in ("", None):
display_value = Markup(" ")
elif is_url(str(display_value).strip()):
display_value = Markup(
'<a href="{url}">{url}</a>'.format(
url=escape(value.strip())))
elif isinstance(display_value, bytes):
blob_url = path_with_format(
request=request,
format="blob",
extra_qs={
"_blob_column":
column,
"_blob_hash":
hashlib.sha256(display_value).hexdigest(),
},
)
display_value = Markup(
'<a class="blob-download" href="{}"><Binary: {} byte{}></a>'
.format(
blob_url,
len(display_value),
"" if len(value) == 1 else "s",
))
display_row.append(display_value)
display_rows.append(display_row)
# Show 'Edit SQL' button only if:
# - User is allowed to execute SQL
# - SQL is an approved SELECT statement
# - No magic parameters, so no :_ in the SQL string
edit_sql_url = None
is_validated_sql = False
try:
validate_sql_select(sql)
is_validated_sql = True
except InvalidSql:
pass
if allow_execute_sql and is_validated_sql and ":_" not in sql:
edit_sql_url = (self.ds.urls.database(database) + "?" +
urlencode({
**{
"sql": sql,
},
**named_parameter_values,
}))
show_hide_hidden = ""
if metadata.get("hide_sql"):
if bool(params.get("_show_sql")):
show_hide_link = path_with_removed_args(
request, {"_show_sql"})
show_hide_text = "hide"
show_hide_hidden = (
'<input type="hidden" name="_show_sql" value="1">')
else:
show_hide_link = path_with_added_args(
request, {"_show_sql": 1})
show_hide_text = "show"
else:
if bool(params.get("_hide_sql")):
show_hide_link = path_with_removed_args(
request, {"_hide_sql"})
show_hide_text = "show"
show_hide_hidden = (
'<input type="hidden" name="_hide_sql" value="1">')
else:
show_hide_link = path_with_added_args(
request, {"_hide_sql": 1})
show_hide_text = "hide"
hide_sql = show_hide_text == "show"
return {
"display_rows": display_rows,
"custom_sql": True,
"named_parameter_values": named_parameter_values,
"editable": editable,
"canned_query": canned_query,
"edit_sql_url": edit_sql_url,
"metadata": metadata,
"settings": self.ds.settings_dict(),
"request": request,
"show_hide_link": show_hide_link,
"show_hide_text": show_hide_text,
"show_hide_hidden": markupsafe.Markup(show_hide_hidden),
"hide_sql": hide_sql,
}
def nl2br(string):
"""Turn newlines into <br/>."""
if not string:
return ''
return markupsafe.Markup('<br/>'.join(
markupsafe.escape(string).splitlines()))
def highlighted(string):
# needs to be marked as safe so we don't need to do it everywhere in the template
# TODO: maybe use something "more semantic" than <b> (CSS needs changing too if so)
return markupsafe.Markup(
string.apply('<b>', '</b>', lambda s: str(markupsafe.escape(s)))
)
def urlparams(*args, **kwargs):
return markupsafe.Markup(utils.urlparams(*args, **kwargs))
def html_sanitize(src, silent=True, sanitize_tags=True, sanitize_attributes=False, sanitize_style=False, sanitize_form=True, strip_style=False, strip_classes=False):
if not src:
return src
src = ustr(src, errors='replace')
# html: remove encoding attribute inside tags
doctype = re.compile(r'(<[^>]*\s)(encoding=(["\'][^"\']*?["\']|[^\s\n\r>]+)(\s[^>]*|/)?>)', re.IGNORECASE | re.DOTALL)
src = doctype.sub(u"", src)
logger = logging.getLogger(__name__ + '.html_sanitize')
# html encode mako tags <% ... %> to decode them later and keep them alive, otherwise they are stripped by the cleaner
src = src.replace(u'<%', misc.html_escape(u'<%'))
src = src.replace(u'%>', misc.html_escape(u'%>'))
kwargs = {
'page_structure': True,
'style': strip_style, # True = remove style tags/attrs
'sanitize_style': sanitize_style, # True = sanitize styling
'forms': sanitize_form, # True = remove form tags
'remove_unknown_tags': False,
'comments': False,
'processing_instructions': False
}
if sanitize_tags:
kwargs['allow_tags'] = allowed_tags
if etree.LXML_VERSION >= (2, 3, 1):
# kill_tags attribute has been added in version 2.3.1
kwargs.update({
'kill_tags': tags_to_kill,
'remove_tags': tags_to_remove,
})
else:
kwargs['remove_tags'] = tags_to_kill + tags_to_remove
if sanitize_attributes and etree.LXML_VERSION >= (3, 1, 0): # lxml < 3.1.0 does not allow to specify safe_attrs. We keep all attributes in order to keep "style"
if strip_classes:
current_safe_attrs = safe_attrs - frozenset(['class'])
else:
current_safe_attrs = safe_attrs
kwargs.update({
'safe_attrs_only': True,
'safe_attrs': current_safe_attrs,
})
else:
kwargs.update({
'safe_attrs_only': False, # keep oe-data attributes + style
'strip_classes': strip_classes, # remove classes, even when keeping other attributes
})
try:
# some corner cases make the parser crash (such as <SCRIPT/XSS SRC=\"http://ha.ckers.org/xss.js\"></SCRIPT> in test_mail)
cleaner = _Cleaner(**kwargs)
cleaned = cleaner.clean_html(src)
assert isinstance(cleaned, str)
# MAKO compatibility: $, { and } inside quotes are escaped, preventing correct mako execution
cleaned = cleaned.replace(u'%24', u'$')
cleaned = cleaned.replace(u'%7B', u'{')
cleaned = cleaned.replace(u'%7D', u'}')
cleaned = cleaned.replace(u'%20', u' ')
cleaned = cleaned.replace(u'%5B', u'[')
cleaned = cleaned.replace(u'%5D', u']')
cleaned = cleaned.replace(u'%7C', u'|')
cleaned = cleaned.replace(u'<%', u'<%')
cleaned = cleaned.replace(u'%>', u'%>')
# html considerations so real html content match database value
cleaned.replace(u'\xa0', u' ')
except etree.ParserError as e:
if 'empty' in str(e):
return u""
if not silent:
raise
logger.warning(u'ParserError obtained when sanitizing %r', src, exc_info=True)
cleaned = u'<p>ParserError when sanitizing</p>'
except Exception:
if not silent:
raise
logger.warning(u'unknown error obtained when sanitizing %r', src, exc_info=True)
cleaned = u'<p>Unknown error when sanitizing</p>'
# this is ugly, but lxml/etree tostring want to put everything in a 'div' that breaks the editor -> remove that
if cleaned.startswith(u'<div>') and cleaned.endswith(u'</div>'):
cleaned = cleaned[5:-6]
return markupsafe.Markup(cleaned)
def impala_paginator(pager):
t = loader.get_template('amo/impala/paginator.html')
return markupsafe.Markup(t.render({'pager': pager}))
def markdown(text):
text = FS_RE.sub(fs_replace, text)
return markupsafe.Markup(hoedown.html(
text, extensions=MARKDOWN_EXTENSIONS, render_flags=MARKDOWN_RENDER_FLAGS))
def meta_tags(context, model=None):
request = context.get('request', None)
if not model:
model = context.get('page', None)
return markupsafe.Markup(tags.meta_tags(request, model))
def validate_new_password(self, field):
if self._breach_service.check_password(field.data,
tags=["method:new_password"]):
raise wtforms.validators.ValidationError(
markupsafe.Markup(self._breach_service.failure_message))
def jinja2_filter_markdown(text):
return markupsafe.Markup(md2html.reset().convert(text))