def check(self, app_configs=None, tags=None, display_num_errors=False,
include_deployment_checks=False):
"""
Uses the system check framework to validate entire Django project.
Raises CommandError for any serious message (error or critical errors).
If there are only light messages (like warnings), they are printed to
stderr and no exception is raised.
"""
all_issues = checks.run_checks(
app_configs=app_configs,
tags=tags,
include_deployment_checks=include_deployment_checks,
)
msg = ""
visible_issue_count = 0 # excludes silenced warnings
if all_issues:
debugs = [e for e in all_issues if e.level < checks.INFO and not e.is_silenced()]
infos = [e for e in all_issues if checks.INFO <= e.level < checks.WARNING and not e.is_silenced()]
warnings = [e for e in all_issues if checks.WARNING <= e.level < checks.ERROR and not e.is_silenced()]
errors = [e for e in all_issues if checks.ERROR <= e.level < checks.CRITICAL]
criticals = [e for e in all_issues if checks.CRITICAL <= e.level]
sorted_issues = [
(criticals, 'CRITICALS'),
(errors, 'ERRORS'),
(warnings, 'WARNINGS'),
(infos, 'INFOS'),
(debugs, 'DEBUGS'),
]
for issues, group_name in sorted_issues:
if issues:
visible_issue_count += len(issues)
formatted = (
color_style().ERROR(force_str(e))
if e.is_serious()
else color_style().WARNING(force_str(e))
for e in issues)
formatted = "\n".join(sorted(formatted))
msg += '\n%s:\n%s\n' % (group_name, formatted)
if msg:
msg = "System check identified some issues:\n%s" % msg
if display_num_errors:
if msg:
msg += '\n'
msg += "System check identified %s (%s silenced)." % (
"no issues" if visible_issue_count == 0 else
"1 issue" if visible_issue_count == 1 else
"%s issues" % visible_issue_count,
len(all_issues) - visible_issue_count,
)
if any(e.is_serious() and not e.is_silenced() for e in all_issues):
raise CommandError(msg)
elif msg and visible_issue_count:
self.stderr.write(msg)
elif msg:
self.stdout.write(msg)
def get_option_value(optionset_admin, db_option, current_only, as_stored):
"""
Given an AppOption object, return its value for the current language.
"""
if not db_option:
return None
name = force_str(db_option.name)
if not name in optionset_admin.options:
return None
field = optionset_admin.options[name]
if not db_option.lang_dependant:
if as_stored:
return db_option.value
return field.to_python(db_option.value) if db_option.value else ''
value_dict = {}
for key, value in json.loads(db_option.value).items():
value_dict[force_str(key)] = value
if current_only:
curr_lang = get_language()
if curr_lang in value_dict:
if as_stored:
return value_dict[curr_lang]
return field.to_python(value_dict[curr_lang]) if value_dict[curr_lang] else ''
else:
for key in value_dict:
value_dict[key] = field.to_python(value_dict[key]) if as_stored else \
value_dict[key]
return value_dict
def lfs_get_object_or_404(klass, *args, **kwargs):
"""
Uses get() to return an object, or raises a Http404 exception if the object
does not exist.
klass may be a Model, Manager, or QuerySet object. All other passed
arguments and keyword arguments are used in the get() query.
Note: Like with get(), an MultipleObjectsReturned will be raised if more than one
object is found.
"""
cache_key = "%s-%s-%s" % (force_str(settings.CACHE_MIDDLEWARE_KEY_PREFIX), klass.__name__.lower(),
force_str(kwargs.values()[0]))
cache_key = hashlib.md5(cache_key).hexdigest()
object = cache.get(cache_key)
if object is not None:
return object
queryset = _get_queryset(klass)
try:
object = queryset.get(*args, **kwargs)
except queryset.model.DoesNotExist:
raise Http404('No %s matches the given query.' % queryset.model._meta.object_name)
else:
cache.set(cache_key, object)
return object
def test_accent(self):
# The pgpass temporary file needs to be encoded using the system locale.
encoding = locale.getpreferredencoding()
username = '******'
password = '******'
try:
username_str = force_str(username, encoding)
password_str = force_str(password, encoding)
pgpass_bytes = force_bytes(
'somehost:444:dbname:%s:%s\n' % (username, password),
encoding=encoding,
)
except UnicodeEncodeError:
if six.PY2:
self.skipTest("Your locale can't run this test.")
self.assertEqual(
self._run_it({
'NAME': 'dbname',
'USER': username_str,
'PASSWORD': password_str,
'HOST': 'somehost',
'PORT': 444,
}), (
['psql', '-U', username_str, '-h', 'somehost', '-p', '444', 'dbname'],
pgpass_bytes,
)
)
def items_for_result(cl, result, form):
"""
Generates the actual list of data.
@jjdelc:
This has been shamelessly copied from original
django.contrib.admin.templatetags.admin_list.items_for_result
in order to alter the dispay for the first element
"""
first = True
pk = cl.lookup_opts.pk.attname
for field_name in cl.list_display:
result_repr, row_class = get_result_and_row_class(cl, field_name,
result)
# If list_display_links not defined, add the link tag to the
# first field
if (first and not cl.list_display_links) or \
field_name in cl.list_display_links:
table_tag = {True: 'th', False: 'td'}[first]
# This spacer indents the nodes based on their depth
spacer = get_spacer(first, result)
# This shows a collapse or expand link for nodes with childs
collapse = get_collapse(result)
# Add a <td/> before the first col to show the drag handler
drag_handler = get_drag_handler(first)
first = False
url = cl.url_for_result(result)
# Convert the pk to something that can be used in Javascript.
# Problem cases are long ints (23L) and non-ASCII strings.
if cl.to_field:
attr = str(cl.to_field)
else:
attr = pk
value = result.serializable_value(attr)
onclickstr = format_html("""
onclick="opener.dismissRelatedLookupPopup(window, '{}'); return false;"
""", mark_safe(value))
yield mark_safe(
u('%s<%s%s>%s %s <a href="%s"%s>%s</a></%s>') % (
drag_handler, table_tag, row_class, spacer, collapse, url,
(cl.is_popup and onclickstr or ''),
conditional_escape(result_repr), table_tag))
else:
# By default the fields come from ModelAdmin.list_editable, but if
# we pull the fields out of the form instead of list_editable
# custom admins can provide fields on a per request basis
if (
form and
field_name in form.fields and
not (
field_name == cl.model._meta.pk.name and
form[cl.model._meta.pk.name].is_hidden
)
):
bf = form[field_name]
result_repr = mark_safe(force_str(bf.errors) + force_str(bf))
yield format_html(u('<td{0}>{1}</td>'), row_class, result_repr)
if form and not form[cl.model._meta.pk.name].is_hidden:
yield format_html(u('<td>{0}</td>'),
force_str(form[cl.model._meta.pk.name]))
def _node_wrapper(self, obj, parent, label, value):
if obj == value:
checked = 'true'
self.display = force_str(label)
else:
checked = 'false'
return dict(id=obj, pId=parent, name=force_str(label), checked=force_str(checked))
def render(self, name, value, attrs=None, choices=()):
cts = ContentType.objects.all()
tree = [dict(id=self.app_base+index, pId=0, name=force_str(app), checked='false', open='true') for index, app in enumerate(SYSTEM_APPS)]
for index, ct in enumerate(cts):
if ct.app_label in SYSTEM_APPS:
model_id = self.model_base + index
tree.append(dict(id=model_id, pId=SYSTEM_APPS.index(ct.app_label)+self.app_base,
name=force_str(ct.model), checked='false'))
model_permissions = ct.permission_set.all()
for mp in model_permissions:
tree.append(dict(id=mp.id, pId=model_id,
name=force_str(mp.name), checked='false'))
final_attrs = self.build_attrs(attrs, name=name, value=value, type='hidden')
display_attrs = {
'id': "display_{0}".format(name),
'class': 'form-control',
'value': self.display
}
output = """
<input {attrs} />
<div id="down_{name}" class="input-group" style="display:none">
<input {display_attrs}>
<div class="input-group-addon"><i class="glyphicon glyphicon-chevron-down"></i></div>
</div>
<div id="zTree_{name}" class="zTreeBackground left">
<ul id="tree_{name}" class="ztree"></ul>
</div>
{script}
"""
script = loader.render_to_string('group_tree.html',
{'nodes': mark_safe(json.dumps(tree)), 'name': name, 'toggle': 'false'})
return format_html(output, attrs=flatatt(final_attrs), display_attrs=flatatt(display_attrs),
name=force_text(name), script=mark_safe(script))
def render(self, name, value, attrs=None, choices=()):
if value is None:
value = []
has_id = attrs and 'id' in attrs
final_attrs = self.build_attrs(attrs, name=name)
output = []
# Normalize to strings
str_values = set([force_str(v) for v in value])
for i, (option_value, option_label) in enumerate(chain(self.choices, choices)):
# If an ID attribute was given, add a numeric index as a suffix,
# so that the checkboxes don't all have the same ID attribute.
if has_id:
final_attrs = dict(final_attrs, id='%s_%s' % (attrs['id'], i))
label_for = u' for="%s"' % final_attrs['id']
else:
label_for = ''
cb = forms.CheckboxInput(
final_attrs, check_test=lambda value: value in str_values)
option_value = force_str(option_value)
rendered_cb = cb.render(name, option_value)
option_label = conditional_escape(force_str(option_label))
if final_attrs.get('inline', False):
output.append(u'<label%s class="checkbox-inline">%s %s</label>' % (label_for, rendered_cb, option_label))
else:
output.append(u'<div class="checkbox"><label%s>%s %s</label></div>' % (label_for, rendered_cb, option_label))
return mark_safe(u'\n'.join(output))
def get_context(self):
context = super(RevisionListView, self).get_context()
opts = self.opts
action_list = [
{
"revision": version.revision,
"url": self.model_admin_url('revision', quote(version.object_id), version.id),
"version": version
}
for version
in self._order_version_queryset(self.revision_manager.get_for_object_reference(
self.model,
self.obj.pk,
).select_related("revision__user"))
]
context.update({
'title': _('Change history: %s') % force_str(self.obj),
'action_list': action_list,
'model_name': capfirst(force_str(opts.verbose_name_plural)),
'object': self.obj,
'app_label': opts.app_label,
"changelist_url": self.model_admin_url("changelist"),
"update_url": self.model_admin_url("change", self.obj.pk),
'opts': opts,
})
return context
def _mjml_render_by_cmd(mjml_code):
if 'cmd_args' not in _cache:
cmd_args = copy.copy(mjml_settings.MJML_EXEC_CMD)
if not isinstance(cmd_args, list):
cmd_args = [cmd_args]
for ca in ('-i', '-s'):
if ca not in cmd_args:
cmd_args.append(ca)
_cache['cmd_args'] = cmd_args
else:
cmd_args = _cache['cmd_args']
with tempfile.SpooledTemporaryFile(max_size=(5 * 1024 * 1024)) as stdout_tmp_f:
try:
p = subprocess.Popen(cmd_args, stdin=subprocess.PIPE, stdout=stdout_tmp_f, stderr=subprocess.PIPE)
stderr = p.communicate(force_bytes(mjml_code))[1]
except (IOError, OSError) as e:
raise RuntimeError(
'Problem to run command "{}"\n'.format(' '.join(cmd_args)) +
'{}\n'.format(e) +
'Check that mjml is installed and allow permissions for execute.\n' +
'See https://github.com/mjmlio/mjml#installation'
)
stdout_tmp_f.seek(0)
stdout = stdout_tmp_f.read()
if stderr:
raise RuntimeError('MJML stderr is not empty: {}.'.format(force_str(stderr)))
return force_str(stdout)
def _begin(self, connection, filterargs=(), escape=True):
"""
Begins an asynchronous search and returns the message id to retrieve
the results.
filterargs is an object that will be used for expansion of the filter
string. If escape is True, values in filterargs will be escaped.
"""
if escape:
filterargs = self._escape_filterargs(filterargs)
try:
filterstr = self.filterstr % filterargs
msgid = connection.search(
force_str(self.base_dn), self.scope, force_str(filterstr),
self.attrlist
)
except ldap.LDAPError as e:
msgid = None
logger.error(
"search('{}', {}, '{}') raised {}".format(
self.base_dn, self.scope, filterstr, pprint.pformat(e)
)
)
return msgid
def test_delete_user(self):
self.reset_ldap_directory()
user = core_factories.UserFactory(
username=self.username,
first_name="Test",
last_name="LDAP",
groups=("SimpleUsers",)
)
user.delete()
ldap_record = self.conn.search_s(
force_str(self.dn), ldap.SCOPE_SUBTREE,
force_str("(&(objectClass=inetOrgPerson))")
)
password = ldap_record[0][1]["userPassword"][0].split(b"}")[1]
self.assertTrue(password.startswith(b"#"))
with self.assertRaises(ldap.INVALID_CREDENTIALS):
lib.get_connection(self.config, self.dn, "toto")
user = core_factories.UserFactory(
username=self.username,
first_name="Test",
last_name="LDAP",
groups=("SimpleUsers",)
)
self.set_global_parameter(
"ldap_sync_delete_remote_account", True, app="core")
user.delete()
self.assertFalse(lib.check_if_dn_exists(self.conn, self.dn))
def execute(self, connection, filterargs=(), escape=True):
"""
Executes the search on the given connection (an LDAPObject). filterargs
is an object that will be used for expansion of the filter string.
If escape is True, values in filterargs will be escaped.
The python-ldap library returns utf8-encoded strings. For the sake of
sanity, this method will decode all result strings and return them as
Unicode.
"""
if escape:
filterargs = self._escape_filterargs(filterargs)
try:
filterstr = self.filterstr % filterargs
results = connection.search_s(force_str(self.base_dn),
self.scope,
force_str(filterstr),
self.attrlist)
except ldap.LDAPError as e:
results = []
logger.error(
"search_s('{}', {}, '{}') raised {}".format(
self.base_dn, self.scope, filterstr, pprint.pformat(e)
)
)
return self._process_results(results)
def __init__(self, request, verbose_name, icon='caret-right', attrs=None):
self.request = request
self.verbose_name = force_str(force_unicode(verbose_name))
self._icon = icon if isinstance(icon, Icon) else Icon(force_str(icon), prefix='fa-', attrs={
'class': 'fa fa-lg fa-fw'
})
self.attrs = attrs or {}
def get_context(self):
"""
Prepare the context for templates.
"""
self.title = _('%s List') % force_str(self.opts.verbose_name)
model_fields = [(f, f.name in self.list_display, self.get_check_field_url(f))
for f in (list(self.opts.fields) + self.get_model_method_fields()) if f.name not in self.list_exclude]
new_context = {
'model_name': force_str(self.opts.verbose_name_plural),
'title': self.title,
'cl': self,
'model_fields': model_fields,
'clean_select_field_url': self.get_query_string(remove=[COL_LIST_VAR]),
'has_add_permission': self.has_add_permission(),
'app_label': self.app_label,
'brand_name': self.opts.verbose_name_plural,
'brand_icon': self.get_model_icon(self.model),
'add_url': self.model_admin_url('add'),
'result_headers': self.result_headers(),
'results': self.results()
}
context = super(ListAdminView, self).get_context()
context.update(new_context)
return context
def test_force_str_DjangoUnicodeDecodeError(self):
msg = (
"'utf-8' codec can't decode byte 0xff in position 0: invalid "
"start byte. You passed in b'\\xff' (<class 'bytes'>)"
)
with self.assertRaisesMessage(DjangoUnicodeDecodeError, msg):
force_str(b'\xff')
def smart_urlquote(url):
"Quotes a URL if it isn't already quoted."
def unquote_quote(segment):
segment = unquote(force_str(segment))
# Tilde is part of RFC3986 Unreserved Characters
# http://tools.ietf.org/html/rfc3986#section-2.3
# See also http://bugs.python.org/issue16285
segment = quote(segment, safe=RFC3986_SUBDELIMS + RFC3986_GENDELIMS + str('~'))
return force_text(segment)
# Handle IDN before quoting.
try:
scheme, netloc, path, query, fragment = urlsplit(url)
except ValueError:
# invalid IPv6 URL (normally square brackets in hostname part).
return unquote_quote(url)
try:
netloc = netloc.encode('idna').decode('ascii') # IDN -> ACE
except UnicodeError: # invalid domain part
return unquote_quote(url)
if query:
# Separately unquoting key/value, so as to not mix querystring separators
# included in query values. See #22267.
query_parts = [(unquote(force_str(q[0])), unquote(force_str(q[1])))
for q in parse_qsl(query, keep_blank_values=True)]
# urlencode will take care of quoting
query = urlencode(query_parts)
path = unquote_quote(path)
fragment = unquote_quote(fragment)
return urlunsplit((scheme, netloc, path, query, fragment))
def open(self):
"""
Ensures we have a connection to the email server. Returns whether or
not a new connection was required (True or False).
"""
if self.connection:
# Nothing to do if the connection is already open.
return False
# If local_hostname is not specified, socket.getfqdn() gets used.
# For performance, we use the cached FQDN for local_hostname.
connection_params = {'local_hostname': DNS_NAME.get_fqdn()}
if self.timeout is not None:
connection_params['timeout'] = self.timeout
if self.use_ssl:
connection_params.update({
'keyfile': self.ssl_keyfile,
'certfile': self.ssl_certfile,
})
try:
self.connection = self.connection_class(self.host, self.port, **connection_params)
# TLS/SSL are mutually exclusive, so only attempt TLS over
# non-secure connections.
if not self.use_ssl and self.use_tls:
self.connection.ehlo()
self.connection.starttls(keyfile=self.ssl_keyfile, certfile=self.ssl_certfile)
self.connection.ehlo()
if self.username and self.password:
self.connection.login(force_str(self.username), force_str(self.password))
return True
except smtplib.SMTPException:
if not self.fail_silently:
raise
def test_permission_required(self):
# Adrian can change his book
self.assertTrue(self.client.login(username='******', password='******'))
response = self.client.get(reverse('change_book', args=(1,)))
self.assertEqual(response.status_code, 200)
self.assertEqual(force_str(response.content), 'OK')
# Martin can change Adrian's book
self.assertTrue(self.client.login(username='******', password='******'))
response = self.client.get(reverse('change_book', args=(1,)))
self.assertEqual(response.status_code, 200)
self.assertEqual(force_str(response.content), 'OK')
# Adrian can delete his book
self.assertTrue(self.client.login(username='******', password='******'))
response = self.client.get(reverse('delete_book', args=(1,)))
self.assertEqual(response.status_code, 200)
self.assertEqual(force_str(response.content), 'OK')
# Martin can *not* create a book
# Up to Django v2.1, the response was a redirect to login
self.assertTrue(self.client.login(username='******', password='******'))
response = self.client.get(reverse('cbv.create_book'))
self.assertIn(response.status_code, [302, 403])
# Martin can *not* delete Adrian's book and is redirected to login
self.assertTrue(self.client.login(username='******', password='******'))
response = self.client.get(reverse('delete_book', args=(1,)))
self.assertEqual(response.status_code, 302)
# Martin can *not* delete Adrian's book and an PermissionDenied is raised
self.assertTrue(self.client.login(username='******', password='******'))
response = self.client.get(reverse('view_that_raises', args=(1,)))
self.assertEqual(response.status_code, 403)
# Test views that require a list of permissions
# Adrian has both permissions
self.assertTrue(self.client.login(username='******', password='******'))
response = self.client.get(reverse('view_with_permission_list', args=(1,)))
self.assertEqual(response.status_code, 200)
self.assertEqual(force_str(response.content), 'OK')
# Martin does not have delete permission
self.assertTrue(self.client.login(username='******', password='******'))
response = self.client.get(reverse('view_with_permission_list', args=(1,)))
self.assertEqual(response.status_code, 302)
# Test views that accept a static object as argument
# fn is passed to has_perm as-is
self.assertTrue(self.client.login(username='******', password='******'))
response = self.client.get(reverse('view_with_object', args=(1,)))
self.assertEqual(response.status_code, 200)
self.assertEqual(force_str(response.content), 'OK')
self.assertTrue(self.client.login(username='******', password='******'))
response = self.client.get(reverse('view_with_object', args=(1,)))
self.assertEqual(response.status_code, 302)
def urlquote_plus(url, safe=''):
"""
A version of Python's urllib.quote_plus() function that can operate on
unicode strings. The url is first UTF-8 encoded before quoting. The
returned string can safely be used as part of an argument to a subsequent
iri_to_uri() call without double-quoting occurring.
"""
return force_text(quote_plus(force_str(url), force_str(safe)))
def as_dict(self):
template = force_str(self.template) if self.template else None
return dict(
code=force_str(self.get_slug()),
name=force_str(self.get_name()),
type=force_str(self.kind),
template=template,
)
def post_response(self):
self.message_user(_('The %(name)s "%(obj)s" was deleted successfully.') %
{'name': force_str(self.opts.verbose_name), 'obj': force_str(self.obj)}, 'success')
if not self.has_view_permission():
return self.get_admin_url('index')
return self.model_admin_url('changelist')
def __init__(self, key=None, sep=':', salt=None):
# Use of native strings in all versions of Python
self.key = key or settings.SECRET_KEY
self.sep = force_str(sep)
if _SEP_UNSAFE.match(self.sep):
warnings.warn('Unsafe Signer separator: %r (cannot be empty or consist of only A-z0-9-_=)' % sep,
RemovedInDjango110Warning)
self.salt = force_str(salt or
'%s.%s' % (self.__class__.__module__, self.__class__.__name__))
def compile_source(self, source):
try:
compiled = sass.compile(string=source, indented=self.indented)
except sass.CompileError as e:
raise exceptions.StaticCompilationError(encoding.force_str(e))
compiled = encoding.force_str(compiled)
return compiled
def _get_conn(self, dn, password):
"""Get a connection from the server."""
conn = ldap.initialize(self.server_uri, bytes_mode=six.PY2)
conn.set_option(ldap.OPT_X_TLS_DEMAND, True)
conn.set_option(ldap.OPT_DEBUG_LEVEL, 255)
conn.simple_bind_s(
force_str(dn), force_str(password)
)
return conn
def gen_bbcode_tag_klass(klass_attrs, options_attrs={}):
# Construc the inner Options class
options_klass = type(force_str('Options'), (), options_attrs)
# Construct the outer BBCodeTag class
tag_klass_attrs = klass_attrs
tag_klass_attrs['Options'] = options_klass
tag_klass = type(
force_str('{}Tag'.format(tag_klass_attrs['name'])), (BBCodeTag, ), tag_klass_attrs)
return tag_klass
def _get_path(self, parsed):
path = force_str(parsed[2])
# If there are parameters, add them
if parsed[3]:
path += str(";") + force_str(parsed[3])
path = uri_to_iri(path).encode(UTF_8)
# Under Python 3, non-ASCII values in the WSGI environ are arbitrarily
# decoded with ISO-8859-1. We replicate this behavior here.
# Refs comment in `get_bytes_from_wsgi()`.
return path.decode(ISO_8859_1) if six.PY3 else path
def _get_path(self, parsed):
path = force_str(parsed[2])
# If there are parameters, add them
if parsed[3]:
path += str(";") + force_str(parsed[3])
path = unquote(path)
# WSGI requires latin-1 encoded strings. See get_path_info().
if six.PY3:
path = path.encode("utf-8").decode("iso-8859-1")
return path
def replace_query_param(url, key, val):
"""
Given a URL and a key/val pair, set or replace an item in the query
parameters of the URL, and return the new URL.
"""
(scheme, netloc, path, query, fragment) = parse.urlsplit(force_str(url))
query_dict = parse.parse_qs(query, keep_blank_values=True)
query_dict[force_str(key)] = [force_str(val)]
query = parse.urlencode(sorted(list(query_dict.items())), doseq=True)
return parse.urlunsplit((scheme, netloc, path, query, fragment))
def default(self, obj):
if isinstance(obj, Translation):
return force_str(obj)
return super(AMOJSONEncoder, self).default(obj)
def prep_for_like_query(self, x):
"""Prepares a value for use in a LIKE query."""
# http://msdn2.microsoft.com/en-us/library/ms179859.aspx
return force_str(x).replace('\\', '\\\\').replace('[', '[[]').replace(
'%', '[%]').replace('_', '[_]')
def msg_dict(self):
return {
'name': force_str(self.model._meta.verbose_name),
'obj': force_str(self.object)
}
def urlunquote_plus(quoted_url):
"""
A wrapper for Python's urllib.unquote_plus() function that can operate on
the result of django.utils.http.urlquote_plus().
"""
return force_text(unquote_plus(force_str(quoted_url)))
def __repr__(self):
return force_str(
'<%s %s %s>' %
(self.__class__.__name__, self.name, self.regex.pattern))
def render(self, value, obj=None):
return force_str(value)
def process_import(request):
supported_extensions = get_supported_extensions()
from_encoding = "utf-8"
form_kwargs = {}
form = ConfirmImportForm(DEFAULT_FORMATS, request.POST or None,
request.FILES or None, **form_kwargs)
is_confirm_form_valid = form.is_valid()
import_formats = get_import_formats()
input_format = import_formats[int(form.cleaned_data["input_format"])]()
FileStorage = get_file_storage()
file_storage = FileStorage(name=form.cleaned_data["import_file_name"])
if not is_confirm_form_valid:
data = file_storage.read(input_format.get_read_mode())
if not input_format.is_binary() and from_encoding:
data = force_str(data, from_encoding)
dataset = input_format.create_dataset(data)
initial = {
"import_file_name": file_storage.name,
"original_file_name": form.cleaned_data["import_file_name"],
}
return render(
request,
"wagtailredirects/confirm_import.html",
{
"form":
ConfirmImportForm(
dataset.headers,
request.POST or None,
request.FILES or None,
initial=initial,
),
"dataset":
dataset,
},
)
data = file_storage.read(input_format.get_read_mode())
if not input_format.is_binary() and from_encoding:
data = force_str(data, from_encoding)
dataset = input_format.create_dataset(data)
import_summary = create_redirects_from_dataset(
dataset,
{
"from_index": int(form.cleaned_data["from_index"]),
"to_index": int(form.cleaned_data["to_index"]),
"permanent": form.cleaned_data["permanent"],
"site": form.cleaned_data["site"],
},
)
file_storage.remove()
if import_summary["errors_count"] > 0:
return render(
request,
"wagtailredirects/import_summary.html",
{
"form": ImportForm(supported_extensions),
"import_summary": import_summary,
},
)
total = import_summary["total"]
messages.success(
request,
ngettext("Imported %(total)d redirect", "Imported %(total)d redirects",
total) % {"total": total},
)
return redirect("wagtailredirects:index")
def get_action_label(self, action):
from wagtail.admin.log_action_registry import registry as log_action_registry
return force_str(log_action_registry.get_action_label(action))
def fmt_diff(value, diff, idx):
"""Format diff if there is any."""
if diff is None:
return escape(value)
return html_diff(force_str(diff[idx]), value)
def last_executed_query(self, cursor, sql, params):
# With MySQLdb, cursor objects have an (undocumented) "_executed"
# attribute where the exact query sent to the database is saved.
# See MySQLdb/cursors.py in the source distribution.
# MySQLdb returns string, PyMySQL bytes.
return force_str(getattr(cursor, '_executed', None), errors='replace')
def format_translation(
value,
language,
plural=None,
diff=None,
search_match=None,
simple=False,
num_plurals=2,
unit=None,
match="search",
):
"""Nicely formats translation text possibly handling plurals or diff."""
# Split plurals to separate strings
plurals = split_plural(value)
if plural is None:
plural = language.plural
# Show plurals?
if int(num_plurals) <= 1:
plurals = plurals[-1:]
# Newline concatenator
newline = SPACE_NL.format(gettext("New line"))
# Split diff plurals
if diff is not None:
diff = split_plural(diff)
# Previous message did not have to be a plural
while len(diff) < len(plurals):
diff.append(diff[0])
# We will collect part for each plural
parts = []
has_content = False
for idx, raw_value in enumerate(plurals):
# HTML escape
value = force_str(raw_value)
# Content of the Copy to clipboard button
copy = escape(value)
# Format diff if there is any
value = fmt_diff(value, diff, idx)
# Create span for checks highlights
value = fmt_highlights(raw_value, value, unit)
# Format search term
value = fmt_search(value, search_match, match)
# Normalize newlines
value = NEWLINES_RE.sub("\n", value)
# Split string
paras = value.split("\n")
# Format whitespace in each paragraph
paras = [fmt_whitespace(p) for p in paras]
# Show label for plural (if there are any)
title = ""
if len(plurals) > 1:
title = plural.get_plural_name(idx)
# Join paragraphs
content = mark_safe(newline.join(paras))
parts.append({"title": title, "content": content, "copy": copy})
has_content |= bool(content)
return {
"simple": simple,
"items": parts,
"language": language,
"unit": unit,
"has_content": has_content,
}
def __call__(self, environ, start_response):
"""
Hijack the main loop from the original thread and listen on events on the Redis
and the Websocket filedescriptors.
"""
websocket = None
subscriber = self.Subscriber(self._redis_connection)
try:
self.assure_protocol_requirements(environ)
request = WSGIRequest(environ)
if callable(settings.FLOW_WS_PROCESS_REQUEST):
settings.FLOW_WS_PROCESS_REQUEST(request)
else:
self.process_request(request)
channels, echo_message = self.process_subscriptions(request)
if callable(settings.FLOW_WS_ALLOWED_CHANNELS):
channels = list(
settings.FLOW_WS_ALLOWED_CHANNELS(request, channels))
websocket = self.upgrade_websocket(environ, start_response)
logger.debug('Subscribed to channels: {0}'.format(
', '.join(channels)))
subscriber.set_pubsub_channels(request, channels)
websocket_fd = websocket.get_file_descriptor()
listening_fds = [websocket_fd]
redis_fd = subscriber.get_file_descriptor()
if redis_fd:
listening_fds.append(redis_fd)
subscriber.send_persited_messages(websocket)
recvmsg = None
while websocket and not websocket.closed:
ready = self.select(listening_fds, [], [], 4.0)[0]
if not ready:
# flush empty socket
websocket.flush()
for fd in ready:
if fd == websocket_fd:
recvmsg = RedisMessage(websocket.receive())
if recvmsg:
subscriber.publish_message(recvmsg)
elif fd == redis_fd:
sendmsg = RedisMessage(subscriber.parse_response())
if sendmsg and (echo_message or sendmsg != recvmsg):
websocket.send(sendmsg)
else:
logger.error('Invalid file descriptor: {0}'.format(fd))
if settings.FLOW_WS_HEARTBEAT:
websocket.send(settings.FLOW_WS_HEARTBEAT)
except WebSocketError as excpt:
logger.warning('WebSocketError: {}'.format(excpt),
exc_info=sys.exc_info())
response = http.HttpResponse(status=1001,
content='Websocket Closed')
except UpgradeRequiredError as excpt:
logger.info('Websocket upgrade required')
response = http.HttpResponseBadRequest(status=426, content=excpt)
except HandshakeError as excpt:
logger.warning('HandshakeError: {}'.format(excpt),
exc_info=sys.exc_info())
response = http.HttpResponseBadRequest(content=excpt)
except PermissionDenied as excpt:
logger.warning('PermissionDenied: {}'.format(excpt),
exc_info=sys.exc_info())
response = http.HttpResponseForbidden(content=excpt)
except Exception as excpt:
logger.error('Other Exception: {}'.format(excpt),
exc_info=sys.exc_info())
response = http.HttpResponseServerError(content=excpt)
else:
response = http.HttpResponse()
finally:
subscriber.release()
if websocket:
websocket.close(code=1001, message='Websocket Closed')
else:
logger.warning('Starting late response on websocket')
status_text = responses.get(response.status_code,
'UNKNOWN STATUS CODE')
status = '{0} {1}'.format(response.status_code, status_text)
start_response(force_str(status), response._headers.values())
logger.info(
'Finish non-websocket response with status code: {}'.
format(response.status_code))
return response
def __repr__(self):
try:
u = six.text_type(self)
except (UnicodeEncodeError, UnicodeDecodeError):
u = '[Bad Unicode data]'
return force_str('<%s: %s>' % (self.__class__.__name__, u))
def get_dataset_groups(dataset):
if not isinstance(dataset, Dataset):
dataset = Dataset.objects.get(dataset_id=force_str(dataset))
return {g.name for g in get_groups_with_perms(dataset)}
def start_import(request):
supported_extensions = get_supported_extensions()
from_encoding = "utf-8"
query_string = request.GET.get("q", "")
if request.POST or request.FILES:
form_kwargs = {}
form = ImportForm(
supported_extensions,
request.POST or None,
request.FILES or None,
**form_kwargs,
)
else:
form = ImportForm(supported_extensions)
if not request.FILES or not form.is_valid():
return render(
request,
"wagtailredirects/choose_import_file.html",
{
"search_form":
SearchForm(
data={"q": query_string} if query_string else None,
placeholder=_("Search redirects"),
),
"form":
form,
},
)
import_file = form.cleaned_data["import_file"]
_name, extension = os.path.splitext(import_file.name)
extension = extension.lstrip(".")
if extension not in supported_extensions:
messages.error(
request,
_('File format of type "{}" is not supported').format(extension))
return redirect("wagtailredirects:start_import")
import_format_cls = get_format_cls_by_extension(extension)
input_format = import_format_cls()
file_storage = write_to_file_storage(import_file, input_format)
try:
data = file_storage.read(input_format.get_read_mode())
if not input_format.is_binary() and from_encoding:
data = force_str(data, from_encoding)
dataset = input_format.create_dataset(data)
except UnicodeDecodeError as e:
messages.error(request,
_("Imported file has a wrong encoding: %s") % e)
return redirect("wagtailredirects:start_import")
except Exception as e: # pragma: no cover
messages.error(
request,
_("%(error)s encountered while trying to read file: %(filename)s")
% {
"error": type(e).__name__,
"filename": import_file.name
},
)
return redirect("wagtailredirects:start_import")
initial = {
"import_file_name": file_storage.name,
"original_file_name": import_file.name,
"input_format": get_import_formats().index(import_format_cls),
}
return render(
request,
"wagtailredirects/confirm_import.html",
{
"form": ConfirmImportForm(dataset.headers, initial=initial),
"dataset": dataset,
},
)
def __str__(self):
return force_str(self.label)
def check(self,
app_configs=None,
tags=None,
display_num_errors=False,
include_deployment_checks=False):
"""
Uses the system check framework to validate entire Django project.
Raises CommandError for any serious message (error or critical errors).
If there are only light messages (like warnings), they are printed to
stderr and no exception is raised.
"""
all_issues = checks.run_checks(
app_configs=app_configs,
tags=tags,
include_deployment_checks=include_deployment_checks,
)
header, body, footer = "", "", ""
visible_issue_count = 0 # excludes silenced warnings
if all_issues:
debugs = [
e for e in all_issues
if e.level < checks.INFO and not e.is_silenced()
]
infos = [
e for e in all_issues
if checks.INFO <= e.level < checks.WARNING
and not e.is_silenced()
]
warnings = [
e for e in all_issues
if checks.WARNING <= e.level < checks.ERROR
and not e.is_silenced()
]
errors = [
e for e in all_issues
if checks.ERROR <= e.level < checks.CRITICAL
]
criticals = [e for e in all_issues if checks.CRITICAL <= e.level]
sorted_issues = [
(criticals, 'CRITICALS'),
(errors, 'ERRORS'),
(warnings, 'WARNINGS'),
(infos, 'INFOS'),
(debugs, 'DEBUGS'),
]
for issues, group_name in sorted_issues:
if issues:
visible_issue_count += len(issues)
formatted = (self.style.ERROR(force_str(e))
if e.is_serious() else self.style.WARNING(
force_str(e)) for e in issues)
formatted = "\n".join(sorted(formatted))
body += '\n%s:\n%s\n' % (group_name, formatted)
if visible_issue_count:
header = "System check identified some issues:\n"
if display_num_errors:
if visible_issue_count:
footer += '\n'
footer += "System check identified %s (%s silenced)." % (
"no issues" if visible_issue_count == 0 else
"1 issue" if visible_issue_count == 1 else "%s issues" %
visible_issue_count,
len(all_issues) - visible_issue_count,
)
if any(e.is_serious() and not e.is_silenced() for e in all_issues):
msg = self.style.ERROR(
"SystemCheckError: %s" % header) + body + footer
raise SystemCheckError(msg)
else:
msg = header + body + footer
if msg:
if visible_issue_count:
self.stderr.write(msg, lambda x: x)
else:
self.stdout.write(msg)
def clean(self, value, row=None, *args, **kwargs):
if self.is_empty(value):
return None
return Decimal(force_str(value))
def write(self, msg, style_func=None, ending=None):
ending = self.ending if ending is None else ending
if ending and not msg.endswith(ending):
msg += ending
style_func = style_func or self.style_func
self._out.write(force_str(style_func(msg)))
def list_to_str(value):
return force_str(", ".join(value))
def add(self, context, error):
self.errors.append((context, error))
self.outfile.write(
self.style.ERROR(force_str("%s: %s\n" % (context, error))))
# Base these on the user's SITE_ROOT.
LOGIN_URL = SITE_ROOT + 'account/login/'
LOGIN_REDIRECT_URL = SITE_ROOT + 'dashboard/'
# Static media setup
if RUNNING_TEST:
PIPELINE_COMPILERS = []
else:
PIPELINE_COMPILERS = [
'djblets.pipeline.compilers.es6.ES6Compiler',
'djblets.pipeline.compilers.less.LessCompiler',
]
NODE_PATH = os.path.join(REVIEWBOARD_ROOT, '..', 'node_modules')
os.environ[str('NODE_PATH')] = force_str(NODE_PATH)
PIPELINE = {
# On production (site-installed) builds, we always want to use the
# pre-compiled versions. We want this regardless of the DEBUG setting
# (since they may turn DEBUG on in order to get better error output).
'PIPELINE_ENABLED': (PRODUCTION or not DEBUG
or os.getenv('FORCE_BUILD_MEDIA', '')),
'COMPILERS':
PIPELINE_COMPILERS,
'JAVASCRIPT':
PIPELINE_JAVASCRIPT,
'STYLESHEETS':
PIPELINE_STYLESHEETS,
'JS_COMPRESSOR':
'pipeline.compressors.uglifyjs.UglifyJSCompressor',
def strptime(self, value, format):
dt = datetime.datetime.strptime(force_str(value), format)
if '%Y' not in format:
dt = datetime.datetime.combine(datetime.date.today(), dt.time())
return dt
def __repr__(self):
return force_str(self._tzname)
def __repr__(self):
rep = "<%s: %r>" % (self.__class__.__name__, self.s[:25])
return force_str(rep, 'ascii', errors='replace')
def send_mail(
subject,
message,
from_email=None,
recipient_list=None,
use_deny_list=True,
perm_setting=None,
manage_url=None,
headers=None,
cc=None,
real_email=False,
html_message=None,
attachments=None,
max_retries=3,
reply_to=None,
countdown=None,
):
"""
A wrapper around django.core.mail.EmailMessage.
Adds deny checking and error logging.
"""
from olympia.amo.templatetags.jinja_helpers import absolutify
from olympia.amo.tasks import send_email
from olympia.users import notifications
if not recipient_list:
return True
if isinstance(recipient_list, str):
raise ValueError('recipient_list should be a list, not a string.')
# Check against user notification settings
if perm_setting:
if isinstance(perm_setting, str):
perm_setting = notifications.NOTIFICATIONS_BY_SHORT[perm_setting]
perms = dict(
UserNotification.objects.filter(
user__email__in=recipient_list, notification_id=perm_setting.id
).values_list('user__email', 'enabled')
)
d = perm_setting.default_checked
recipient_list = [e for e in recipient_list if e and perms.setdefault(e, d)]
# Prune denied emails.
if use_deny_list:
white_list = []
for email in recipient_list:
if email and email.lower() in settings.EMAIL_DENY_LIST:
log.info('Blacklisted email removed from list: %s' % email)
else:
white_list.append(email)
else:
white_list = recipient_list
if not from_email:
from_email = settings.DEFAULT_FROM_EMAIL
if cc:
# If not str, assume it is already a list.
if isinstance(cc, str):
cc = [cc]
if not headers:
headers = {}
# Avoid auto-replies per rfc 3834 and the Microsoft variant
headers['X-Auto-Response-Suppress'] = 'RN, NRN, OOF, AutoReply'
headers['Auto-Submitted'] = 'auto-generated'
def send(recipients, message, **options):
kwargs = {
'attachments': attachments,
'cc': cc,
'from_email': from_email,
'headers': headers,
'html_message': html_message,
'max_retries': max_retries,
'real_email': real_email,
'reply_to': reply_to,
'countdown': countdown,
}
kwargs.update(options)
# Email subject *must not* contain newlines
args = (list(recipients), ' '.join(subject.splitlines()), message)
return send_email.delay(*args, **kwargs)
if white_list:
if perm_setting:
html_template = loader.get_template('amo/emails/unsubscribe.html')
text_template = loader.get_template('amo/emails/unsubscribe.ltxt')
if not manage_url:
manage_url = urlparams(
absolutify(reverse('users.edit', add_prefix=False)), 'acct-notify'
)
for recipient in white_list:
# Add unsubscribe link to footer.
token, hash = UnsubscribeCode.create(recipient)
unsubscribe_url = absolutify(
reverse(
'users.unsubscribe',
args=[force_str(token), hash, perm_setting.short],
add_prefix=False,
)
)
context = {
'message': message,
'manage_url': manage_url,
'unsubscribe_url': unsubscribe_url,
'perm_setting': perm_setting.label,
'SITE_URL': settings.SITE_URL,
'mandatory': perm_setting.mandatory,
}
# Render this template in the default locale until
# bug 635840 is fixed.
with translation.override(settings.LANGUAGE_CODE):
message_with_unsubscribe = text_template.render(context)
if html_message:
context['message'] = html_message
with translation.override(settings.LANGUAGE_CODE):
html_with_unsubscribe = html_template.render(context)
result = send(
[recipient],
message_with_unsubscribe,
html_message=html_with_unsubscribe,
attachments=attachments,
)
else:
result = send(
[recipient], message_with_unsubscribe, attachments=attachments
)
else:
result = send(
recipient_list,
message=message,
html_message=html_message,
attachments=attachments,
)
else:
result = True
return result
def strptime(self, value, format):
return datetime.datetime.strptime(force_str(value), format)
def handle(self, *args, **options):
username = options.get(self.UserModel.USERNAME_FIELD)
database = options.get('database')
# If not provided, create the user with an unusable password
password = None
user_data = {}
# Do quick and dirty validation if --noinput
if not options['interactive']:
try:
if not username:
raise CommandError("You must use --%s with --noinput." %
self.UserModel.USERNAME_FIELD)
username = self.username_field.clean(username, None)
for field_name in self.UserModel.REQUIRED_FIELDS:
if options.get(field_name):
field = self.UserModel._meta.get_field(field_name)
user_data[field_name] = field.clean(
options[field_name], None)
else:
raise CommandError(
"You must use --%s with --noinput." % field_name)
except exceptions.ValidationError as e:
raise CommandError('; '.join(e.messages))
else:
# Prompt for username/password, and any other required fields.
# Enclose this whole thing in a try/except to catch
# KeyboardInterrupt and exit gracefully.
default_username = get_default_username()
try:
if hasattr(self.stdin, 'isatty') and not self.stdin.isatty():
raise NotRunningInTTYException("Not running in a TTY")
# Get a username
verbose_field_name = self.username_field.verbose_name
while username is None:
input_msg = capfirst(verbose_field_name)
if default_username:
input_msg += " (leave blank to use '%s')" % default_username
username_rel = self.username_field.remote_field
input_msg = force_str(
'%s%s: ' %
(input_msg, ' (%s.%s)' %
(username_rel.model._meta.object_name,
username_rel.field_name) if username_rel else ''))
username = self.get_input_data(self.username_field,
input_msg, default_username)
if not username:
continue
if self.username_field.unique:
try:
self.UserModel._default_manager.db_manager(
database).get_by_natural_key(username)
except self.UserModel.DoesNotExist:
pass
else:
self.stderr.write(
"Error: That %s is already taken." %
verbose_field_name)
username = None
for field_name in self.UserModel.REQUIRED_FIELDS:
field = self.UserModel._meta.get_field(field_name)
user_data[field_name] = options.get(field_name)
while user_data[field_name] is None:
message = force_str('%s%s: ' % (
capfirst(field.verbose_name),
' (%s.%s)' % (
field.remote_field.model._meta.object_name,
field.remote_field.field_name,
) if field.remote_field else '',
))
user_data[field_name] = self.get_input_data(
field, message)
# Get a password
while password is None:
if not password:
password = getpass.getpass()
password2 = getpass.getpass(
force_str('Password (again): '))
if password != password2:
self.stderr.write(
"Error: Your passwords didn't match.")
password = None
continue
if password.strip() == '':
self.stderr.write(
"Error: Blank passwords aren't allowed.")
password = None
continue
except KeyboardInterrupt:
self.stderr.write("\nOperation cancelled.")
sys.exit(1)
except NotRunningInTTYException:
self.stdout.write(
"Superuser creation skipped due to not running in a TTY. "
"You can run `manage.py createsuperuser` in your project "
"to create one manually.")
if username:
user_data[self.UserModel.USERNAME_FIELD] = username
user_data['password'] = password
self.UserModel._default_manager.db_manager(
database).create_superuser(**user_data)
if options['verbosity'] >= 1:
self.stdout.write("Superuser created successfully.")
def render_basic(self, value, context=None):
"""
Return a text rendering of 'value', suitable for display on templates. render() will fall back on
this if the block does not define a 'template' property.
"""
return force_str(value)
def replace_english(value, language):
return value.replace("English", force_str(language))
def post_response(self):
"""
Determines the HttpResponse for the add_view stage.
"""
request = self.request
msg = _(
'The %(name)s "%(obj)s" was added successfully.') % {'name': force_str(self.opts.verbose_name),
'obj': "<a class='alert-link' href='%s'>%s</a>" % (self.model_admin_url('change', self.new_obj._get_pk_val()), force_str(self.new_obj))}
if "_continue" in request.REQUEST:
self.message_user(
msg + ' ' + _("You may edit it again below."), 'success')
return self.model_admin_url('change', self.new_obj._get_pk_val())
if "_addanother" in request.REQUEST:
self.message_user(msg + ' ' + (_("You may add another %s below.") % force_str(self.opts.verbose_name)), 'success')
return request.path
else:
self.message_user(msg, 'success')
# Figure out where to redirect. If the user has change permission,
# redirect to the change-list page for this object. Otherwise,
# redirect to the admin index.
if "_redirect" in request.REQUEST:
return request.REQUEST["_redirect"]
elif self.has_view_permission():
return self.model_admin_url('changelist')
else:
return self.get_admin_url('index')
