def check_placeholders(dt, dp):
# Go though each page, look at the placeholders in the page
# template and compare to what it currently has.
for page in sorted(Page.objects.all(), key=lambda x: x.get_title()):
try:
template = page.get_template()
# list() is needed with Python3, why? Cf. https://www.quora.com/unanswered/Why-map-content-is-vanishing-outside-of-a-try-with-Python3
template_placeholders = list(
map(lambda x: x.slot, get_placeholders(template)))
except TemplateDoesNotExist:
print('** "%s" has template "%s" which could not be found **' % (
page.get_title(),
page.template,
))
continue
for language in ['fr', 'en']:
papa = page.site.name + ':' + page.get_absolute_url(
language=language)
if not page.is_published(language):
papa += ' (unpublished)'
papa += ' ' + str(page.id)
for placeholder in template_placeholders:
dp[placeholder].add(papa)
dt[template].add(papa)
for placeholder in page.placeholders.all():
if not placeholder.slot in template_placeholders:
for language in ["fr", "en"]:
plugs = placeholder.get_plugins(language=language)
if len(plugs) > 0:
print(
'"%s" id %d has "%s" placeholder slot with not in %s'
%
(page.get_absolute_url(language=language), page.id,
placeholder.slot, list(template_placeholders)))
def test_super_extends_not_corrupted_by_placeholder_scan(self):
"""
This is the test for the symptom of the bug: because the block
context now contains two copies of the inherited block, that block
will be executed twice, and if it adds content to {{block.super}},
that content will be added twice.
"""
template = get_template("placeholder_tests/test_super_extends_2.html")
output = template.render({})
self.assertEqual(['Whee'],
[o for o in output.split('\n') if 'Whee' in o])
get_placeholders("placeholder_tests/test_super_extends_2.html")
template = get_template("placeholder_tests/test_super_extends_2.html")
output = template.render({})
self.assertEqual(['Whee'],
[o for o in output.split('\n') if 'Whee' in o])
def test_super_extends_not_corrupted_by_placeholder_scan(self):
"""
This is the test for the symptom of the bug: because the block
context now contains two copies of the inherited block, that block
will be executed twice, and if it adds content to {{block.super}},
that content will be added twice.
"""
template = get_template("placeholder_tests/test_super_extends_2.html")
output = template.render({})
self.assertEqual(['Whee'], [o for o in output.split('\n')
if 'Whee' in o])
get_placeholders("placeholder_tests/test_super_extends_2.html")
template = get_template("placeholder_tests/test_super_extends_2.html")
output = template.render({})
self.assertEqual(['Whee'], [o for o in output.split('\n')
if 'Whee' in o])
def get_plugins_for_page(request, page, lang=None):
if not page:
return []
lang = lang or get_language_from_request(request)
if not hasattr(page, '_%s_plugins_cache' % lang):
slots = get_placeholders(page.get_template())
setattr(page, '_%s_plugins_cache' % lang, get_cmsplugin_queryset(request).filter(
placeholder__page=page, placeholder__slot__in=slots, language=lang, parent__isnull=True
).order_by('placeholder', 'position').select_related())
return getattr(page, '_%s_plugins_cache' % lang)
def get_plugins_for_page(request, page, lang=None):
if not page:
return []
lang = lang or get_language_from_request(request)
if not hasattr(page, '_%s_plugins_cache' % lang):
slots = [pl.slot for pl in get_placeholders(page.get_template())]
setattr(page, '_%s_plugins_cache' % lang, get_cmsplugin_queryset(request).filter(
placeholder__page=page, placeholder__slot__in=slots, language=lang, parent__isnull=True
).order_by('placeholder', 'position').select_related())
return getattr(page, '_%s_plugins_cache' % lang)
def test_cached_template_not_corrupted_by_placeholder_scan(self):
"""
This is the test for the low-level code that caused the bug:
the placeholder scan corrupts the nodelist of the extends node,
which is retained by the cached template loader, and future
renders of that template will render the super block twice.
"""
nodelist = _get_nodelist(get_template("placeholder_tests/test_super_extends_2.html"))
self.assertNotIn('one',
nodelist[0].blocks.keys(),
"test_super_extends_1.html contains a block called 'one', "
"but _2.html does not.")
get_placeholders("placeholder_tests/test_super_extends_2.html")
nodelist = _get_nodelist(get_template("placeholder_tests/test_super_extends_2.html"))
self.assertNotIn('one',
nodelist[0].blocks.keys(),
"test_super_extends_1.html still should not contain a block "
"called 'one' after rescanning placeholders.")
def test_cached_template_not_corrupted_by_placeholder_scan(self):
"""
This is the test for the low-level code that caused the bug:
the placeholder scan corrupts the nodelist of the extends node,
which is retained by the cached template loader, and future
renders of that template will render the super block twice.
"""
nodelist = _get_nodelist(
get_template("placeholder_tests/test_super_extends_2.html"))
self.assertNotIn(
'one', nodelist[0].blocks.keys(),
"test_super_extends_1.html contains a block called 'one', "
"but _2.html does not.")
get_placeholders("placeholder_tests/test_super_extends_2.html")
nodelist = _get_nodelist(
get_template("placeholder_tests/test_super_extends_2.html"))
self.assertNotIn(
'one', nodelist[0].blocks.keys(),
"test_super_extends_1.html still should not contain a block "
"called 'one' after rescanning placeholders.")
def test_placeholder_scanning_sekizai_double_extend(self):
placeholders = get_placeholders(
'placeholder_tests/test_three_sekizai.html')
self.assertEqual(sorted(placeholders),
sorted([u'new_one', u'two', u'new_three']))
def test_placeholder_scanning_include(self):
placeholders = get_placeholders('placeholder_tests/test_two.html')
self.assertEqual(sorted(placeholders), sorted([u'child', u'three']))
def test_placeholder_scanning_nested_super(self):
placeholders = get_placeholders(
'placeholder_tests/nested_super_level1.html')
self.assertEqual(sorted(placeholders),
sorted([u'level1', u'level2', u'level3', u'level4']))
def test_placeholder_scanning_nested(self):
placeholders = get_placeholders('placeholder_tests/test_six.html')
self.assertEqual(sorted(placeholders),
sorted([u'new_one', u'new_two', u'new_three']))
def test_placeholder_scanning_complex(self):
placeholders = get_placeholders('placeholder_tests/test_four.html')
self.assertEqual(sorted(placeholders), sorted([u'new_one', u'child', u'four']))
def test_placeholder_scanning_sekizai_double_extend(self):
placeholders = get_placeholders('placeholder_tests/test_three_sekizai.html')
self.assertEqual(sorted(placeholders), sorted([u'new_one', u'two', u'new_three']))
def test_placeholder_scanning_include(self):
placeholders = get_placeholders('placeholder_tests/test_two.html')
self.assertEqual(sorted(placeholders), sorted([u'child', u'three']))
def test_placeholder_scanning_nested_super(self):
placeholders = get_placeholders('placeholder_tests/nested_super_level1.html')
self.assertEqual(sorted(placeholders), sorted([u'level1', u'level2', u'level3', u'level4']))
def test_placeholder_scanning_sekizai_extend_outside_block_nested(self):
placeholders = get_placeholders('placeholder_tests/outside_nested_sekizai.html')
self.assertEqual(sorted(placeholders), sorted([u'new_one', u'two', u'base_outside']))
def test_placeholder_scanning_complex(self):
placeholders = get_placeholders('placeholder_tests/test_four.html')
self.assertEqual(sorted(placeholders),
sorted([u'new_one', u'child', u'four']))
def test_placeholder_scanning_super(self):
placeholders = get_placeholders('placeholder_tests/test_five.html')
self.assertEqual(sorted(placeholders), sorted([u'one', u'extra_one', u'two', u'three']))
def test_placeholder_scanning_super(self):
placeholders = get_placeholders('placeholder_tests/test_five.html')
self.assertEqual(sorted(placeholders),
sorted([u'one', u'extra_one', u'two', u'three']))
def test_placeholder_scanning_nested(self):
placeholders = get_placeholders('placeholder_tests/test_six.html')
self.assertEqual(sorted(placeholders), sorted([u'new_one', u'new_two', u'new_three']))
def _get_placeholder_slots(template):
return [pl.slot for pl in get_placeholders(template)]
def handle(self, *args, **options):
self.stdout.write("Start link check...")
"""
We're only interested in link plugins that are either not on any page or
are on a published page.
NOTE: This could be a large set.
"""
verify_exists = options['verify_exists']
scheme = options['scheme']
netloc = options['netloc']
bad_links = []
unknown_plugin_classes = []
count_all_links = 0
if options['only_page_reverse_id'] is not None:
pages = Page.objects.filter(
reverse_id=options["only_page_reverse_id"],
publisher_is_draft=False)
self.stdout.write("Check only page: {}".format(
pages.first().get_title(LANGUAGE_CODE)))
elif options['only_page_id'] is not None:
pages = Page.objects.filter(id=options["only_page_id"])
self.stdout.write("Check only page: {}".format(
pages.first().get_title(LANGUAGE_CODE)))
else:
pages = Page.objects.all()
if options['only_placeholder_id'] is None:
self.stdout.write("Search for placeholders to exclude...")
# Find ghosts placeholders ie placeholders created
# by a template that is no longer used by a page
excluded_placeholders = []
if options['only_page_reverse_id'] is not None:
pages = pages.filter(
reverse_id=options['only_page_reverse_id'])
elif options['only_page_id'] is not None:
pages = pages.filter(id=options['only_page_id'])
for page in pages:
try:
template_placeholders = list(
map(lambda x: x.slot,
get_placeholders(page.get_template())))
except TemplateDoesNotExist:
self.stdout.write(
'** "{}" has template "{}" which could not be found **'
.format(
page.get_title(),
page.template,
))
continue
for placeholder in page.placeholders.all():
if not placeholder.slot in template_placeholders:
excluded_placeholders.append(placeholder)
self.stdout.write("Found {} placeholders to exclude".format(
len(excluded_placeholders)))
link_plugins = CMSPlugin.objects.filter(
plugin_type__in=link_manager_pool.get_link_plugin_types())
if options['only_page_reverse_id'] is not None:
link_plugins = link_plugins.filter(
placeholder__page__reverse_id=options['only_page_reverse_id']
).exclude(placeholder__in=excluded_placeholders)
elif options['only_page_id'] is not None:
link_plugins = link_plugins.filter(
placeholder__page__id=options['only_page_id']).exclude(
placeholder__in=excluded_placeholders)
elif options['only_placeholder_id'] is not None:
link_plugins = link_plugins.filter(
placeholder__id=options['only_placeholder_id'])
else:
# Check only plugins contained in placeholders which
# - are on a published page
# - or are on no page at all (PlaceholderFields).
# - and are not ghost placeholder
link_plugins = link_plugins.filter(
Q(placeholder__page__isnull=True)
| Q(placeholder__page__publisher_is_draft=False)).exclude(
placeholder__in=excluded_placeholders)
self.stdout.write('Will check {} Plugins'.format(link_plugins.count()))
count = 0
for link_plugin in link_plugins.iterator():
count += 1
if not (count % 1000):
self.stdout.write(' Checked {} plugins...'.format(count))
plugin_inst, plugin_class = link_plugin.get_plugin_instance()
link_manager = self.get_link_manager(plugin_inst.plugin_type,
scheme=scheme,
netloc=netloc)
if link_manager:
link_reports = link_manager.check_link(
plugin_inst,
verify_exists=verify_exists,
)
# Convert to a list if only a single item was returned.
try:
iter(link_reports)
except TypeError:
# Result is not a list
link_reports = [link_reports]
for link_report in link_reports:
count_all_links += 1
if not link_report.valid:
slot = link_plugin.placeholder.slot
slot_name = get_placeholder_conf('name', slot)
if slot_name is None:
slot_name = slot
page = getattr(link_plugin.placeholder, 'page', None)
if page:
try:
page_url = 'https://{}{}'.format(
page.node.site.domain,
page.get_absolute_url(
plugin_inst.language),
)
except NoReverseMatch:
page_url = ''
else:
infos = self.handle_placeholder_outside_cms(
link_plugin)
if infos is None:
# ignore orphaned placeholders
continue
page = infos['title']
page_url = infos['url']
bad_link = {
'cls': plugin_inst.plugin_type,
'page': page,
'page_url': page_url,
'pk': plugin_inst.pk,
'slot': slot_name,
'label': link_report.text,
'url': link_report.url,
'instance': plugin_inst,
}
self.stdout.write(
'Broken link "{url}" on "{page_url}" plugin.id:{pk} placeholder:{slot}'
.format(**bad_link))
bad_links.append(bad_link)
else:
if plugin_inst.plugin_type not in unknown_plugin_classes:
unknown_plugin_classes.append(plugin_inst.plugin_type)
template = get_template(options['template'])
report = template.render({
'bad_links':
bad_links,
'count_all_links':
count_all_links,
'options':
options,
'timestamp':
now(),
'unknown_plugin_classes':
unknown_plugin_classes,
})
if options['mail_managers']:
try:
mail_managers(_('Broken link report: {0}').format(now()),
report,
fail_silently=False)
self.stdout.write(
'Successfully sent broken link report via email')
except Exception as exception:
self.stderr.write(
'ERROR: Report could not be sent via mail: {0}'.format(
exception))
else:
self.stdout.write(report)
def test_placeholder_scanning_sekizai_extend_outside_block_nested(self):
placeholders = get_placeholders(
'placeholder_tests/outside_nested_sekizai.html')
self.assertEqual(sorted(placeholders),
sorted([u'new_one', u'two', u'base_outside']))
def _get_placeholder_slots(template):
return [pl.slot for pl in get_placeholders(template)]